diff --git a/crates/ds4-dist/src/lib.rs b/crates/ds4-dist/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..d6b2126159b74f371f057125859179f68ba19ee6 --- /dev/null +++ b/crates/ds4-dist/src/lib.rs @@ -0,0 +1,121 @@ +//! Distributed coordinator/worker wire and blocking runtime. Explicit integer +//! codecs only — do not `#[repr(C)]` records onto the socket. + +mod activation; +mod assemble; +mod codec; +mod coordinator; +mod exec; +mod forward; +mod hash; +mod hops; +mod memory_snapshot; +mod native_snapshot; +mod options; +mod plan; +mod prefetch; +mod prefetch_local; +mod reconnect; +mod relay; +mod route; +mod serve_local; +mod snapshot; +mod snapshot_gather; +mod snapshot_merge; +mod snapshot_temp; +mod transport; +mod work; +mod worker; +mod worker_snapshot; + +pub use activation::{ + bits_or_default, bits_valid, decode_activation, encode_activation, f16_to_f32, f32_to_f16, + f32_to_f8_e4m3, f8_e4m3_to_f32, values_from_wire_bytes, wire_bytes, wire_bytes_from_f32_bytes, + BITS_DEFAULT, +}; +pub use assemble::{ + assemble_worker, print_worker_listen_banner, slice_meta, worker_hello, worker_listen_banner, + worker_listen_port, worker_model_name, worker_plan, AssembledWorker, SliceMeta, + WorkerListenBanner, WorkerPlan, UNKNOWN_MODEL_NAME, +}; +pub use codec::{ + bytes_have_nul, decode_frame_header, decode_hello_payload, decode_snapshot_begin_body, + decode_snapshot_chunk_body, decode_snapshot_done_body, decode_snapshot_load_begin_body, + decode_tokens_be, encode_error_frame, encode_frame_header, encode_hello_payload, + encode_snapshot_begin_body, encode_snapshot_chunk_body, encode_snapshot_done_body, + encode_tokens_be, get_u32_be, put_u32_be, u64_from_halves, u64_to_halves, CodecError, + FrameHeader, Hello, ResultHdr, Route, RouteReturn, SnapshotBegin, SnapshotChunk, SnapshotDone, + SnapshotReq, Telemetry, Work, FRAME_HEADER_BYTES, HELLO_FIXED_BYTES, MAGIC, MAX_MODEL_NAME, + MSG_ERROR, MSG_HELLO, MSG_RESULT, MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, MSG_SNAPSHOT_DONE, + MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK, RESULT_ACK, RESULT_FIXED_BYTES, + RESULT_HIDDEN_STATE, RESULT_LOGITS, ROUTE_FIXED_BYTES, ROUTE_F_OUTPUT_LOGITS, + ROUTE_RETURN_FIXED_BYTES, ROUTE_RETURN_UPSTREAM, SNAPSHOT_BEGIN_FIXED_BYTES, + SNAPSHOT_CHUNK_BYTES, SNAPSHOT_CHUNK_FIXED_BYTES, SNAPSHOT_DONE_FIXED_BYTES, + SNAPSHOT_REQ_FIXED_BYTES, TELEMETRY_FIXED_BYTES, WORK_FIXED_BYTES, WORK_F_ACK_ONLY, + WORK_F_INPUT_HC, WORK_F_OUTPUT_LOGITS, WORK_F_RESET_SESSION, WORK_F_VALID_MASK, +}; +pub use coordinator::{ + accept_data_client, accept_loop, dispatch_eval, format_telemetry_line, listen, + open_data_listener, recv_hello_only, token_span_hashes, Coordinator, EvalOutcome, + RegisteredWorker, SharedCoordinator, ERR_DATA_LISTEN_PORT, +}; +pub use exec::{SliceExec, WorkOutput, WorkRequest}; +pub use forward::{ + forward_window, forward_window_from, opened_forwarder_message, PendingQueue, PendingRequest, + ERR_CLOSED_WHILE_RESULT, ERR_FORWARD, ERR_FORWARD_HIDDEN, ERR_INVALID_RESULT, ERR_NEXT_CLOSED, + ERR_OOM_FORWARDER, ERR_OOM_TRACK, ERR_RELAY_THREAD, ERR_RESULT_METADATA, ERR_RESULT_TOO_LARGE, + ERR_TELEMETRY_TOO_LARGE, FORWARD_WINDOW_DEFAULT, FORWARD_WINDOW_MAX, FORWARD_WINDOW_MIN, +}; +pub use hash::{token_hash_prefix, token_hash_update, token_hash_update_span, TOKEN_HASH_INIT}; +pub use hops::ForwarderPool; +pub use native_snapshot::{ + apply_snapshot_load, copy_chunked, create_temp, dispatch_worker_snapshot, + prepare_snapshot_save, MemorySnapshotStore, SnapshotLoad, SnapshotSave, SnapshotStore, + TempShard, LOAD_PREFIX, SAVE_PREFIX, +}; +pub use options::{ + parse_cli, parse_cli_arg, parse_layers, parse_role, prepare_engine_options, resolved_layer_end, + validate_layers_for_model, validate_options, CliResult, Layers, Options, Role, USAGE, +}; +pub use plan::{build_route_plan, register_worker, CoordinatorView, RoutePlan, WorkerInfo}; +pub use prefetch::{ + prefetch_depth, prefetch_depth_from, prefetch_disabled, prefetch_disabled_from, + prefetch_enabled_message, JobQueue, ERR_OOM_QUEUE, ERR_OOM_READ, PREFETCH_DEPTH_DEFAULT, + PREFETCH_DEPTH_MAX, PREFETCH_DEPTH_MIN, +}; +pub use prefetch_local::{ + local_prefetch_enabled, local_prefetch_enabled_from, serve_prefetch_local_with, PrefetchJob, +}; +pub use reconnect::{ + cleared_sessions_message, connect_endpoint, connect_endpoint_once, connect_error, + connect_retryable, connected_message, disconnected_message, hello_failed_message, peer_name, + reconnect_local, reconnect_with, retrying_message, sleep_reconnect, LocalReconnect, + CONNECT_RETRY_ATTEMPTS, CONNECT_RETRY_DELAY, RECONNECT_SLEEP, +}; +pub use relay::{ + forward_work_blocking, local_work_telemetry, now_sec, prepend_telemetry, usec_since, Forwarder, +}; +pub use route::{ + decode_route_blob, encode_route_blob, validate_route_blob, ReturnTarget, RouteEntry, +}; +pub use snapshot::{coordinator_load_snapshot, coordinator_save_snapshot, SnapshotMeta}; +pub use snapshot_gather::{ + gather_kv_shards, scatter_kv_shards, GatherShard, GatheredShards, ERR_EMPTY_SHARD, + ERR_NO_TIMELINE, +}; +pub use snapshot_merge::{ + kv_state_bytes, layer_tensor_bytes, merge_session_payload, parse_layer_payload, DistKvLayout, + ParsedLayerShard, ERR_METADATA, ERR_TENSOR_BYTES, LAYER_MAGIC, SESSION_MAGIC, +}; +pub use transport::{read_frame, write_frame}; +pub use work::{ + decode_logits_payload, decode_result_body, decode_work_body, encode_logits_payload, + encode_result_body, encode_result_frame, encode_work_body, encode_work_frame, + error_result_frame, ok_result_hdr, result_hash, result_request_id, work_with_ids, ResultBody, + WorkBody, +}; +pub use worker::{recv_hello, send_hello, Worker}; +pub use worker_snapshot::{ + worker_handle_snapshot_load, worker_handle_snapshot_load_restore, worker_handle_snapshot_save, + WorkerLoadOffer, WorkerSnapshotIdentity, +}; diff --git a/crates/ds4-dist/src/memory_snapshot.rs b/crates/ds4-dist/src/memory_snapshot.rs new file mode 100644 index 0000000000000000000000000000000000000000..4537d0dc37f22c229a478afbb283940fdfb0287f --- /dev/null +++ b/crates/ds4-dist/src/memory_snapshot.rs @@ -0,0 +1,186 @@ +//! In-memory SnapshotStore with C worker status strings. + +use std::collections::HashMap; +use std::io::{self, Read, Seek, SeekFrom, Write}; + +use crate::hash::token_hash_prefix; +use crate::snapshot_temp::{copy_chunked, create_temp, TempShard, LOAD_PREFIX, SAVE_PREFIX}; + +pub struct SnapshotSave { + pub session_id: u64, + pub token_count: u32, + pub token_hash: u64, +} + +pub struct SnapshotLoad<'a> { + pub session_id: u64, + pub tokens: &'a [i32], + pub token_hash: u64, + pub payload_bytes: u64, +} + +pub trait SnapshotStore { + type SaveReader: Read; + fn save(&mut self, req: SnapshotSave) -> Result<(Self::SaveReader, u64), String>; + fn load(&mut self, req: SnapshotLoad<'_>, payload: &mut dyn Read) -> Result<(), String>; +} + +struct StoredSession { + tokens: Vec, + shard: TempShard, + token_hash: u64, + token_hash_valid: bool, +} + +pub struct MemorySnapshotStore { + sessions: HashMap, + fail_restore: bool, +} + +impl MemorySnapshotStore { + pub fn new() -> Self { + Self { + sessions: HashMap::new(), + fail_restore: false, + } + } + + pub fn insert( + &mut self, + session_id: u64, + tokens: Vec, + shard: &[u8], + ) -> Result<(), String> { + let mut tmp = create_temp(SAVE_PREFIX) + .map_err(|_| "failed to create worker snapshot temp file".to_string())?; + copy_chunked(&mut io::Cursor::new(shard), &mut tmp, shard.len() as u64) + .map_err(|_| "failed to save worker KV shard".to_string())?; + tmp.flush() + .map_err(|_| "failed to flush worker KV shard".to_string())?; + tmp.rewind() + .map_err(|_| "failed to rewind worker KV shard".to_string())?; + let token_hash = token_hash_prefix(&tokens); + self.sessions.insert( + session_id, + StoredSession { + tokens, + shard: tmp, + token_hash, + token_hash_valid: true, + }, + ); + Ok(()) + } + + pub fn fail_restore(&mut self, fail: bool) { + self.fail_restore = fail; + } + + pub fn token_hash_valid(&self, session_id: u64) -> bool { + self.sessions + .get(&session_id) + .is_some_and(|s| s.token_hash_valid) + } + + pub fn token_hash(&self, session_id: u64) -> Option { + self.sessions.get(&session_id).map(|s| s.token_hash) + } + + pub fn tokens(&self, session_id: u64) -> Option<&[i32]> { + self.sessions.get(&session_id).map(|s| s.tokens.as_slice()) + } + + pub fn read_shard(&mut self, session_id: u64) -> Result, String> { + let session = self + .sessions + .get_mut(&session_id) + .ok_or_else(|| "worker has no distributed session to snapshot".to_string())?; + session + .shard + .rewind() + .map_err(|_| "failed to rewind worker KV shard".to_string())?; + let mut out = Vec::new(); + session + .shard + .read_to_end(&mut out) + .map_err(|_| "failed to read worker KV shard".to_string())?; + session + .shard + .rewind() + .map_err(|_| "failed to rewind worker KV shard".to_string())?; + Ok(out) + } +} + +impl Default for MemorySnapshotStore { + fn default() -> Self { + Self::new() + } +} + +impl SnapshotStore for MemorySnapshotStore { + type SaveReader = TempShard; + + fn save(&mut self, req: SnapshotSave) -> Result<(TempShard, u64), String> { + let session = self + .sessions + .get_mut(&req.session_id) + .ok_or_else(|| "worker has no distributed session to snapshot".to_string())?; + if session.tokens.len() as u32 != req.token_count { + return Err("worker snapshot token count mismatch".into()); + } + if token_hash_prefix(&session.tokens) != req.token_hash { + return Err("worker snapshot token hash mismatch".into()); + } + let mut tmp = create_temp(SAVE_PREFIX) + .map_err(|_| "failed to create worker snapshot temp file".to_string())?; + session + .shard + .rewind() + .map_err(|_| "failed to rewind worker KV shard".to_string())?; + let len = session + .shard + .seek(SeekFrom::End(0)) + .map_err(|_| "failed to measure worker KV shard".to_string())?; + session + .shard + .rewind() + .map_err(|_| "failed to rewind worker KV shard".to_string())?; + copy_chunked(&mut session.shard, &mut tmp, len) + .map_err(|_| "failed to save worker KV shard".to_string())?; + tmp.flush() + .map_err(|_| "failed to flush worker KV shard".to_string())?; + let payload_bytes = tmp + .stream_position() + .map_err(|_| "failed to measure worker KV shard".to_string())?; + tmp.rewind() + .map_err(|_| "failed to rewind worker KV shard".to_string())?; + Ok((tmp, payload_bytes)) + } + + fn load(&mut self, req: SnapshotLoad<'_>, payload: &mut dyn Read) -> Result<(), String> { + if self.fail_restore { + if let Some(session) = self.sessions.get_mut(&req.session_id) { + session.token_hash_valid = false; + } + return Err("failed to restore worker KV shard".into()); + } + let mut shard = create_temp(LOAD_PREFIX) + .map_err(|_| "failed to restore worker KV shard".to_string())?; + copy_chunked(payload, &mut shard, req.payload_bytes) + .map_err(|_| "failed to restore worker KV shard".to_string())?; + shard + .rewind() + .map_err(|_| "failed to restore worker KV shard".to_string())?; + self.sessions.insert( + req.session_id, + StoredSession { + tokens: req.tokens.to_vec(), + shard, + token_hash: req.token_hash, + token_hash_valid: true, + }, + ); + Ok(()) + } +} diff --git a/crates/ds4-dist/src/native_snapshot.rs b/crates/ds4-dist/src/native_snapshot.rs new file mode 100644 index 0000000000000000000000000000000000000000..669c25455233c4d39bb5f729d367a80909b7d081 --- /dev/null +++ b/crates/ds4-dist/src/native_snapshot.rs @@ -0,0 +1,126 @@ +//! Bind snapshot frames to a SnapshotStore with C restore-before-DONE order. + +use std::fs::File; +use std::io::{self, Read, Write}; + +use crate::codec::{ + u64_from_halves, SnapshotReq, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, + SNAPSHOT_REQ_FIXED_BYTES, +}; +use crate::worker_snapshot::{ + worker_handle_snapshot_load_restore, worker_handle_snapshot_save, WorkerLoadOffer, + WorkerSnapshotIdentity, +}; + +pub use crate::memory_snapshot::{MemorySnapshotStore, SnapshotLoad, SnapshotSave, SnapshotStore}; +pub use crate::snapshot_temp::{copy_chunked, create_temp, TempShard, LOAD_PREFIX, SAVE_PREFIX}; + +pub fn prepare_snapshot_save( + store: &mut Store, + identity: WorkerSnapshotIdentity, + req_body: &[u8], +) -> Result<(Store::SaveReader, u64), String> { + if req_body.len() != SNAPSHOT_REQ_FIXED_BYTES { + return Err(String::new()); + } + let req = SnapshotReq::decode(req_body).map_err(|e| e.to_string())?; + if !identity.accepts( + req.model_id, + req.layer_start, + req.layer_end, + req.token_count, + ) { + return Err(String::new()); + } + store.save(SnapshotSave { + session_id: u64_from_halves(req.session_hi, req.session_lo), + token_count: req.token_count, + token_hash: u64_from_halves(req.token_hash_hi, req.token_hash_lo), + }) +} + +pub fn apply_snapshot_load( + stream: &mut S, + identity: WorkerSnapshotIdentity, + begin_body: &[u8], + vocab_size: u32, + store: &mut Store, +) -> Result +where + S: Read + Write, + Store: SnapshotStore, +{ + match create_temp(LOAD_PREFIX) { + Ok(mut tmp) => { + let path = tmp.path.clone(); + worker_handle_snapshot_load_restore( + stream, + identity, + begin_body, + vocab_size, + &mut tmp, + |offer| { + let mut replay = File::open(&path) + .map_err(|_| "failed to rewind worker KV shard restore file".to_string())?; + store.load( + SnapshotLoad { + session_id: offer.session_id, + tokens: &offer.tokens, + token_hash: offer.token_hash, + payload_bytes: offer.payload_bytes, + }, + &mut replay, + ) + }, + ) + } + Err(_) => { + let mut sink = io::sink(); + worker_handle_snapshot_load_restore( + stream, + identity, + begin_body, + vocab_size, + &mut sink, + |_| Err("failed to create worker snapshot restore temp file".into()), + ) + } + } +} + +pub fn dispatch_worker_snapshot( + stream: &mut Stream, + typ: u32, + body: &[u8], + identity: WorkerSnapshotIdentity, + vocab: u32, + store: &mut Store, +) -> Result, String> +where + Stream: Read + Write, + Store: SnapshotStore, +{ + match typ { + MSG_SNAPSHOT_SAVE_REQ => { + match prepare_snapshot_save(store, identity, body) { + Ok((mut reader, n)) => { + worker_handle_snapshot_save(stream, identity, body, Ok((&mut reader, n)))?; + } + Err(e) => { + worker_handle_snapshot_save::<_, io::Cursor>>( + stream, + identity, + body, + Err(e), + )?; + } + } + Ok(None) + } + MSG_SNAPSHOT_LOAD_BEGIN => { + let offer = apply_snapshot_load(stream, identity, body, vocab, store)?; + Ok(Some((offer.session_id, offer.token_hash))) + } + _ => Err(format!("rejected unsupported frame type {typ}")), + } +} diff --git a/crates/ds4-dist/src/options.rs b/crates/ds4-dist/src/options.rs new file mode 100644 index 0000000000000000000000000000000000000000..9d653334b83ef8bc95383ff0a985003f7e7a47d5 --- /dev/null +++ b/crates/ds4-dist/src/options.rs @@ -0,0 +1,351 @@ +//! CLI + option validation matching `ds4_dist_parse_cli_arg` / `dist_validate_options`. + +use crate::activation::bits_valid; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Role { + None = 0, + Coordinator = 1, + Worker = 2, +} + +impl Role { + pub fn name(self) -> &'static str { + match self { + Role::None => "none", + Role::Coordinator => "coordinator", + Role::Worker => "worker", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Layers { + pub start: u32, + pub end: u32, + pub has_output: bool, + pub set: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Options { + pub role: Role, + pub layers: Layers, + pub listen_host: Option, + pub listen_port: i32, + pub coordinator_host: Option, + pub coordinator_port: i32, + pub prefill_chunk: u32, + pub prefill_window: u32, + pub activation_bits: u32, + pub replay_check: bool, + pub debug: bool, +} + +impl Default for Role { + fn default() -> Self { + Role::None + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CliResult { + Error, + NotMatched, + Matched, +} + +pub fn parse_role(s: &str) -> Option { + match s { + "none" => Some(Role::None), + "coordinator" => Some(Role::Coordinator), + "worker" => Some(Role::Worker), + _ => None, + } +} + +pub fn parse_layers(s: &str) -> Result { + let Some(colon) = s.find(':') else { + return Err("expected A:B or A:output".into()); + }; + if colon == 0 || colon + 1 == s.len() { + return Err("expected A:B or A:output".into()); + } + if s[colon + 1..].contains(':') { + return Err("layer range has too many ':' separators".into()); + } + let start = + parse_u32_component(&s[..colon]).ok_or_else(|| format!("invalid start layer in {s}"))?; + let end_s = &s[colon + 1..]; + if end_s == "output" { + return Ok(Layers { + start, + end: u32::MAX, + has_output: true, + set: true, + }); + } + let end = parse_u32_component(end_s).ok_or_else(|| format!("invalid end layer in {s}"))?; + if end < start { + return Err(format!("layer range end precedes start in {s}")); + } + Ok(Layers { + start, + end, + has_output: false, + set: true, + }) +} + +fn parse_u32_component(s: &str) -> Option { + if s.is_empty() || s.len() >= 32 { + return None; + } + if s.as_bytes().iter().any(|c| !c.is_ascii_digit()) { + return None; + } + s.parse().ok() +} + +fn parse_port(s: &str, arg: &str) -> Result { + let v: i64 = s + .parse() + .map_err(|_| format!("invalid value for {arg}: {s}"))?; + if v <= 0 || v > 65535 { + return Err(format!("invalid value for {arg}: {s}")); + } + Ok(v as i32) +} + +fn parse_positive_u32(s: &str, name: &str) -> Result { + if s.is_empty() { + return Err(format!("{name} requires a positive integer")); + } + let v: u64 = s + .parse() + .map_err(|_| format!("invalid value for {name}: {s}"))?; + if v == 0 || v > u64::from(u32::MAX) { + return Err(format!("invalid value for {name}: {s}")); + } + Ok(v as u32) +} + +pub fn parse_cli_arg( + arg: &str, + rest: &mut impl Iterator, + opt: &mut Options, +) -> Result { + match arg { + "--role" => { + let role = rest + .next() + .ok_or_else(|| "--role requires an argument".to_string())?; + let parsed = parse_role(&role).ok_or_else(|| { + format!("invalid distributed role: {role} (valid roles: none, coordinator, worker)") + })?; + opt.role = parsed; + Ok(CliResult::Matched) + } + "--layers" => { + let layers = rest + .next() + .ok_or_else(|| "--layers requires an argument".to_string())?; + match parse_layers(&layers) { + Ok(l) => { + opt.layers = l; + Ok(CliResult::Matched) + } + Err(detail) => Err(format!("invalid --layers {layers}: {detail}")), + } + } + "--listen" => { + if opt.listen_host.is_some() || opt.listen_port != 0 { + return Err("specify --listen only once".into()); + } + let host = rest + .next() + .ok_or_else(|| "--listen requires an argument".to_string())?; + let port = rest + .next() + .ok_or_else(|| "--listen requires an argument".to_string())?; + opt.listen_port = parse_port(&port, "--listen")?; + opt.listen_host = Some(host); + Ok(CliResult::Matched) + } + "--coordinator" => { + if opt.coordinator_host.is_some() || opt.coordinator_port != 0 { + return Err("specify --coordinator only once".into()); + } + let host = rest + .next() + .ok_or_else(|| "--coordinator requires an argument".to_string())?; + let port = rest + .next() + .ok_or_else(|| "--coordinator requires an argument".to_string())?; + opt.coordinator_port = parse_port(&port, "--coordinator")?; + opt.coordinator_host = Some(host); + Ok(CliResult::Matched) + } + "--dist-prefill-chunk" => { + let value = rest + .next() + .ok_or_else(|| "--dist-prefill-chunk requires an argument".to_string())?; + opt.prefill_chunk = parse_positive_u32(&value, "--dist-prefill-chunk")?; + Ok(CliResult::Matched) + } + "--dist-prefill-window" => { + let value = rest + .next() + .ok_or_else(|| "--dist-prefill-window requires an argument".to_string())?; + opt.prefill_window = parse_positive_u32(&value, "--dist-prefill-window")?; + if opt.prefill_window > 64 { + return Err("--dist-prefill-window must be <= 64".into()); + } + Ok(CliResult::Matched) + } + "--dist-activation-bits" => { + let value = rest + .next() + .ok_or_else(|| "--dist-activation-bits requires an argument".to_string())?; + let bits = parse_positive_u32(&value, "--dist-activation-bits")?; + if !bits_valid(bits) { + return Err("--dist-activation-bits must be 32, 16, or 8".into()); + } + opt.activation_bits = bits; + Ok(CliResult::Matched) + } + "--dist-replay-check" => { + opt.replay_check = true; + Ok(CliResult::Matched) + } + "--debug" => { + opt.debug = true; + Ok(CliResult::Matched) + } + _ => Ok(CliResult::NotMatched), + } +} + +pub fn parse_cli(args: &[String]) -> Result<(Options, Vec), String> { + let mut opt = Options::default(); + let mut unmatched = Vec::new(); + let mut rest = args.iter().cloned().collect::>().into_iter(); + while let Some(arg) = rest.next() { + match parse_cli_arg(&arg, &mut rest, &mut opt)? { + CliResult::Matched => {} + CliResult::NotMatched => unmatched.push(arg), + CliResult::Error => unreachable!(), + } + } + Ok((opt, unmatched)) +} + +pub fn validate_options(opt: &Options) -> Result<(), String> { + if opt.role == Role::None { + if opt.layers.set + || opt.listen_host.is_some() + || opt.listen_port != 0 + || opt.coordinator_host.is_some() + || opt.coordinator_port != 0 + || opt.prefill_chunk != 0 + || opt.prefill_window != 0 + || opt.activation_bits != 0 + { + return Err("distributed options require --role coordinator or --role worker".into()); + } + return Ok(()); + } + if !opt.layers.set { + return Err(format!("--role {} requires --layers", opt.role.name())); + } + if opt.prefill_window > 64 { + return Err("--dist-prefill-window must be <= 64".into()); + } + if opt.activation_bits != 0 && !bits_valid(opt.activation_bits) { + return Err("--dist-activation-bits must be 32, 16, or 8".into()); + } + match opt.role { + Role::Coordinator => { + if opt.listen_host.is_none() || opt.listen_port <= 0 { + return Err("--role coordinator requires --listen HOST PORT".into()); + } + if opt.coordinator_host.is_some() || opt.coordinator_port != 0 { + return Err("--role coordinator must not use --coordinator".into()); + } + Ok(()) + } + Role::Worker => { + if opt.coordinator_host.is_none() || opt.coordinator_port <= 0 { + return Err("--role worker requires --coordinator HOST PORT".into()); + } + if opt.prefill_chunk != 0 { + return Err("--dist-prefill-chunk requires --role coordinator".into()); + } + if opt.prefill_window != 0 { + return Err("--dist-prefill-window requires --role coordinator".into()); + } + if opt.activation_bits != 0 { + return Err("--dist-activation-bits requires --role coordinator".into()); + } + Ok(()) + } + Role::None => Ok(()), + } +} + +pub fn validate_layers_for_model(opt: &Options, n_layers: u32) -> Result<(), String> { + if opt.role == Role::None || !opt.layers.set { + return Ok(()); + } + if n_layers == 0 { + return Err("model reports no layers".into()); + } + let last = n_layers - 1; + if opt.layers.start > last { + return Err(format!("layer range starts past final model layer {last}")); + } + if !opt.layers.has_output && opt.layers.end > last { + return Err(format!("layer range ends past final model layer {last}")); + } + if opt.role == Role::Coordinator && opt.layers.start != 0 { + return Err("coordinator layer range must start at layer 0".into()); + } + Ok(()) +} + +pub fn resolved_layer_end(layers: &Layers, n_layers: u32) -> u32 { + if layers.has_output { + n_layers.saturating_sub(1) + } else { + layers.end + } +} + +pub fn prepare_engine_options(opt: &Options) -> Result<(), String> { + validate_options(opt)?; + if opt.replay_check && opt.role != Role::Coordinator { + return Err("--dist-replay-check requires --role coordinator".into()); + } + Ok(()) +} + +pub const USAGE: &str = "\ + --role ROLE\n\ + Distributed role: coordinator or worker.\n\ + --layers A:B\n\ + Inclusive distributed layer slice, e.g. 10:20 or 21:output.\n\ + --listen HOST PORT\n\ + Coordinator TCP listen address. Workers may later use it to force their data listener.\n\ + --coordinator HOST PORT\n\ + Coordinator TCP address for --role worker.\n\ + --dist-prefill-chunk N\n\ + Coordinator prefill pipeline chunk size. Default: session cap, normally 4096.\n\ + Non-default values are experimental and can change logits unless validated.\n\ + --dist-prefill-window N\n\ + Coordinator max end-to-end prefill chunks in flight. Default: workers+2, capped at 8.\n\ + --dist-activation-bits N\n\ + Coordinator hidden-state transport width: 32, 16, or 8. Default: 32.\n\ + --dist-replay-check\n\ + Coordinator diagnostic: reset and replay the prompt, then compare logits.\n\ + --debug\n\ + Print coordinator route/debug logs. Workers keep their normal logs without this.\n"; diff --git a/crates/ds4-dist/src/plan.rs b/crates/ds4-dist/src/plan.rs new file mode 100644 index 0000000000000000000000000000000000000000..c4763a7b130eb9474403e0471677ddb0b501ac73 --- /dev/null +++ b/crates/ds4-dist/src/plan.rs @@ -0,0 +1,157 @@ +//! Worker registry and route search. Matches `dist_coordinator_build_route_plan`. + +use crate::codec::{ROUTE_F_OUTPUT_LOGITS, ROUTE_RETURN_UPSTREAM}; +use crate::route::{encode_route_blob, ReturnTarget, RouteEntry}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WorkerInfo { + pub peer_host: String, + pub listen_port: u32, + pub model_id: u32, + pub quant_bits: u32, + pub layer_start: u32, + pub layer_end: u32, + pub has_output: bool, + pub has_hidden: bool, +} + +#[derive(Debug, Clone)] +pub struct CoordinatorView { + pub n_layers: u32, + pub local_start: u32, + pub local_end: u32, + pub local_has_output: bool, + pub local_can_output_head: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RoutePlan { + pub entries: Vec, + pub blob: Vec, +} + +fn worker_cmp(a: &WorkerInfo, b: &WorkerInfo) -> std::cmp::Ordering { + a.layer_start + .cmp(&b.layer_start) + .then_with(|| b.has_output.cmp(&a.has_output)) + .then_with(|| b.layer_end.cmp(&a.layer_end)) +} + +fn candidate_ok(view: &CoordinatorView, w: &WorkerInfo, last: u32) -> bool { + let needs_hidden = w.layer_end < last || !w.has_output; + if needs_hidden && !w.has_hidden { + return false; + } + if w.layer_end >= last && !w.has_output && !view.local_can_output_head { + return false; + } + true +} + +fn search( + view: &CoordinatorView, + workers: &[WorkerInfo], + next: u32, + last: u32, + path: &mut Vec, + missing: &mut u32, +) -> bool { + let mut saw_start = false; + for (i, w) in workers.iter().enumerate() { + if w.layer_start < next { + continue; + } + if w.layer_start > next { + break; + } + saw_start = true; + if !candidate_ok(view, w, last) { + continue; + } + path.push(i); + if w.layer_end >= last { + return true; + } + let mut child_missing = w.layer_end + 1; + if search(view, workers, child_missing, last, path, &mut child_missing) { + return true; + } + if child_missing > *missing { + *missing = child_missing; + } + path.pop(); + } + if !saw_start && next > *missing { + *missing = next; + } + false +} + +pub fn build_route_plan( + view: &CoordinatorView, + workers: &[WorkerInfo], +) -> Result { + if view.local_start != 0 { + return Err("coordinator route does not start at layer 0".into()); + } + let last = view.n_layers.saturating_sub(1); + if view.local_end == last && (view.local_has_output || view.local_can_output_head) { + return Ok(RoutePlan { + entries: Vec::new(), + blob: Vec::new(), + }); + } + + let mut sorted = workers.to_vec(); + sorted.sort_by(worker_cmp); + let next = view.local_end + 1; + let mut path = Vec::new(); + let mut missing = next; + if !search(view, &sorted, next, last, &mut path, &mut missing) { + return Err(format!( + "distributed route incomplete: missing layer {missing}" + )); + } + + let mut entries = Vec::new(); + for idx in path { + let w = &sorted[idx]; + entries.push(RouteEntry { + host: w.peer_host.clone(), + port: w.listen_port, + layer_start: w.layer_start, + layer_end: w.layer_end, + flags: if w.has_output { + ROUTE_F_OUTPUT_LOGITS + } else { + 0 + }, + }); + } + let blob = if entries.is_empty() { + Vec::new() + } else { + encode_route_blob( + &entries, + &ReturnTarget { + kind: ROUTE_RETURN_UPSTREAM, + host: String::new(), + port: 0, + }, + ) + .map_err(|e| e.to_string())? + }; + Ok(RoutePlan { entries, blob }) +} + +/// Replace a stale worker with the same host/model/layers/output (C prepends). +pub fn register_worker(list: &mut Vec, neu: WorkerInfo) { + list.retain(|old| { + !(old.peer_host == neu.peer_host + && old.model_id == neu.model_id + && old.layer_start == neu.layer_start + && old.layer_end == neu.layer_end + && old.has_output == neu.has_output) + }); + list.insert(0, neu); +} diff --git a/crates/ds4-dist/src/prefetch.rs b/crates/ds4-dist/src/prefetch.rs new file mode 100644 index 0000000000000000000000000000000000000000..91b3ad49c70056add32eca2cc7fb943563d30a02 --- /dev/null +++ b/crates/ds4-dist/src/prefetch.rs @@ -0,0 +1,278 @@ +//! Worker receive prefetch: `DS4_DIST_WORKER_PREFETCH_DEPTH` and the C job queue. + +use std::collections::VecDeque; +use std::io::{self, Write}; +use std::net::TcpStream; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread; + +use crate::codec::{MSG_ERROR, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK}; +use crate::exec::SliceExec; +use crate::native_snapshot::SnapshotStore; +use crate::transport::{read_frame, write_frame}; +use crate::worker::Worker; + +pub const PREFETCH_DEPTH_DEFAULT: u32 = 2; +pub const PREFETCH_DEPTH_MIN: u32 = 1; +pub const PREFETCH_DEPTH_MAX: u32 = 8; +pub const ERR_OOM_QUEUE: &str = "out of memory queueing distributed WORK"; +pub const ERR_OOM_READ: &str = "out of memory reading distributed WORK frame"; + +pub fn prefetch_depth_from(env: Option<&str>) -> u32 { + let Some(raw) = env else { + return PREFETCH_DEPTH_DEFAULT; + }; + if raw.is_empty() { + return PREFETCH_DEPTH_DEFAULT; + } + match parse_strtol(raw) { + Some(v) if v >= i64::from(PREFETCH_DEPTH_MIN) && v <= i64::from(PREFETCH_DEPTH_MAX) => { + v as u32 + } + _ => PREFETCH_DEPTH_DEFAULT, + } +} + +pub fn prefetch_depth() -> u32 { + prefetch_depth_from( + std::env::var("DS4_DIST_WORKER_PREFETCH_DEPTH") + .ok() + .as_deref(), + ) +} + +pub fn prefetch_disabled_from(env: Option<&std::ffi::OsStr>) -> bool { + env.is_some() +} + +pub fn prefetch_disabled() -> bool { + prefetch_disabled_from(std::env::var_os("DS4_DIST_DISABLE_WORKER_PREFETCH").as_deref()) +} + +pub fn prefetch_enabled_message(depth: u32) -> String { + format!("ds4: distributed worker: receive prefetch depth {depth} enabled") +} + +fn parse_strtol(s: &str) -> Option { + let s = s.trim_start_matches(|c: char| c.is_ascii_whitespace()); + if s.is_empty() { + return None; + } + let (neg, digits) = match s.as_bytes()[0] { + b'+' => (false, &s[1..]), + b'-' => (true, &s[1..]), + _ => (false, s), + }; + if digits.is_empty() || !digits.as_bytes()[0].is_ascii_digit() { + return None; + } + if !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let v = digits.parse::().ok()?; + Some(if neg { -v } else { v }) +} + +struct QueueInner { + depth: u32, + queued: u32, + jobs: VecDeque, + closed: bool, + canceled: bool, +} + +pub struct JobQueue> { + inner: Mutex>, + not_empty: Condvar, + not_full: Condvar, +} + +impl JobQueue { + pub fn new(depth: u32) -> Self { + let depth = depth.max(1); + Self { + inner: Mutex::new(QueueInner { + depth, + queued: 0, + jobs: VecDeque::new(), + closed: false, + canceled: false, + }), + not_empty: Condvar::new(), + not_full: Condvar::new(), + } + } + + pub fn depth(&self) -> u32 { + self.inner.lock().expect("prefetch queue").depth + } + + pub fn queued(&self) -> u32 { + self.inner.lock().expect("prefetch queue").queued + } + + pub fn enqueue(&self, job: T) -> bool { + let mut g = self.inner.lock().expect("prefetch queue"); + while !g.closed && !g.canceled && g.queued >= g.depth { + g = self.not_full.wait(g).expect("prefetch queue"); + } + if g.closed || g.canceled { + return false; + } + g.jobs.push_back(job); + g.queued += 1; + self.not_empty.notify_one(); + true + } + + pub fn wait_until_queued(&self, n: u32) { + let mut g = self.inner.lock().expect("prefetch queue"); + while g.queued < n && !g.closed && !g.canceled { + g = self.not_empty.wait(g).expect("prefetch queue"); + } + } + + fn take_job(g: &mut QueueInner, not_full: &Condvar) -> Option { + if g.canceled || g.jobs.is_empty() { + return None; + } + let job = g.jobs.pop_front(); + g.queued = g.queued.saturating_sub(1); + not_full.notify_one(); + job + } + + pub fn try_pop(&self) -> Option { + let mut g = self.inner.lock().expect("prefetch queue"); + Self::take_job(&mut g, &self.not_full) + } + + pub fn pop(&self) -> Option { + let mut g = self.inner.lock().expect("prefetch queue"); + while g.jobs.is_empty() && !g.closed && !g.canceled { + g = self.not_empty.wait(g).expect("prefetch queue"); + } + Self::take_job(&mut g, &self.not_full) + } + + pub fn finish(&self) { + let mut g = self.inner.lock().expect("prefetch queue"); + g.closed = true; + self.not_empty.notify_all(); + self.not_full.notify_all(); + } + + pub fn cancel(&self) { + let mut g = self.inner.lock().expect("prefetch queue"); + g.closed = true; + g.canceled = true; + g.jobs.clear(); + g.queued = 0; + self.not_empty.notify_all(); + self.not_full.notify_all(); + } +} + +pub(crate) struct MutexWrite<'a>(pub &'a Mutex); + +impl Write for MutexWrite<'_> { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.lock().expect("worker write").write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.0.lock().expect("worker write").flush() + } +} + +impl Worker { + pub fn serve_prefetch(&mut self, stream: &mut TcpStream) -> io::Result<()> { + let depth = prefetch_depth(); + let queue = Arc::new(JobQueue::>::new(depth)); + let worker = Mutex::new(&mut *self); + let writer = Arc::new(Mutex::new(stream.try_clone()?)); + let mut reader = stream.try_clone()?; + worker + .lock() + .expect("worker hops") + .bind_hops(Arc::clone(&writer)); + eprintln!("{}", prefetch_enabled_message(depth)); + + let rc = thread::scope(|scope| { + let eval = scope.spawn(|| -> io::Result<()> { + while let Some(payload) = queue.pop() { + let reply = { + let mut w = worker.lock().expect("worker eval"); + w.process_work(&payload, &mut MutexWrite(writer.as_ref()))? + }; + if let Some(frame) = reply { + writer.lock().expect("worker write").write_all(&frame)?; + } + } + Ok(()) + }); + + let mut loop_err: Option = None; + loop { + let (typ, body) = match read_frame(&mut reader) { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => break, + Err(e) => { + loop_err = Some(e); + break; + } + }; + if typ == MSG_ERROR { + loop_err = Some(io::Error::new( + io::ErrorKind::Other, + format!("coordinator error: {}", String::from_utf8_lossy(&body)), + )); + break; + } + if typ == MSG_WORK { + if !queue.enqueue(body) { + loop_err = Some(io::Error::new( + io::ErrorKind::Interrupted, + "prefetch queue canceled", + )); + break; + } + continue; + } + if typ == MSG_SNAPSHOT_SAVE_REQ || typ == MSG_SNAPSHOT_LOAD_BEGIN { + let mut w = worker.lock().expect("worker snapshot"); + let mut out = writer.lock().expect("worker snapshot"); + w.apply_snapshot(&mut *out, typ, &body)?; + continue; + } + { + let mut out = writer.lock().expect("worker write"); + write_frame( + &mut *out, + MSG_ERROR, + b"unsupported distributed worker frame", + )?; + } + loop_err = Some(io::Error::new( + io::ErrorKind::InvalidData, + format!("rejected unsupported frame type {typ}"), + )); + break; + } + + if loop_err.is_some() { + queue.cancel(); + } else { + queue.finish(); + } + let eval_rc = eval.join().expect("prefetch eval"); + match (loop_err, eval_rc) { + (Some(e), _) => Err(e), + (None, Err(e)) => Err(e), + (None, Ok(())) => Ok(()), + } + }); + self.shutdown_hops(); + rc + } +} diff --git a/crates/ds4-dist/src/prefetch_local.rs b/crates/ds4-dist/src/prefetch_local.rs new file mode 100644 index 0000000000000000000000000000000000000000..921257a72a4e016a37f55406f2c07b750f34a287 --- /dev/null +++ b/crates/ds4-dist/src/prefetch_local.rs @@ -0,0 +1,253 @@ +//! !Send receive-prefetch: queue on a helper thread, eval on the session thread. + +use std::io::{self, ErrorKind, Write}; +use std::net::{Shutdown, TcpStream}; +use std::sync::mpsc::{self, Receiver}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; + +use crate::codec::{MSG_ERROR, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK}; +use crate::exec::SliceExec; +use crate::native_snapshot::SnapshotStore; +use crate::prefetch::{prefetch_depth, prefetch_enabled_message, JobQueue, MutexWrite}; +use crate::serve_local::{drive_ready, Drive, IDLE}; +use crate::transport::{read_frame, write_frame}; +use crate::worker::Worker; + +pub enum PrefetchJob { + Work(Vec), + Snapshot { typ: u32, body: Vec }, +} + +pub fn local_prefetch_enabled_from(env: Option<&std::ffi::OsStr>) -> bool { + !crate::prefetch::prefetch_disabled_from(env) +} + +pub fn local_prefetch_enabled() -> bool { + local_prefetch_enabled_from(std::env::var_os("DS4_DIST_DISABLE_WORKER_PREFETCH").as_deref()) +} + +fn recv_prefetch( + mut reader: TcpStream, + queue: &JobQueue, + writer: &Mutex, + mut on_job: impl FnMut(), +) -> Option { + let err = loop { + match read_frame(&mut reader) { + Ok((typ, body)) if typ == MSG_WORK => { + if !queue.enqueue(PrefetchJob::Work(body)) { + break Some(io::Error::new( + ErrorKind::Interrupted, + "prefetch queue canceled", + )); + } + on_job(); + } + Ok((typ, body)) if typ == MSG_SNAPSHOT_SAVE_REQ || typ == MSG_SNAPSHOT_LOAD_BEGIN => { + if !queue.enqueue(PrefetchJob::Snapshot { typ, body }) { + break Some(io::Error::new( + ErrorKind::Interrupted, + "prefetch queue canceled", + )); + } + on_job(); + } + Ok((typ, body)) if typ == MSG_ERROR => { + break Some(io::Error::new( + ErrorKind::Other, + format!("coordinator error: {}", String::from_utf8_lossy(&body)), + )); + } + Ok((typ, _)) => { + let _ = write_frame( + &mut *writer.lock().expect("worker write"), + MSG_ERROR, + b"unsupported distributed worker frame", + ); + break Some(io::Error::new( + ErrorKind::InvalidData, + format!("rejected unsupported frame type {typ}"), + )); + } + Err(e) if e.kind() == ErrorKind::UnexpectedEof => break None, + Err(e) + if matches!( + e.kind(), + ErrorKind::TimedOut | ErrorKind::WouldBlock | ErrorKind::Interrupted + ) => + { + continue; + } + Err(e) => break Some(e), + } + }; + if err.is_some() { + queue.cancel(); + } else { + queue.finish(); + } + on_job(); + err +} + +fn apply_prefetch_job( + worker: &mut Worker, + writer: &Arc>, + job: PrefetchJob, +) -> io::Result<()> +where + E: SliceExec, + S: SnapshotStore, +{ + match job { + PrefetchJob::Work(payload) => { + let reply = worker.process_work(&payload, &mut MutexWrite(writer))?; + if let Some(frame) = reply { + writer.lock().expect("worker write").write_all(&frame)?; + } + Ok(()) + } + PrefetchJob::Snapshot { typ, body } => { + let mut out = writer.lock().expect("worker snapshot"); + worker.apply_snapshot(&mut *out, typ, &body) + } + } +} + +pub fn serve_prefetch_local_with( + worker: &mut Worker, + stream: &mut TcpStream, + queue: &JobQueue, +) -> io::Result<()> +where + E: SliceExec, + S: SnapshotStore, +{ + let writer = Arc::new(Mutex::new(stream.try_clone()?)); + let reader = stream.try_clone()?; + worker.bind_hops(Arc::clone(&writer)); + eprintln!("{}", prefetch_enabled_message(queue.depth())); + let rc = thread::scope(|scope| { + let recv = scope.spawn(|| recv_prefetch(reader, queue, &writer, || {})); + let mut eval_err = None; + while let Some(job) = queue.pop() { + if let Err(e) = apply_prefetch_job(worker, &writer, job) { + eval_err = Some(e); + queue.cancel(); + break; + } + } + match (recv.join().expect("prefetch recv"), eval_err) { + (Some(e), _) => Err(e), + (None, Some(e)) => Err(e), + (None, None) => Ok(()), + } + }); + worker.shutdown_hops(); + rc +} + +struct PrefetchConn { + queue: Arc>, + writer: Arc>, + recv: Option>, +} + +fn start_prefetch_recv( + stream: TcpStream, + wake: mpsc::Sender, + idx: usize, +) -> io::Result { + let _ = stream.set_nodelay(true); + let writer = Arc::new(Mutex::new(stream.try_clone()?)); + let queue = Arc::new(JobQueue::::new(prefetch_depth())); + let q = Arc::clone(&queue); + let w = Arc::clone(&writer); + eprintln!("{}", prefetch_enabled_message(queue.depth())); + let recv = thread::Builder::new() + .name("ds4-dist-prefetch-recv".into()) + .spawn(move || { + let _ = recv_prefetch(stream, &q, &w, || { + let _ = wake.send(idx); + }); + })?; + Ok(PrefetchConn { + queue, + writer, + recv: Some(recv), + }) +} + +fn shutdown_prefetch_conns(conns: &mut [PrefetchConn]) { + for conn in conns.iter() { + conn.queue.cancel(); + let _ = conn + .writer + .lock() + .expect("worker write") + .shutdown(Shutdown::Both); + } + for conn in conns.iter_mut() { + if let Some(handle) = conn.recv.take() { + let _ = handle.join(); + } + } +} + +fn drain_conn(worker: &mut Worker, conn: &PrefetchConn) -> io::Result<()> +where + E: SliceExec, + S: SnapshotStore, +{ + worker.bind_hops(Arc::clone(&conn.writer)); + while let Some(job) = conn.queue.try_pop() { + apply_prefetch_job(worker, &conn.writer, job)?; + } + Ok(()) +} + +pub(crate) fn serve_coordinator_and_hops_prefetch( + worker: &mut Worker, + coordinator: &mut TcpStream, + hops: Option<&Receiver>, + should_stop: &mut Stop, +) -> io::Result<()> +where + E: SliceExec, + S: SnapshotStore, + Stop: FnMut() -> bool, +{ + coordinator.set_read_timeout(Some(IDLE))?; + let (wake_tx, wake_rx) = mpsc::channel(); + let mut conns: Vec = Vec::new(); + let rc = 'run: loop { + if should_stop() { + break Ok(()); + } + if let Some(rx) = hops { + while let Ok(hop) = rx.try_recv() { + let idx = conns.len(); + match start_prefetch_recv(hop, wake_tx.clone(), idx) { + Ok(conn) => conns.push(conn), + Err(e) => break 'run Err(e), + } + } + } + while let Ok(idx) = wake_rx.try_recv() { + if let Some(conn) = conns.get(idx) { + if let Err(e) = drain_conn(worker, conn) { + break 'run Err(e); + } + } + } + match drive_ready(worker, coordinator) { + Drive::Continue => {} + Drive::Closed => break Ok(()), + Drive::Failed(e) => break Err(e), + } + }; + shutdown_prefetch_conns(&mut conns); + worker.shutdown_hops(); + rc +} diff --git a/crates/ds4-dist/src/reconnect.rs b/crates/ds4-dist/src/reconnect.rs new file mode 100644 index 0000000000000000000000000000000000000000..9eb3e3ee8a8626ddda3366df3c15d13349d7174d --- /dev/null +++ b/crates/ds4-dist/src/reconnect.rs @@ -0,0 +1,217 @@ +//! Worker reconnect: connect retries, HELLO, session clear, 1s backoff. + +use std::io::{self, ErrorKind}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::thread; +use std::time::Duration; + +use crate::codec::Hello; +use crate::exec::SliceExec; +use crate::native_snapshot::SnapshotStore; +use crate::prefetch::prefetch_disabled; +use crate::prefetch_local::{local_prefetch_enabled, serve_coordinator_and_hops_prefetch}; +use crate::serve_local::{serve_coordinator_and_hops, spawn_hop_accept, HopAccept}; +use crate::worker::{send_hello, Worker}; + +pub const RECONNECT_SLEEP: Duration = Duration::from_secs(1); +pub const CONNECT_RETRY_DELAY: Duration = Duration::from_millis(25); +pub const CONNECT_RETRY_ATTEMPTS: u32 = 200; + +pub fn sleep_reconnect() { + thread::sleep(RECONNECT_SLEEP); +} + +pub fn connect_retryable(err: &io::Error) -> bool { + matches!( + err.kind(), + ErrorKind::ConnectionRefused + | ErrorKind::HostUnreachable + | ErrorKind::NetworkUnreachable + | ErrorKind::TimedOut + | ErrorKind::AddrNotAvailable + ) +} + +pub fn connect_error(host: &str, port: u16, err: &io::Error) -> String { + format!("unable to connect to {host}:{port}: {err}") +} + +pub fn retrying_message(err: &str) -> String { + format!("ds4: distributed worker: {err}; retrying") +} + +pub fn connected_message(host: &str, port: &str) -> String { + format!("ds4: distributed worker: connected to coordinator {host}:{port}") +} + +pub fn hello_failed_message(err: &io::Error) -> String { + format!("ds4: distributed worker: failed to send HELLO: {err}") +} + +pub fn cleared_sessions_message(n: u32) -> String { + format!("ds4: distributed worker: cleared {n} sessions after coordinator disconnect") +} + +pub fn disconnected_message(after_error: bool) -> String { + if after_error { + "ds4: distributed worker: coordinator disconnected after error; reconnecting".into() + } else { + "ds4: distributed worker: coordinator disconnected; reconnecting".into() + } +} + +pub fn connect_endpoint_once(host: &str, port: u16) -> io::Result { + let stream = TcpStream::connect((host, port))?; + let _ = stream.set_nodelay(true); + Ok(stream) +} + +pub fn connect_endpoint(host: &str, port: u16) -> io::Result { + let mut last = None; + for _ in 0..CONNECT_RETRY_ATTEMPTS { + match connect_endpoint_once(host, port) { + Ok(s) => return Ok(s), + Err(e) => { + let retry = connect_retryable(&e); + last = Some(e); + if !retry { + break; + } + thread::sleep(CONNECT_RETRY_DELAY); + } + } + } + Err(last.unwrap_or_else(|| io::Error::new(ErrorKind::Other, "unable to connect"))) +} + +pub fn peer_name(stream: &TcpStream) -> (String, String) { + match stream.peer_addr() { + Ok(SocketAddr::V4(a)) => (a.ip().to_string(), a.port().to_string()), + Ok(SocketAddr::V6(a)) => (a.ip().to_string(), a.port().to_string()), + Err(_) => ("?".into(), "?".into()), + } +} + +pub struct LocalReconnect<'a, Connect, Sleep, Stop> { + pub connect: Connect, + pub hello: &'a Hello, + pub model_name: &'a str, + pub sleep: Sleep, + pub should_stop: Stop, + pub listener: Option<&'a TcpListener>, +} + +fn reconnect_loop( + worker: &mut Worker, + connect: &mut Connect, + hello: &Hello, + model_name: &str, + sleep: &mut Sleep, + should_stop: &mut Stop, + mut serve: Serve, +) -> io::Result<()> +where + E: SliceExec, + S: SnapshotStore, + Connect: FnMut() -> io::Result, + Sleep: FnMut(), + Stop: FnMut() -> bool, + Serve: FnMut(&mut Worker, &mut TcpStream, &mut Stop) -> io::Result<()>, +{ + while !should_stop() { + let mut stream = match connect() { + Ok(s) => s, + Err(e) => { + eprintln!("{}", retrying_message(&e.to_string())); + sleep(); + continue; + } + }; + let (peer_host, peer_port) = peer_name(&stream); + eprintln!("{}", connected_message(&peer_host, &peer_port)); + if let Err(e) = send_hello(&mut stream, hello, model_name) { + eprintln!("{}", hello_failed_message(&e)); + sleep(); + continue; + } + let rc = serve(worker, &mut stream, should_stop); + let after_error = rc.is_err(); + let dropped = worker.clear_sessions(); + if dropped != 0 { + eprintln!("{}", cleared_sessions_message(dropped)); + } + eprintln!("{}", disconnected_message(after_error)); + if should_stop() { + return rc; + } + sleep(); + if after_error { + let _ = rc; + } + } + Ok(()) +} + +pub fn reconnect_with( + worker: &mut Worker, + mut connect: Connect, + hello: &Hello, + model_name: &str, + mut sleep: Sleep, + mut should_stop: Stop, + use_prefetch: bool, +) -> io::Result<()> +where + E: SliceExec + Send, + S: SnapshotStore + Send, + Connect: FnMut() -> io::Result, + Sleep: FnMut(), + Stop: FnMut() -> bool, +{ + reconnect_loop( + worker, + &mut connect, + hello, + model_name, + &mut sleep, + &mut should_stop, + |w, stream, _stop| { + if use_prefetch && !prefetch_disabled() { + w.serve_prefetch(stream) + } else { + w.serve(stream) + } + }, + ) +} + +/// Reconnect a `!Send` worker, polling the coordinator and accepted hop streams. +pub fn reconnect_local( + worker: &mut Worker, + mut spec: LocalReconnect<'_, Connect, Sleep, Stop>, +) -> io::Result<()> +where + E: SliceExec, + S: SnapshotStore, + Connect: FnMut() -> io::Result, + Sleep: FnMut(), + Stop: FnMut() -> bool, +{ + let hops = spec.listener.map(spawn_hop_accept).transpose()?; + let rx = hops.as_ref().and_then(HopAccept::receiver); + reconnect_loop( + worker, + &mut spec.connect, + spec.hello, + spec.model_name, + &mut spec.sleep, + &mut spec.should_stop, + |w, stream, stop| { + if local_prefetch_enabled() { + serve_coordinator_and_hops_prefetch(w, stream, rx, stop) + } else { + serve_coordinator_and_hops(w, stream, rx, stop) + } + }, + ) +} diff --git a/crates/ds4-dist/src/relay.rs b/crates/ds4-dist/src/relay.rs new file mode 100644 index 0000000000000000000000000000000000000000..31df54f53bc446c0078c5408c9299549ff9eac49 --- /dev/null +++ b/crates/ds4-dist/src/relay.rs @@ -0,0 +1,217 @@ +//! Persistent worker-to-worker forwarder and RESULT relay thread. + +use std::io::{self, Write}; +use std::net::TcpStream; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::Instant; + +use crate::codec::{Telemetry, MSG_RESULT}; +use crate::forward::{ + opened_forwarder_message, PendingQueue, PendingRequest, ERR_FORWARD, ERR_INVALID_RESULT, + ERR_NEXT_CLOSED, ERR_RESULT_METADATA, ERR_RESULT_TOO_LARGE, ERR_TELEMETRY_TOO_LARGE, +}; +use crate::reconnect::connect_endpoint; +use crate::route::RouteEntry; +use crate::transport::read_frame; +use crate::work::{ + decode_result_body, encode_result_frame, encode_work_frame, result_request_id, WorkBody, +}; + +pub fn now_sec() -> f64 { + static START: std::sync::OnceLock = std::sync::OnceLock::new(); + START.get_or_init(Instant::now).elapsed().as_secs_f64() +} + +pub fn local_work_telemetry( + work: &crate::codec::Work, + eval_usec: u32, + output_bytes: u32, +) -> Telemetry { + Telemetry { + layer_start: work.layer_start, + layer_end: work.layer_end, + route_index: work.route_index, + pos0: work.pos0, + n_tokens: work.n_tokens, + eval_usec, + downstream_wait_usec: 0, + forward_send_usec: 0, + input_bytes: work.token_bytes.saturating_add(work.input_hc_bytes), + output_bytes, + } +} + +pub fn usec_since(t0: f64, t1: f64) -> u32 { + if t1 <= t0 { + return 0; + } + let usec = (t1 - t0) * 1_000_000.0; + if usec >= f64::from(u32::MAX) { + u32::MAX + } else { + (usec + 0.5) as u32 + } +} + +pub fn prepend_telemetry(result_payload: &[u8], local: Telemetry) -> Result, &'static str> { + let mut body = decode_result_body(result_payload).map_err(|_| ERR_RESULT_METADATA)?; + if body.telemetry.len() >= u32::MAX as usize { + return Err(ERR_TELEMETRY_TOO_LARGE); + } + body.telemetry.push(local); + encode_result_frame(&body).map_err(|_| ERR_RESULT_TOO_LARGE) +} + +pub struct Forwarder { + pub host: String, + pub port: u32, + writer: Mutex, + queue: Arc, + stop: Arc, + relay: Option>, +} + +impl Forwarder { + pub fn connect( + host: &str, + port: u32, + window: u32, + on_result: impl Fn(Vec) + Send + 'static, + on_error: impl Fn(u64, &'static str) + Send + 'static, + ) -> Result { + let stream = connect_endpoint(host, port as u16).map_err(|e| e.to_string())?; + let mut reader = stream.try_clone().map_err(|e| e.to_string())?; + let queue = Arc::new(PendingQueue::new(window)); + let stop = Arc::new(AtomicBool::new(false)); + let relay_q = Arc::clone(&queue); + let relay_stop = Arc::clone(&stop); + let relay = thread::Builder::new() + .name(format!("ds4-dist-relay-{host}:{port}")) + .spawn(move || relay_main(&mut reader, &relay_q, &relay_stop, on_result, on_error)) + .map_err(|e| e.to_string())?; + eprintln!("{}", opened_forwarder_message(host, port, window)); + Ok(Self { + host: host.to_string(), + port, + writer: Mutex::new(stream), + queue, + stop, + relay: Some(relay), + }) + } + + pub fn send_work(&self, request: PendingRequest, frame: &[u8]) -> Result<(), &'static str> { + let request_id = request.request_id; + self.queue.enqueue(request)?; + let send_t0 = now_sec(); + let write_rc = self.writer.lock().expect("forward send").write_all(frame); + let send_t1 = now_sec(); + self.queue + .note_send_done(request_id, usec_since(send_t0, send_t1), send_t1); + if write_rc.is_err() { + self.queue.remove(request_id); + let _ = self + .writer + .lock() + .expect("forward send") + .shutdown(std::net::Shutdown::Both); + return Err(ERR_FORWARD); + } + Ok(()) + } + + pub fn shutdown(&mut self) { + self.stop.store(true, Ordering::SeqCst); + self.queue.close(); + if let Ok(w) = self.writer.lock() { + let _ = w.shutdown(std::net::Shutdown::Both); + } + if let Some(relay) = self.relay.take() { + let _ = relay.join(); + } + self.queue.clear(); + } +} + +impl Drop for Forwarder { + fn drop(&mut self) { + self.shutdown(); + } +} + +fn relay_main( + reader: &mut TcpStream, + queue: &PendingQueue, + stop: &AtomicBool, + on_result: impl Fn(Vec), + on_error: impl Fn(u64, &'static str), +) { + while !stop.load(Ordering::Relaxed) { + let (typ, body) = match read_frame(reader) { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof || stop.load(Ordering::Relaxed) => { + if let Some(pending) = queue.pop() { + on_error(pending.request_id, ERR_NEXT_CLOSED); + } + break; + } + Err(_) => { + if let Some(pending) = queue.pop() { + on_error(pending.request_id, ERR_NEXT_CLOSED); + } + break; + } + }; + if typ != MSG_RESULT { + if let Some(pending) = queue.pop() { + on_error(pending.request_id, ERR_INVALID_RESULT); + } + break; + } + let decoded = match decode_result_body(&body) { + Ok(v) => v, + Err(_) => { + if let Some(pending) = queue.pop() { + on_error(pending.request_id, ERR_RESULT_METADATA); + } + break; + } + }; + let Some(mut pending) = queue.pop() else { + break; + }; + if result_request_id(&decoded.hdr) != pending.request_id { + on_error(pending.request_id, ERR_RESULT_METADATA); + break; + } + pending.telemetry.downstream_wait_usec = usec_since(pending.downstream_t0, now_sec()); + match prepend_telemetry(&body, pending.telemetry) { + Ok(frame) => on_result(frame), + Err(msg) => on_error(pending.request_id, msg), + } + } + queue.close(); +} + +pub fn forward_work_blocking( + next: &RouteEntry, + body: &WorkBody, + upstream: &mut W, + mut local: Telemetry, +) -> Result<(), String> { + let mut hop = connect_endpoint(&next.host, next.port as u16).map_err(|_| ERR_FORWARD)?; + let frame = encode_work_frame(body).map_err(|e| e.to_string())?; + let send_t0 = now_sec(); + hop.write_all(&frame).map_err(|_| ERR_FORWARD)?; + let send_t1 = now_sec(); + local.forward_send_usec = usec_since(send_t0, send_t1); + let (typ, reply) = read_frame(&mut hop).map_err(|_| ERR_FORWARD)?; + if typ != MSG_RESULT { + return Err(ERR_INVALID_RESULT.into()); + } + local.downstream_wait_usec = usec_since(send_t1, now_sec()); + let prepended = prepend_telemetry(&reply, local).map_err(|e| e.to_string())?; + upstream.write_all(&prepended).map_err(|e| e.to_string()) +} diff --git a/crates/ds4-dist/src/route.rs b/crates/ds4-dist/src/route.rs new file mode 100644 index 0000000000000000000000000000000000000000..886bcc39cc95cb6a738f736fd1bd17d0511ce5db --- /dev/null +++ b/crates/ds4-dist/src/route.rs @@ -0,0 +1,162 @@ +//! Route blob: `[Route + host]*` then `RouteReturn + host`. No struct memcpy. + +use crate::codec::{ + bytes_have_nul, CodecError, Route, RouteReturn, NI_MAXHOST, ROUTE_FIXED_BYTES, + ROUTE_F_OUTPUT_LOGITS, ROUTE_RETURN_FIXED_BYTES, ROUTE_RETURN_UPSTREAM, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RouteEntry { + pub host: String, + pub port: u32, + pub layer_start: u32, + pub layer_end: u32, + pub flags: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReturnTarget { + pub kind: u32, + pub host: String, + pub port: u32, +} + +pub fn encode_route_blob( + entries: &[RouteEntry], + ret: &ReturnTarget, +) -> Result, CodecError> { + let mut out = Vec::new(); + for e in entries { + let host = e.host.as_bytes(); + if host.is_empty() || host.len() >= NI_MAXHOST || bytes_have_nul(host) { + return Err(CodecError::Invalid("invalid route host length")); + } + let rec = Route { + host_len: host.len() as u32, + port: e.port, + layer_start: e.layer_start, + layer_end: e.layer_end, + flags: e.flags, + }; + out.extend_from_slice(&rec.encode()); + out.extend_from_slice(host); + } + let rhost = ret.host.as_bytes(); + if rhost.len() >= NI_MAXHOST || bytes_have_nul(rhost) { + return Err(CodecError::Invalid( + "invalid route final destination host length", + )); + } + let rec = RouteReturn { + kind: ret.kind, + host_len: rhost.len() as u32, + port: ret.port, + }; + out.extend_from_slice(&rec.encode()); + out.extend_from_slice(rhost); + Ok(out) +} + +pub fn decode_route_blob( + blob: &[u8], + route_count: u32, +) -> Result<(Vec, ReturnTarget), CodecError> { + let mut remaining = blob; + let mut entries = Vec::new(); + for _ in 0..route_count { + if remaining.len() < ROUTE_FIXED_BYTES { + return Err(CodecError::Invalid("truncated route entry")); + } + let fixed = Route::decode(remaining)?; + remaining = &remaining[ROUTE_FIXED_BYTES..]; + if fixed.host_len == 0 + || fixed.host_len as usize >= NI_MAXHOST + || fixed.host_len as usize > remaining.len() + { + return Err(CodecError::Invalid("invalid route host length")); + } + let host = &remaining[..fixed.host_len as usize]; + if bytes_have_nul(host) { + return Err(CodecError::Invalid("route host contains NUL bytes")); + } + entries.push(RouteEntry { + host: String::from_utf8_lossy(host).into_owned(), + port: fixed.port, + layer_start: fixed.layer_start, + layer_end: fixed.layer_end, + flags: fixed.flags, + }); + remaining = &remaining[fixed.host_len as usize..]; + } + if remaining.len() < ROUTE_RETURN_FIXED_BYTES { + return Err(CodecError::Invalid( + "route payload missing final destination", + )); + } + let ret = RouteReturn::decode(remaining)?; + remaining = &remaining[ROUTE_RETURN_FIXED_BYTES..]; + if ret.host_len as usize >= NI_MAXHOST || ret.host_len as usize > remaining.len() { + return Err(CodecError::Invalid( + "invalid route final destination host length", + )); + } + let host = &remaining[..ret.host_len as usize]; + if bytes_have_nul(host) { + return Err(CodecError::Invalid( + "route final destination host contains NUL bytes", + )); + } + remaining = &remaining[ret.host_len as usize..]; + if !remaining.is_empty() { + return Err(CodecError::Invalid("route payload has trailing bytes")); + } + Ok(( + entries, + ReturnTarget { + kind: ret.kind, + host: String::from_utf8_lossy(host).into_owned(), + port: ret.port, + }, + )) +} + +pub fn validate_route_blob(blob: &[u8], route_count: u32, n_layers: u32) -> Result<(), CodecError> { + if route_count == 0 { + return if blob.is_empty() { + Ok(()) + } else { + Err(CodecError::Invalid( + "route payload has entries without a route count", + )) + }; + } + let (entries, ret) = decode_route_blob(blob, route_count)?; + let mut prev_end = u32::MAX; + for (i, e) in entries.iter().enumerate() { + if e.port == 0 || e.port > 65535 { + return Err(CodecError::Invalid("invalid route port")); + } + if e.layer_start >= n_layers || e.layer_end >= n_layers || e.layer_end < e.layer_start { + return Err(CodecError::Invalid("invalid route layer range")); + } + if (e.flags & !ROUTE_F_OUTPUT_LOGITS) != 0 { + return Err(CodecError::Invalid("invalid route flags")); + } + if (e.flags & ROUTE_F_OUTPUT_LOGITS) != 0 && e.layer_end + 1 != n_layers { + return Err(CodecError::Invalid("route logits require final layer")); + } + if i != 0 && e.layer_start != prev_end + 1 { + return Err(CodecError::Invalid("route layer ranges are not contiguous")); + } + prev_end = e.layer_end; + } + if ret.kind != ROUTE_RETURN_UPSTREAM { + return Err(CodecError::Invalid("unsupported route final destination")); + } + if !ret.host.is_empty() || ret.port != 0 { + return Err(CodecError::Invalid( + "invalid upstream route final destination", + )); + } + Ok(()) +} diff --git a/crates/ds4-dist/src/serve_local.rs b/crates/ds4-dist/src/serve_local.rs new file mode 100644 index 0000000000000000000000000000000000000000..7e45e8e4f8266ee33a4e143c4d1a95feafa09e03 --- /dev/null +++ b/crates/ds4-dist/src/serve_local.rs @@ -0,0 +1,127 @@ +//! Same-thread coordinator + hop mux. Accept thread only accepts. + +use std::io::{self, ErrorKind}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::sync::mpsc::{self, Receiver}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use crate::coordinator::accept_data_client; +use crate::exec::SliceExec; +use crate::native_snapshot::SnapshotStore; +use crate::worker::Worker; + +pub(crate) const IDLE: Duration = Duration::from_millis(25); + +pub(crate) struct HopAccept { + rx: Option>, + wakeup: SocketAddr, + handle: Option>, +} + +impl HopAccept { + pub(crate) fn receiver(&self) -> Option<&Receiver> { + self.rx.as_ref() + } +} + +impl Drop for HopAccept { + fn drop(&mut self) { + self.rx.take(); + let _ = TcpStream::connect(self.wakeup); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +pub(crate) fn spawn_hop_accept(listener: &TcpListener) -> io::Result { + let listener = listener.try_clone()?; + let wakeup = listener.local_addr()?; + let (tx, rx) = mpsc::channel(); + let handle = thread::Builder::new() + .name("ds4-dist-hop-accept".into()) + .spawn(move || loop { + match accept_data_client(&listener) { + Ok(stream) => { + if tx.send(stream).is_err() { + break; + } + } + Err(_) => break, + } + })?; + Ok(HopAccept { + rx: Some(rx), + wakeup, + handle: Some(handle), + }) +} + +pub(crate) enum Drive { + Continue, + Closed, + Failed(io::Error), +} + +pub(crate) fn drive_ready(worker: &mut Worker, stream: &mut TcpStream) -> Drive +where + E: SliceExec, + S: SnapshotStore, +{ + match worker.serve_once(stream) { + Ok(()) => Drive::Continue, + Err(e) => match e.kind() { + ErrorKind::TimedOut | ErrorKind::WouldBlock | ErrorKind::Interrupted => Drive::Continue, + ErrorKind::UnexpectedEof | ErrorKind::InvalidData => Drive::Closed, + _ => Drive::Failed(e), + }, + } +} + +pub(crate) fn serve_coordinator_and_hops( + worker: &mut Worker, + coordinator: &mut TcpStream, + hops: Option<&Receiver>, + should_stop: &mut Stop, +) -> io::Result<()> +where + E: SliceExec, + S: SnapshotStore, + Stop: FnMut() -> bool, +{ + coordinator.set_read_timeout(Some(IDLE))?; + let mut accepted = Vec::new(); + let rc = loop { + if should_stop() { + break Ok(()); + } + if let Some(rx) = hops { + while let Ok(hop) = rx.try_recv() { + let _ = hop.set_nodelay(true); + let _ = hop.set_read_timeout(Some(IDLE)); + accepted.push(hop); + } + } + let mut i = 0; + while i < accepted.len() { + if let Ok(clone) = accepted[i].try_clone() { + worker.bind_hops(Arc::new(Mutex::new(clone))); + } + match drive_ready(worker, &mut accepted[i]) { + Drive::Continue => i += 1, + Drive::Closed | Drive::Failed(_) => { + accepted.swap_remove(i); + } + } + } + match drive_ready(worker, coordinator) { + Drive::Continue => {} + Drive::Closed => break Ok(()), + Drive::Failed(e) => break Err(e), + } + }; + worker.shutdown_hops(); + rc +} diff --git a/crates/ds4-dist/src/snapshot.rs b/crates/ds4-dist/src/snapshot.rs new file mode 100644 index 0000000000000000000000000000000000000000..a7648ea6b2be423e2d71bdd935011396ba584202 --- /dev/null +++ b/crates/ds4-dist/src/snapshot.rs @@ -0,0 +1,337 @@ +//! Coordinator-side DS4D snapshot save/load transport. + +use std::io::{self, Read, Write}; + +use crate::codec::{ + decode_frame_header, encode_frame_header, u64_from_halves, u64_to_halves, FrameHeader, + SnapshotBegin, SnapshotChunk, SnapshotDone, SnapshotReq, FRAME_HEADER_BYTES, + MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, MSG_SNAPSHOT_DONE, MSG_SNAPSHOT_LOAD_BEGIN, + MSG_SNAPSHOT_SAVE_REQ, SNAPSHOT_BEGIN_FIXED_BYTES, SNAPSHOT_CHUNK_BYTES, + SNAPSHOT_CHUNK_FIXED_BYTES, SNAPSHOT_DONE_FIXED_BYTES, +}; +use crate::transport::write_frame; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SnapshotMeta { + pub model_id: u32, + pub session_id: u64, + pub request_id: u64, + pub token_hash: u64, + pub layer_start: u32, + pub layer_end: u32, +} + +fn token_count(tokens: &[i32]) -> Result { + u32::try_from(tokens.len()).map_err(|_| "invalid distributed snapshot token length".to_string()) +} + +fn c_message(message: &[u8]) -> String { + let end = message + .iter() + .position(|&b| b == 0) + .unwrap_or(message.len()); + String::from_utf8_lossy(&message[..end]).into_owned() +} + +fn read_header(r: &mut R, closed: &'static str) -> Result { + let mut buf = [0; FRAME_HEADER_BYTES]; + r.read_exact(&mut buf).map_err(|e| { + if e.kind() == io::ErrorKind::UnexpectedEof { + closed.to_string() + } else { + format!("failed to read frame header: {e}") + } + })?; + decode_frame_header(&buf).map_err(|e| e.to_string()) +} + +fn read_or(r: &mut R, buf: &mut [u8], message: &'static str) -> Result<(), String> { + r.read_exact(buf).map_err(|_| message.to_string()) +} + +fn discard(r: &mut R, mut bytes: u32) -> io::Result<()> { + let mut buf = [0; 4096]; + while bytes != 0 { + let n = bytes.min(buf.len() as u32) as usize; + r.read_exact(&mut buf[..n])?; + bytes -= n as u32; + } + Ok(()) +} + +fn read_message( + r: &mut R, + bytes: u32, + read_error: &'static str, + discard_error: &'static str, +) -> Result, String> { + let copied = bytes.min(255) as usize; + let mut message = vec![0; copied]; + read_or(r, &mut message, read_error)?; + discard(r, bytes - copied as u32).map_err(|_| discard_error.to_string())?; + Ok(message) +} + +fn read_begin(r: &mut R) -> Result<(SnapshotBegin, Vec), String> { + let header = read_header(r, "distributed worker closed snapshot connection")?; + if header.typ != MSG_SNAPSHOT_BEGIN || header.bytes < SNAPSHOT_BEGIN_FIXED_BYTES as u32 { + let _ = discard(r, header.bytes); + return Err("distributed worker returned invalid snapshot frame".into()); + } + + let mut fixed = [0; SNAPSHOT_BEGIN_FIXED_BYTES]; + read_or(r, &mut fixed, "failed to read distributed snapshot header")?; + let begin = SnapshotBegin::decode(&fixed) + .map_err(|_| "invalid distributed snapshot response header".to_string())?; + let mut body = header.bytes - SNAPSHOT_BEGIN_FIXED_BYTES as u32; + let expected_token_bytes = u64::from(begin.token_count) * 4; + if expected_token_bytes > u64::from(u32::MAX) + || begin.token_bytes != expected_token_bytes as u32 + || begin.token_bytes > body + || begin.message_bytes > body - begin.token_bytes + { + let _ = discard(r, body); + return Err("invalid distributed snapshot response header".into()); + } + + discard(r, begin.token_bytes) + .map_err(|_| "failed to discard distributed snapshot response tokens".to_string())?; + body -= begin.token_bytes; + let message = read_message( + r, + begin.message_bytes, + "failed to read distributed snapshot response message", + "failed to discard distributed snapshot response message", + )?; + body -= begin.message_bytes; + discard(r, body).map_err(|_| { + "failed to discard trailing distributed snapshot response bytes".to_string() + })?; + Ok((begin, message)) +} + +fn read_done(r: &mut R, request_id: u64) -> Result<(), String> { + let header = read_header(r, "distributed worker closed before snapshot completion")?; + if header.typ != MSG_SNAPSHOT_DONE || header.bytes < SNAPSHOT_DONE_FIXED_BYTES as u32 { + let _ = discard(r, header.bytes); + return Err("distributed worker returned invalid snapshot completion frame".into()); + } + + let mut fixed = [0; SNAPSHOT_DONE_FIXED_BYTES]; + read_or( + r, + &mut fixed, + "failed to read distributed snapshot completion", + )?; + let done = SnapshotDone::decode(&fixed) + .map_err(|_| "invalid distributed snapshot completion message".to_string())?; + let mut body = header.bytes - SNAPSHOT_DONE_FIXED_BYTES as u32; + if done.message_bytes > body { + let _ = discard(r, body); + return Err("invalid distributed snapshot completion message".into()); + } + let message = read_message( + r, + done.message_bytes, + "failed to read distributed snapshot completion message", + "failed to discard distributed snapshot completion message", + )?; + body -= done.message_bytes; + discard(r, body).map_err(|_| { + "failed to discard trailing distributed snapshot completion bytes".to_string() + })?; + + if u64_from_halves(done.request_hi, done.request_lo) != request_id { + return Err("distributed snapshot completion request mismatch".into()); + } + if done.status != 0 { + let message = c_message(&message); + return Err(if message.is_empty() { + "distributed worker failed snapshot request".into() + } else { + message + }); + } + Ok(()) +} + +/// Ask a connected worker for one KV shard and stream it into `payload`. +pub fn coordinator_save_snapshot( + stream: &mut S, + meta: SnapshotMeta, + tokens: &[i32], + payload: &mut W, +) -> Result { + let token_count = token_count(tokens)?; + let (session_hi, session_lo) = u64_to_halves(meta.session_id); + let (request_hi, request_lo) = u64_to_halves(meta.request_id); + let (token_hash_hi, token_hash_lo) = u64_to_halves(meta.token_hash); + let request = SnapshotReq { + model_id: meta.model_id, + session_hi, + session_lo, + request_hi, + request_lo, + token_hash_hi, + token_hash_lo, + token_count, + layer_start: meta.layer_start, + layer_end: meta.layer_end, + }; + write_frame(stream, MSG_SNAPSHOT_SAVE_REQ, &request.encode()) + .map_err(|_| "failed to request distributed KV shard".to_string())?; + + let (begin, message) = read_begin(stream)?; + if begin.status != 0 { + let message = c_message(&message); + return Err(if message.is_empty() { + "distributed worker refused KV snapshot".into() + } else { + message + }); + } + + let payload_bytes = u64_from_halves(begin.payload_hi, begin.payload_lo); + if begin.model_id != meta.model_id + || u64_from_halves(begin.session_hi, begin.session_lo) != meta.session_id + || u64_from_halves(begin.request_hi, begin.request_lo) != meta.request_id + || u64_from_halves(begin.token_hash_hi, begin.token_hash_lo) != meta.token_hash + || (begin.token_count != 0 && begin.token_count != token_count) + || begin.layer_start != meta.layer_start + || begin.layer_end != meta.layer_end + { + return Err("distributed KV shard metadata mismatch".into()); + } + + let mut buf = Vec::new(); + if payload_bytes != 0 { + buf.try_reserve_exact(SNAPSHOT_CHUNK_BYTES) + .map_err(|_| "out of memory receiving distributed KV shard".to_string())?; + buf.resize(SNAPSHOT_CHUNK_BYTES, 0); + } + let mut received = 0u64; + while received < payload_bytes { + let header = read_header(stream, "distributed worker closed while sending KV shard")?; + if header.typ != MSG_SNAPSHOT_CHUNK || header.bytes < SNAPSHOT_CHUNK_FIXED_BYTES as u32 { + let _ = discard(stream, header.bytes); + return Err("expected distributed KV shard chunk".into()); + } + + let mut fixed = [0; SNAPSHOT_CHUNK_FIXED_BYTES]; + read_or( + stream, + &mut fixed, + "failed to read distributed KV shard chunk header", + )?; + let chunk = SnapshotChunk::decode(&fixed) + .map_err(|_| "invalid distributed KV shard chunk".to_string())?; + let chunk_bytes = header.bytes - SNAPSHOT_CHUNK_FIXED_BYTES as u32; + if u64_from_halves(chunk.request_hi, chunk.request_lo) != meta.request_id + || chunk.chunk_bytes != chunk_bytes + || chunk_bytes as usize > SNAPSHOT_CHUNK_BYTES + || u64::from(chunk_bytes) > payload_bytes - received + { + let _ = discard(stream, chunk_bytes); + return Err("invalid distributed KV shard chunk".into()); + } + let chunk_bytes = chunk_bytes as usize; + read_or( + stream, + &mut buf[..chunk_bytes], + "failed to read distributed KV shard chunk", + )?; + payload + .write_all(&buf[..chunk_bytes]) + .map_err(|_| "failed to write distributed KV shard".to_string())?; + received += chunk_bytes as u64; + } + read_done(stream, meta.request_id)?; + Ok(payload_bytes) +} + +/// Stream one caller-owned KV shard payload to a connected worker. +pub fn coordinator_load_snapshot( + stream: &mut S, + meta: SnapshotMeta, + tokens: &[i32], + payload: &mut R, + payload_bytes: u64, +) -> Result<(), String> { + let token_count = token_count(tokens)?; + let token_bytes = token_count + .checked_mul(4) + .ok_or_else(|| "invalid distributed snapshot token length".to_string())?; + let (session_hi, session_lo) = u64_to_halves(meta.session_id); + let (request_hi, request_lo) = u64_to_halves(meta.request_id); + let (token_hash_hi, token_hash_lo) = u64_to_halves(meta.token_hash); + let (payload_hi, payload_lo) = u64_to_halves(payload_bytes); + let begin = SnapshotBegin { + model_id: meta.model_id, + session_hi, + session_lo, + request_hi, + request_lo, + token_hash_hi, + token_hash_lo, + token_count, + layer_start: meta.layer_start, + layer_end: meta.layer_end, + payload_hi, + payload_lo, + status: 0, + token_bytes, + message_bytes: 0, + }; + let frame_bytes = SNAPSHOT_BEGIN_FIXED_BYTES as u64 + u64::from(token_bytes); + if frame_bytes > u64::from(u32::MAX) { + return Err("distributed snapshot begin frame is too large".into()); + } + let send_begin = || "failed to send distributed KV shard restore request".to_string(); + stream + .write_all(&encode_frame_header( + MSG_SNAPSHOT_LOAD_BEGIN, + frame_bytes as u32, + )) + .map_err(|_| send_begin())?; + stream + .write_all(&begin.encode()) + .map_err(|_| send_begin())?; + for &token in tokens { + stream + .write_all(&(token as u32).to_be_bytes()) + .map_err(|_| send_begin())?; + } + + let mut buf = Vec::new(); + if payload_bytes != 0 { + buf.try_reserve_exact(SNAPSHOT_CHUNK_BYTES) + .map_err(|_| "failed to send distributed KV shard restore payload".to_string())?; + buf.resize(SNAPSHOT_CHUNK_BYTES, 0); + } + let mut remaining = payload_bytes; + while remaining != 0 { + let n = remaining.min(SNAPSHOT_CHUNK_BYTES as u64) as usize; + read_or( + payload, + &mut buf[..n], + "failed to send distributed KV shard restore payload", + )?; + let (request_hi, request_lo) = u64_to_halves(meta.request_id); + let chunk = SnapshotChunk { + request_hi, + request_lo, + chunk_bytes: n as u32, + }; + let frame_bytes = SNAPSHOT_CHUNK_FIXED_BYTES as u32 + n as u32; + let send_chunk = || "failed to send distributed KV shard restore payload".to_string(); + stream + .write_all(&encode_frame_header(MSG_SNAPSHOT_CHUNK, frame_bytes)) + .map_err(|_| send_chunk())?; + stream + .write_all(&chunk.encode()) + .map_err(|_| send_chunk())?; + stream.write_all(&buf[..n]).map_err(|_| send_chunk())?; + remaining -= n as u64; + } + read_done(stream, meta.request_id) +} diff --git a/crates/ds4-dist/src/snapshot_gather.rs b/crates/ds4-dist/src/snapshot_gather.rs new file mode 100644 index 0000000000000000000000000000000000000000..4d603254274b25f52d923c4fe46963d809903d83 --- /dev/null +++ b/crates/ds4-dist/src/snapshot_gather.rs @@ -0,0 +1,186 @@ +//! Coordinator KV gather order from `ds4_dist_session_save_payload`. +//! +//! Host owns shard walk, empty-shard refusal, and token hashing. Layer +//! GPU bytes stay native; DSV4 header merge lives in `snapshot_merge`. + +use crate::hash::token_hash_prefix; + +pub const ERR_NO_TIMELINE: &str = "distributed session has no valid token timeline"; +pub const ERR_EMPTY_SHARD: &str = "distributed KV shard is empty"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GatherShard { + pub layer_start: u32, + pub layer_end: u32, + pub local: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GatheredShards { + pub token_count: u32, + pub token_hash: u64, + pub shards: Vec<(GatherShard, Vec)>, +} + +/// C save walk: shard 0 is the owner layer payload; later shards are +/// remote. Empty shard bytes fail before any DSV4 merge. +pub fn gather_kv_shards( + tokens: &[i32], + plan: &[GatherShard], + mut fetch: F, +) -> Result +where + F: FnMut(GatherShard) -> Result, String>, +{ + if tokens.len() > u32::MAX as usize { + return Err(ERR_NO_TIMELINE.into()); + } + if plan.is_empty() { + return Err(ERR_EMPTY_SHARD.into()); + } + let token_count = tokens.len() as u32; + let token_hash = token_hash_prefix(tokens); + let mut shards = Vec::with_capacity(plan.len()); + for (i, spec) in plan.iter().copied().enumerate() { + if i == 0 && !spec.local { + return Err("distributed KV shard 0 must be the owner layer".into()); + } + if i != 0 && spec.local { + return Err("distributed KV remote shard marked local".into()); + } + let bytes = fetch(spec)?; + if bytes.is_empty() { + return Err(ERR_EMPTY_SHARD.into()); + } + shards.push((spec, bytes)); + } + Ok(GatheredShards { + token_count, + token_hash, + shards, + }) +} + +/// C load walk: shard 0 is the owner layer payload; later shards are +/// remote. Host owns the order. Layer apply and DSV4 parse stay native. +pub fn scatter_kv_shards( + tokens: &[i32], + plan: &[GatherShard], + shards: &[Vec], + mut apply: F, +) -> Result +where + F: FnMut(GatherShard, &[u8]) -> Result<(), String>, +{ + if tokens.len() > u32::MAX as usize { + return Err(ERR_NO_TIMELINE.into()); + } + if plan.is_empty() || plan.len() != shards.len() { + return Err(ERR_EMPTY_SHARD.into()); + } + for (i, (spec, bytes)) in plan.iter().copied().zip(shards.iter()).enumerate() { + if i == 0 && !spec.local { + return Err("distributed KV shard 0 must be the owner layer".into()); + } + if i != 0 && spec.local { + return Err("distributed KV remote shard marked local".into()); + } + if bytes.is_empty() { + return Err(ERR_EMPTY_SHARD.into()); + } + apply(spec, bytes)?; + } + Ok(token_hash_prefix(tokens)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn two_shard_plan() -> [GatherShard; 2] { + [ + GatherShard { + layer_start: 0, + layer_end: 10, + local: true, + }, + GatherShard { + layer_start: 10, + layer_end: 20, + local: false, + }, + ] + } + + #[test] + fn gather_walks_owner_then_remote() { + let tokens = [1i32, 2, 3]; + let got = gather_kv_shards(&tokens, &two_shard_plan(), |spec| { + Ok(vec![spec.layer_start as u8, spec.layer_end as u8]) + }) + .unwrap(); + assert_eq!(got.token_count, 3); + assert_eq!(got.token_hash, token_hash_prefix(&tokens)); + assert_eq!(got.shards.len(), 2); + assert!(got.shards[0].0.local); + assert!(!got.shards[1].0.local); + assert_eq!(got.shards[0].1, vec![0, 10]); + assert_eq!(got.shards[1].1, vec![10, 20]); + } + + #[test] + fn gather_refuses_empty_shard() { + let err = gather_kv_shards(&[1], &two_shard_plan(), |spec| { + if spec.local { + Ok(vec![1]) + } else { + Ok(vec![]) + } + }) + .unwrap_err(); + assert_eq!(err, ERR_EMPTY_SHARD); + } + + #[test] + fn gather_requires_owner_first() { + let plan = [GatherShard { + layer_start: 0, + layer_end: 1, + local: false, + }]; + let err = gather_kv_shards(&[1], &plan, |_| Ok(vec![1])).unwrap_err(); + assert!(err.contains("owner layer")); + } + + #[test] + fn gather_accepts_empty_token_timeline() { + let got = gather_kv_shards(&[], &two_shard_plan(), |_| Ok(vec![1])).unwrap(); + assert_eq!(got.token_count, 0); + assert_eq!(got.token_hash, token_hash_prefix(&[])); + } + + #[test] + fn scatter_walks_owner_then_remote() { + let tokens = [7i32]; + let mut seen = Vec::new(); + let hash = scatter_kv_shards( + &tokens, + &two_shard_plan(), + &[vec![1], vec![2]], + |spec, bytes| { + seen.push((spec.local, bytes.to_vec())); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(hash, token_hash_prefix(&tokens)); + assert_eq!(seen, vec![(true, vec![1]), (false, vec![2])]); + } + + #[test] + fn scatter_refuses_empty_shard() { + let err = scatter_kv_shards(&[1], &two_shard_plan(), &[vec![1], vec![]], |_, _| Ok(())) + .unwrap_err(); + assert_eq!(err, ERR_EMPTY_SHARD); + } +} diff --git a/crates/ds4-dist/src/snapshot_merge.rs b/crates/ds4-dist/src/snapshot_merge.rs new file mode 100644 index 0000000000000000000000000000000000000000..2e8f9ead6e7d39e6f78b734c99c4c60b975f6455 --- /dev/null +++ b/crates/ds4-dist/src/snapshot_merge.rs @@ -0,0 +1,415 @@ +//! Coordinator DSV4 assembly from `ds4_dist_session_save_payload`. +//! +//! Host owns layer-header parse, layout match, tensor-size arithmetic, and +//! write order. Layer GPU bytes and logits stay opaque blobs. + +use crate::snapshot_gather::ERR_EMPTY_SHARD; + +pub const LAYER_MAGIC: u32 = 0x4C56_5344; /* "DSVL" */ +pub const LAYER_VERSION: u32 = 1; +pub const LAYER_U32_FIELDS: usize = 14; +pub const SESSION_MAGIC: u32 = 0x3456_5344; /* "DSV4" */ +pub const SESSION_VERSION: u32 = 3; +pub const SESSION_U32_FIELDS: usize = 13; + +pub const ERR_UNSUPPORTED_LAYER: &str = "unsupported distributed KV layer payload"; +pub const ERR_RANGE: &str = "distributed KV shard range mismatch"; +pub const ERR_LAYOUT: &str = "distributed KV shard layout is invalid"; +pub const ERR_LAYOUTS_DIFFER: &str = "distributed KV shards use different layouts"; +pub const ERR_COMP_ROWS: &str = "distributed KV shard has invalid compressed row count"; +pub const ERR_INDEX_ROWS: &str = "distributed KV shard has invalid indexer row count"; +pub const ERR_TENSOR_OVERFLOW: &str = "distributed KV shard tensor size overflow"; +pub const ERR_TENSOR_BYTES: &str = "distributed KV shard tensor byte count mismatch"; +pub const ERR_METADATA: &str = "distributed KV shard metadata mismatch"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DistKvLayout { + pub ctx: u32, + pub prefill_cap: u32, + pub raw_cap: u32, + pub raw_window: u32, + pub comp_cap: u32, + pub token_count: u32, + pub n_layers: u32, + pub head_dim: u32, + pub indexer_head_dim: u32, + pub vocab: u32, + pub raw_live: u32, +} + +impl DistKvLayout { + pub fn matches_core(&self, other: &Self) -> bool { + self.ctx == other.ctx + && self.prefill_cap == other.prefill_cap + && self.raw_cap == other.raw_cap + && self.raw_window == other.raw_window + && self.comp_cap == other.comp_cap + && self.token_count == other.token_count + && self.n_layers == other.n_layers + && self.head_dim == other.head_dim + && self.indexer_head_dim == other.indexer_head_dim + && self.raw_live == other.raw_live + } + + pub fn raw_live_valid(&self) -> bool { + if self.raw_window == 0 || self.raw_cap == 0 { + return false; + } + let expected = self.token_count.min(self.raw_window); + self.raw_live == expected && self.raw_live <= self.raw_cap + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedLayerShard { + pub layout: DistKvLayout, + pub layer_start: u32, + pub layer_end: u32, + pub n_comp: Vec, + pub n_index_comp: Vec, + pub tensor: Vec, +} + +fn get_u32(bytes: &[u8]) -> Option { + bytes.get(..4)?.try_into().ok().map(u32::from_le_bytes) +} + +fn put_u32(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_le_bytes()); +} + +/// C `dist_kv_state_bytes`. +pub fn kv_state_bytes(ratio: u32, head_dim: u32) -> Option { + let coff = if ratio == 4 { 2u64 } else { 1u64 }; + coff.checked_mul(u64::from(head_dim))? + .checked_mul(coff)? + .checked_mul(u64::from(ratio))? + .checked_mul(4) +} + +/// C `dist_kv_layer_tensor_bytes` once the host compress ratio is known. +pub fn layer_tensor_bytes( + layout: &DistKvLayout, + ratio: u32, + n_comp: u32, + n_index_comp: u32, +) -> Option { + let mut bytes = u64::from(layout.raw_live) + .checked_mul(u64::from(layout.head_dim))? + .checked_mul(4)?; + if ratio == 0 { + return Some(bytes); + } + bytes = bytes.checked_add( + u64::from(n_comp) + .checked_mul(u64::from(layout.head_dim))? + .checked_mul(4)?, + )?; + let attn = kv_state_bytes(ratio, layout.head_dim)?; + bytes = bytes.checked_add(attn)?.checked_add(attn)?; + if ratio == 4 { + bytes = bytes.checked_add( + u64::from(n_index_comp) + .checked_mul(u64::from(layout.indexer_head_dim))? + .checked_mul(4)?, + )?; + let index = kv_state_bytes(ratio, layout.indexer_head_dim)?; + bytes = bytes.checked_add(index)?.checked_add(index)?; + } + Some(bytes) +} + +pub fn parse_layer_payload( + bytes: &[u8], + expected_start: u32, + expected_end: u32, + compress: &[u32], +) -> Result { + if bytes.len() < LAYER_U32_FIELDS * 4 { + return Err(ERR_UNSUPPORTED_LAYER.into()); + } + let mut h = [0u32; LAYER_U32_FIELDS]; + for (i, slot) in h.iter_mut().enumerate() { + *slot = get_u32(&bytes[i * 4..]).ok_or(ERR_UNSUPPORTED_LAYER)?; + } + if h[0] != LAYER_MAGIC || h[1] != LAYER_VERSION { + return Err(ERR_UNSUPPORTED_LAYER.into()); + } + let layout = DistKvLayout { + ctx: h[2], + prefill_cap: h[3], + raw_cap: h[4], + raw_window: h[5], + comp_cap: h[6], + token_count: h[7], + n_layers: h[8], + head_dim: h[9], + indexer_head_dim: h[10], + vocab: 0, + raw_live: h[13], + }; + let layer_start = h[11]; + let layer_end = h[12]; + if layer_start != expected_start + || layer_end != expected_end + || layer_start > layer_end + || layer_end >= layout.n_layers + { + return Err(ERR_RANGE.into()); + } + if !layout.raw_live_valid() { + return Err(ERR_LAYOUT.into()); + } + let slice = (layer_end - layer_start + 1) as usize; + let mut off = LAYER_U32_FIELDS * 4; + let mut n_comp = vec![0u32; layout.n_layers as usize]; + let mut n_index_comp = vec![0u32; layout.n_layers as usize]; + for i in 0..slice { + let il = (layer_start as usize) + i; + let v = get_u32(bytes.get(off..).unwrap_or(&[])).ok_or(ERR_UNSUPPORTED_LAYER)?; + if v > layout.comp_cap { + return Err(ERR_COMP_ROWS.into()); + } + n_comp[il] = v; + off += 4; + } + for i in 0..slice { + let il = (layer_start as usize) + i; + let v = get_u32(bytes.get(off..).unwrap_or(&[])).ok_or(ERR_UNSUPPORTED_LAYER)?; + if v > layout.comp_cap { + return Err(ERR_INDEX_ROWS.into()); + } + n_index_comp[il] = v; + off += 4; + } + let mut expected_tensor = 0u64; + for il in layer_start..=layer_end { + let ratio = compress.get(il as usize).copied().unwrap_or(0); + let layer_bytes = layer_tensor_bytes( + &layout, + ratio, + n_comp[il as usize], + n_index_comp[il as usize], + ) + .ok_or(ERR_TENSOR_OVERFLOW)?; + expected_tensor = expected_tensor + .checked_add(layer_bytes) + .ok_or(ERR_TENSOR_OVERFLOW)?; + } + let tensor = bytes.get(off..).unwrap_or(&[]).to_vec(); + if tensor.len() as u64 != expected_tensor { + return Err(ERR_TENSOR_BYTES.into()); + } + Ok(ParsedLayerShard { + layout, + layer_start, + layer_end, + n_comp, + n_index_comp, + tensor, + }) +} + +/// C save merge after shards are gathered: header, tokens, logits, +/// n_comp, n_index_comp, then shard tensor tails in walk order. +pub fn merge_session_payload( + tokens: &[i32], + logits: &[f32], + n_layers: u32, + shards: &[ParsedLayerShard], + compress: &[u32], +) -> Result, String> { + if shards.is_empty() { + return Err(ERR_EMPTY_SHARD.into()); + } + let token_count = u32::try_from(tokens.len()).map_err(|_| ERR_METADATA.to_string())?; + let vocab = u32::try_from(logits.len()).map_err(|_| ERR_METADATA.to_string())?; + let mut layout = shards[0].layout; + layout.vocab = vocab; + if layout.token_count != token_count || layout.n_layers != n_layers || !layout.raw_live_valid() + { + return Err(ERR_METADATA.into()); + } + let mut n_comp = vec![0u32; n_layers as usize]; + let mut n_index_comp = vec![0u32; n_layers as usize]; + let mut covered = vec![false; n_layers as usize]; + for shard in shards { + if !layout.matches_core(&shard.layout) { + return Err(ERR_LAYOUTS_DIFFER.into()); + } + let mut shard_bytes = 0u64; + for il in shard.layer_start..=shard.layer_end { + let i = il as usize; + if covered[i] { + return Err(ERR_RANGE.into()); + } + covered[i] = true; + n_comp[i] = shard.n_comp[i]; + n_index_comp[i] = shard.n_index_comp[i]; + let ratio = compress.get(i).copied().unwrap_or(0); + let layer_bytes = layer_tensor_bytes(&layout, ratio, n_comp[i], n_index_comp[i]) + .ok_or(ERR_TENSOR_OVERFLOW)?; + shard_bytes = shard_bytes + .checked_add(layer_bytes) + .ok_or(ERR_TENSOR_OVERFLOW)?; + } + if shard.tensor.len() as u64 != shard_bytes { + return Err(ERR_TENSOR_BYTES.into()); + } + } + if covered.iter().any(|c| !*c) { + return Err(ERR_METADATA.into()); + } + let mut out = Vec::new(); + for f in [ + SESSION_MAGIC, + SESSION_VERSION, + layout.ctx, + layout.prefill_cap, + layout.raw_cap, + layout.raw_window, + layout.comp_cap, + layout.token_count, + layout.n_layers, + layout.head_dim, + layout.indexer_head_dim, + layout.vocab, + layout.raw_live, + ] { + put_u32(&mut out, f); + } + for &tok in tokens { + put_u32(&mut out, tok as u32); + } + for &logit in logits { + out.extend_from_slice(&logit.to_le_bytes()); + } + for v in &n_comp { + put_u32(&mut out, *v); + } + for v in &n_index_comp { + put_u32(&mut out, *v); + } + for shard in shards { + out.extend_from_slice(&shard.tensor); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn layout() -> DistKvLayout { + DistKvLayout { + ctx: 16, + prefill_cap: 8, + raw_cap: 16, + raw_window: 4, + comp_cap: 8, + token_count: 2, + n_layers: 2, + head_dim: 4, + indexer_head_dim: 2, + vocab: 0, + raw_live: 2, + } + } + + fn encode_layer( + start: u32, + end: u32, + n_comp: &[u32], + n_index: &[u32], + tensor: &[u8], + ) -> Vec { + let l = layout(); + let mut out = Vec::new(); + for f in [ + LAYER_MAGIC, + LAYER_VERSION, + l.ctx, + l.prefill_cap, + l.raw_cap, + l.raw_window, + l.comp_cap, + l.token_count, + l.n_layers, + l.head_dim, + l.indexer_head_dim, + start, + end, + l.raw_live, + ] { + put_u32(&mut out, f); + } + for il in start..=end { + put_u32(&mut out, n_comp[il as usize]); + } + for il in start..=end { + put_u32(&mut out, n_index[il as usize]); + } + out.extend_from_slice(tensor); + out + } + + #[test] + fn layer_tensor_bytes_ratio_zero_is_raw_only() { + let l = layout(); + assert_eq!(layer_tensor_bytes(&l, 0, 3, 9), Some(2 * 4 * 4)); + } + + #[test] + fn parse_and_merge_two_uncompressed_shards() { + let compress = [0u32, 0]; + let t0 = vec![1u8; 32]; + let t1 = vec![2u8; 32]; + let s0 = parse_layer_payload(&encode_layer(0, 0, &[0, 0], &[0, 0], &t0), 0, 0, &compress) + .unwrap(); + let s1 = parse_layer_payload(&encode_layer(1, 1, &[0, 0], &[0, 0], &t1), 1, 1, &compress) + .unwrap(); + let tokens = [7i32, 8]; + let logits = [0.5f32, -1.0]; + let merged = merge_session_payload(&tokens, &logits, 2, &[s0, s1], &compress).unwrap(); + assert_eq!(&merged[..4], &SESSION_MAGIC.to_le_bytes()); + assert_eq!(&merged[4..8], &SESSION_VERSION.to_le_bytes()); + let header = SESSION_U32_FIELDS * 4; + assert_eq!(&merged[header..header + 4], &7u32.to_le_bytes()); + assert_eq!(&merged[header + 4..header + 8], &8u32.to_le_bytes()); + let after_tokens = header + 8; + assert_eq!( + &merged[after_tokens..after_tokens + 4], + &0.5f32.to_le_bytes() + ); + let after_logits = after_tokens + 8; + let after_counts = after_logits + 16; + assert_eq!(&merged[after_counts..after_counts + 32], &t0[..]); + assert_eq!(&merged[after_counts + 32..after_counts + 64], &t1[..]); + } + + #[test] + fn parse_refuses_tensor_byte_mismatch() { + let compress = [0u32, 0]; + let err = parse_layer_payload( + &encode_layer(0, 0, &[0, 0], &[0, 0], &[1, 2, 3]), + 0, + 0, + &compress, + ) + .unwrap_err(); + assert_eq!(err, ERR_TENSOR_BYTES); + } + + #[test] + fn merge_refuses_layout_mismatch() { + let compress = [0u32, 0]; + let t = vec![1u8; 32]; + let s0 = parse_layer_payload(&encode_layer(0, 0, &[0, 0], &[0, 0], &t), 0, 0, &compress) + .unwrap(); + let mut other = encode_layer(1, 1, &[0, 0], &[0, 0], &t); + other[2 * 4..2 * 4 + 4].copy_from_slice(&32u32.to_le_bytes()); + let s1 = parse_layer_payload(&other, 1, 1, &compress).unwrap(); + let err = merge_session_payload(&[7, 8], &[0.0, 0.0], 2, &[s0, s1], &compress).unwrap_err(); + assert_eq!(err, ERR_LAYOUTS_DIFFER); + } +} diff --git a/crates/ds4-dist/src/snapshot_temp.rs b/crates/ds4-dist/src/snapshot_temp.rs new file mode 100644 index 0000000000000000000000000000000000000000..1d3ddda79042021286ee20de9515026ad78951fe --- /dev/null +++ b/crates/ds4-dist/src/snapshot_temp.rs @@ -0,0 +1,99 @@ +//! Bounded temp files for worker KV shard save/load. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::path::PathBuf; + +use crate::codec::SNAPSHOT_CHUNK_BYTES; + +pub const SAVE_PREFIX: &str = "ds4-dist-save"; +pub const LOAD_PREFIX: &str = "ds4-dist-load"; + +pub struct TempShard { + pub(crate) file: File, + pub(crate) path: PathBuf, +} + +impl TempShard { + pub fn path(&self) -> &std::path::Path { + &self.path + } +} + +impl Drop for TempShard { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} + +impl Read for TempShard { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.file.read(buf) + } +} + +impl Write for TempShard { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.file.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.file.flush() + } +} + +impl Seek for TempShard { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + self.file.seek(pos) + } +} + +fn temp_root() -> PathBuf { + #[cfg(unix)] + { + PathBuf::from("/tmp") + } + #[cfg(not(unix))] + { + std::env::temp_dir() + } +} + +pub fn create_temp(prefix: &str) -> io::Result { + let root = temp_root(); + for n in 0u32..1024 { + let path = root.join(format!("{prefix}.{}.{:04x}", std::process::id(), n)); + match OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + { + Ok(file) => return Ok(TempShard { file, path }), + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e), + } + } + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "temp name space exhausted", + )) +} + +pub fn copy_chunked( + src: &mut dyn Read, + dest: &mut dyn Write, + mut remaining: u64, +) -> io::Result<()> { + if remaining == 0 { + return Ok(()); + } + let mut buf = vec![0u8; SNAPSHOT_CHUNK_BYTES]; + while remaining != 0 { + let n = remaining.min(SNAPSHOT_CHUNK_BYTES as u64) as usize; + src.read_exact(&mut buf[..n])?; + dest.write_all(&buf[..n])?; + remaining -= n as u64; + } + Ok(()) +} diff --git a/crates/ds4-dist/src/transport.rs b/crates/ds4-dist/src/transport.rs new file mode 100644 index 0000000000000000000000000000000000000000..514797edb48a92f50be685d634a927282008c0d6 --- /dev/null +++ b/crates/ds4-dist/src/transport.rs @@ -0,0 +1,30 @@ +//! Blocking frame I/O. Same layout as `dist_write_frame_header` / `dist_read_frame_header`. + +use std::io::{self, Read, Write}; + +use crate::codec::{decode_frame_header, encode_frame_header, CodecError, FRAME_HEADER_BYTES}; + +pub fn write_frame(w: &mut W, typ: u32, payload: &[u8]) -> io::Result<()> { + let bytes = u32::try_from(payload.len()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "distributed frame too large"))?; + w.write_all(&encode_frame_header(typ, bytes))?; + w.write_all(payload)?; + Ok(()) +} + +pub fn read_frame(r: &mut R) -> io::Result<(u32, Vec)> { + let mut hdr = [0u8; FRAME_HEADER_BYTES]; + r.read_exact(&mut hdr)?; + let h = decode_frame_header(&hdr).map_err(|e| match e { + CodecError::BadMagic(m) => io::Error::new( + io::ErrorKind::InvalidData, + format!("bad frame magic 0x{m:08x}"), + ), + other => io::Error::new(io::ErrorKind::InvalidData, other.to_string()), + })?; + let mut payload = vec![0u8; h.bytes as usize]; + if h.bytes != 0 { + r.read_exact(&mut payload)?; + } + Ok((h.typ, payload)) +} diff --git a/crates/ds4-dist/src/work.rs b/crates/ds4-dist/src/work.rs new file mode 100644 index 0000000000000000000000000000000000000000..d709f1656a7b2c3dd8ce720c7137868fd09a9172 --- /dev/null +++ b/crates/ds4-dist/src/work.rs @@ -0,0 +1,207 @@ +//! WORK / RESULT frame bodies. Tokens are BE u32; activations use the packers. + +use crate::activation::{bits_or_default, decode_activation, encode_activation}; +use crate::codec::{ + decode_tokens_be, encode_frame_header, encode_tokens_be, u64_from_halves, u64_to_halves, + CodecError, ResultHdr, Telemetry, Work, MSG_RESULT, MSG_WORK, RESULT_FIXED_BYTES, + TELEMETRY_FIXED_BYTES, WORK_FIXED_BYTES, +}; + +#[derive(Debug, Clone)] +pub struct WorkBody { + pub work: Work, + pub tokens: Vec, + pub input_hc: Vec, + pub route_blob: Vec, +} + +pub fn encode_work_body(body: &WorkBody) -> Result, CodecError> { + if body.tokens.is_empty() { + return Err(CodecError::Invalid("WORK frame has no tokens")); + } + let mut work = body.work; + work.n_tokens = body.tokens.len() as u32; + work.token_bytes = work.n_tokens * 4; + let hc_bits = bits_or_default(work.input_hc_bits); + let hc_wire = if body.input_hc.is_empty() { + Vec::new() + } else { + encode_activation(&body.input_hc, hc_bits) + .ok_or(CodecError::Invalid("invalid distributed activation width"))? + }; + work.input_hc_bytes = hc_wire.len() as u32; + work.input_hc_bits = hc_bits; + work.route_bytes = body.route_blob.len() as u32; + let mut out = work.encode().to_vec(); + out.extend_from_slice(&encode_tokens_be(&body.tokens)); + out.extend_from_slice(&hc_wire); + out.extend_from_slice(&body.route_blob); + Ok(out) +} + +pub fn encode_work_frame(body: &WorkBody) -> Result, CodecError> { + let payload = encode_work_body(body)?; + let mut frame = encode_frame_header(MSG_WORK, payload.len() as u32).to_vec(); + frame.extend_from_slice(&payload); + Ok(frame) +} + +pub fn decode_work_body(buf: &[u8]) -> Result { + if buf.len() < WORK_FIXED_BYTES { + return Err(CodecError::Invalid("truncated distributed WORK frame")); + } + let work = Work::decode(buf)?; + let rest = &buf[WORK_FIXED_BYTES..]; + let expected = + work.token_bytes as usize + work.input_hc_bytes as usize + work.route_bytes as usize; + if work.token_bytes as usize != work.n_tokens as usize * 4 || rest.len() != expected { + return Err(CodecError::Invalid( + "invalid distributed WORK payload sizes", + )); + } + let (tok, rest) = rest.split_at(work.token_bytes as usize); + let tokens = decode_tokens_be(tok)?; + let (hc, rest) = rest.split_at(work.input_hc_bytes as usize); + let input_hc = if hc.is_empty() { + Vec::new() + } else { + decode_activation(hc, work.input_hc_bits) + .ok_or(CodecError::Invalid("invalid distributed activation width"))? + }; + Ok(WorkBody { + work, + tokens, + input_hc, + route_blob: rest.to_vec(), + }) +} + +#[derive(Debug, Clone)] +pub struct ResultBody { + pub hdr: ResultHdr, + pub telemetry: Vec, + pub payload: Vec, +} + +pub fn encode_result_body(body: &ResultBody) -> Result, CodecError> { + let mut hdr = body.hdr; + hdr.telemetry_count = body.telemetry.len() as u32; + hdr.telemetry_bytes = hdr.telemetry_count * TELEMETRY_FIXED_BYTES as u32; + hdr.payload_bytes = body.payload.len() as u32; + let mut out = hdr.encode().to_vec(); + for t in &body.telemetry { + out.extend_from_slice(&t.encode()); + } + out.extend_from_slice(&body.payload); + Ok(out) +} + +pub fn encode_result_frame(body: &ResultBody) -> Result, CodecError> { + let payload = encode_result_body(body)?; + let mut frame = encode_frame_header(MSG_RESULT, payload.len() as u32).to_vec(); + frame.extend_from_slice(&payload); + Ok(frame) +} + +pub fn decode_result_body(buf: &[u8]) -> Result { + if buf.len() < RESULT_FIXED_BYTES { + return Err(CodecError::Invalid( + "distributed worker returned invalid frame", + )); + } + let hdr = ResultHdr::decode(buf)?; + let rest = &buf[RESULT_FIXED_BYTES..]; + if hdr.telemetry_bytes as usize % TELEMETRY_FIXED_BYTES != 0 + || hdr.telemetry_count as usize != hdr.telemetry_bytes as usize / TELEMETRY_FIXED_BYTES + || hdr.telemetry_bytes as usize > rest.len() + || hdr.payload_bytes as usize != rest.len() - hdr.telemetry_bytes as usize + { + return Err(CodecError::Invalid( + "distributed result telemetry metadata mismatch", + )); + } + let (tel, rest) = rest.split_at(hdr.telemetry_bytes as usize); + let mut telemetry = Vec::new(); + for chunk in tel.chunks_exact(TELEMETRY_FIXED_BYTES) { + telemetry.push(Telemetry::decode(chunk)?); + } + if rest.len() != hdr.payload_bytes as usize { + return Err(CodecError::Truncated); + } + Ok(ResultBody { + hdr, + telemetry, + payload: rest.to_vec(), + }) +} + +pub fn result_request_id(hdr: &ResultHdr) -> u64 { + u64_from_halves(hdr.request_hi, hdr.request_lo) +} + +pub fn result_hash(hdr: &ResultHdr) -> u64 { + u64_from_halves(hdr.result_hash_hi, hdr.result_hash_lo) +} + +pub fn encode_logits_payload(logits: &[f32]) -> Vec { + encode_activation(logits, 32).unwrap_or_default() +} + +pub fn decode_logits_payload(buf: &[u8]) -> Result, CodecError> { + decode_activation(buf, 32).ok_or(CodecError::Invalid("invalid logits payload")) +} + +pub fn work_with_ids(mut work: Work, session: u64, request: u64, prefix: u64, result: u64) -> Work { + let (shi, slo) = u64_to_halves(session); + let (rhi, rlo) = u64_to_halves(request); + let (phi, plo) = u64_to_halves(prefix); + let (ehi, elo) = u64_to_halves(result); + work.session_hi = shi; + work.session_lo = slo; + work.request_hi = rhi; + work.request_lo = rlo; + work.prefix_hash_hi = phi; + work.prefix_hash_lo = plo; + work.result_hash_hi = ehi; + work.result_hash_lo = elo; + work +} + +pub fn ok_result_hdr(request: u64, hash: u64, kind: u32, bits: u32) -> ResultHdr { + let (rhi, rlo) = u64_to_halves(request); + let (hhi, hlo) = u64_to_halves(hash); + ResultHdr { + request_hi: rhi, + request_lo: rlo, + result_hash_hi: hhi, + result_hash_lo: hlo, + status: 0, + result_kind: kind, + telemetry_count: 0, + telemetry_bytes: 0, + payload_bytes: 0, + payload_bits: bits, + } +} + +pub fn error_result_frame(request: u64, msg: &str) -> Vec { + let (rhi, rlo) = u64_to_halves(request); + let payload = msg.as_bytes().to_vec(); + let body = ResultBody { + hdr: ResultHdr { + request_hi: rhi, + request_lo: rlo, + result_hash_hi: 0, + result_hash_lo: 0, + status: 1, + result_kind: 0, + telemetry_count: 0, + telemetry_bytes: 0, + payload_bytes: payload.len() as u32, + payload_bits: 0, + }, + telemetry: Vec::new(), + payload, + }; + encode_result_frame(&body).unwrap_or_else(|_| crate::codec::encode_error_frame(msg)) +} diff --git a/crates/ds4-dist/src/worker.rs b/crates/ds4-dist/src/worker.rs new file mode 100644 index 0000000000000000000000000000000000000000..593fdeae84b0022f2faeaef7a2b24a99d4625bd3 --- /dev/null +++ b/crates/ds4-dist/src/worker.rs @@ -0,0 +1,424 @@ +//! Worker control loop: HELLO, WORK parse/validate, slice eval, RESULT. + +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::sync::{Arc, Mutex}; + +use crate::activation::{bits_or_default, wire_bytes}; +use crate::codec::{ + decode_hello_payload, encode_hello_payload, u64_from_halves, CodecError, Hello, Work, + MSG_ERROR, MSG_HELLO, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK, RESULT_ACK, + RESULT_HIDDEN_STATE, RESULT_LOGITS, ROUTE_F_OUTPUT_LOGITS, ROUTE_RETURN_UPSTREAM, + WORK_FIXED_BYTES, WORK_F_ACK_ONLY, WORK_F_INPUT_HC, WORK_F_OUTPUT_LOGITS, WORK_F_RESET_SESSION, + WORK_F_VALID_MASK, +}; +use crate::exec::{SliceExec, WorkRequest}; +use crate::forward::ERR_FORWARD_HIDDEN; +use crate::hash::{token_hash_update_span, TOKEN_HASH_INIT}; +use crate::hops::ForwarderPool; +use crate::native_snapshot::{dispatch_worker_snapshot, MemorySnapshotStore, SnapshotStore}; +use crate::relay::{forward_work_blocking, local_work_telemetry, now_sec, usec_since}; +use crate::route::{decode_route_blob, validate_route_blob}; +use crate::transport::{read_frame, write_frame}; +use crate::work::{ + decode_work_body, encode_logits_payload, encode_result_frame, error_result_frame, + ok_result_hdr, ResultBody, WorkBody, +}; +use crate::worker_snapshot::WorkerSnapshotIdentity; + +pub fn send_hello(w: &mut W, hello: &Hello, model_name: &str) -> std::io::Result<()> { + let payload = encode_hello_payload(hello, model_name) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e.to_string()))?; + write_frame(w, MSG_HELLO, &payload) +} + +pub fn recv_hello(r: &mut R) -> std::io::Result<(Hello, String)> { + let (typ, body) = read_frame(r)?; + if typ != MSG_HELLO { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("expected HELLO frame, got type {typ}"), + )); + } + decode_hello_payload(&body) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) +} + +pub struct Worker { + pub exec: E, + sessions: HashMap, + store: S, + hops: ForwarderPool, +} + +impl Worker { + pub fn new(exec: E) -> Self { + Self::with_store(exec, MemorySnapshotStore::new()) + } +} + +impl Worker { + pub fn with_store(exec: E, store: S) -> Self { + Self { + exec, + sessions: HashMap::new(), + store, + hops: ForwarderPool::new(), + } + } + + pub fn store(&self) -> &S { + &self.store + } + + pub fn store_mut(&mut self) -> &mut S { + &mut self.store + } + + pub fn session_count(&self) -> usize { + self.sessions.len() + } + + pub fn clear_sessions(&mut self) -> u32 { + let n = u32::try_from(self.sessions.len()).unwrap_or(u32::MAX); + self.sessions.clear(); + n + } + + pub(crate) fn apply_snapshot( + &mut self, + stream: &mut Stream, + typ: u32, + body: &[u8], + ) -> std::io::Result<()> { + match dispatch_worker_snapshot( + stream, + typ, + body, + self.snapshot_identity(), + self.exec.vocab(), + &mut self.store, + ) { + Ok(Some((session_id, token_hash))) => { + self.sessions.insert(session_id, token_hash); + } + Ok(None) => {} + Err(_) => {} + } + Ok(()) + } + + fn snapshot_identity(&self) -> WorkerSnapshotIdentity { + WorkerSnapshotIdentity { + model_id: self.exec.model_id(), + layer_start: self.exec.layer_start(), + layer_end: self.exec.layer_end(), + ctx_size: self.exec.ctx_size(), + } + } + + pub fn hello(&self, listen_port: u32, ctx_size: u32, model_name: &str) -> (Hello, String) { + ( + Hello { + model_id: self.exec.model_id(), + quant_bits: 0, + layer_start: self.exec.layer_start(), + layer_end: self.exec.layer_end(), + has_output: if self.exec.has_output() { 1 } else { 0 }, + has_hidden: 1, + ctx_size, + n_layers: self.exec.n_layers(), + listen_port, + model_name_len: 0, + }, + model_name.to_string(), + ) + } + + pub(crate) fn bind_hops(&mut self, upstream: Arc>) { + self.hops.bind(upstream); + } + + pub(crate) fn shutdown_hops(&mut self) { + self.hops.shutdown(); + } + + pub fn serve(&mut self, stream: &mut TcpStream) -> std::io::Result<()> { + if let Ok(clone) = stream.try_clone() { + self.bind_hops(Arc::new(Mutex::new(clone))); + } + let rc = self.drive(stream); + self.shutdown_hops(); + rc + } + + pub fn serve_once(&mut self, stream: &mut TcpStream) -> std::io::Result<()> { + self.drive_one(stream) + } + + fn drive(&mut self, stream: &mut Stream) -> std::io::Result<()> { + loop { + match self.drive_one(stream) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()), + Err(e) => return Err(e), + } + } + } + + fn drive_one(&mut self, stream: &mut Stream) -> std::io::Result<()> { + let (typ, body) = read_frame(stream)?; + if typ == MSG_ERROR { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("coordinator error: {}", String::from_utf8_lossy(&body)), + )); + } + match typ { + MSG_WORK => { + let reply = self.process_work(&body, stream)?; + if let Some(frame) = reply { + stream.write_all(&frame)?; + } + Ok(()) + } + MSG_SNAPSHOT_SAVE_REQ | MSG_SNAPSHOT_LOAD_BEGIN => { + self.apply_snapshot(stream, typ, &body) + } + _ => { + write_frame(stream, MSG_ERROR, b"unsupported distributed worker frame")?; + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("rejected unsupported frame type {typ}"), + )) + } + } + } + + pub(crate) fn process_work( + &mut self, + payload: &[u8], + stream: &mut W, + ) -> std::io::Result>> { + if payload.len() < WORK_FIXED_BYTES { + return Ok(Some(error_result_frame( + 0, + "truncated distributed WORK frame", + ))); + } + let request_id = match Work::decode(payload) { + Ok(w) => u64_from_halves(w.request_hi, w.request_lo), + Err(_) => 0, + }; + let body = match decode_work_body(payload) { + Ok(b) => b, + Err(e) => { + let msg = match e { + CodecError::Invalid(s) => s, + _ => "truncated distributed WORK frame", + }; + return Ok(Some(error_result_frame(request_id, msg))); + } + }; + match self.handle_decoded(body, stream) { + Ok(frame) => Ok(frame), + Err(msg) => Ok(Some(error_result_frame(request_id, &msg))), + } + } + + fn handle_decoded( + &mut self, + body: WorkBody, + stream: &mut W, + ) -> Result>, String> { + let work = body.work; + let request_id = u64_from_halves(work.request_hi, work.request_lo); + let session_id = u64_from_halves(work.session_hi, work.session_lo); + let prefix_hash = u64_from_halves(work.prefix_hash_hi, work.prefix_hash_lo); + let result_hash = u64_from_halves(work.result_hash_hi, work.result_hash_lo); + + let fail = |m: String| -> Result>, String> { + Ok(Some(error_result_frame(request_id, &m))) + }; + + if work.route_count == 0 { + return fail("WORK frame is missing distributed route".into()); + } + if work.route_index >= work.route_count { + return fail("invalid distributed WORK route metadata".into()); + } + if work.model_id != self.exec.model_id() { + return fail(format!( + "model id mismatch: work={} worker={}", + work.model_id, + self.exec.model_id() + )); + } + if work.layer_start != self.exec.layer_start() || work.layer_end != self.exec.layer_end() { + return fail(format!( + "worker is assigned layers {}:{} but request asked for {}:{}", + self.exec.layer_start(), + self.exec.layer_end(), + work.layer_start, + work.layer_end + )); + } + if (work.flags & !WORK_F_VALID_MASK) != 0 { + return fail("invalid distributed WORK flags".into()); + } + if body.tokens.is_empty() { + return fail("WORK frame has no tokens".into()); + } + let ctx = self.exec.ctx_size(); + if work.pos0 > ctx || work.n_tokens > ctx.saturating_sub(work.pos0) { + return fail("WORK token span exceeds worker context".into()); + } + let output_logits = (work.flags & WORK_F_OUTPUT_LOGITS) != 0; + let input_hc_present = (work.flags & WORK_F_INPUT_HC) != 0; + let ack_only = (work.flags & WORK_F_ACK_ONLY) != 0; + if input_hc_present && work.layer_start == 0 { + return fail("layer 0 WORK must not provide input hidden-state".into()); + } + if !input_hc_present && work.layer_start != 0 { + return fail("nonzero layer WORK requires input hidden-state".into()); + } + if output_logits && !self.exec.has_output() { + return fail("worker was not assigned the output head".into()); + } + if output_logits && work.layer_end + 1 != self.exec.n_layers() { + return fail("WORK logits require final transformer layer".into()); + } + for &t in &body.tokens { + if t < 0 || (t as u32) >= self.exec.vocab() { + return fail("WORK token id is outside the model vocabulary".into()); + } + } + if token_hash_update_span(prefix_hash, &body.tokens) != result_hash { + return fail("WORK token prefix hash metadata mismatch".into()); + } + + let hc_values = self.exec.hidden_values(); + let expected_hc = (work.n_tokens as u64).saturating_mul(hc_values) as usize; + if input_hc_present && body.input_hc.len() != expected_hc { + return fail("input hidden-state size does not match token span".into()); + } + if !input_hc_present && !body.input_hc.is_empty() { + return fail("WORK frame has hidden bytes without input flag".into()); + } + + if let Err(e) = + validate_route_blob(&body.route_blob, work.route_count, self.exec.n_layers()) + { + return fail(e.to_string()); + } + let (entries, ret) = + decode_route_blob(&body.route_blob, work.route_count).map_err(|e| e.to_string())?; + let current = entries + .get(work.route_index as usize) + .ok_or_else(|| "invalid route entry index".to_string())?; + if current.layer_start != work.layer_start || current.layer_end != work.layer_end { + return fail("WORK layer range does not match route entry".into()); + } + let route_out = (current.flags & ROUTE_F_OUTPUT_LOGITS) != 0; + if route_out != output_logits { + return fail("WORK logits flag does not match route entry".into()); + } + let has_next = (work.route_index + 1) < work.route_count; + if has_next && output_logits { + return fail("non-final route entry requested logits".into()); + } + if !has_next && ret.kind != ROUTE_RETURN_UPSTREAM { + return fail("unsupported final result destination".into()); + } + + let reset = (work.flags & WORK_F_RESET_SESSION) != 0; + if reset { + self.sessions.insert(session_id, TOKEN_HASH_INIT); + } + let sess_hash = *self.sessions.entry(session_id).or_insert(TOKEN_HASH_INIT); + if sess_hash != prefix_hash { + return fail("worker KV prefix hash mismatch".into()); + } + + let final_ack_only = ack_only && !has_next; + let local_output = output_logits && !has_next && !final_ack_only; + let produce_hidden = !local_output && !final_ack_only; + let eval_t0 = now_sec(); + let out = match self.exec.eval(&WorkRequest { + session_id, + request_id, + tokens: body.tokens.clone(), + pos0: work.pos0, + layer_start: work.layer_start, + layer_end: work.layer_end, + reset, + produce_hidden, + produce_logits: local_output, + input_hc: body.input_hc.clone(), + }) { + Ok(v) => v, + Err(e) => return fail(e), + }; + let eval_usec = usec_since(eval_t0, now_sec()); + self.sessions.insert(session_id, result_hash); + + if has_next { + let next = entries[(work.route_index + 1) as usize].clone(); + let mut fwd = body.clone(); + fwd.work.layer_start = next.layer_start; + fwd.work.layer_end = next.layer_end; + fwd.work.route_index = work.route_index + 1; + fwd.work.flags |= WORK_F_INPUT_HC; + if (next.flags & ROUTE_F_OUTPUT_LOGITS) != 0 { + fwd.work.flags |= WORK_F_OUTPUT_LOGITS; + } else { + fwd.work.flags &= !WORK_F_OUTPUT_LOGITS; + } + let hidden = out.hidden.unwrap_or_default(); + let output_bytes = + match wire_bytes(bits_or_default(work.input_hc_bits), hidden.len() as u64) { + Some(n) => n, + None => return fail(ERR_FORWARD_HIDDEN.into()), + }; + let telemetry = local_work_telemetry(&work, eval_usec, output_bytes); + fwd.input_hc = hidden; + let hop = if self.hops.is_bound() { + self.hops.forward(&next, &fwd, telemetry) + } else { + forward_work_blocking(&next, &fwd, stream, telemetry) + }; + if let Err(msg) = hop { + return fail(msg); + } + return Ok(None); + } + + let (kind, payload, bits) = if final_ack_only { + (RESULT_ACK, Vec::new(), 0u32) + } else if local_output { + let logits = out + .logits + .ok_or_else(|| "slice exec returned no logits".to_string())?; + (RESULT_LOGITS, encode_logits_payload(&logits), 32) + } else { + let hidden = out + .hidden + .ok_or_else(|| "slice exec returned no hidden-state".to_string())?; + let bits = bits_or_default(work.input_hc_bits); + let wire = crate::activation::encode_activation(&hidden, bits) + .ok_or_else(|| "invalid output hidden-state size".to_string())?; + (RESULT_HIDDEN_STATE, wire, bits) + }; + let mut hdr = ok_result_hdr(request_id, result_hash, kind, bits); + hdr.payload_bytes = payload.len() as u32; + let telemetry = local_work_telemetry(&work, eval_usec, payload.len() as u32); + let frame = encode_result_frame(&ResultBody { + hdr, + telemetry: vec![telemetry], + payload, + }) + .map_err(|e| e.to_string())?; + Ok(Some(frame)) + } +} diff --git a/crates/ds4-dist/src/worker_snapshot.rs b/crates/ds4-dist/src/worker_snapshot.rs new file mode 100644 index 0000000000000000000000000000000000000000..d1b397206d8ccc4e26d26c915cede144fb92055d --- /dev/null +++ b/crates/ds4-dist/src/worker_snapshot.rs @@ -0,0 +1,291 @@ +//! Worker-side DS4D snapshot save/load frame handling. + +use std::io::{Read, Write}; + +use crate::codec::{ + decode_snapshot_chunk_body, decode_snapshot_load_begin_body, encode_snapshot_begin_body, + encode_snapshot_chunk_body, encode_snapshot_done_body, u64_from_halves, u64_to_halves, + SnapshotBegin, SnapshotReq, MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, MSG_SNAPSHOT_DONE, + SNAPSHOT_BEGIN_FIXED_BYTES, SNAPSHOT_CHUNK_BYTES, SNAPSHOT_REQ_FIXED_BYTES, +}; +use crate::hash::token_hash_prefix; +use crate::transport::{read_frame, write_frame}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WorkerSnapshotIdentity { + pub model_id: u32, + pub layer_start: u32, + pub layer_end: u32, + pub ctx_size: u32, +} + +fn write_begin( + stream: &mut S, + begin: &SnapshotBegin, + tokens: &[i32], + message: &[u8], +) -> Result<(), String> { + let body = encode_snapshot_begin_body(begin, tokens, message).map_err(|e| e.to_string())?; + write_frame(stream, MSG_SNAPSHOT_BEGIN, &body).map_err(|e| e.to_string()) +} + +fn write_error_begin( + stream: &mut S, + identity: WorkerSnapshotIdentity, + request_id: u64, + session_id: u64, + message: &str, +) -> Result<(), String> { + let (session_hi, session_lo) = u64_to_halves(session_id); + let (request_hi, request_lo) = u64_to_halves(request_id); + let begin = SnapshotBegin { + model_id: identity.model_id, + session_hi, + session_lo, + request_hi, + request_lo, + token_hash_hi: 0, + token_hash_lo: 0, + token_count: 0, + layer_start: identity.layer_start, + layer_end: identity.layer_end, + payload_hi: 0, + payload_lo: 0, + status: 1, + token_bytes: 0, + message_bytes: message.len() as u32, + }; + write_begin(stream, &begin, &[], message.as_bytes()) +} + +fn write_done( + stream: &mut S, + request_id: u64, + status: u32, + message: &str, +) -> Result<(), String> { + let body = encode_snapshot_done_body(request_id, status, message.as_bytes()) + .map_err(|e| e.to_string())?; + write_frame(stream, MSG_SNAPSHOT_DONE, &body).map_err(|e| e.to_string()) +} + +fn write_chunks( + stream: &mut S, + request_id: u64, + payload: &mut R, + mut remaining: u64, +) -> Result<(), String> { + let mut buf = Vec::new(); + if remaining != 0 { + buf.try_reserve_exact(SNAPSHOT_CHUNK_BYTES) + .map_err(|_| "out of memory sending worker KV shard".to_string())?; + buf.resize(SNAPSHOT_CHUNK_BYTES, 0); + } + while remaining != 0 { + let n = remaining.min(SNAPSHOT_CHUNK_BYTES as u64) as usize; + payload + .read_exact(&mut buf[..n]) + .map_err(|_| "failed to read worker KV shard".to_string())?; + let body = encode_snapshot_chunk_body(request_id, &buf[..n]).map_err(|e| e.to_string())?; + write_frame(stream, MSG_SNAPSHOT_CHUNK, &body).map_err(|e| e.to_string())?; + remaining -= n as u64; + } + Ok(()) +} + +impl WorkerSnapshotIdentity { + pub fn accepts(self, model_id: u32, start: u32, end: u32, tokens: u32) -> bool { + model_id == self.model_id + && start == self.layer_start + && end == self.layer_end + && tokens <= self.ctx_size + } +} + +/// Answer one SAVE_REQ body with BEGIN+CHUNK+DONE, or a C error BEGIN. +pub fn worker_handle_snapshot_save( + stream: &mut S, + identity: WorkerSnapshotIdentity, + req_body: &[u8], + payload: Result<(&mut R, u64), String>, +) -> Result<(), String> +where + S: Write, + R: Read, +{ + if req_body.len() != SNAPSHOT_REQ_FIXED_BYTES { + return write_error_begin( + stream, + identity, + 0, + 0, + "invalid distributed snapshot save request", + ); + } + let req = SnapshotReq::decode(req_body).map_err(|e| e.to_string())?; + let request_id = u64_from_halves(req.request_hi, req.request_lo); + let session_id = u64_from_halves(req.session_hi, req.session_lo); + if !identity.accepts( + req.model_id, + req.layer_start, + req.layer_end, + req.token_count, + ) { + return write_error_begin( + stream, + identity, + request_id, + session_id, + "snapshot save request does not match worker state", + ); + } + let (payload, payload_bytes) = match payload { + Ok(ok) => ok, + Err(message) => { + return write_error_begin(stream, identity, request_id, session_id, &message); + } + }; + let (payload_hi, payload_lo) = u64_to_halves(payload_bytes); + let begin = SnapshotBegin { + model_id: identity.model_id, + session_hi: req.session_hi, + session_lo: req.session_lo, + request_hi: req.request_hi, + request_lo: req.request_lo, + token_hash_hi: req.token_hash_hi, + token_hash_lo: req.token_hash_lo, + token_count: 0, + layer_start: identity.layer_start, + layer_end: identity.layer_end, + payload_hi, + payload_lo, + status: 0, + token_bytes: 0, + message_bytes: 0, + }; + write_begin(stream, &begin, &[], b"")?; + write_chunks(stream, request_id, payload, payload_bytes)?; + write_done(stream, request_id, 0, "") +} + +/// Consume one LOAD_BEGIN body plus following CHUNKs, then send DONE. +pub fn worker_handle_snapshot_load( + stream: &mut S, + identity: WorkerSnapshotIdentity, + begin_body: &[u8], + vocab_size: u32, + payload: &mut W, +) -> Result +where + S: Read + Write, + W: Write, +{ + worker_handle_snapshot_load_restore(stream, identity, begin_body, vocab_size, payload, |_| { + Ok(()) + }) +} + +/// Same as [`worker_handle_snapshot_load`], but runs `restore` before DONE. +pub fn worker_handle_snapshot_load_restore( + stream: &mut S, + identity: WorkerSnapshotIdentity, + begin_body: &[u8], + vocab_size: u32, + payload: &mut W, + restore: F, +) -> Result +where + S: Read + Write, + W: Write, + F: FnOnce(&WorkerLoadOffer) -> Result<(), String>, +{ + if begin_body.len() < SNAPSHOT_BEGIN_FIXED_BYTES { + return Err("invalid distributed snapshot load header".into()); + } + + let Ok((begin, tokens, _)) = decode_snapshot_load_begin_body(begin_body) else { + write_done(stream, 0, 1, "invalid distributed snapshot load header")?; + return Err("invalid distributed snapshot load header".into()); + }; + let request_id = u64_from_halves(begin.request_hi, begin.request_lo); + let session_id = u64_from_halves(begin.session_hi, begin.session_lo); + let token_hash = u64_from_halves(begin.token_hash_hi, begin.token_hash_lo); + let payload_bytes = u64_from_halves(begin.payload_hi, begin.payload_lo); + let mut err = if tokens + .iter() + .any(|token| *token < 0 || (*token as u32) >= vocab_size) + { + "snapshot token id is outside the model vocabulary" + } else if token_hash_prefix(&tokens) != token_hash { + "snapshot load token hash mismatch" + } else if !identity.accepts( + begin.model_id, + begin.layer_start, + begin.layer_end, + begin.token_count, + ) { + "snapshot load request does not match worker state" + } else { + "" + } + .to_string(); + + let mut received = 0u64; + while err.is_empty() && received < payload_bytes { + let (typ, body) = match read_frame(stream) { + Ok(frame) => frame, + Err(_) => { + let _ = write_done(stream, request_id, 1, "expected distributed snapshot chunk"); + return Err("expected distributed snapshot chunk".into()); + } + }; + if typ != MSG_SNAPSHOT_CHUNK { + err = "expected distributed snapshot chunk".into(); + break; + } + match decode_snapshot_chunk_body(&body, request_id, payload_bytes - received) { + Ok((_, chunk)) => { + if payload.write_all(chunk).is_err() { + err = "failed to write worker KV shard temp file".into(); + break; + } + received += chunk.len() as u64; + } + Err(_) => { + err = "invalid distributed snapshot chunk".into(); + break; + } + } + } + + if err.is_empty() && payload.flush().is_err() { + err = "failed to flush worker KV shard restore file".into(); + } + let offer = WorkerLoadOffer { + session_id, + request_id, + token_hash, + tokens, + payload_bytes, + }; + if err.is_empty() { + if let Err(message) = restore(&offer) { + err = message; + } + } + let status = u32::from(!err.is_empty()); + write_done(stream, request_id, status, &err)?; + if !err.is_empty() { + return Err(err); + } + Ok(offer) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WorkerLoadOffer { + pub session_id: u64, + pub request_id: u64, + pub token_hash: u64, + pub tokens: Vec, + pub payload_bytes: u64, +} diff --git a/crates/ds4-dist/tests/forward.rs b/crates/ds4-dist/tests/forward.rs new file mode 100644 index 0000000000000000000000000000000000000000..b6b7a5ea5f99496b364d88b2a10fc0257e7e3408 --- /dev/null +++ b/crates/ds4-dist/tests/forward.rs @@ -0,0 +1,128 @@ +//! Forward-window env parse and pending-request queue. + +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use ds4_dist::{ + forward_window_from, opened_forwarder_message, PendingQueue, PendingRequest, Telemetry, + ERR_CLOSED_WHILE_RESULT, ERR_FORWARD, ERR_FORWARD_HIDDEN, ERR_INVALID_RESULT, ERR_NEXT_CLOSED, + ERR_OOM_FORWARDER, ERR_OOM_TRACK, ERR_RELAY_THREAD, ERR_RESULT_METADATA, ERR_RESULT_TOO_LARGE, + ERR_TELEMETRY_TOO_LARGE, FORWARD_WINDOW_DEFAULT, FORWARD_WINDOW_MAX, FORWARD_WINDOW_MIN, +}; + +fn pending(id: u64) -> PendingRequest { + PendingRequest { + request_id: id, + telemetry: Telemetry { + layer_start: 0, + layer_end: 1, + route_index: 0, + pos0: 0, + n_tokens: 1, + eval_usec: 0, + downstream_wait_usec: 0, + forward_send_usec: 0, + input_bytes: 0, + output_bytes: 0, + }, + downstream_t0: 0.0, + } +} + +#[test] +fn forward_window_matches_c() { + assert_eq!(forward_window_from(None), FORWARD_WINDOW_DEFAULT); + assert_eq!(forward_window_from(Some("")), FORWARD_WINDOW_DEFAULT); + assert_eq!(forward_window_from(Some("4")), 4); + assert_eq!(forward_window_from(Some("1")), FORWARD_WINDOW_MIN); + assert_eq!(forward_window_from(Some("64")), FORWARD_WINDOW_MAX); + assert_eq!(forward_window_from(Some("0")), FORWARD_WINDOW_DEFAULT); + assert_eq!(forward_window_from(Some("65")), FORWARD_WINDOW_DEFAULT); + assert_eq!(forward_window_from(Some("-1")), FORWARD_WINDOW_DEFAULT); + assert_eq!(forward_window_from(Some("8 ")), FORWARD_WINDOW_DEFAULT); + assert_eq!(forward_window_from(Some(" 8")), 8); + assert_eq!(forward_window_from(Some("+16")), 16); +} + +#[test] +fn forwarder_strings_match_c() { + assert_eq!( + opened_forwarder_message("10.0.0.2", 7100, 4), + "ds4: distributed worker: opened pipelined worker-to-worker connection to 10.0.0.2:7100 (window 4)" + ); + assert_eq!(ERR_NEXT_CLOSED, "next worker closed connection"); + assert_eq!( + ERR_INVALID_RESULT, + "next worker did not return valid RESULT" + ); + assert_eq!( + ERR_CLOSED_WHILE_RESULT, + "next worker closed while returning RESULT" + ); + assert_eq!(ERR_RESULT_METADATA, "next worker RESULT metadata mismatch"); + assert_eq!( + ERR_TELEMETRY_TOO_LARGE, + "distributed telemetry chain is too large" + ); + assert_eq!( + ERR_RESULT_TOO_LARGE, + "distributed RESULT frame is too large" + ); + assert_eq!( + ERR_OOM_FORWARDER, + "out of memory creating worker-to-worker forwarder" + ); + assert_eq!( + ERR_RELAY_THREAD, + "failed to start worker-to-worker relay thread" + ); + assert_eq!(ERR_OOM_TRACK, "out of memory tracking forwarded request"); + assert_eq!(ERR_FORWARD, "failed to forward distributed work"); + assert_eq!(ERR_FORWARD_HIDDEN, "invalid forwarded hidden-state size"); +} + +#[test] +fn pending_queue_blocks_at_window_and_pop_unblocks() { + let q = PendingQueue::new(2); + assert_eq!(q.depth(), 2); + assert!(q.enqueue(pending(1)).is_ok()); + assert!(q.enqueue(pending(2)).is_ok()); + let (tx, rx) = mpsc::channel(); + thread::scope(|s| { + s.spawn(|| { + assert!(q.enqueue(pending(3)).is_ok()); + tx.send(()).unwrap(); + }); + thread::sleep(Duration::from_millis(30)); + assert!(rx.try_recv().is_err()); + assert_eq!(q.pop().unwrap().request_id, 1); + rx.recv_timeout(Duration::from_secs(2)).unwrap(); + }); + assert_eq!(q.pop().unwrap().request_id, 2); + assert_eq!(q.pop().unwrap().request_id, 3); + assert!(q.pop().is_none()); +} + +#[test] +fn pending_queue_remove_and_note_send_done() { + let q = PendingQueue::new(4); + assert!(q.enqueue(pending(9)).is_ok()); + assert!(q.enqueue(pending(10)).is_ok()); + q.note_send_done(10, 42, 1.5); + assert!(q.remove(9)); + let got = q.pop().unwrap(); + assert_eq!(got.request_id, 10); + assert_eq!(got.telemetry.forward_send_usec, 42); + assert_eq!(got.downstream_t0, 1.5); + assert!(!q.remove(9)); +} + +#[test] +fn pending_queue_close_rejects_enqueue() { + let q = PendingQueue::new(2); + q.close(); + assert_eq!(q.enqueue(pending(1)), Err(ERR_NEXT_CLOSED)); + q.clear(); + assert!(q.pop().is_none()); +} diff --git a/crates/ds4-dist/tests/prefetch.rs b/crates/ds4-dist/tests/prefetch.rs new file mode 100644 index 0000000000000000000000000000000000000000..385688ce76d87f3b6a0ff3d1c9affcd68861d3a7 --- /dev/null +++ b/crates/ds4-dist/tests/prefetch.rs @@ -0,0 +1,267 @@ +//! Prefetch depth, job queue, reconnect strings, and pipelined serve. + +use std::io; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; + +use ds4_dist::{ + cleared_sessions_message, connect_retryable, connected_message, disconnected_message, + hello_failed_message, prefetch_depth_from, prefetch_disabled_from, prefetch_enabled_message, + reconnect_with, retrying_message, send_hello, JobQueue, SliceExec, WorkOutput, WorkRequest, + Worker, ERR_OOM_QUEUE, ERR_OOM_READ, PREFETCH_DEPTH_DEFAULT, PREFETCH_DEPTH_MAX, + PREFETCH_DEPTH_MIN, +}; + +#[derive(Clone)] +struct MockExec { + model_id: u32, + n_layers: u32, + vocab: u32, + ctx_size: u32, + hidden_values: u64, + has_output: bool, + layer_start: u32, + layer_end: u32, + hidden: Vec, + logits: Vec, +} + +impl SliceExec for MockExec { + fn model_id(&self) -> u32 { + self.model_id + } + fn n_layers(&self) -> u32 { + self.n_layers + } + fn vocab(&self) -> u32 { + self.vocab + } + fn ctx_size(&self) -> u32 { + self.ctx_size + } + fn hidden_values(&self) -> u64 { + self.hidden_values + } + fn has_output(&self) -> bool { + self.has_output + } + fn layer_start(&self) -> u32 { + self.layer_start + } + fn layer_end(&self) -> u32 { + self.layer_end + } + fn eval(&mut self, req: &WorkRequest) -> Result { + Ok(WorkOutput { + hidden: if req.produce_hidden { + Some(self.hidden.clone()) + } else { + None + }, + logits: if req.produce_logits { + Some(self.logits.clone()) + } else { + None + }, + }) + } +} + +fn mock() -> MockExec { + MockExec { + model_id: 3, + n_layers: 4, + vocab: 8, + ctx_size: 128, + hidden_values: 2, + has_output: true, + layer_start: 2, + layer_end: 3, + hidden: vec![1.0, 2.0], + logits: vec![0.1, 0.2, 0.7, 0.0], + } +} + +fn tune(s: &TcpStream) { + let _ = s.set_nodelay(true); + let _ = s.set_read_timeout(Some(Duration::from_secs(5))); + let _ = s.set_write_timeout(Some(Duration::from_secs(5))); +} + +#[test] +fn prefetch_depth_matches_c() { + assert_eq!(prefetch_depth_from(None), PREFETCH_DEPTH_DEFAULT); + assert_eq!(prefetch_depth_from(Some("")), PREFETCH_DEPTH_DEFAULT); + assert_eq!(prefetch_depth_from(Some("2")), 2); + assert_eq!(prefetch_depth_from(Some("1")), PREFETCH_DEPTH_MIN); + assert_eq!(prefetch_depth_from(Some("8")), PREFETCH_DEPTH_MAX); + assert_eq!(prefetch_depth_from(Some("0")), PREFETCH_DEPTH_DEFAULT); + assert_eq!(prefetch_depth_from(Some("9")), PREFETCH_DEPTH_DEFAULT); + assert_eq!(prefetch_depth_from(Some("-1")), PREFETCH_DEPTH_DEFAULT); + assert_eq!(prefetch_depth_from(Some("3 ")), PREFETCH_DEPTH_DEFAULT); + assert_eq!(prefetch_depth_from(Some(" 4")), 4); + assert_eq!(prefetch_depth_from(Some("+5")), 5); + assert_eq!(prefetch_depth_from(Some("2foo")), PREFETCH_DEPTH_DEFAULT); +} + +#[test] +fn prefetch_disable_is_presence() { + assert!(!prefetch_disabled_from(None)); + assert!(prefetch_disabled_from(Some(std::ffi::OsStr::new("")))); + assert!(prefetch_disabled_from(Some(std::ffi::OsStr::new("1")))); +} + +#[test] +fn prefetch_and_oom_strings_match_c() { + assert_eq!( + prefetch_enabled_message(2), + "ds4: distributed worker: receive prefetch depth 2 enabled" + ); + assert_eq!(ERR_OOM_QUEUE, "out of memory queueing distributed WORK"); + assert_eq!(ERR_OOM_READ, "out of memory reading distributed WORK frame"); +} + +#[test] +fn job_queue_blocks_at_depth_and_finish_drains() { + let q = JobQueue::new(2); + assert_eq!(q.depth(), 2); + assert!(q.enqueue(vec![1])); + assert!(q.enqueue(vec![2])); + let (tx, rx) = mpsc::channel(); + thread::scope(|s| { + s.spawn(|| { + assert!(q.enqueue(vec![3])); + tx.send(()).unwrap(); + }); + thread::sleep(Duration::from_millis(30)); + assert!(rx.try_recv().is_err()); + assert_eq!(q.pop(), Some(vec![1])); + rx.recv_timeout(Duration::from_secs(2)).unwrap(); + }); + assert_eq!(q.pop(), Some(vec![2])); + q.finish(); + assert_eq!(q.pop(), Some(vec![3])); + assert_eq!(q.pop(), None); +} + +#[test] +fn job_queue_cancel_drops_queued() { + let q = JobQueue::new(2); + assert!(q.enqueue(vec![9])); + q.cancel(); + assert!(!q.enqueue(vec![8])); + assert_eq!(q.pop(), None); +} + +#[test] +fn reconnect_messages_match_c() { + assert_eq!( + retrying_message("unable to connect to 127.0.0.1:9: Connection refused"), + "ds4: distributed worker: unable to connect to 127.0.0.1:9: Connection refused; retrying" + ); + assert_eq!( + connected_message("127.0.0.1", "7000"), + "ds4: distributed worker: connected to coordinator 127.0.0.1:7000" + ); + let err = io::Error::from_raw_os_error(32); + assert!( + hello_failed_message(&err).starts_with("ds4: distributed worker: failed to send HELLO:") + ); + assert_eq!( + cleared_sessions_message(3), + "ds4: distributed worker: cleared 3 sessions after coordinator disconnect" + ); + assert_eq!( + disconnected_message(false), + "ds4: distributed worker: coordinator disconnected; reconnecting" + ); + assert_eq!( + disconnected_message(true), + "ds4: distributed worker: coordinator disconnected after error; reconnecting" + ); +} + +#[test] +fn connect_retryable_matches_c_errno_set() { + assert!(connect_retryable(&io::Error::from_raw_os_error(111))); // ECONNREFUSED + assert!(connect_retryable(&io::Error::from_raw_os_error(113))); // EHOSTUNREACH + assert!(connect_retryable(&io::Error::from_raw_os_error(101))); // ENETUNREACH + assert!(connect_retryable(&io::Error::from_raw_os_error(110))); // ETIMEDOUT + assert!(connect_retryable(&io::Error::from_raw_os_error(99))); // EADDRNOTAVAIL + assert!(!connect_retryable(&io::Error::from_raw_os_error(13))); // EACCES +} + +#[test] +fn clear_sessions_returns_count() { + let mut worker = Worker::new(mock()); + assert_eq!(worker.session_count(), 0); + assert_eq!(worker.clear_sessions(), 0); +} + +#[test] +fn reconnect_retries_then_serves_and_clears() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let attempts = AtomicU32::new(0); + let (ready_tx, ready_rx) = mpsc::channel(); + let worker_thread = thread::spawn(move || { + let mut worker = Worker::new(mock()); + let (hello, name) = worker.hello(7000, 128, "mock"); + let mut sleeps = 0u32; + reconnect_with( + &mut worker, + || { + let n = attempts.fetch_add(1, Ordering::SeqCst); + if n == 0 { + Err(io::Error::new(io::ErrorKind::ConnectionRefused, "refused")) + } else { + let s = TcpStream::connect(addr)?; + tune(&s); + Ok(s) + } + }, + &hello, + &name, + || { + sleeps += 1; + }, + || attempts.load(Ordering::SeqCst) > 1 && ready_rx.try_recv().is_ok(), + false, + ) + .unwrap(); + assert_eq!(worker.session_count(), 0); + assert!(sleeps >= 1); + }); + + let (mut stream, _) = listener.accept().unwrap(); + tune(&stream); + let _ = ds4_dist::recv_hello(&mut stream).unwrap(); + ready_tx.send(()).unwrap(); + drop(stream); + worker_thread.join().unwrap(); +} + +#[test] +fn serve_prefetch_hello_then_eof() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let worker_thread = thread::spawn(move || { + let mut stream = TcpStream::connect(addr).unwrap(); + tune(&stream); + let mut worker = Worker::new(mock()); + let (hello, name) = worker.hello(7000, 128, "mock"); + send_hello(&mut stream, &hello, &name).unwrap(); + worker.serve_prefetch(&mut stream).unwrap(); + assert_eq!(worker.clear_sessions(), 0); + }); + let (mut stream, _) = listener.accept().unwrap(); + tune(&stream); + let (hello, name) = ds4_dist::recv_hello(&mut stream).unwrap(); + assert_eq!(hello.model_id, 3); + assert_eq!(name, "mock"); + drop(stream); + worker_thread.join().unwrap(); +} diff --git a/crates/ds4-dist/tests/relay.rs b/crates/ds4-dist/tests/relay.rs new file mode 100644 index 0000000000000000000000000000000000000000..b320e37c076c218836b9103f5d784087b85f7bed --- /dev/null +++ b/crates/ds4-dist/tests/relay.rs @@ -0,0 +1,296 @@ +//! Persistent forwarder relay and telemetry prepend. + +use std::io::Write; +use std::net::{TcpListener, TcpStream}; +use std::sync::{mpsc, Arc, Mutex}; +use std::thread; +use std::time::Duration; + +use ds4_dist::{ + decode_result_body, encode_result_frame, forward_work_blocking, local_work_telemetry, + ok_result_hdr, prepend_telemetry, read_frame, result_request_id, usec_since, Forwarder, + ForwarderPool, PendingRequest, ResultBody, RouteEntry, Telemetry, Work, WorkBody, + ERR_NEXT_CLOSED, FRAME_HEADER_BYTES, MSG_RESULT, RESULT_ACK, +}; + +fn pending(id: u64) -> PendingRequest { + PendingRequest { + request_id: id, + telemetry: Telemetry { + layer_start: 2, + layer_end: 3, + route_index: 1, + pos0: 0, + n_tokens: 1, + eval_usec: 11, + downstream_wait_usec: 0, + forward_send_usec: 0, + input_bytes: 8, + output_bytes: 4, + }, + downstream_t0: 0.0, + } +} + +fn result_frame(request_id: u64, payload: &[u8]) -> Vec { + let mut hdr = ok_result_hdr(request_id, 0, RESULT_ACK, 0); + hdr.payload_bytes = payload.len() as u32; + encode_result_frame(&ResultBody { + hdr, + telemetry: Vec::new(), + payload: payload.to_vec(), + }) + .unwrap() +} + +fn tune(s: &std::net::TcpStream) { + let _ = s.set_nodelay(true); + let _ = s.set_read_timeout(Some(Duration::from_secs(2))); + let _ = s.set_write_timeout(Some(Duration::from_secs(2))); +} + +#[test] +fn local_work_telemetry_matches_c_fields() { + let work = Work { + layer_start: 2, + layer_end: 3, + route_index: 1, + pos0: 4, + n_tokens: 2, + token_bytes: 8, + input_hc_bytes: 16, + ..Work::default() + }; + let tel = local_work_telemetry(&work, 12, 32); + assert_eq!(tel.layer_start, 2); + assert_eq!(tel.layer_end, 3); + assert_eq!(tel.route_index, 1); + assert_eq!(tel.pos0, 4); + assert_eq!(tel.n_tokens, 2); + assert_eq!(tel.eval_usec, 12); + assert_eq!(tel.downstream_wait_usec, 0); + assert_eq!(tel.forward_send_usec, 0); + assert_eq!(tel.input_bytes, 24); + assert_eq!(tel.output_bytes, 32); +} + +#[test] +fn usec_since_matches_c_rounding() { + assert_eq!(usec_since(1.0, 1.0), 0); + assert_eq!(usec_since(1.0, 0.5), 0); + assert_eq!(usec_since(0.0, 0.000_001), 1); + assert_eq!(usec_since(0.0, 1.5), 1_500_000); +} + +#[test] +fn prepend_telemetry_appends_local_record() { + let frame = result_frame(9, b"ok"); + let body = &frame[FRAME_HEADER_BYTES..]; + let local = pending(9).telemetry; + let out = prepend_telemetry(body, local).unwrap(); + let (typ, payload) = { + let mut cur = std::io::Cursor::new(out); + read_frame(&mut cur).unwrap() + }; + assert_eq!(typ, MSG_RESULT); + let decoded = decode_result_body(&payload).unwrap(); + assert_eq!(result_request_id(&decoded.hdr), 9); + assert_eq!(decoded.payload, b"ok"); + assert_eq!(decoded.telemetry.len(), 1); + assert_eq!(decoded.telemetry[0], local); +} + +#[test] +fn forwarder_relays_result_and_prepends_telemetry() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = mpsc::channel(); + let err_tx = tx.clone(); + let mut forwarder = Forwarder::connect( + "127.0.0.1", + addr.port() as u32, + 4, + move |frame| { + let _ = tx.send(Ok(frame)); + }, + move |id, msg| { + let _ = err_tx.send(Err((id, msg))); + }, + ) + .unwrap(); + + let next = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + tune(&stream); + let (_typ, body) = read_frame(&mut stream).unwrap(); + assert_eq!(body, b"WORK"); + stream.write_all(&result_frame(7, b"logits")).unwrap(); + }); + + forwarder + .send_work(pending(7), &{ + let mut frame = ds4_dist::encode_frame_header(ds4_dist::MSG_WORK, 4).to_vec(); + frame.extend_from_slice(b"WORK"); + frame + }) + .unwrap(); + let frame = rx + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .expect("relayed RESULT"); + let mut cur = std::io::Cursor::new(frame); + let (typ, payload) = read_frame(&mut cur).unwrap(); + assert_eq!(typ, MSG_RESULT); + let decoded = decode_result_body(&payload).unwrap(); + assert_eq!(decoded.payload, b"logits"); + assert_eq!(decoded.telemetry.len(), 1); + assert_eq!(decoded.telemetry[0].layer_start, 2); + assert_eq!(decoded.telemetry[0].eval_usec, 11); + forwarder.shutdown(); + next.join().unwrap(); +} + +fn work_frame(payload: &[u8]) -> Vec { + let mut frame = + ds4_dist::encode_frame_header(ds4_dist::MSG_WORK, payload.len() as u32).to_vec(); + frame.extend_from_slice(payload); + frame +} + +#[test] +fn forwarder_reports_closed_connection() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = mpsc::channel(); + let mut forwarder = Forwarder::connect( + "127.0.0.1", + addr.port() as u32, + 2, + move |_| {}, + move |id, msg| { + let _ = tx.send((id, msg)); + }, + ) + .unwrap(); + let (stream, _) = listener.accept().unwrap(); + tune(&stream); + forwarder.send_work(pending(3), &work_frame(&[1])).unwrap(); + drop(stream); + let (id, msg) = rx.recv_timeout(Duration::from_secs(2)).unwrap(); + assert_eq!(id, 3); + assert_eq!(msg, ERR_NEXT_CLOSED); + forwarder.shutdown(); +} + +#[test] +fn blocking_forward_prepends_local_telemetry() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let next = RouteEntry { + host: "127.0.0.1".into(), + port: addr.port() as u32, + layer_start: 2, + layer_end: 3, + flags: 0, + }; + let work = Work { + layer_start: 1, + layer_end: 1, + route_index: 0, + pos0: 4, + n_tokens: 1, + token_bytes: 4, + request_lo: 7, + ..Work::default() + }; + let body = WorkBody { + work, + tokens: vec![1], + input_hc: vec![1.0, 2.0], + route_blob: Vec::new(), + }; + let local = local_work_telemetry(&work, 11, 8); + let next_thread = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + tune(&stream); + let (_typ, _body) = read_frame(&mut stream).unwrap(); + stream.write_all(&result_frame(7, b"logits")).unwrap(); + }); + let mut upstream = Vec::new(); + forward_work_blocking(&next, &body, &mut upstream, local).unwrap(); + let mut cur = std::io::Cursor::new(upstream); + let (typ, payload) = read_frame(&mut cur).unwrap(); + assert_eq!(typ, MSG_RESULT); + let decoded = decode_result_body(&payload).unwrap(); + assert_eq!(decoded.payload, b"logits"); + assert_eq!(decoded.telemetry.len(), 1); + assert_eq!(decoded.telemetry[0].eval_usec, 11); + assert_eq!(decoded.telemetry[0].layer_start, 1); + assert_eq!(decoded.telemetry[0].input_bytes, 4); + assert_eq!(decoded.telemetry[0].output_bytes, 8); + next_thread.join().unwrap(); +} + +#[test] +fn pool_reuses_forwarder_and_prepends_telemetry() { + let next_l = TcpListener::bind("127.0.0.1:0").unwrap(); + let up_l = TcpListener::bind("127.0.0.1:0").unwrap(); + let next_addr = next_l.local_addr().unwrap(); + let up_addr = up_l.local_addr().unwrap(); + let upstream = TcpStream::connect(up_addr).unwrap(); + tune(&upstream); + let (mut up_peer, _) = up_l.accept().unwrap(); + tune(&up_peer); + let mut pool = ForwarderPool::new(); + pool.bind(Arc::new(Mutex::new(upstream))); + let next = RouteEntry { + host: "127.0.0.1".into(), + port: next_addr.port() as u32, + layer_start: 2, + layer_end: 3, + flags: 0, + }; + let work = Work { + layer_start: 1, + layer_end: 1, + n_tokens: 1, + token_bytes: 4, + request_lo: 7, + ..Work::default() + }; + let body = WorkBody { + work, + tokens: vec![1], + input_hc: vec![1.0, 2.0], + route_blob: Vec::new(), + }; + let next_thread = thread::spawn(move || { + let (mut stream, _) = next_l.accept().unwrap(); + tune(&stream); + let (_typ, _body) = read_frame(&mut stream).unwrap(); + stream.write_all(&result_frame(7, b"logits")).unwrap(); + let (_typ, _body) = read_frame(&mut stream).unwrap(); + stream.write_all(&result_frame(8, b"again")).unwrap(); + }); + pool.forward(&next, &body, local_work_telemetry(&work, 11, 8)) + .unwrap(); + assert_eq!(pool.len(), 1); + let mut body2 = body; + body2.work.request_lo = 8; + pool.forward(&next, &body2, local_work_telemetry(&work, 12, 8)) + .unwrap(); + assert_eq!(pool.len(), 1); + let (typ, payload) = read_frame(&mut up_peer).unwrap(); + assert_eq!(typ, MSG_RESULT); + let decoded = decode_result_body(&payload).unwrap(); + assert_eq!(decoded.payload, b"logits"); + assert_eq!(decoded.telemetry.len(), 1); + assert_eq!(decoded.telemetry[0].eval_usec, 11); + let (typ, payload) = read_frame(&mut up_peer).unwrap(); + assert_eq!(typ, MSG_RESULT); + let decoded = decode_result_body(&payload).unwrap(); + assert_eq!(decoded.payload, b"again"); + assert_eq!(decoded.telemetry[0].eval_usec, 12); + pool.shutdown(); + next_thread.join().unwrap(); +} diff --git a/crates/ds4-dist/tests/runtime.rs b/crates/ds4-dist/tests/runtime.rs new file mode 100644 index 0000000000000000000000000000000000000000..48573ddb8a708c05bdc2cf815db28570ad625b0b --- /dev/null +++ b/crates/ds4-dist/tests/runtime.rs @@ -0,0 +1,1257 @@ +//! Coordinator/worker runtime: CLI strings, route search, HELLO/WORK/RESULT. + +use ds4_dist::{ + build_route_plan, decode_logits_payload, decode_result_body, dispatch_eval, encode_route_blob, + encode_work_frame, format_telemetry_line, parse_cli, parse_layers, parse_role, + prefetch_disabled_from, prepare_engine_options, read_frame, reconnect_local, register_worker, + resolved_layer_end, send_hello, serve_prefetch_local_with, token_hash_update_span, + token_span_hashes, validate_layers_for_model, validate_options, work_with_ids, write_frame, + Coordinator, CoordinatorView, EvalOutcome, JobQueue, LocalReconnect, PrefetchJob, ReturnTarget, + RouteEntry, SliceExec, Telemetry, Work, WorkBody, WorkOutput, WorkRequest, Worker, WorkerInfo, + ERR_NEXT_CLOSED, MSG_ERROR, MSG_RESULT, RESULT_LOGITS, ROUTE_F_OUTPUT_LOGITS, + ROUTE_RETURN_UPSTREAM, TOKEN_HASH_INIT, WORK_F_INPUT_HC, WORK_F_OUTPUT_LOGITS, + WORK_F_RESET_SESSION, +}; +use std::io::{Read, Write}; +use std::marker::PhantomData; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::sync::{mpsc, Arc, Condvar, Mutex}; +use std::thread; +use std::time::Duration; + +fn oracle() -> std::path::PathBuf { + if let Ok(p) = std::env::var("DS4_DIST_C_ORACLE") { + return std::path::PathBuf::from(p); + } + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/dist_c_oracle") +} + +fn require_oracle() -> std::path::PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/dist_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_out(args: &[&str]) -> String { + let out = std::process::Command::new(require_oracle()) + .args(args) + .output() + .expect("run dist_c_oracle"); + assert!( + out.status.success(), + "oracle failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +fn rust_cli(args: &[&str]) -> Result<(), String> { + let owned: Vec = args.iter().map(|s| (*s).to_string()).collect(); + let (opt, unmatched) = parse_cli(&owned)?; + if let Some(u) = unmatched.first() { + return Err(format!("unmatched {u}")); + } + validate_options(&opt) +} + +fn rust_cli_layers(n_layers: u32, args: &[&str]) -> Result<(), String> { + let owned: Vec = args.iter().map(|s| (*s).to_string()).collect(); + let (opt, unmatched) = parse_cli(&owned)?; + if let Some(u) = unmatched.first() { + return Err(format!("unmatched {u}")); + } + validate_options(&opt)?; + validate_layers_for_model(&opt, n_layers) +} + +fn assert_cli(args: &[&str]) { + let mut cargs = vec!["cli"]; + cargs.extend_from_slice(args); + let c = c_out(&cargs); + let r = rust_cli(args); + if c == "OK" { + r.expect("rust rejected a CLI C accepts"); + } else { + let msg = c.strip_prefix("ERROR:").unwrap_or(&c); + let err = r.expect_err("rust accepted a CLI C rejects"); + assert_eq!(err, msg, "args={args:?}"); + } +} + +#[derive(Clone)] +struct MockExec { + model_id: u32, + n_layers: u32, + vocab: u32, + ctx_size: u32, + hidden_values: u64, + has_output: bool, + layer_start: u32, + layer_end: u32, + hidden: Vec, + logits: Vec, +} + +impl SliceExec for MockExec { + fn model_id(&self) -> u32 { + self.model_id + } + fn n_layers(&self) -> u32 { + self.n_layers + } + fn vocab(&self) -> u32 { + self.vocab + } + fn ctx_size(&self) -> u32 { + self.ctx_size + } + fn hidden_values(&self) -> u64 { + self.hidden_values + } + fn has_output(&self) -> bool { + self.has_output + } + fn layer_start(&self) -> u32 { + self.layer_start + } + fn layer_end(&self) -> u32 { + self.layer_end + } + fn eval(&mut self, req: &WorkRequest) -> Result { + Ok(WorkOutput { + hidden: if req.produce_hidden { + Some(self.hidden.clone()) + } else { + None + }, + logits: if req.produce_logits { + Some(self.logits.clone()) + } else { + None + }, + }) + } +} + +fn tune(s: &TcpStream) { + let _ = s.set_nodelay(true); + let _ = s.set_read_timeout(Some(Duration::from_secs(5))); + let _ = s.set_write_timeout(Some(Duration::from_secs(5))); +} + +#[test] +fn telemetry_line_matches_c_format() { + let tel = Telemetry { + layer_start: 2, + layer_end: 3, + route_index: 1, + pos0: 4, + n_tokens: 2, + eval_usec: 1500, + downstream_wait_usec: 250, + forward_send_usec: 100, + input_bytes: 2 * 1024 * 1024, + output_bytes: 512 * 1024, + }; + assert_eq!( + format_telemetry_line(9, 0, &tel), + "ds4: distributed telemetry: request=9 hop=0 layers=2:3 route=1 pos=4 tokens=2 eval=1.500ms downstream_wait=0.250ms forward_send=0.100ms input=2.00MiB output=0.50MiB\n" + ); +} + +#[test] +fn layers_and_role_match_c() { + assert_eq!(c_out(&["layers", "0:1"]), "start=0 end=1 has_output=0"); + assert_eq!( + c_out(&["layers", "21:output"]), + "start=21 end=4294967295 has_output=1" + ); + let l = parse_layers("21:output").unwrap(); + assert_eq!(l.start, 21); + assert!(l.has_output); + assert_eq!(resolved_layer_end(&l, 53), 52); + assert_eq!(c_out(&["layers", "foo"]), "ERROR:expected A:B or A:output"); + assert_eq!(parse_layers("foo").unwrap_err(), "expected A:B or A:output"); + assert_eq!( + c_out(&["layers", "x:1"]), + "ERROR:invalid start layer in x:1" + ); + assert_eq!( + parse_layers("x:1").unwrap_err(), + "invalid start layer in x:1" + ); + assert_eq!( + c_out(&["layers", "3:1"]), + "ERROR:layer range end precedes start in 3:1" + ); + assert_eq!(c_out(&["role", "coordinator"]), "coordinator"); + assert_eq!(parse_role("coordinator"), Some(ds4_dist::Role::Coordinator)); + assert!(c_out(&["role", "boss"]).starts_with("ERROR:invalid distributed role: boss")); + assert!(parse_role("boss").is_none()); +} + +#[test] +fn cli_validate_matches_c() { + assert_cli(&[]); + assert_cli(&["--listen", "127.0.0.1", "7000"]); + assert_cli(&["--role", "coordinator"]); + assert_cli(&[ + "--role", + "coordinator", + "--layers", + "0:1", + "--listen", + "127.0.0.1", + "7000", + ]); + assert_cli(&[ + "--role", + "coordinator", + "--layers", + "0:1", + "--listen", + "127.0.0.1", + "7000", + "--coordinator", + "10.0.0.1", + "9", + ]); + assert_cli(&["--role", "worker", "--layers", "2:output"]); + assert_cli(&[ + "--role", + "worker", + "--layers", + "2:output", + "--coordinator", + "127.0.0.1", + "7000", + ]); + assert_cli(&[ + "--role", + "worker", + "--layers", + "2:output", + "--coordinator", + "127.0.0.1", + "7000", + "--dist-prefill-chunk", + "128", + ]); + assert_cli(&["--role", "nope"]); + assert_cli(&["--layers", "0:1:2"]); + assert_cli(&["--dist-prefill-window", "99"]); + assert_cli(&[ + "--role", + "coordinator", + "--layers", + "0:1", + "--listen", + "127.0.0.1", + "0", + ]); +} + +#[test] +fn layers_for_model_match_c() { + let args = [ + "--role", + "coordinator", + "--layers", + "2:3", + "--listen", + "127.0.0.1", + "7000", + ]; + let c = c_out(&[ + "layers-for-model", + "4", + "--role", + "coordinator", + "--layers", + "2:3", + "--listen", + "127.0.0.1", + "7000", + ]); + let r = rust_cli_layers(4, &args); + assert_eq!(c, "ERROR:coordinator layer range must start at layer 0"); + assert_eq!( + r.unwrap_err(), + "coordinator layer range must start at layer 0" + ); + + let past = rust_cli_layers( + 4, + &[ + "--role", + "worker", + "--layers", + "9:9", + "--coordinator", + "127.0.0.1", + "1", + ], + ); + assert_eq!( + past.unwrap_err(), + "layer range starts past final model layer 3" + ); +} + +#[test] +fn replay_check_requires_coordinator() { + let owned = vec![ + "--role".into(), + "worker".into(), + "--layers".into(), + "2:output".into(), + "--coordinator".into(), + "127.0.0.1".into(), + "7000".into(), + "--dist-replay-check".into(), + ]; + let (opt, _) = parse_cli(&owned).unwrap(); + validate_options(&opt).unwrap(); + assert_eq!( + prepare_engine_options(&opt).unwrap_err(), + "--dist-replay-check requires --role coordinator" + ); +} + +fn worker(start: u32, end: u32, output: bool) -> WorkerInfo { + WorkerInfo { + peer_host: format!("10.0.0.{end}"), + listen_port: 7000 + end, + model_id: 3, + quant_bits: 2, + layer_start: start, + layer_end: end, + has_output: output, + has_hidden: true, + } +} + +#[test] +fn route_plan_local_complete_is_empty() { + let view = CoordinatorView { + n_layers: 4, + local_start: 0, + local_end: 3, + local_has_output: true, + local_can_output_head: false, + }; + let plan = build_route_plan(&view, &[]).unwrap(); + assert!(plan.entries.is_empty()); + assert!(plan.blob.is_empty()); +} + +#[test] +fn route_plan_requires_layer_zero() { + let view = CoordinatorView { + n_layers: 4, + local_start: 1, + local_end: 1, + local_has_output: false, + local_can_output_head: false, + }; + assert_eq!( + build_route_plan(&view, &[]).unwrap_err(), + "coordinator route does not start at layer 0" + ); +} + +#[test] +fn route_plan_missing_layer_message() { + let view = CoordinatorView { + n_layers: 4, + local_start: 0, + local_end: 1, + local_has_output: false, + local_can_output_head: false, + }; + assert_eq!( + build_route_plan(&view, &[]).unwrap_err(), + "distributed route incomplete: missing layer 2" + ); +} + +#[test] +fn route_plan_prefers_output_then_longer_end() { + let view = CoordinatorView { + n_layers: 6, + local_start: 0, + local_end: 1, + local_has_output: false, + local_can_output_head: false, + }; + let workers = vec![worker(2, 3, false), worker(2, 5, true), worker(2, 4, false)]; + let plan = build_route_plan(&view, &workers).unwrap(); + assert_eq!(plan.entries.len(), 1); + assert_eq!(plan.entries[0].layer_end, 5); + assert_ne!(plan.entries[0].flags, 0); +} + +#[test] +fn register_worker_replaces_stale_and_prepends() { + let mut list = vec![worker(2, 5, true)]; + list[0].peer_host = "10.0.0.9".into(); + let neu = WorkerInfo { + peer_host: "10.0.0.9".into(), + listen_port: 7101, + model_id: 3, + quant_bits: 4, + layer_start: 2, + layer_end: 5, + has_output: true, + has_hidden: true, + }; + register_worker(&mut list, neu.clone()); + assert_eq!(list.len(), 1); + assert_eq!(list[0].listen_port, 7101); + let other = worker(2, 3, false); + register_worker(&mut list, other); + assert_eq!(list.len(), 2); + assert_eq!(list[0].layer_end, 3); +} + +#[test] +fn single_hop_hello_work_logits() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let logits = vec![0.1f32, 0.2, 0.7, 0.0]; + let hidden = vec![1.0f32, 2.0, 3.0, 4.0]; + let exec = MockExec { + model_id: 3, + n_layers: 4, + vocab: 8, + ctx_size: 128, + hidden_values: 2, + has_output: true, + layer_start: 2, + layer_end: 3, + hidden: hidden.clone(), + logits: logits.clone(), + }; + let worker_thread = thread::spawn(move || { + let mut stream = TcpStream::connect(addr).unwrap(); + tune(&stream); + let mut worker = Worker::new(exec); + let (hello, name) = worker.hello(7000, 128, "mock"); + send_hello(&mut stream, &hello, &name).unwrap(); + worker.serve(&mut stream).unwrap(); + }); + + let (mut stream, _) = listener.accept().unwrap(); + tune(&stream); + let view = CoordinatorView { + n_layers: 4, + local_start: 0, + local_end: 1, + local_has_output: false, + local_can_output_head: false, + }; + let mut coord = Coordinator::new(view, 3, 32); + coord.accept_hello(&mut stream, "127.0.0.1").unwrap(); + let tokens = [1i32, 2]; + let out = dispatch_eval( + &coord, + &mut stream, + &tokens, + 0, + 9, + 11, + true, + TOKEN_HASH_INIT, + &hidden, + ) + .unwrap(); + match out { + EvalOutcome::Logits(v) => assert_eq!(v, logits), + other => panic!("expected logits, got {other:?}"), + } + drop(stream); + worker_thread.join().unwrap(); +} + +#[test] +fn two_hop_blocking_forward() { + let data = TcpListener::bind("127.0.0.1:0").unwrap(); + let data_port = data.local_addr().unwrap().port() as u32; + let coord_l = TcpListener::bind("127.0.0.1:0").unwrap(); + let coord_addr = coord_l.local_addr().unwrap(); + let (ready_tx, ready_rx) = mpsc::channel::<()>(); + + let tail = MockExec { + model_id: 3, + n_layers: 4, + vocab: 8, + ctx_size: 128, + hidden_values: 2, + has_output: true, + layer_start: 2, + layer_end: 3, + hidden: vec![9.0, 8.0], + logits: vec![0.0, 1.0, 0.0, 0.0], + }; + let mid = MockExec { + model_id: 3, + n_layers: 4, + vocab: 8, + ctx_size: 128, + hidden_values: 2, + has_output: false, + layer_start: 1, + layer_end: 1, + hidden: vec![0.5, 1.5], + logits: vec![], + }; + + let tail_thread = thread::spawn(move || { + let mut hello_s = TcpStream::connect(coord_addr).unwrap(); + tune(&hello_s); + let mut worker = Worker::new(tail); + let (hello, name) = worker.hello(data_port, 128, "tail"); + send_hello(&mut hello_s, &hello, &name).unwrap(); + ready_tx.send(()).unwrap(); + let (mut work_s, _) = data.accept().unwrap(); + tune(&work_s); + worker.serve(&mut work_s).unwrap(); + }); + ready_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + + let mid_thread = thread::spawn(move || { + let mut stream = TcpStream::connect(coord_addr).unwrap(); + tune(&stream); + let mut worker = Worker::new(mid); + let (hello, name) = worker.hello(7100, 128, "mid"); + send_hello(&mut stream, &hello, &name).unwrap(); + worker.serve(&mut stream).unwrap(); + }); + + let mut first = None; + let view = CoordinatorView { + n_layers: 4, + local_start: 0, + local_end: 0, + local_has_output: false, + local_can_output_head: false, + }; + let mut coord = Coordinator::new(view, 3, 32); + for _ in 0..2 { + let (mut s, _) = coord_l.accept().unwrap(); + tune(&s); + let (hello, _) = coord.accept_hello(&mut s, "127.0.0.1").unwrap(); + if hello.layer_start == 1 { + first = Some(s); + } + } + let mut stream = first.expect("mid worker stream"); + let tokens = [3i32]; + let hc = vec![0.25f32, 0.75]; + let out = dispatch_eval( + &coord, + &mut stream, + &tokens, + 0, + 1, + 2, + true, + TOKEN_HASH_INIT, + &hc, + ) + .unwrap(); + match out { + EvalOutcome::Logits(v) => assert_eq!(v, vec![0.0, 1.0, 0.0, 0.0]), + other => panic!("expected logits, got {other:?}"), + } + drop(stream); + mid_thread.join().unwrap(); + tail_thread.join().unwrap(); +} + +#[test] +fn worker_rejects_model_mismatch_with_c_string() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let exec = MockExec { + model_id: 3, + n_layers: 4, + vocab: 8, + ctx_size: 128, + hidden_values: 2, + has_output: true, + layer_start: 2, + layer_end: 3, + hidden: vec![0.0; 4], + logits: vec![1.0; 4], + }; + let worker_thread = thread::spawn(move || { + let mut stream = TcpStream::connect(addr).unwrap(); + tune(&stream); + let mut worker = Worker::new(exec); + let (hello, name) = worker.hello(7000, 128, "mock"); + send_hello(&mut stream, &hello, &name).unwrap(); + worker.serve(&mut stream).unwrap(); + }); + let (mut stream, _) = listener.accept().unwrap(); + tune(&stream); + let view = CoordinatorView { + n_layers: 4, + local_start: 0, + local_end: 1, + local_has_output: false, + local_can_output_head: false, + }; + let mut coord = Coordinator::new(view, 99, 32); + coord.accept_hello(&mut stream, "127.0.0.1").unwrap(); + let err = dispatch_eval( + &coord, + &mut stream, + &[1], + 0, + 1, + 1, + true, + TOKEN_HASH_INIT, + &[0.0, 0.0], + ) + .unwrap_err(); + assert_eq!(err, "model id mismatch: work=99 worker=3"); + drop(stream); + worker_thread.join().unwrap(); +} + +#[test] +fn prefix_hash_continues_committed_timeline() { + let committed = [7i32, 8]; + let span = [9i32]; + let (prefix, result) = token_span_hashes(&committed, &span); + assert_eq!(prefix, ds4_dist::token_hash_prefix(&committed)); + assert_eq!(result, ds4_dist::token_hash_update_span(prefix, &span)); + assert_ne!(prefix, TOKEN_HASH_INIT); +} + +#[test] +fn usage_text_mentions_roles() { + assert!(ds4_dist::USAGE.contains("--role ROLE")); + assert!(ds4_dist::USAGE.contains("21:output")); +} + +#[test] +fn open_data_listener_assigns_ephemeral_port() { + let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + assert_ne!(port, 0); + assert_eq!(listener.local_addr().unwrap().port(), port); + let _stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); +} + +#[test] +fn open_data_listener_accept_data_client_returns_client() { + let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + let connector = thread::spawn(move || TcpStream::connect(("127.0.0.1", port)).unwrap()); + let client = ds4_dist::accept_data_client(&listener).unwrap(); + let _peer = connector.join().unwrap(); + assert!(client.nodelay().unwrap()); +} + +#[test] +fn open_data_listener_accept_data_client_errors_after_close() { + use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd}; + + let (listener, _port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + let raw = listener.into_raw_fd(); + // SAFETY: `raw` is exclusively owned; OwnedFd closes it. + drop(unsafe { OwnedFd::from_raw_fd(raw) }); + // SAFETY: `raw` is a closed fd. Reconstruct only so accept observes the + // error, then forget so Drop does not close twice. + let listener = unsafe { TcpListener::from_raw_fd(raw) }; + let err = ds4_dist::accept_data_client(&listener); + std::mem::forget(listener); + assert!(err.is_err()); +} + +fn hop_logits_exec() -> (MockExec, Vec) { + let logits = vec![0.1f32, 0.2, 0.7, 0.0]; + ( + MockExec { + model_id: 3, + n_layers: 4, + vocab: 8, + ctx_size: 128, + hidden_values: 2, + has_output: true, + layer_start: 2, + layer_end: 3, + hidden: vec![1.0, 2.0, 3.0, 4.0], + logits: logits.clone(), + }, + logits, + ) +} + +fn hop_work_frame(port: u16) -> Vec { + hop_work_frame_with_id(port, 11) +} + +fn hop_work_frame_with_id(port: u16, request_id: u64) -> Vec { + let tokens = [1i32, 2]; + let prefix = TOKEN_HASH_INIT; + let result = token_hash_update_span(prefix, &tokens); + let route_blob = encode_route_blob( + &[RouteEntry { + host: "127.0.0.1".into(), + port: u32::from(port), + layer_start: 2, + layer_end: 3, + flags: ROUTE_F_OUTPUT_LOGITS, + }], + &ReturnTarget { + kind: ROUTE_RETURN_UPSTREAM, + host: String::new(), + port: 0, + }, + ) + .unwrap(); + let work = work_with_ids( + Work { + model_id: 3, + pos0: 0, + n_tokens: tokens.len() as u32, + layer_start: 2, + layer_end: 3, + flags: WORK_F_INPUT_HC | WORK_F_OUTPUT_LOGITS | WORK_F_RESET_SESSION, + input_hc_bits: 32, + route_count: 1, + route_index: 0, + ..Work::default() + }, + 9, + request_id, + prefix, + result, + ); + encode_work_frame(&WorkBody { + work, + tokens: tokens.to_vec(), + input_hc: vec![1.0, 2.0, 3.0, 4.0], + route_blob, + }) + .unwrap() +} + +#[test] +fn accepted_hop_serve_once_returns_result_for_work() { + // Given: a data listener and a MockExec worker that owns the final hop + let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + let frame = hop_work_frame(port); + let client = thread::spawn(move || { + let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); + tune(&stream); + stream.write_all(&frame).unwrap(); + read_frame(&mut stream).unwrap() + }); + + // When: accept the hop and serve one WORK + let mut hop = ds4_dist::accept_data_client(&listener).unwrap(); + tune(&hop); + let (exec, logits) = hop_logits_exec(); + let mut worker = Worker::new(exec); + worker.serve_once(&mut hop).unwrap(); + + // Then: the client reads a RESULT with the mock logits + let (typ, reply) = client.join().unwrap(); + assert_eq!(typ, MSG_RESULT); + let result = decode_result_body(&reply).unwrap(); + assert_eq!(result.hdr.status, 0); + assert_eq!(result.hdr.result_kind, RESULT_LOGITS); + assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits); +} + +#[test] +fn accepted_hop_serve_once_unknown_frame_yields_error() { + // Given: a data listener and a MockExec worker + let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + let client = thread::spawn(move || { + let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); + tune(&stream); + write_frame(&mut stream, 99, b"not-a-work-frame").unwrap(); + read_frame(&mut stream).unwrap() + }); + + // When: accept the hop and serve one unknown frame + let mut hop = ds4_dist::accept_data_client(&listener).unwrap(); + tune(&hop); + let (exec, _) = hop_logits_exec(); + let mut worker = Worker::new(exec); + let err = worker.serve_once(&mut hop).unwrap_err(); + + // Then: the worker rejects the type and the client reads an ERROR frame + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + let (typ, body) = client.join().unwrap(); + assert_eq!(typ, MSG_ERROR); + assert_eq!(body, b"unsupported distributed worker frame"); +} + +#[test] +fn reconnect_local_keeps_coordinator_open_while_hop_work_completes() { + // Given: a coordinator TCP and a worker data listener + let coord_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let coord_addr = coord_listener.local_addr().unwrap(); + let (data_listener, data_port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + let frame = hop_work_frame(data_port); + let (stop_tx, stop_rx) = mpsc::channel(); + let (exec, logits) = hop_logits_exec(); + + let worker_thread = thread::spawn(move || { + let mut worker = Worker::new(exec); + let (hello, name) = worker.hello(u32::from(data_port), 128, "mock"); + let mut stopped = false; + reconnect_local( + &mut worker, + LocalReconnect { + connect: || { + let stream = TcpStream::connect(coord_addr)?; + tune(&stream); + Ok(stream) + }, + hello: &hello, + model_name: &name, + sleep: || {}, + should_stop: || { + stopped |= stop_rx.try_recv().is_ok(); + stopped + }, + listener: Some(&data_listener), + }, + ) + }); + + // When: HELLO stays on the coordinator while a hop WORK is served + let (mut coord, _) = coord_listener.accept().unwrap(); + tune(&coord); + let _ = ds4_dist::recv_hello(&mut coord).unwrap(); + + let mut hop = TcpStream::connect(("127.0.0.1", data_port)).unwrap(); + tune(&hop); + hop.write_all(&frame).unwrap(); + let (typ, reply) = read_frame(&mut hop).unwrap(); + + // Then: hop RESULT arrives and the coordinator TCP is still open + assert_eq!(typ, MSG_RESULT); + let result = decode_result_body(&reply).unwrap(); + assert_eq!(result.hdr.status, 0); + assert_eq!(result.hdr.result_kind, RESULT_LOGITS); + assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits); + assert!(coord.peer_addr().is_ok()); + coord + .set_read_timeout(Some(Duration::from_millis(50))) + .unwrap(); + let mut probe = [0u8; 1]; + let idle = coord.read(&mut probe); + assert!( + matches!(&idle, Err(e) if e.kind() == std::io::ErrorKind::TimedOut + || e.kind() == std::io::ErrorKind::WouldBlock), + "coordinator must stay open (got {idle:?})" + ); + + stop_tx.send(()).unwrap(); + drop(coord); + worker_thread.join().unwrap().unwrap(); +} + +fn middle_exec() -> MockExec { + MockExec { + model_id: 3, + n_layers: 4, + vocab: 8, + ctx_size: 128, + hidden_values: 2, + has_output: false, + layer_start: 0, + layer_end: 1, + hidden: vec![1.0, 2.0, 3.0, 4.0], + logits: Vec::new(), + } +} + +fn two_hop_work_frame(middle_port: u16, next_port: u16) -> Vec { + let tokens = [1i32, 2]; + let prefix = TOKEN_HASH_INIT; + let result = token_hash_update_span(prefix, &tokens); + let route_blob = encode_route_blob( + &[ + RouteEntry { + host: "127.0.0.1".into(), + port: u32::from(middle_port), + layer_start: 0, + layer_end: 1, + flags: 0, + }, + RouteEntry { + host: "127.0.0.1".into(), + port: u32::from(next_port), + layer_start: 2, + layer_end: 3, + flags: ROUTE_F_OUTPUT_LOGITS, + }, + ], + &ReturnTarget { + kind: ROUTE_RETURN_UPSTREAM, + host: String::new(), + port: 0, + }, + ) + .unwrap(); + let work = work_with_ids( + Work { + model_id: 3, + pos0: 0, + n_tokens: tokens.len() as u32, + layer_start: 0, + layer_end: 1, + flags: WORK_F_RESET_SESSION, + input_hc_bits: 32, + route_count: 2, + route_index: 0, + ..Work::default() + }, + 9, + 11, + prefix, + result, + ); + encode_work_frame(&WorkBody { + work, + tokens: tokens.to_vec(), + input_hc: Vec::new(), + route_blob, + }) + .unwrap() +} + +fn spawn_reconnect_worker( + exec: MockExec, + coord_addr: SocketAddr, + data_listener: TcpListener, + data_port: u16, + stop_rx: mpsc::Receiver<()>, +) -> thread::JoinHandle> { + thread::spawn(move || { + let mut worker = Worker::new(exec); + let (hello, name) = worker.hello(u32::from(data_port), 128, "mock"); + let mut stopped = false; + reconnect_local( + &mut worker, + LocalReconnect { + connect: || { + let stream = TcpStream::connect(coord_addr)?; + tune(&stream); + Ok(stream) + }, + hello: &hello, + model_name: &name, + sleep: || {}, + should_stop: || { + stopped |= stop_rx.try_recv().is_ok(); + stopped + }, + listener: Some(&data_listener), + }, + ) + }) +} + +fn accept_hello(listener: &TcpListener) -> TcpStream { + let (stream, _) = listener.accept().unwrap(); + tune(&stream); + let mut stream = stream; + let _ = ds4_dist::recv_hello(&mut stream).unwrap(); + stream +} + +fn assert_c_shaped_telemetry(request_id: u64, hop: u32, tel: &Telemetry) { + let line = format_telemetry_line(request_id, hop, tel); + assert!( + line.starts_with("ds4: distributed telemetry: "), + "C telemetry prefix missing: {line:?}" + ); + assert!(line.contains(&format!("request={request_id}"))); + assert!(line.contains(&format!("hop={hop}"))); + assert!(line.contains(&format!("layers={}:{}", tel.layer_start, tel.layer_end))); + assert!(line.contains("eval=")); + assert!(line.contains("downstream_wait=")); + assert!(line.contains("forward_send=")); + assert!(line.contains("input=")); + assert!(line.contains("output=")); + assert!(line.contains("MiB")); + assert!(line.ends_with('\n')); +} + +#[test] +fn reconnect_local_two_hop_result_includes_c_shaped_telemetry() { + // Given: production reconnect_local mux on a middle worker and a final hop + let coord_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let coord_addr = coord_listener.local_addr().unwrap(); + let (middle_listener, middle_port) = + ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + let (final_listener, final_port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + let (stop_mid_tx, stop_mid_rx) = mpsc::channel(); + let (stop_fin_tx, stop_fin_rx) = mpsc::channel(); + let (final_exec, logits) = hop_logits_exec(); + + let final_thread = spawn_reconnect_worker( + final_exec, + coord_addr, + final_listener, + final_port, + stop_fin_rx, + ); + let middle_thread = spawn_reconnect_worker( + middle_exec(), + coord_addr, + middle_listener, + middle_port, + stop_mid_rx, + ); + let coord_final = accept_hello(&coord_listener); + let coord_middle = accept_hello(&coord_listener); + + // When: a two-hop WORK is sent on the middle worker data port + let frame = two_hop_work_frame(middle_port, final_port); + let mut hop = TcpStream::connect(("127.0.0.1", middle_port)).unwrap(); + tune(&hop); + hop.write_all(&frame).unwrap(); + let (typ, reply) = read_frame(&mut hop).unwrap(); + + // Then: hop RESULT carries both hops and format_telemetry_line is C-shaped + assert_eq!(typ, MSG_RESULT); + let result = decode_result_body(&reply).unwrap(); + assert_eq!(result.hdr.status, 0); + assert_eq!(result.hdr.result_kind, RESULT_LOGITS); + assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits); + assert_eq!(result.telemetry.len(), 2); + assert_eq!(result.telemetry[0].layer_start, 2); + assert_eq!(result.telemetry[0].layer_end, 3); + assert_eq!(result.telemetry[0].route_index, 1); + assert_eq!(result.telemetry[1].layer_start, 0); + assert_eq!(result.telemetry[1].layer_end, 1); + assert_eq!(result.telemetry[1].route_index, 0); + assert_c_shaped_telemetry(11, 0, &result.telemetry[0]); + assert_c_shaped_telemetry(11, 1, &result.telemetry[1]); + + stop_mid_tx.send(()).unwrap(); + stop_fin_tx.send(()).unwrap(); + drop(hop); + drop(coord_final); + drop(coord_middle); + middle_thread.join().unwrap().unwrap(); + final_thread.join().unwrap().unwrap(); +} + +#[test] +fn reconnect_local_missing_next_hop_keeps_c_forward_error() { + // Given: a middle worker whose next hop accepts then closes + let coord_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let coord_addr = coord_listener.local_addr().unwrap(); + let (middle_listener, middle_port) = + ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + let next_listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let next_port = next_listener.local_addr().unwrap().port(); + let (stop_tx, stop_rx) = mpsc::channel(); + let middle_thread = spawn_reconnect_worker( + middle_exec(), + coord_addr, + middle_listener, + middle_port, + stop_rx, + ); + let coord = accept_hello(&coord_listener); + let next_thread = thread::spawn(move || { + let (stream, _) = next_listener.accept().unwrap(); + drop(stream); + }); + + // When: two-hop WORK is forwarded to the closed next hop + let frame = two_hop_work_frame(middle_port, next_port); + let mut hop = TcpStream::connect(("127.0.0.1", middle_port)).unwrap(); + tune(&hop); + hop.write_all(&frame).unwrap(); + let (typ, reply) = read_frame(&mut hop).unwrap(); + + // Then: RESULT uses the existing C forward-error string + assert_eq!(typ, MSG_RESULT); + let result = decode_result_body(&reply).unwrap(); + assert_ne!(result.hdr.status, 0); + let msg = String::from_utf8_lossy(&result.payload); + assert_eq!(msg, ERR_NEXT_CLOSED); + assert!(!msg.contains("telemetry")); + next_thread.join().unwrap(); + + stop_tx.send(()).unwrap(); + drop(hop); + drop(coord); + middle_thread.join().unwrap().unwrap(); +} + +struct EvalGate { + started: Mutex, + started_cv: Condvar, + release: Mutex, + release_cv: Condvar, +} + +impl EvalGate { + fn new() -> Arc { + Arc::new(Self { + started: Mutex::new(0), + started_cv: Condvar::new(), + release: Mutex::new(false), + release_cv: Condvar::new(), + }) + } + + fn wait_started(&self) { + let mut n = self.started.lock().expect("eval gate"); + while *n == 0 { + n = self.started_cv.wait(n).expect("eval gate"); + } + } + + fn release(&self) { + *self.release.lock().expect("eval gate") = true; + self.release_cv.notify_all(); + } +} + +struct NotSendExec { + inner: MockExec, + gate: Arc, + _not_send: PhantomData<*const ()>, +} + +impl SliceExec for NotSendExec { + fn model_id(&self) -> u32 { + self.inner.model_id() + } + fn n_layers(&self) -> u32 { + self.inner.n_layers() + } + fn vocab(&self) -> u32 { + self.inner.vocab() + } + fn ctx_size(&self) -> u32 { + self.inner.ctx_size() + } + fn hidden_values(&self) -> u64 { + self.inner.hidden_values() + } + fn has_output(&self) -> bool { + self.inner.has_output() + } + fn layer_start(&self) -> u32 { + self.inner.layer_start() + } + fn layer_end(&self) -> u32 { + self.inner.layer_end() + } + fn eval(&mut self, req: &WorkRequest) -> Result { + { + let mut n = self.gate.started.lock().expect("eval gate"); + *n += 1; + self.gate.started_cv.notify_all(); + } + let mut released = self.gate.release.lock().expect("eval gate"); + while !*released { + released = self.gate.release_cv.wait(released).expect("eval gate"); + } + self.inner.eval(req) + } +} + +fn assert_local_prefetch_accepts_not_send( + worker: &mut Worker, + stream: &mut TcpStream, + queue: &JobQueue, +) -> std::io::Result<()> +where + E: SliceExec, + S: ds4_dist::SnapshotStore, +{ + serve_prefetch_local_with(worker, stream, queue) +} + +#[test] +fn prefetch_env_unset_enables_local_prefetch() { + // Given: C default is prefetch on unless DS4_DIST_DISABLE_WORKER_PREFETCH is present + // When: the disable env is unset + // Then: local reconnect must take the prefetch path + assert!(!prefetch_disabled_from(None)); + assert!(ds4_dist::local_prefetch_enabled_from(None)); +} + +#[test] +fn prefetch_env_set_disables_local_prefetch() { + // Given: C turns prefetch off when the disable env exists (any value) + // When: the disable env is present + // Then: local reconnect must not use prefetch + assert!(prefetch_disabled_from(Some(std::ffi::OsStr::new("1")))); + assert!(!ds4_dist::local_prefetch_enabled_from(Some( + std::ffi::OsStr::new("1") + ))); +} + +#[test] +fn serve_prefetch_local_depth_2_queues_second_work_during_not_send_eval() { + // Given: a !Send exec and a depth-2 local prefetch queue + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let (inner, logits) = hop_logits_exec(); + let gate = EvalGate::new(); + let queue = Arc::new(JobQueue::::new(2)); + let frame1 = hop_work_frame_with_id(port, 11); + let frame2 = hop_work_frame_with_id(port, 12); + let client_gate = Arc::clone(&gate); + let client_queue = Arc::clone(&queue); + + let client = thread::spawn(move || { + let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); + tune(&stream); + stream.write_all(&frame1).unwrap(); + client_gate.wait_started(); + stream.write_all(&frame2).unwrap(); + client_queue.wait_until_queued(1); + let queued_during_eval = client_queue.queued(); + client_gate.release(); + let first = read_frame(&mut stream).unwrap(); + let second = read_frame(&mut stream).unwrap(); + (queued_during_eval, first, second) + }); + + // When: session-thread prefetch serves both WORKs on a !Send worker + let mut stream = listener.accept().unwrap().0; + tune(&stream); + let mut worker = Worker::new(NotSendExec { + inner, + gate, + _not_send: PhantomData, + }); + assert_local_prefetch_accepts_not_send(&mut worker, &mut stream, &queue).unwrap(); + + // Then: WORK2 was queued while eval1 was still in progress + let (queued_during_eval, first, second) = client.join().unwrap(); + assert!( + queued_during_eval >= 1, + "depth 2 must queue WORK2 during eval1, queued={queued_during_eval}" + ); + for (typ, reply) in [first, second] { + assert_eq!(typ, MSG_RESULT); + let result = decode_result_body(&reply).unwrap(); + assert_eq!(result.hdr.status, 0); + assert_eq!(result.hdr.result_kind, RESULT_LOGITS); + assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits); + } +} diff --git a/crates/ds4-dist/tests/snapshot_store.rs b/crates/ds4-dist/tests/snapshot_store.rs new file mode 100644 index 0000000000000000000000000000000000000000..bda7195f5d3af62e9c6595534e033fd2b2ebbef5 --- /dev/null +++ b/crates/ds4-dist/tests/snapshot_store.rs @@ -0,0 +1,214 @@ +use std::io::Cursor; +use std::net::{TcpListener, TcpStream}; +use std::thread; +use std::time::Duration; + +use ds4_dist::{ + coordinator_load_snapshot, coordinator_save_snapshot, token_hash_prefix, MemorySnapshotStore, + SliceExec, SnapshotLoad, SnapshotMeta, SnapshotSave, SnapshotStore, WorkOutput, WorkRequest, + Worker, +}; + +#[derive(Clone)] +struct MockExec { + model_id: u32, + n_layers: u32, + vocab: u32, + ctx_size: u32, + layer_start: u32, + layer_end: u32, +} + +impl SliceExec for MockExec { + fn model_id(&self) -> u32 { + self.model_id + } + fn n_layers(&self) -> u32 { + self.n_layers + } + fn vocab(&self) -> u32 { + self.vocab + } + fn ctx_size(&self) -> u32 { + self.ctx_size + } + fn hidden_values(&self) -> u64 { + 2 + } + fn has_output(&self) -> bool { + true + } + fn layer_start(&self) -> u32 { + self.layer_start + } + fn layer_end(&self) -> u32 { + self.layer_end + } + fn eval(&mut self, _req: &WorkRequest) -> Result { + Ok(WorkOutput { + hidden: None, + logits: Some(vec![0.0; 4]), + }) + } +} + +fn exec() -> MockExec { + MockExec { + model_id: 3, + n_layers: 16, + vocab: 32, + ctx_size: 128, + layer_start: 8, + layer_end: 12, + } +} + +fn meta(tokens: &[i32]) -> SnapshotMeta { + SnapshotMeta { + model_id: 3, + session_id: 0x11_2233, + request_id: 0x44_5566, + token_hash: token_hash_prefix(tokens), + layer_start: 8, + layer_end: 12, + } +} + +fn tune(s: &TcpStream) { + let _ = s.set_nodelay(true); + let _ = s.set_read_timeout(Some(Duration::from_secs(5))); + let _ = s.set_write_timeout(Some(Duration::from_secs(5))); +} + +#[test] +fn store_save_reports_missing_session_with_c_text() { + let mut store = MemorySnapshotStore::new(); + let err = store + .save(SnapshotSave { + session_id: 7, + token_count: 1, + token_hash: 1, + }) + .err() + .unwrap(); + assert_eq!(err, "worker has no distributed session to snapshot"); +} + +#[test] +fn store_save_reports_token_count_mismatch_with_c_text() { + let mut store = MemorySnapshotStore::new(); + store.insert(7, vec![1, 2], b"shard").unwrap(); + let err = store + .save(SnapshotSave { + session_id: 7, + token_count: 1, + token_hash: token_hash_prefix(&[1, 2]), + }) + .err() + .unwrap(); + assert_eq!(err, "worker snapshot token count mismatch"); +} + +#[test] +fn store_save_reports_token_hash_mismatch_with_c_text() { + let mut store = MemorySnapshotStore::new(); + store.insert(7, vec![1, 2], b"shard").unwrap(); + let err = store + .save(SnapshotSave { + session_id: 7, + token_count: 2, + token_hash: token_hash_prefix(&[1, 2]) ^ 1, + }) + .err() + .unwrap(); + assert_eq!(err, "worker snapshot token hash mismatch"); +} + +#[test] +fn store_load_sets_hash_valid_and_clears_it_on_restore_failure() { + let mut store = MemorySnapshotStore::new(); + store.insert(7, vec![1], b"old").unwrap(); + assert!(store.token_hash_valid(7)); + + let tokens = [3, 4]; + let hash = token_hash_prefix(&tokens); + store + .load( + SnapshotLoad { + session_id: 7, + tokens: &tokens, + token_hash: hash, + payload_bytes: 4, + }, + &mut Cursor::new(b"next"), + ) + .unwrap(); + assert!(store.token_hash_valid(7)); + assert_eq!(store.token_hash(7), Some(hash)); + assert_eq!(store.tokens(7), Some(tokens.as_slice())); + assert_eq!(store.read_shard(7).unwrap(), b"next"); + + store.fail_restore(true); + let err = store + .load( + SnapshotLoad { + session_id: 7, + tokens: &tokens, + token_hash: hash, + payload_bytes: 3, + }, + &mut Cursor::new(b"bad"), + ) + .unwrap_err(); + assert_eq!(err, "failed to restore worker KV shard"); + assert!(!store.token_hash_valid(7)); +} + +#[test] +fn worker_serve_save_load_round_trips_memory_shard() { + let tokens = [7i32, 8]; + let meta = meta(&tokens); + let first = b"kv-shard-one".to_vec(); + let second = b"kv-shard-two".to_vec(); + let mut store = MemorySnapshotStore::new(); + store + .insert(meta.session_id, tokens.to_vec(), &first) + .unwrap(); + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let worker_thread = thread::spawn(move || { + let mut stream = TcpStream::connect(addr).unwrap(); + tune(&stream); + let mut worker = Worker::with_store(exec(), store); + worker.serve(&mut stream).unwrap(); + }); + + let (mut stream, _) = listener.accept().unwrap(); + tune(&stream); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut stream, meta, &tokens, &mut saved).unwrap(), + first.len() as u64 + ); + assert_eq!(saved, first); + + coordinator_load_snapshot( + &mut stream, + meta, + &tokens, + &mut Cursor::new(&second), + second.len() as u64, + ) + .unwrap(); + + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut stream, meta, &tokens, &mut saved).unwrap(), + second.len() as u64 + ); + assert_eq!(saved, second); + + drop(stream); + worker_thread.join().unwrap(); +} diff --git a/crates/ds4-dist/tests/snapshot_transport.rs b/crates/ds4-dist/tests/snapshot_transport.rs new file mode 100644 index 0000000000000000000000000000000000000000..b38056da4647c2d3c5550aaa9f34114ea3c96224 --- /dev/null +++ b/crates/ds4-dist/tests/snapshot_transport.rs @@ -0,0 +1,646 @@ +use std::io::{self, Cursor, Read, Write}; + +use ds4_dist::{ + coordinator_load_snapshot, coordinator_save_snapshot, decode_snapshot_chunk_body, + decode_snapshot_load_begin_body, encode_frame_header, encode_snapshot_begin_body, + encode_snapshot_chunk_body, encode_snapshot_done_body, read_frame, token_hash_prefix, + worker_handle_snapshot_load, worker_handle_snapshot_save, write_frame, SnapshotBegin, + SnapshotMeta, SnapshotReq, WorkerSnapshotIdentity, MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, + MSG_SNAPSHOT_DONE, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, SNAPSHOT_CHUNK_BYTES, + SNAPSHOT_REQ_FIXED_BYTES, +}; + +#[derive(Default)] +struct ScriptedStream { + read: Cursor>, + written: Vec, +} + +impl ScriptedStream { + fn new(read: Vec) -> Self { + Self { + read: Cursor::new(read), + written: Vec::new(), + } + } +} + +impl Read for ScriptedStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.read.read(buf) + } +} + +impl Write for ScriptedStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.written.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +struct RejectLargeReads { + read: Cursor>, + written: Vec, + largest_read: usize, +} + +impl RejectLargeReads { + fn new(read: Vec) -> Self { + Self { + read: Cursor::new(read), + written: Vec::new(), + largest_read: 0, + } + } +} + +impl Read for RejectLargeReads { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.largest_read = self.largest_read.max(buf.len()); + assert!(buf.len() <= SNAPSHOT_CHUNK_BYTES); + self.read.read(buf) + } +} + +impl Write for RejectLargeReads { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.written.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn meta() -> SnapshotMeta { + SnapshotMeta { + model_id: 3, + session_id: 0x11_2233, + request_id: 0x44_5566, + token_hash: 0x77_8899, + layer_start: 8, + layer_end: 12, + } +} + +fn begin(meta: SnapshotMeta, token_count: u32, payload_bytes: u64) -> SnapshotBegin { + let (session_hi, session_lo) = ds4_dist::u64_to_halves(meta.session_id); + let (request_hi, request_lo) = ds4_dist::u64_to_halves(meta.request_id); + let (token_hash_hi, token_hash_lo) = ds4_dist::u64_to_halves(meta.token_hash); + let (payload_hi, payload_lo) = ds4_dist::u64_to_halves(payload_bytes); + SnapshotBegin { + model_id: meta.model_id, + session_hi, + session_lo, + request_hi, + request_lo, + token_hash_hi, + token_hash_lo, + token_count, + layer_start: meta.layer_start, + layer_end: meta.layer_end, + payload_hi, + payload_lo, + status: 0, + token_bytes: token_count * 4, + message_bytes: 0, + } +} + +fn push_frame(out: &mut Vec, typ: u32, body: &[u8]) { + write_frame(out, typ, body).unwrap(); +} + +#[test] +fn coordinator_save_follows_c_frame_order_and_lengths() { + let meta = meta(); + let tokens = [7, 8]; + let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3]; + payload[SNAPSHOT_CHUNK_BYTES] = 1; + payload[SNAPSHOT_CHUNK_BYTES + 1] = 2; + payload[SNAPSHOT_CHUNK_BYTES + 2] = 3; + let mut scripted = Vec::new(); + + let mut begin_body = encode_snapshot_begin_body( + &begin(meta, tokens.len() as u32, payload.len() as u64), + &tokens, + b"", + ) + .unwrap(); + begin_body.extend_from_slice(b"ignored trailing begin bytes"); + push_frame(&mut scripted, MSG_SNAPSHOT_BEGIN, &begin_body); + push_frame( + &mut scripted, + MSG_SNAPSHOT_CHUNK, + &encode_snapshot_chunk_body(meta.request_id, &payload[..SNAPSHOT_CHUNK_BYTES]).unwrap(), + ); + push_frame( + &mut scripted, + MSG_SNAPSHOT_CHUNK, + &encode_snapshot_chunk_body(meta.request_id, &payload[SNAPSHOT_CHUNK_BYTES..]).unwrap(), + ); + let mut done = encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(); + done.extend_from_slice(b"ignored trailing done bytes"); + push_frame(&mut scripted, MSG_SNAPSHOT_DONE, &done); + + let mut stream = ScriptedStream::new(scripted); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut stream, meta, &tokens, &mut saved).unwrap(), + payload.len() as u64 + ); + assert_eq!(saved, payload); + + let mut written = Cursor::new(stream.written); + let (typ, body) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_SAVE_REQ); + let request = SnapshotReq::decode(&body).unwrap(); + assert_eq!(request.model_id, meta.model_id); + assert_eq!( + ds4_dist::u64_from_halves(request.session_hi, request.session_lo), + meta.session_id + ); + assert_eq!( + ds4_dist::u64_from_halves(request.request_hi, request.request_lo), + meta.request_id + ); + assert_eq!(request.token_count, tokens.len() as u32); + assert!(read_frame(&mut written).is_err()); +} + +#[test] +fn coordinator_save_rejects_status_metadata_and_frame_sequence_errors() { + let meta = meta(); + let tokens = [7, 8]; + + let mut refused = begin(meta, 0, 0); + refused.status = 1; + refused.message_bytes = 12; + let body = encode_snapshot_begin_body(&refused, &[], b"refused\0tail").unwrap(); + let mut frames = Vec::new(); + push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,) + .unwrap_err(), + "refused" + ); + + let message = vec![b'x'; 300]; + refused.message_bytes = message.len() as u32; + let body = encode_snapshot_begin_body(&refused, &[], &message).unwrap(); + let mut frames = Vec::new(); + push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,) + .unwrap_err(), + "x".repeat(255) + ); + + let mut wrong = begin(meta, tokens.len() as u32, 0); + wrong.layer_end += 1; + let body = encode_snapshot_begin_body(&wrong, &tokens, b"").unwrap(); + let mut frames = Vec::new(); + push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,) + .unwrap_err(), + "distributed KV shard metadata mismatch" + ); + + let body = encode_snapshot_begin_body(&begin(meta, 0, 4), &[], b"").unwrap(); + let mut frames = Vec::new(); + push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body); + push_frame( + &mut frames, + MSG_SNAPSHOT_CHUNK, + &encode_snapshot_chunk_body(meta.request_id, b"ab").unwrap(), + ); + push_frame( + &mut frames, + MSG_SNAPSHOT_DONE, + &encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(), + ); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,) + .unwrap_err(), + "expected distributed KV shard chunk" + ); + + let body = encode_snapshot_begin_body(&begin(meta, 0, 2), &[], b"").unwrap(); + let mut frames = Vec::new(); + push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body); + push_frame( + &mut frames, + MSG_SNAPSHOT_CHUNK, + &encode_snapshot_chunk_body(meta.request_id, b"abc").unwrap(), + ); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,) + .unwrap_err(), + "invalid distributed KV shard chunk" + ); + + let body = encode_snapshot_begin_body(&begin(meta, 0, 1), &[], b"").unwrap(); + let mut frames = Vec::new(); + push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body); + push_frame( + &mut frames, + MSG_SNAPSHOT_CHUNK, + &encode_snapshot_chunk_body(meta.request_id + 1, b"a").unwrap(), + ); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,) + .unwrap_err(), + "invalid distributed KV shard chunk" + ); + + let body = encode_snapshot_begin_body(&begin(meta, 0, 1), &[], b"").unwrap(); + let mut frames = Vec::new(); + push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body); + for _ in 0..2 { + push_frame( + &mut frames, + MSG_SNAPSHOT_CHUNK, + &encode_snapshot_chunk_body(meta.request_id, b"a").unwrap(), + ); + } + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,) + .unwrap_err(), + "distributed worker returned invalid snapshot completion frame" + ); + + let body = encode_snapshot_begin_body(&begin(meta, 0, 0), &[], b"").unwrap(); + let mut frames = Vec::new(); + push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body); + push_frame( + &mut frames, + MSG_SNAPSHOT_DONE, + &encode_snapshot_done_body(meta.request_id + 1, 0, b"").unwrap(), + ); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,) + .unwrap_err(), + "distributed snapshot completion request mismatch" + ); +} + +#[test] +fn coordinator_save_bounds_peer_lengths_and_reports_partial_stages() { + let meta = meta(); + + let mut stream = + RejectLargeReads::new(encode_frame_header(MSG_SNAPSHOT_BEGIN, u32::MAX).to_vec()); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut stream, meta, &[], &mut saved).unwrap_err(), + "failed to read distributed snapshot header" + ); + assert!(stream.largest_read <= 60); + + let mut partial_done = Vec::new(); + push_frame( + &mut partial_done, + MSG_SNAPSHOT_BEGIN, + &encode_snapshot_begin_body(&begin(meta, 0, 0), &[], b"").unwrap(), + ); + partial_done.extend_from_slice(&encode_frame_header(MSG_SNAPSHOT_DONE, 16)); + partial_done.extend_from_slice(&[0; 8]); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot( + &mut ScriptedStream::new(partial_done), + meta, + &[], + &mut saved, + ) + .unwrap_err(), + "failed to read distributed snapshot completion" + ); +} + +#[test] +fn coordinator_load_chunks_payload_and_checks_completion() { + let meta = meta(); + let tokens = vec![7, 8]; + let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3]; + payload[SNAPSHOT_CHUNK_BYTES] = 1; + payload[SNAPSHOT_CHUNK_BYTES + 1] = 2; + payload[SNAPSHOT_CHUNK_BYTES + 2] = 3; + + let mut reply = Vec::new(); + let mut done = encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(); + done.extend_from_slice(b"trailing"); + push_frame(&mut reply, MSG_SNAPSHOT_DONE, &done); + let mut stream = ScriptedStream::new(reply); + let payload_len = payload.len() as u64; + coordinator_load_snapshot( + &mut stream, + meta, + &tokens, + &mut Cursor::new(&payload), + payload_len, + ) + .unwrap(); + + let mut written = Cursor::new(stream.written); + let (typ, body) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN); + let (header, got_tokens, message) = decode_snapshot_load_begin_body(&body).unwrap(); + assert_eq!(got_tokens, tokens); + assert!(message.is_empty()); + assert_eq!( + ds4_dist::u64_from_halves(header.payload_hi, header.payload_lo), + payload.len() as u64 + ); + + let (typ, body) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_CHUNK); + let (_, chunk) = + decode_snapshot_chunk_body(&body, meta.request_id, payload.len() as u64).unwrap(); + assert_eq!(chunk.len(), SNAPSHOT_CHUNK_BYTES); + let (typ, body) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_CHUNK); + let (_, chunk) = decode_snapshot_chunk_body(&body, meta.request_id, 3).unwrap(); + assert_eq!(chunk, [1, 2, 3]); + assert!(read_frame(&mut written).is_err()); + + let mut failed = Vec::new(); + push_frame( + &mut failed, + MSG_SNAPSHOT_DONE, + &encode_snapshot_done_body(meta.request_id, 1, b"restore failed").unwrap(), + ); + assert_eq!( + coordinator_load_snapshot( + &mut ScriptedStream::new(failed), + meta, + &tokens, + &mut Cursor::new(b"payload"), + 7, + ) + .unwrap_err(), + "restore failed" + ); +} + +#[test] +fn coordinator_zero_payload_does_not_require_or_send_chunks() { + let meta = meta(); + + let mut save_reply = Vec::new(); + push_frame( + &mut save_reply, + MSG_SNAPSHOT_BEGIN, + &encode_snapshot_begin_body(&begin(meta, 0, 0), &[], b"").unwrap(), + ); + push_frame( + &mut save_reply, + MSG_SNAPSHOT_DONE, + &encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(), + ); + let mut saved = Vec::new(); + assert_eq!( + coordinator_save_snapshot(&mut ScriptedStream::new(save_reply), meta, &[], &mut saved,) + .unwrap(), + 0 + ); + assert!(saved.is_empty()); + + let mut reply = Vec::new(); + push_frame( + &mut reply, + MSG_SNAPSHOT_DONE, + &encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(), + ); + let mut stream = ScriptedStream::new(reply); + let mut source = Cursor::new(b"not part of the declared payload"); + coordinator_load_snapshot(&mut stream, meta, &[], &mut source, 0).unwrap(); + assert_eq!(source.position(), 0); + + let mut written = Cursor::new(stream.written); + let (typ, _) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN); + assert!(read_frame(&mut written).is_err()); +} + +fn identity(meta: SnapshotMeta) -> WorkerSnapshotIdentity { + WorkerSnapshotIdentity { + model_id: meta.model_id, + layer_start: meta.layer_start, + layer_end: meta.layer_end, + ctx_size: 128, + } +} + +fn hashed_meta(tokens: &[i32]) -> SnapshotMeta { + let mut meta = meta(); + meta.token_hash = token_hash_prefix(tokens); + meta +} + +#[cfg(unix)] +fn duplex_save(meta: SnapshotMeta, tokens: &[i32], payload: Vec) -> (Vec, u64) { + let (mut coord, mut worker) = std::os::unix::net::UnixStream::pair().unwrap(); + let identity = identity(meta); + let sent = payload.clone(); + let handle = std::thread::spawn(move || { + let (typ, body) = read_frame(&mut worker).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_SAVE_REQ); + let mut src = Cursor::new(sent); + let len = src.get_ref().len() as u64; + worker_handle_snapshot_save(&mut worker, identity, &body, Ok((&mut src, len))).unwrap(); + }); + let mut saved = Vec::new(); + let n = coordinator_save_snapshot(&mut coord, meta, tokens, &mut saved).unwrap(); + handle.join().unwrap(); + (saved, n) +} + +#[cfg(unix)] +fn duplex_load(meta: SnapshotMeta, tokens: &[i32], payload: Vec) -> Vec { + let (mut coord, mut worker) = std::os::unix::net::UnixStream::pair().unwrap(); + let identity = identity(meta); + let handle = std::thread::spawn(move || { + let (typ, body) = read_frame(&mut worker).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN); + let mut restored = Vec::new(); + let offer = + worker_handle_snapshot_load(&mut worker, identity, &body, 32, &mut restored).unwrap(); + (restored, offer) + }); + coordinator_load_snapshot( + &mut coord, + meta, + tokens, + &mut Cursor::new(&payload), + payload.len() as u64, + ) + .unwrap(); + let (restored, offer) = handle.join().unwrap(); + assert_eq!(offer.session_id, meta.session_id); + assert_eq!(offer.request_id, meta.request_id); + assert_eq!(offer.token_hash, meta.token_hash); + assert_eq!(offer.tokens, tokens); + restored +} + +#[cfg(unix)] +#[test] +fn worker_save_answers_coordinator_with_c_frame_order() { + let tokens = [7, 8]; + let meta = hashed_meta(&tokens); + let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3]; + payload[SNAPSHOT_CHUNK_BYTES] = 1; + payload[SNAPSHOT_CHUNK_BYTES + 1] = 2; + payload[SNAPSHOT_CHUNK_BYTES + 2] = 3; + let (saved, n) = duplex_save(meta, &tokens, payload.clone()); + assert_eq!(n, payload.len() as u64); + assert_eq!(saved, payload); +} + +#[cfg(unix)] +#[test] +fn worker_load_accepts_coordinator_chunks_and_reports_tokens() { + let tokens = [7, 8]; + let meta = hashed_meta(&tokens); + let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3]; + payload[SNAPSHOT_CHUNK_BYTES] = 9; + let restored = duplex_load(meta, &tokens, payload.clone()); + assert_eq!(restored, payload); +} + +#[cfg(unix)] +#[test] +fn worker_zero_payload_round_trips_without_chunks() { + let meta = hashed_meta(&[]); + let (saved, n) = duplex_save(meta, &[], Vec::new()); + assert_eq!(n, 0); + assert!(saved.is_empty()); + assert!(duplex_load(meta, &[], Vec::new()).is_empty()); +} + +#[test] +fn worker_save_rejects_bad_size_and_identity_with_c_text() { + let meta = meta(); + let identity = identity(meta); + let mut stream = ScriptedStream::new(Vec::new()); + worker_handle_snapshot_save::<_, Cursor>>( + &mut stream, + identity, + &[0; SNAPSHOT_REQ_FIXED_BYTES - 1], + Err("unused".into()), + ) + .unwrap(); + let mut written = Cursor::new(stream.written); + let (typ, body) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_BEGIN); + let (begin, _, message) = ds4_dist::decode_snapshot_begin_body(&body).unwrap(); + assert_eq!(begin.status, 1); + assert_eq!(message, b"invalid distributed snapshot save request"); + + let mut request = ScriptedStream::new(Vec::new()); + let mut unused = Vec::new(); + coordinator_save_snapshot(&mut request, meta, &[7], &mut unused).unwrap_err(); + let mut written = Cursor::new(request.written); + let (typ, body) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_SAVE_REQ); + let mut wrong = identity; + wrong.layer_end += 1; + let mut reply = ScriptedStream::new(Vec::new()); + worker_handle_snapshot_save::<_, Cursor>>( + &mut reply, + wrong, + &body, + Err("unused".into()), + ) + .unwrap(); + let mut written = Cursor::new(reply.written); + let (typ, body) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_BEGIN); + let (begin, _, message) = ds4_dist::decode_snapshot_begin_body(&body).unwrap(); + assert_eq!(begin.status, 1); + assert_eq!( + std::str::from_utf8(&message).unwrap(), + "snapshot save request does not match worker state" + ); +} + +#[test] +fn worker_save_forwards_session_error_text() { + let meta = meta(); + let mut request = ScriptedStream::new(Vec::new()); + let mut unused = Vec::new(); + coordinator_save_snapshot(&mut request, meta, &[7], &mut unused).unwrap_err(); + let mut written = Cursor::new(request.written); + let (_, body) = read_frame(&mut written).unwrap(); + let mut reply = ScriptedStream::new(Vec::new()); + worker_handle_snapshot_save::<_, Cursor>>( + &mut reply, + identity(meta), + &body, + Err("worker has no distributed session to snapshot".into()), + ) + .unwrap(); + let mut written = Cursor::new(reply.written); + let (_, body) = read_frame(&mut written).unwrap(); + let (begin, _, message) = ds4_dist::decode_snapshot_begin_body(&body).unwrap(); + assert_eq!(begin.status, 1); + assert_eq!( + std::str::from_utf8(&message).unwrap(), + "worker has no distributed session to snapshot" + ); +} + +#[test] +fn worker_load_rejects_header_hash_and_identity_with_c_text() { + let identity = identity(meta()); + let mut stream = ScriptedStream::new(Vec::new()); + assert_eq!( + worker_handle_snapshot_load(&mut stream, identity, &[0; 8], 32, &mut Vec::new()) + .unwrap_err(), + "invalid distributed snapshot load header" + ); + assert!(stream.written.is_empty()); + + let tokens = [7, 8]; + let mut meta = hashed_meta(&tokens); + meta.token_hash ^= 1; + let mut request = ScriptedStream::new({ + let mut done = Vec::new(); + push_frame( + &mut done, + MSG_SNAPSHOT_DONE, + &encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(), + ); + done + }); + coordinator_load_snapshot(&mut request, meta, &tokens, &mut Cursor::new(b""), 0).unwrap(); + let mut written = Cursor::new(request.written); + let (typ, body) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN); + let mut reply = ScriptedStream::new(Vec::new()); + assert_eq!( + worker_handle_snapshot_load(&mut reply, identity, &body, 32, &mut Vec::new()).unwrap_err(), + "snapshot load token hash mismatch" + ); + let mut written = Cursor::new(reply.written); + let (typ, body) = read_frame(&mut written).unwrap(); + assert_eq!(typ, MSG_SNAPSHOT_DONE); + let (done, message) = ds4_dist::decode_snapshot_done_body(&body, meta.request_id).unwrap(); + assert_eq!(done.status, 1); + assert_eq!(message, b"snapshot load token hash mismatch"); +} diff --git a/crates/ds4-dist/tests/wire_oracle.rs b/crates/ds4-dist/tests/wire_oracle.rs new file mode 100644 index 0000000000000000000000000000000000000000..722a99e4d4110300a9ab06061a13bf10e687429c --- /dev/null +++ b/crates/ds4-dist/tests/wire_oracle.rs @@ -0,0 +1,398 @@ +//! C↔Rust DS4D wire bytes. No `#[repr(C)]` on the Rust side. + +use ds4_dist::{ + decode_frame_header, decode_hello_payload, decode_snapshot_begin_body, + decode_snapshot_chunk_body, decode_snapshot_done_body, decode_snapshot_load_begin_body, + encode_activation, encode_frame_header, encode_hello_payload, encode_route_blob, + encode_snapshot_begin_body, encode_snapshot_chunk_body, encode_snapshot_done_body, + encode_tokens_be, f32_to_f16, f32_to_f8_e4m3, read_frame, token_hash_prefix, + validate_route_blob, write_frame, Hello, ResultHdr, ReturnTarget, RouteEntry, SnapshotBegin, + SnapshotChunk, SnapshotDone, SnapshotReq, Work, FRAME_HEADER_BYTES, HELLO_FIXED_BYTES, MAGIC, + MSG_HELLO, MSG_WORK, RESULT_FIXED_BYTES, ROUTE_RETURN_UPSTREAM, SNAPSHOT_CHUNK_BYTES, + WORK_FIXED_BYTES, +}; +use std::path::PathBuf; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_DIST_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/dist_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/dist_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_out(args: &[&str]) -> String { + let out = Command::new(require_oracle()) + .args(args) + .output() + .expect("run dist_c_oracle"); + assert!( + out.status.success(), + "oracle failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[test] +fn struct_sizes_match_c() { + let line = c_out(&["sizes"]); + assert!(line.contains("frame 12"), "{line}"); + assert!(line.contains("hello 40"), "{line}"); + assert!(line.contains("work 80"), "{line}"); + assert!(line.contains(&format!("work {WORK_FIXED_BYTES}")), "{line}"); + assert_eq!(FRAME_HEADER_BYTES, 12); + assert_eq!(HELLO_FIXED_BYTES, 40); + assert_eq!(RESULT_FIXED_BYTES, 40); +} + +#[test] +fn frame_header_matches_c() { + let rust = encode_frame_header(MSG_WORK, 80); + assert_eq!(hex(&rust), c_out(&["frame", "3", "80"])); + let h = decode_frame_header(&rust).unwrap(); + assert_eq!(h.typ, MSG_WORK); + assert_eq!(h.bytes, 80); + assert_eq!(u32::from_be_bytes(rust[0..4].try_into().unwrap()), MAGIC); +} + +#[test] +fn hello_matches_c() { + let rec = Hello { + model_id: 3, + quant_bits: 2, + layer_start: 2, + layer_end: 51, + has_output: 1, + has_hidden: 1, + ctx_size: 8192, + n_layers: 53, + listen_port: 7000, + model_name_len: 0, + }; + let rust = encode_hello_payload(&rec, "motif-3").unwrap(); + assert_eq!( + hex(&rust), + c_out(&["hello", "3", "2", "2", "51", "1", "1", "8192", "53", "7000", "motif-3"]) + ); + let (back, name) = decode_hello_payload(&rust).unwrap(); + assert_eq!(name, "motif-3"); + assert_eq!(back.model_id, 3); + assert_eq!(back.listen_port, 7000); +} + +#[test] +fn work_and_tokens_match_c() { + let w = Work { + model_id: 3, + session_hi: 0, + session_lo: 9, + request_hi: 0, + request_lo: 11, + prefix_hash_hi: 1, + prefix_hash_lo: 2, + result_hash_hi: 3, + result_hash_lo: 4, + pos0: 16, + n_tokens: 3, + layer_start: 2, + layer_end: 10, + flags: 1, + token_bytes: 12, + input_hc_bytes: 0, + input_hc_bits: 32, + route_count: 1, + route_index: 0, + route_bytes: 0, + }; + assert_eq!( + hex(&w.encode()), + c_out(&[ + "work", "3", "0", "9", "0", "11", "1", "2", "3", "4", "16", "3", "2", "10", "1", "12", + "0", "32", "1", "0", "0" + ]) + ); + let tokens = [7i32, -1, 256]; + assert_eq!( + hex(&encode_tokens_be(&tokens)), + c_out(&["tokens", "7", "-1", "256"]) + ); +} + +#[test] +fn token_hash_matches_c() { + assert_eq!( + format!("{:016x}", token_hash_prefix(&[1, 2, 3, 99])), + c_out(&["token-hash", "1", "2", "3", "99"]) + ); +} + +#[test] +fn route_blob_round_trip() { + let entries = [RouteEntry { + host: "10.0.0.2".into(), + port: 7100, + layer_start: 2, + layer_end: 20, + flags: 0, + }]; + let ret = ReturnTarget { + kind: ROUTE_RETURN_UPSTREAM, + host: String::new(), + port: 0, + }; + let blob = encode_route_blob(&entries, &ret).unwrap(); + assert_eq!( + hex(&blob[..20 + 8]), + c_out(&["route", "10.0.0.2", "7100", "2", "20", "0"]) + ); + validate_route_blob(&blob, 1, 53).unwrap(); +} + +#[test] +fn result_matches_c() { + let r = ResultHdr { + request_hi: 0, + request_lo: 11, + result_hash_hi: 3, + result_hash_lo: 4, + status: 0, + result_kind: 2, + telemetry_count: 1, + telemetry_bytes: 40, + payload_bytes: 16, + payload_bits: 32, + }; + assert_eq!( + hex(&r.encode()), + c_out(&["result", "0", "11", "3", "4", "0", "2", "1", "40", "16", "32"]) + ); +} + +#[test] +fn f16_f8_match_c() { + let samples = [ + 0x00000000u32, + 0x3f800000, + 0xbf800000, + 0x3f000000, + 0x7f800000, + 0x7fc00000, + 0x00000001, + 0x3fc00000, + ]; + for bits in samples { + let hex_bits = format!("{bits:08x}"); + let f = f32::from_bits(bits); + assert_eq!( + hex(&f32_to_f16(f).to_le_bytes()), + c_out(&["f16", &hex_bits]), + "f16 {hex_bits}" + ); + assert_eq!( + hex(&[f32_to_f8_e4m3(f)]), + c_out(&["f8", &hex_bits]), + "f8 {hex_bits}" + ); + } +} + +#[test] +fn blocking_hello_round_trip() { + use std::net::{TcpListener, TcpStream}; + use std::thread; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let rec = Hello { + model_id: 1, + quant_bits: 4, + layer_start: 0, + layer_end: 11, + has_output: 0, + has_hidden: 1, + ctx_size: 2048, + n_layers: 48, + listen_port: 9, + model_name_len: 0, + }; + let payload = encode_hello_payload(&rec, "solar").unwrap(); + let reply = payload.clone(); + let server = thread::spawn(move || { + let (mut s, _) = listener.accept().unwrap(); + let (typ, body) = read_frame(&mut s).unwrap(); + assert_eq!(typ, MSG_HELLO); + let (h, name) = decode_hello_payload(&body).unwrap(); + assert_eq!(name, "solar"); + assert_eq!(h.n_layers, 48); + write_frame(&mut s, MSG_HELLO, &reply).unwrap(); + }); + let mut client = TcpStream::connect(addr).unwrap(); + write_frame(&mut client, MSG_HELLO, &payload).unwrap(); + let (typ, body) = read_frame(&mut client).unwrap(); + assert_eq!(typ, MSG_HELLO); + let (_, name) = decode_hello_payload(&body).unwrap(); + assert_eq!(name, "solar"); + server.join().unwrap(); +} + +#[test] +fn activation_f32_is_le_host_layout() { + let src = [1.0f32, -2.5]; + let wire = encode_activation(&src, 32).unwrap(); + let mut expect = Vec::new(); + expect.extend_from_slice(&1.0f32.to_bits().to_le_bytes()); + expect.extend_from_slice(&(-2.5f32).to_bits().to_le_bytes()); + assert_eq!(wire, expect); +} + +#[test] +fn snapshot_records_match_c() { + let req = SnapshotReq { + model_id: 3, + session_hi: 1, + session_lo: 2, + request_hi: 3, + request_lo: 4, + token_hash_hi: 5, + token_hash_lo: 6, + token_count: 7, + layer_start: 8, + layer_end: 9, + }; + assert_eq!( + hex(&req.encode()), + c_out(&[ + "snapshot-req", + "3", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9" + ]) + ); + + let begin = SnapshotBegin { + model_id: 3, + session_hi: 1, + session_lo: 2, + request_hi: 3, + request_lo: 4, + token_hash_hi: 5, + token_hash_lo: 6, + token_count: 2, + layer_start: 8, + layer_end: 9, + payload_hi: 10, + payload_lo: 11, + status: 0, + token_bytes: 8, + message_bytes: 0, + }; + assert_eq!( + hex(&begin.encode()), + c_out(&[ + "snapshot-begin", + "3", + "1", + "2", + "3", + "4", + "5", + "6", + "2", + "8", + "9", + "10", + "11", + "0", + "8", + "0" + ]) + ); + + let chunk = SnapshotChunk { + request_hi: 3, + request_lo: 4, + chunk_bytes: 5, + }; + assert_eq!( + hex(&chunk.encode()), + c_out(&["snapshot-chunk", "3", "4", "5"]) + ); + + let done = SnapshotDone { + request_hi: 3, + request_lo: 4, + status: 1, + message_bytes: 5, + }; + assert_eq!( + hex(&done.encode()), + c_out(&["snapshot-done", "3", "4", "1", "5"]) + ); +} + +#[test] +fn snapshot_bodies_enforce_c_lengths_and_ids() { + let begin = SnapshotBegin { + model_id: 3, + session_hi: 0, + session_lo: 2, + request_hi: 0, + request_lo: 4, + token_hash_hi: 0, + token_hash_lo: 6, + token_count: 2, + layer_start: 8, + layer_end: 9, + payload_hi: 0, + payload_lo: 5, + status: 0, + token_bytes: 8, + message_bytes: 0, + }; + let body = encode_snapshot_begin_body(&begin, &[7, 8], b"").unwrap(); + let (back, tokens, message) = decode_snapshot_load_begin_body(&body).unwrap(); + assert_eq!(back, begin); + assert_eq!(tokens, [7, 8]); + assert!(message.is_empty()); + + let mut bad = begin; + bad.token_bytes = 4; + assert!(encode_snapshot_begin_body(&bad, &[7, 8], b"").is_err()); + assert!(decode_snapshot_begin_body(&body[..body.len() - 1]).is_err()); + + let payload = b"kv-shard"; + let chunk = encode_snapshot_chunk_body(4, payload).unwrap(); + let (_, decoded) = decode_snapshot_chunk_body(&chunk, 4, payload.len() as u64).unwrap(); + assert_eq!(decoded, payload); + assert!(decode_snapshot_chunk_body(&chunk, 5, payload.len() as u64).is_err()); + assert!(decode_snapshot_chunk_body(&chunk, 4, (payload.len() - 1) as u64).is_err()); + assert!(encode_snapshot_chunk_body(4, &vec![0; SNAPSHOT_CHUNK_BYTES + 1]).is_err()); + + let done = encode_snapshot_done_body(4, 1, b"failed").unwrap(); + let (header, message) = decode_snapshot_done_body(&done, 4).unwrap(); + assert_eq!(header.status, 1); + assert_eq!(message, b"failed"); + assert!(decode_snapshot_done_body(&done, 5).is_err()); +} diff --git a/crates/ds4-kv/Cargo.toml b/crates/ds4-kv/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..e1ee1d4783c05a0da8be939a77c3cf0c888e216b --- /dev/null +++ b/crates/ds4-kv/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ds4-kv" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "KVC checkpoint store (Phase 4)" + +[lib] +path = "src/lib.rs" diff --git a/crates/ds4-kv/src/extensions.rs b/crates/ds4-kv/src/extensions.rs new file mode 100644 index 0000000000000000000000000000000000000000..964f82034d9f1c1a0895e4f1d1d3d763b515d7f9 --- /dev/null +++ b/crates/ds4-kv/src/extensions.rs @@ -0,0 +1,245 @@ +//! Bank thinking bit + KTM extension records. +//! +//! Thinking is `EXT_THINKING_VISIBLE` on the KVC header (no extra magic). +//! The only appended trailer magic is C's `KTM\x01`. A missing or invalid +//! section is ignored the way `kv_tool_map_load_from_pos` returns 0. + +use crate::format::{ + Record, EXT_BANK_REPLAY_V1, EXT_IMAGE_PIXELS_V2, EXT_RESPONSES_VISIBLE, EXT_SESSION_TITLE, + EXT_THINKING_VISIBLE, EXT_TOOL_MAP, +}; + +pub const KTM_MAGIC: [u8; 3] = [b'K', b'T', b'M']; +pub const KTM_VERSION: u8 = 1; +const KTM_HEADER: [u8; 4] = [b'K', b'T', b'M', KTM_VERSION]; +const DEFAULT_MAX_IDS: usize = 100_000; +const MAX_ID_BYTES: usize = 256; +const MAX_DSML_BYTES: usize = 512 * 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExtensionRecord { + pub id: Vec, + pub body: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BankThinkingExtensions { + pub thinking_visible: bool, + pub records: Vec, +} + +impl BankThinkingExtensions { + pub fn ext_flags(&self) -> u8 { + self.wire(0).0 + } + + pub fn encode_trailer(&self) -> Vec { + self.wire(0).1 + } + + fn wire(&self, keep: u8) -> (u8, Vec) { + let trailer = encode_ktm(&self.records); + ( + bank_persist_ext_flags(keep, self.thinking_visible, !trailer.is_empty()), + trailer, + ) + } + + pub fn restore(ext_flags: u8, trailer: &[u8]) -> Self { + Self { + thinking_visible: ext_flags & EXT_THINKING_VISIBLE != 0, + records: decode_ktm(trailer), + } + } +} + +impl Record { + pub fn persist_bank_extensions(&mut self, ext: &BankThinkingExtensions) { + let (flags, trailer) = ext.wire(self.header.ext_flags); + self.header.ext_flags = flags; + self.trailer = trailer; + self.header.text_bytes = self.text.len() as u32; + self.header.payload_bytes = self.payload.len() as u64; + } +} + +/// Bank persist always sets `EXT_BANK_REPLAY_V1`. Responses/title bits on +/// the incoming header survive; thinking and KTM are applied from `ext`. +pub fn bank_persist_ext_flags(keep: u8, thinking_visible: bool, has_trailer: bool) -> u8 { + let mut flags = EXT_BANK_REPLAY_V1 + | (keep & (EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE | EXT_IMAGE_PIXELS_V2)); + if thinking_visible { + flags |= EXT_THINKING_VISIBLE; + } + if has_trailer { + flags |= EXT_TOOL_MAP; + } + flags +} + +fn wire_ok(id: &[u8], body: &[u8]) -> bool { + !id.is_empty() + && !body.is_empty() + && id.len() <= MAX_ID_BYTES + && body.len() <= MAX_DSML_BYTES + && !id.contains(&0) + && !body.contains(&0) +} + +pub fn encode_ktm(records: &[ExtensionRecord]) -> Vec { + let kept: Vec<&ExtensionRecord> = records + .iter() + .filter(|record| wire_ok(&record.id, &record.body)) + .collect(); + if kept.is_empty() { + return Vec::new(); + } + let Ok(count) = u32::try_from(kept.len()) else { + return Vec::new(); + }; + let mut out = Vec::from(KTM_HEADER); + out.extend_from_slice(&count.to_le_bytes()); + for record in kept { + let id_len = record.id.len() as u32; + let body_len = record.body.len() as u32; + out.extend_from_slice(&id_len.to_le_bytes()); + out.extend_from_slice(&body_len.to_le_bytes()); + out.extend_from_slice(&record.id); + out.extend_from_slice(&record.body); + } + out +} + +pub fn decode_ktm(bytes: &[u8]) -> Vec { + decode_ktm_bounded(bytes, DEFAULT_MAX_IDS) +} + +fn decode_ktm_bounded(bytes: &[u8], max_ids: usize) -> Vec { + if bytes.len() < 8 || bytes[..4] != KTM_HEADER { + return Vec::new(); + } + let count = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]); + let max_ids = if max_ids == 0 { + DEFAULT_MAX_IDS + } else { + max_ids + }; + let max_count = (max_ids as u64).saturating_mul(4); + if u64::from(count) > max_count { + return Vec::new(); + } + let mut records = Vec::new(); + let mut pos = 8usize; + for _ in 0..count { + let Some(lens_end) = pos.checked_add(8).filter(|&end| end <= bytes.len()) else { + return records; + }; + let id_len = + u32::from_le_bytes([bytes[pos], bytes[pos + 1], bytes[pos + 2], bytes[pos + 3]]) + as usize; + let body_len = u32::from_le_bytes([ + bytes[pos + 4], + bytes[pos + 5], + bytes[pos + 6], + bytes[pos + 7], + ]) as usize; + if id_len == 0 || id_len > MAX_ID_BYTES || body_len == 0 || body_len > MAX_DSML_BYTES { + return records; + } + pos = lens_end; + let Some(id_end) = pos.checked_add(id_len).filter(|&end| end <= bytes.len()) else { + return records; + }; + let Some(body_end) = id_end + .checked_add(body_len) + .filter(|&end| end <= bytes.len()) + else { + return records; + }; + let id = until_nul(&bytes[pos..id_end]); + let body = until_nul(&bytes[id_end..body_end]); + if !id.is_empty() && !body.is_empty() { + records.push(ExtensionRecord { + id: id.to_vec(), + body: body.to_vec(), + }); + } + pos = body_end; + } + records +} + +fn until_nul(bytes: &[u8]) -> &[u8] { + match bytes.iter().position(|&b| b == 0) { + Some(i) => &bytes[..i], + None => bytes, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_records_write_no_ktm_header() { + let ext = BankThinkingExtensions { + thinking_visible: true, + records: Vec::new(), + }; + assert_eq!(ext.ext_flags(), EXT_BANK_REPLAY_V1 | EXT_THINKING_VISIBLE); + assert!(ext.encode_trailer().is_empty()); + } + + #[test] + fn invalid_records_do_not_set_tool_map() { + let ext = BankThinkingExtensions { + thinking_visible: false, + records: vec![ExtensionRecord { + id: Vec::new(), + body: b"x".to_vec(), + }], + }; + assert_eq!(ext.ext_flags(), EXT_BANK_REPLAY_V1); + assert!(ext.encode_trailer().is_empty()); + } + + #[test] + fn persist_bank_extensions_keeps_identity_responses_and_title() { + // Given: a bank record that already carries Responses/title bits + let mut rec = Record { + header: crate::format::Header { + quant_bits: 2, + reason: crate::format::Reason::BankEvict, + ext_flags: EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE | EXT_IMAGE_PIXELS_V2, + model_id: 1, + tokens: 8, + hits: 0, + ctx_size: 4096, + created_at: 1, + last_used: 1, + payload_bytes: 1, + text_bytes: 4, + }, + text: b"keep".to_vec(), + payload: b"P".to_vec(), + trailer: Vec::new(), + }; + let ext = BankThinkingExtensions { + thinking_visible: true, + records: Vec::new(), + }; + + // When: thinking/KTM extensions are persisted onto that record + rec.persist_bank_extensions(&ext); + + // Then: BANK_REPLAY_V1 + thinking land, and the two prior bits survive + assert_eq!( + rec.header.ext_flags, + EXT_BANK_REPLAY_V1 + | EXT_THINKING_VISIBLE + | EXT_RESPONSES_VISIBLE + | EXT_SESSION_TITLE + | EXT_IMAGE_PIXELS_V2 + ); + } +} diff --git a/crates/ds4-kv/src/format.rs b/crates/ds4-kv/src/format.rs new file mode 100644 index 0000000000000000000000000000000000000000..33555c39cbbc51183a6c277cb6519d84a7d1cf49 --- /dev/null +++ b/crates/ds4-kv/src/format.rs @@ -0,0 +1,781 @@ +//! KVC file envelope: 48-byte header + text length + text + payload. +//! +//! Matches `ds4_kvstore_fill_header` / `ds4_kvstore_read_header` and the +//! store fwrite order in `ds4_kvstore.c`. Do not invent a new format. + +use crate::sha1::sha1_hex; +use std::fs; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub const FIXED_HEADER: usize = 48; +pub const MAGIC: [u8; 3] = [b'K', b'V', b'C']; +pub const VERSION: u8 = 1; +pub const PAYLOAD_ABI: u8 = 2; + +static TEMP_SEQ: AtomicU64 = AtomicU64::new(0); + +pub const EXT_TOOL_MAP: u8 = 1 << 0; +pub const EXT_RESPONSES_VISIBLE: u8 = 1 << 1; +pub const EXT_THINKING_VISIBLE: u8 = 1 << 2; +pub const EXT_SESSION_TITLE: u8 = 1 << 3; +pub const EXT_BANK_REPLAY_V1: u8 = 1 << 4; +pub const EXT_IMAGE_PIXELS_V2: u8 = 1 << 5; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum Reason { + Unknown = 0, + Cold = 1, + Continued = 2, + Evict = 3, + Shutdown = 4, + AgentSystem = 5, + AgentSession = 6, + BankEvict = 7, + BankShutdown = 8, + BankCheckpoint = 9, +} + +impl Reason { + pub fn from_u8(v: u8) -> Self { + match v { + 1 => Self::Cold, + 2 => Self::Continued, + 3 => Self::Evict, + 4 => Self::Shutdown, + 5 => Self::AgentSystem, + 6 => Self::AgentSession, + 7 => Self::BankEvict, + 8 => Self::BankShutdown, + 9 => Self::BankCheckpoint, + _ => Self::Unknown, + } + } + + pub fn from_name(name: &str) -> Self { + match name { + "cold" => Self::Cold, + "continued" => Self::Continued, + "evict" => Self::Evict, + "shutdown" => Self::Shutdown, + "agent-system" => Self::AgentSystem, + "agent-session" => Self::AgentSession, + "bank-evict" => Self::BankEvict, + "bank-shutdown" => Self::BankShutdown, + "bank-checkpoint" => Self::BankCheckpoint, + _ => Self::Unknown, + } + } + + pub fn family(self) -> u8 { + match self as u8 { + 1..=4 => 1, + 5..=6 => 2, + 7..=9 => 3, + _ => 0, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Header { + pub quant_bits: u8, + pub reason: Reason, + pub ext_flags: u8, + pub model_id: u8, + pub tokens: u32, + pub hits: u32, + pub ctx_size: u32, + pub created_at: u64, + pub last_used: u64, + pub payload_bytes: u64, + pub text_bytes: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Record { + pub header: Header, + pub text: Vec, + pub payload: Vec, + pub trailer: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Metadata { + pub(crate) header: Header, + pub(crate) payload_offset: u64, + pub(crate) trailer_bytes: u64, + pub(crate) file_size: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Envelope { + pub header: Header, + pub text: Vec, + pub payload_offset: u64, + pub trailer_bytes: u64, + pub file_size: u64, +} + +#[derive(Debug)] +pub enum FormatError { + Io(io::Error), + BadMagic, + BadVersion, + BadPayloadAbi, + BadQuant, + ZeroTokens, + Truncated, + TextLenMismatch, + TrailerTooLarge, +} + +impl From for FormatError { + fn from(e: io::Error) -> Self { + Self::Io(e) + } +} + +impl std::fmt::Display for FormatError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(e) => write!(f, "{e}"), + Self::BadMagic => write!(f, "not a KVC file"), + Self::BadVersion => write!(f, "unsupported KVC version"), + Self::BadPayloadAbi => write!(f, "unsupported KVC payload ABI"), + Self::BadQuant => write!(f, "quant_bits must be 2 or 4"), + Self::ZeroTokens => write!(f, "token count is zero"), + Self::Truncated => write!(f, "truncated KVC file"), + Self::TextLenMismatch => write!(f, "text length does not match header"), + Self::TrailerTooLarge => write!(f, "KVC trailer exceeds read bound"), + } + } +} + +impl std::error::Error for FormatError {} + +pub fn le_put32(p: &mut [u8], v: u32) { + p[0] = v as u8; + p[1] = (v >> 8) as u8; + p[2] = (v >> 16) as u8; + p[3] = (v >> 24) as u8; +} + +pub fn le_get32(p: &[u8]) -> u32 { + u32::from(p[0]) | (u32::from(p[1]) << 8) | (u32::from(p[2]) << 16) | (u32::from(p[3]) << 24) +} + +pub fn le_put64(p: &mut [u8], v: u64) { + for i in 0..8 { + p[i] = (v >> (8 * i)) as u8; + } +} + +pub fn le_get64(p: &[u8]) -> u64 { + let mut v = 0u64; + for i in (0..8).rev() { + v = (v << 8) | u64::from(p[i]); + } + v +} + +pub fn fill_header( + model_id: u8, + quant_bits: u8, + reason: Reason, + ext_flags: u8, + tokens: u32, + hits: u32, + ctx_size: u32, + created_at: u64, + last_used: u64, + payload_bytes: u64, +) -> [u8; FIXED_HEADER] { + let mut h = [0u8; FIXED_HEADER]; + h[0] = MAGIC[0]; + h[1] = MAGIC[1]; + h[2] = MAGIC[2]; + h[3] = VERSION; + h[4] = quant_bits; + h[5] = reason as u8; + h[6] = ext_flags; + h[7] = model_id; + le_put32(&mut h[8..12], tokens); + le_put32(&mut h[12..16], hits); + le_put32(&mut h[16..20], ctx_size); + h[20] = PAYLOAD_ABI; + le_put64(&mut h[24..32], created_at); + le_put64(&mut h[32..40], last_used); + le_put64(&mut h[40..48], payload_bytes); + h +} + +pub fn parse_header(h: &[u8; FIXED_HEADER], text_bytes: u32) -> Result { + if h[0] != MAGIC[0] || h[1] != MAGIC[1] || h[2] != MAGIC[2] { + return Err(FormatError::BadMagic); + } + if h[3] != VERSION { + return Err(FormatError::BadVersion); + } + if h[20] != PAYLOAD_ABI { + return Err(FormatError::BadPayloadAbi); + } + let tokens = le_get32(&h[8..12]); + let quant_bits = h[4]; + if tokens == 0 { + return Err(FormatError::ZeroTokens); + } + if quant_bits != 2 && quant_bits != 4 { + return Err(FormatError::BadQuant); + } + Ok(Header { + quant_bits, + reason: Reason::from_u8(h[5]), + ext_flags: h[6], + model_id: h[7], + tokens, + hits: le_get32(&h[12..16]), + ctx_size: le_get32(&h[16..20]), + created_at: le_get64(&h[24..32]), + last_used: le_get64(&h[32..40]), + payload_bytes: le_get64(&h[40..48]), + text_bytes, + }) +} + +pub fn encode_file(record: &Record) -> Vec { + let mut h = fill_header( + record.header.model_id, + record.header.quant_bits, + record.header.reason, + record.header.ext_flags, + record.header.tokens, + record.header.hits, + record.header.ctx_size, + record.header.created_at, + record.header.last_used, + record.payload.len() as u64, + ); + // payload_bytes in the header follows the payload vector, matching C. + le_put64(&mut h[40..48], record.payload.len() as u64); + let mut out = Vec::with_capacity( + FIXED_HEADER + 4 + record.text.len() + record.payload.len() + record.trailer.len(), + ); + out.extend_from_slice(&h); + let mut tb = [0u8; 4]; + le_put32(&mut tb, record.text.len() as u32); + out.extend_from_slice(&tb); + out.extend_from_slice(&record.text); + out.extend_from_slice(&record.payload); + out.extend_from_slice(&record.trailer); + out +} + +pub fn decode_file(bytes: &[u8]) -> Result { + if bytes.len() < FIXED_HEADER + 4 { + return Err(FormatError::Truncated); + } + let mut hdr = [0u8; FIXED_HEADER]; + hdr.copy_from_slice(&bytes[..FIXED_HEADER]); + let text_bytes = le_get32(&bytes[FIXED_HEADER..FIXED_HEADER + 4]); + let header = parse_header(&hdr, text_bytes)?; + let text_start = FIXED_HEADER + 4; + let text_end = text_start + .checked_add(text_bytes as usize) + .ok_or(FormatError::Truncated)?; + let payload_end = text_end + .checked_add(header.payload_bytes as usize) + .ok_or(FormatError::Truncated)?; + if bytes.len() < payload_end { + return Err(FormatError::Truncated); + } + Ok(Record { + header, + text: bytes[text_start..text_end].to_vec(), + payload: bytes[text_end..payload_end].to_vec(), + trailer: bytes[payload_end..].to_vec(), + }) +} + +pub fn write_path(path: &Path, record: &Record) -> Result<(), FormatError> { + let bytes = encode_file(record); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let tmp = path.with_extension("kv.tmp"); + { + let mut f = fs::File::create(&tmp)?; + f.write_all(&bytes)?; + f.flush()?; + } + fs::rename(&tmp, path)?; + Ok(()) +} + +#[cfg(test)] +pub(crate) fn write_stream( + path: &Path, + header: &Header, + text: &[u8], + payload: &mut impl Read, + payload_bytes: u64, + trailer: &[u8], +) -> Result<(), FormatError> { + let tmp = stage_stream(path, header, text, payload, payload_bytes, trailer)?; + let result = fs::rename(&tmp, path).map_err(FormatError::Io); + if result.is_err() { + let _ = fs::remove_file(&tmp); + } + result +} + +pub(crate) fn stage_stream( + path: &Path, + header: &Header, + text: &[u8], + payload: &mut impl Read, + payload_bytes: u64, + trailer: &[u8], +) -> Result { + if text.len() > u32::MAX as usize { + return Err(FormatError::TextLenMismatch); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let (tmp, mut f) = loop { + let seq = TEMP_SEQ.fetch_add(1, Ordering::Relaxed); + let tmp = path.with_extension(format!("kv.tmp.{}.{}", std::process::id(), seq)); + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + { + Ok(f) => break (tmp, f), + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(FormatError::Io(e)), + } + }; + let result: Result<(), FormatError> = (|| { + let h = fill_header( + header.model_id, + header.quant_bits, + header.reason, + header.ext_flags, + header.tokens, + header.hits, + header.ctx_size, + header.created_at, + header.last_used, + payload_bytes, + ); + f.write_all(&h)?; + let mut raw_text_bytes = [0u8; 4]; + le_put32(&mut raw_text_bytes, text.len() as u32); + f.write_all(&raw_text_bytes)?; + f.write_all(text)?; + let copied = io::copy(&mut payload.take(payload_bytes), &mut f)?; + if copied != payload_bytes { + return Err(FormatError::Truncated); + } + f.write_all(trailer)?; + f.flush()?; + drop(f); + Ok(()) + })(); + if let Err(error) = result { + let _ = fs::remove_file(&tmp); + return Err(error); + } + Ok(tmp) +} + +pub fn read_path(path: &Path) -> Result { + let mut f = fs::File::open(path)?; + let mut bytes = Vec::new(); + f.read_to_end(&mut bytes)?; + decode_file(&bytes) +} + +pub fn read_envelope(path: &Path) -> Result { + let (metadata, text) = read_text_prefix(path, usize::MAX)?; + if text.len() != metadata.header.text_bytes as usize { + return Err(FormatError::Truncated); + } + Ok(Envelope { + header: metadata.header, + text, + payload_offset: metadata.payload_offset, + trailer_bytes: metadata.trailer_bytes, + file_size: metadata.file_size, + }) +} + +pub fn read_trailer(path: &Path, max_bytes: u64) -> Result<(Header, Vec), FormatError> { + let mut file = fs::File::open(path)?; + let metadata = read_metadata_file(&mut file)?; + if metadata.trailer_bytes > max_bytes { + return Err(FormatError::TrailerTooLarge); + } + let trailer_start = metadata + .payload_offset + .checked_add(metadata.header.payload_bytes) + .ok_or(FormatError::Truncated)?; + let trailer_bytes = + usize::try_from(metadata.trailer_bytes).map_err(|_| FormatError::TrailerTooLarge)?; + file.seek(SeekFrom::Start(trailer_start))?; + let mut trailer = vec![0; trailer_bytes]; + read_exact(&mut file, &mut trailer)?; + Ok((metadata.header, trailer)) +} + +pub(crate) fn read_metadata(path: &Path) -> Result { + let mut f = fs::File::open(path)?; + read_metadata_file(&mut f) +} + +/// The header, and as much of the text as the file still holds, without +/// asking it for the payload behind them. A truncation can take the +/// payload and leave what says which conversation the record was keyed by. +pub(crate) fn read_header_text( + path: &Path, + max_bytes: usize, +) -> Result<(Header, Vec), FormatError> { + let mut f = fs::File::open(path)?; + let mut raw_header = [0u8; FIXED_HEADER]; + read_exact(&mut f, &mut raw_header)?; + let mut raw_text_bytes = [0u8; 4]; + read_exact(&mut f, &mut raw_text_bytes)?; + let header = parse_header(&raw_header, le_get32(&raw_text_bytes))?; + let want = (header.text_bytes as usize).min(max_bytes); + let mut text = Vec::new(); + f.take(want as u64).read_to_end(&mut text)?; + Ok((header, text)) +} + +pub(crate) fn read_text_prefix( + path: &Path, + max_bytes: usize, +) -> Result<(Metadata, Vec), FormatError> { + let mut f = fs::File::open(path)?; + let metadata = read_metadata_file(&mut f)?; + let want = (metadata.header.text_bytes as usize).min(max_bytes); + let mut text = Vec::with_capacity(want); + f.take(want as u64).read_to_end(&mut text)?; + Ok((metadata, text)) +} + +fn read_metadata_file(f: &mut fs::File) -> Result { + let file_size = f.metadata()?.len(); + if file_size < (FIXED_HEADER + 4) as u64 { + return Err(FormatError::Truncated); + } + + let mut raw_header = [0u8; FIXED_HEADER]; + read_exact(f, &mut raw_header)?; + let mut raw_text_bytes = [0u8; 4]; + read_exact(f, &mut raw_text_bytes)?; + let text_bytes = le_get32(&raw_text_bytes); + let header = parse_header(&raw_header, text_bytes)?; + let payload_offset = (FIXED_HEADER + 4) as u64 + u64::from(text_bytes); + let payload_end = payload_offset + .checked_add(header.payload_bytes) + .ok_or(FormatError::Truncated)?; + if file_size < payload_end { + return Err(FormatError::Truncated); + } + + Ok(Metadata { + header, + payload_offset, + trailer_bytes: file_size - payload_end, + file_size, + }) +} + +fn read_exact(f: &mut fs::File, buf: &mut [u8]) -> Result<(), FormatError> { + match f.read_exact(buf) { + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => Err(FormatError::Truncated), + result => result.map_err(FormatError::Io), + } +} + +pub fn text_sha_hex(text: &[u8]) -> String { + sha1_hex(text) +} + +pub fn sha_hex_name(name: &str) -> Option { + if name.len() != 43 || !name.ends_with(".kv") { + return None; + } + let hex = &name[..40]; + if !hex.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + Some(hex.to_ascii_lowercase()) +} + +pub fn path_for_sha(dir: &Path, sha: &str) -> PathBuf { + dir.join(format!("{sha}.kv")) +} + +pub fn key_kind(ext_flags: u8) -> &'static str { + if ext_flags & EXT_RESPONSES_VISIBLE != 0 { + "responses-visible" + } else if ext_flags & EXT_THINKING_VISIBLE != 0 { + "thinking-visible" + } else { + "token-text" + } +} + +pub fn is_bank_replay_v1(reason: Reason, ext_flags: u8) -> bool { + reason.family() == 3 && ext_flags & EXT_BANK_REPLAY_V1 != 0 +} + +pub fn is_automatic_exact_replay(reason: Reason, ext_flags: u8) -> bool { + match reason.family() { + 1 => ext_flags & EXT_BANK_REPLAY_V1 == 0, + _ => is_bank_replay_v1(reason, ext_flags), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct ErrorAfterPrefix { + offset: usize, + } + + impl Read for ErrorAfterPrefix { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + const PREFIX: &[u8] = b"DSV4"; + if self.offset == PREFIX.len() { + return Err(io::Error::other("injected payload read failure")); + } + let n = buf.len().min(PREFIX.len() - self.offset); + buf[..n].copy_from_slice(&PREFIX[self.offset..self.offset + n]); + self.offset += n; + Ok(n) + } + } + + fn sample() -> Record { + Record { + header: Header { + quant_bits: 2, + reason: Reason::Cold, + ext_flags: 0, + model_id: 0, + tokens: 512, + hits: 3, + ctx_size: 2048, + created_at: 1_700_000_000, + last_used: 1_700_000_100, + payload_bytes: 4, + text_bytes: 5, + }, + text: b"hello".to_vec(), + payload: b"ABCD".to_vec(), + trailer: Vec::new(), + } + } + + #[test] + fn header_round_trip() { + let rec = sample(); + let bytes = encode_file(&rec); + let got = decode_file(&bytes).unwrap(); + assert_eq!(got.text, rec.text); + assert_eq!(got.payload, rec.payload); + assert_eq!(got.header.tokens, 512); + assert_eq!(got.header.quant_bits, 2); + assert_eq!(got.header.reason, Reason::Cold); + assert_eq!(got.header.ctx_size, 2048); + assert_eq!(&bytes[..3], b"KVC"); + assert_eq!(bytes[3], 1); + assert_eq!(bytes[20], 2); + } + + #[test] + fn envelope_has_offsets_without_payload() { + let dir = std::env::temp_dir().join(format!("ds4-kv-envelope-{}", std::process::id())); + let path = dir.join("sample.kv"); + let _ = fs::remove_dir_all(&dir); + let mut rec = sample(); + rec.trailer = b"end".to_vec(); + write_path(&path, &rec).unwrap(); + + let got = read_envelope(&path).unwrap(); + assert_eq!(got.header, rec.header); + assert_eq!(got.text, rec.text); + assert_eq!( + got.payload_offset, + (FIXED_HEADER + 4 + rec.text.len()) as u64 + ); + assert_eq!(got.trailer_bytes, rec.trailer.len() as u64); + assert_eq!(got.file_size, fs::metadata(&path).unwrap().len()); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn trailer_read_seeks_past_sparse_payload_and_honors_bound() { + use std::io::{Seek, SeekFrom}; + + const PAYLOAD_BYTES: u64 = 4 * 1024 * 1024 * 1024; + let dir = + std::env::temp_dir().join(format!("ds4-kv-trailer-sparse-{}", std::process::id())); + let path = dir.join("sample.kv"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + + let rec = sample(); + let mut file = fs::File::create(&path).unwrap(); + file.write_all(&fill_header( + rec.header.model_id, + rec.header.quant_bits, + rec.header.reason, + EXT_TOOL_MAP, + rec.header.tokens, + rec.header.hits, + rec.header.ctx_size, + rec.header.created_at, + rec.header.last_used, + PAYLOAD_BYTES, + )) + .unwrap(); + let mut text_bytes = [0u8; 4]; + le_put32(&mut text_bytes, rec.text.len() as u32); + file.write_all(&text_bytes).unwrap(); + file.write_all(&rec.text).unwrap(); + file.seek(SeekFrom::Current(PAYLOAD_BYTES as i64)).unwrap(); + file.write_all(b"\0\xffTR").unwrap(); + drop(file); + + let (header, trailer) = read_trailer(&path, 4).unwrap(); + assert_eq!(header.payload_bytes, PAYLOAD_BYTES); + assert_eq!(header.ext_flags, EXT_TOOL_MAP); + assert_eq!(trailer, b"\0\xffTR"); + assert!(matches!( + read_trailer(&path, 3), + Err(FormatError::TrailerTooLarge) + )); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn rejects_wrong_quant() { + let mut rec = sample(); + rec.header.quant_bits = 8; + let bytes = encode_file(&rec); + assert!(matches!(decode_file(&bytes), Err(FormatError::BadQuant))); + } + + #[test] + fn filename_is_text_sha() { + assert_eq!(sha_hex_name("not-a-kv"), None); + let sha = text_sha_hex(b"hello"); + assert_eq!(sha.len(), 40); + assert_eq!( + sha_hex_name(&format!("{sha}.kv")).as_deref(), + Some(sha.as_str()) + ); + } + + #[test] + fn stream_failure_preserves_destination_and_removes_temp() { + let dir = + std::env::temp_dir().join(format!("ds4-kv-stream-failure-{}", std::process::id())); + let path = dir.join("entry.kv"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::write(&path, b"original KVC bytes").unwrap(); + + let rec = sample(); + let mut payload = ErrorAfterPrefix { offset: 0 }; + let result = write_stream(&path, &rec.header, &rec.text, &mut payload, 8, &rec.trailer); + + assert!(result.is_err()); + assert_eq!(fs::read(&path).unwrap(), b"original KVC bytes"); + let mut names: Vec<_> = fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + names.sort(); + assert_eq!(names, [std::ffi::OsString::from("entry.kv")]); + + let mut short_payload = &b"DSV4"[..]; + let result = write_stream( + &path, + &rec.header, + &rec.text, + &mut short_payload, + 8, + &rec.trailer, + ); + assert!(result.is_err()); + assert_eq!(fs::read(&path).unwrap(), b"original KVC bytes"); + let names: Vec<_> = fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(names, [std::ffi::OsString::from("entry.kv")]); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn stream_layout_matches_buffered_encoder() { + let dir = std::env::temp_dir().join(format!("ds4-kv-stream-layout-{}", std::process::id())); + let path = dir.join("entry.kv"); + let _ = fs::remove_dir_all(&dir); + + let mut rec = sample(); + rec.trailer = b"KVT1 trailer".to_vec(); + let mut payload = rec.payload.as_slice(); + write_stream( + &path, + &rec.header, + &rec.text, + &mut payload, + rec.payload.len() as u64, + &rec.trailer, + ) + .unwrap(); + assert_eq!(fs::read(&path).unwrap(), encode_file(&rec)); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn staged_stream_leaves_destination_until_commit() { + let dir = std::env::temp_dir().join(format!("ds4-kv-stream-stage-{}", std::process::id())); + let path = dir.join("entry.kv"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::write(&path, b"original KVC bytes").unwrap(); + + let rec = sample(); + let mut payload = rec.payload.as_slice(); + let staged = stage_stream( + &path, + &rec.header, + &rec.text, + &mut payload, + rec.payload.len() as u64, + &rec.trailer, + ) + .unwrap(); + assert_eq!(fs::read(&path).unwrap(), b"original KVC bytes"); + assert_eq!(fs::read(&staged).unwrap(), encode_file(&rec)); + + fs::rename(staged, &path).unwrap(); + assert_eq!(fs::read(&path).unwrap(), encode_file(&rec)); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/crates/ds4-kv/src/host.rs b/crates/ds4-kv/src/host.rs new file mode 100644 index 0000000000000000000000000000000000000000..93e68810a19940b24531ec80f2b9546d9898ac9b --- /dev/null +++ b/crates/ds4-kv/src/host.rs @@ -0,0 +1,64 @@ +//! Host-ledger token counts for KV policy. Callers pass these fields; +//! policy never reads a native session or CUDA scratch. + +use crate::policy::{bank_checkpoint_due, continued_store_target, Options}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HostKvView { + pub live_tokens: i32, + pub stored_tokens: i32, +} + +pub fn continued_store_target_from_host(opt: &Options, host: HostKvView) -> i32 { + continued_store_target(opt, host.stored_tokens, host.live_tokens) +} + +pub fn bank_checkpoint_due_from_host(opt: &Options, host: HostKvView) -> bool { + bank_checkpoint_due(opt, host.live_tokens, host.stored_tokens) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn continued_target_uses_host_live_tokens_not_native_pos() { + let opt = Options::default(); + let due = HostKvView { + live_tokens: 10240, + stored_tokens: 0, + }; + let raw_interval = HostKvView { + live_tokens: 10000, + stored_tokens: 0, + }; + assert_eq!(continued_store_target_from_host(&opt, due), 10240); + assert_eq!(continued_store_target_from_host(&opt, raw_interval), 0); + } + + #[test] + fn bank_checkpoint_due_uses_host_committed_frontier() { + let opt = Options::default(); + assert!(bank_checkpoint_due_from_host( + &opt, + HostKvView { + live_tokens: 10240, + stored_tokens: 0, + } + )); + assert!(!bank_checkpoint_due_from_host( + &opt, + HostKvView { + live_tokens: 10000, + stored_tokens: 0, + } + )); + assert!(!bank_checkpoint_due_from_host( + &opt, + HostKvView { + live_tokens: 9999, + stored_tokens: 0, + } + )); + } +} diff --git a/crates/ds4-kv/src/identity.rs b/crates/ds4-kv/src/identity.rs new file mode 100644 index 0000000000000000000000000000000000000000..3f669a8625cf001ddfe73e0ba5644c840853a94d --- /dev/null +++ b/crates/ds4-kv/src/identity.rs @@ -0,0 +1,42 @@ +//! Fixed identity footer after opaque payload and optional KTM records. +//! Reading identity never scans payload bytes or allocates from file lengths. + +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; + +pub(crate) const FLAG: u8 = 1 << 6; +pub(crate) const BYTES: usize = 40; +const MAGIC: &[u8; 4] = b"KVI\x01"; +const END: &[u8; 4] = b"KVI!"; + +pub(crate) fn append(trailer: &mut Vec, identity: &[u8; 32]) { + trailer.extend_from_slice(MAGIC); + trailer.extend_from_slice(identity); + trailer.extend_from_slice(END); +} + +pub(crate) fn read(path: &Path, file_size: u64, trailer_bytes: u64, flags: u8) -> Option<[u8; 32]> { + if flags & FLAG == 0 || trailer_bytes < BYTES as u64 || file_size < BYTES as u64 { + return None; + } + let mut file = File::open(path).ok()?; + if file.metadata().ok()?.len() != file_size { + return None; + } + file.seek(SeekFrom::Start(file_size - BYTES as u64)).ok()?; + let mut footer = [0; BYTES]; + file.read_exact(&mut footer).ok()?; + decode(flags, &footer) +} + +pub(crate) fn decode(flags: u8, trailer: &[u8]) -> Option<[u8; 32]> { + if flags & FLAG == 0 || trailer.len() < BYTES { + return None; + } + let footer = &trailer[trailer.len() - BYTES..]; + if &footer[..4] != MAGIC || &footer[36..] != END { + return None; + } + footer[4..36].try_into().ok() +} diff --git a/crates/ds4-kv/src/lib.rs b/crates/ds4-kv/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..72d3fd27b21c98ec6a669047df44becf11729316 --- /dev/null +++ b/crates/ds4-kv/src/lib.rs @@ -0,0 +1,28 @@ +//! KVC disk store. File format, SHA key, eviction, and prefix lookup +//! match `ds4_kvstore.c` at `v0.6.3-dfm`. Payload bytes stay opaque. + +mod extensions; +mod format; +mod host; +mod identity; +mod policy; +mod sha1; +mod store; + +pub use extensions::{ + bank_persist_ext_flags, decode_ktm, encode_ktm, BankThinkingExtensions, ExtensionRecord, + KTM_MAGIC, KTM_VERSION, +}; +pub use format::{ + decode_file, encode_file, fill_header, key_kind, parse_header, path_for_sha, read_envelope, + read_path, read_trailer, sha_hex_name, text_sha_hex, write_path, Envelope, FormatError, Header, + Reason, Record, EXT_BANK_REPLAY_V1, EXT_IMAGE_PIXELS_V2, EXT_RESPONSES_VISIBLE, + EXT_SESSION_TITLE, EXT_THINKING_VISIBLE, EXT_TOOL_MAP, FIXED_HEADER, PAYLOAD_ABI, VERSION, +}; +pub use host::{bank_checkpoint_due_from_host, continued_store_target_from_host, HostKvView}; +pub use policy::{ + bank_checkpoint_due, chat_anchor_pos, continued_store_target, eviction_score, file_size_fits, + store_len, EvictionContext, Options, ScoreEntry, +}; +pub use sha1::sha1_hex; +pub use store::{Entry, PayloadGuard, PayloadTemp, PrefixAnswer, Store}; diff --git a/crates/ds4-kv/src/policy.rs b/crates/ds4-kv/src/policy.rs new file mode 100644 index 0000000000000000000000000000000000000000..98276c8e67eab5369b1158d1f76b1c4946424706 --- /dev/null +++ b/crates/ds4-kv/src/policy.rs @@ -0,0 +1,340 @@ +//! Store-length, continued pacing, budget, and eviction scoring. +//! Port of the corresponding helpers in `ds4_kvstore.c`. + +use crate::format::{text_sha_hex, Reason}; + +pub const DEFAULT_MB: u64 = 4096; +pub const HIT_HALF_LIFE_SECONDS: u64 = 6 * 60 * 60; +pub const DEFAULT_MIN_TOKENS: i32 = 512; +pub const DEFAULT_COLD_MAX_TOKENS: i32 = 30000; +pub const DEFAULT_BOUNDARY_TRIM_TOKENS: i32 = 32; +pub const DEFAULT_BOUNDARY_ALIGN_TOKENS: i32 = 2048; +pub const DEFAULT_CONTINUED_INTERVAL_TOKENS: i32 = 10000; +const MIN_EFFECTIVE_HITS: f64 = 0.01; +const CONTINUED_PREFIX_MIN_FACTOR: f64 = 0.05; +const CONTINUED_PREFIX_HIT_FACTOR: f64 = 0.45; +const ANCHOR_REASON_SCORE_FACTOR: f64 = 2.0; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Options { + pub min_tokens: i32, + pub cold_max_tokens: i32, + pub continued_interval_tokens: i32, + pub boundary_trim_tokens: i32, + pub boundary_align_tokens: i32, +} + +impl Default for Options { + fn default() -> Self { + Self { + min_tokens: DEFAULT_MIN_TOKENS, + cold_max_tokens: DEFAULT_COLD_MAX_TOKENS, + continued_interval_tokens: DEFAULT_CONTINUED_INTERVAL_TOKENS, + boundary_trim_tokens: DEFAULT_BOUNDARY_TRIM_TOKENS, + boundary_align_tokens: DEFAULT_BOUNDARY_ALIGN_TOKENS, + } + } +} + +#[derive(Clone, Debug)] +pub struct EvictionContext<'a> { + pub text: &'a [u8], + pub model_id: u8, + pub quant_bits: u8, + pub ctx_size: u32, + pub reject_different_quant: bool, +} + +#[derive(Clone, Debug)] +pub struct ScoreEntry<'a> { + pub sha: &'a str, + pub quant_bits: u8, + pub model_id: u8, + pub reason: Reason, + pub tokens: u32, + pub hits: u32, + pub ctx_size: u32, + pub created_at: u64, + pub last_used: u64, + pub text_bytes: u64, + pub file_size: u64, +} + +pub fn store_len(opt: &Options, tokens: i32) -> i32 { + let trim = opt.boundary_trim_tokens; + let align = opt.boundary_align_tokens; + if tokens > opt.min_tokens + trim { + let mut stable = tokens - trim; + if align > 0 { + stable -= stable % align; + } + if stable >= opt.min_tokens { + return stable; + } + } + tokens +} + +pub fn continued_step(opt: &Options) -> i32 { + if opt.continued_interval_tokens <= 0 { + return 0; + } + let mut step = opt.continued_interval_tokens; + let align = opt.boundary_align_tokens; + if align > 0 { + let rounded = + ((i64::from(step) + i64::from(align) - 1) / i64::from(align)) * i64::from(align); + step = i32::try_from(rounded).unwrap_or(0); + } + step +} + +pub fn continued_store_target(opt: &Options, last_store_tokens: i32, live_tokens: i32) -> i32 { + let step = continued_step(opt); + if step <= 0 { + return 0; + } + if live_tokens < opt.min_tokens { + return 0; + } + if live_tokens % step != 0 { + return 0; + } + if live_tokens <= last_store_tokens { + return 0; + } + live_tokens +} + +pub fn bank_checkpoint_due(opt: &Options, committed: i32, stored_tokens: i32) -> bool { + let step = continued_step(opt); + if step <= 0 || committed < opt.min_tokens || committed < stored_tokens + step { + return false; + } + true +} + +pub fn file_size_bytes(text_bytes: u64, payload_bytes: u64, trailer_bytes: u64) -> Option { + const FIXED: u64 = 48 + 4; + FIXED + .checked_add(text_bytes)? + .checked_add(payload_bytes)? + .checked_add(trailer_bytes) +} + +pub fn budget_required(file_bytes: u64) -> Option { + let mut slack = file_bytes / 100; + if file_bytes % 100 != 0 { + slack += 1; + } + file_bytes.checked_add(slack) +} + +pub fn file_size_fits( + budget_bytes: u64, + text_bytes: u64, + payload_bytes: u64, + trailer_bytes: u64, +) -> bool { + let Some(file_bytes) = file_size_bytes(text_bytes, payload_bytes, trailer_bytes) else { + return false; + }; + if budget_bytes == 0 { + return true; + } + match budget_required(file_bytes) { + Some(required) => required <= budget_bytes, + None => false, + } +} + +fn reason_is_anchor(reason: Reason) -> bool { + matches!( + reason, + Reason::Cold + | Reason::Evict + | Reason::Shutdown + | Reason::BankEvict + | Reason::BankShutdown + | Reason::BankCheckpoint + ) +} + +fn incoming_supersedes_continued(e: &ScoreEntry<'_>, incoming: &EvictionContext<'_>) -> bool { + if e.reason != Reason::Continued && e.reason != Reason::BankCheckpoint { + return false; + } + if e.text_bytes == 0 || e.text_bytes >= incoming.text.len() as u64 { + return false; + } + if e.model_id != incoming.model_id { + return false; + } + if incoming.reject_different_quant && e.quant_bits != incoming.quant_bits { + return false; + } + if incoming.ctx_size > e.ctx_size { + return false; + } + let prefix = &incoming.text[..e.text_bytes as usize]; + text_sha_hex(prefix) == e.sha +} + +pub fn eviction_score(e: &ScoreEntry<'_>, now: u64, incoming: Option<&EvictionContext<'_>>) -> f64 { + if e.file_size == 0 { + return 0.0; + } + let mut effective_hits = e.hits as f64; + let used_at = if e.last_used != 0 { + e.last_used + } else { + e.created_at + }; + if used_at == 0 { + effective_hits = 0.0; + } else if now > used_at { + let elapsed = (now - used_at) as f64; + effective_hits *= (-elapsed / HIT_HALF_LIFE_SECONDS as f64).exp2(); + if effective_hits < MIN_EFFECTIVE_HITS { + effective_hits = 0.0; + } + } + let mut score = (effective_hits + 1.0) * (e.tokens as f64) / (e.file_size as f64); + if reason_is_anchor(e.reason) { + score *= ANCHOR_REASON_SCORE_FACTOR; + } + if let Some(incoming) = incoming { + if incoming_supersedes_continued(e, incoming) { + let h = if effective_hits > 0.0 { + effective_hits / (effective_hits + 1.0) + } else { + 0.0 + }; + score *= CONTINUED_PREFIX_MIN_FACTOR + CONTINUED_PREFIX_HIT_FACTOR * h; + } + } + score +} + +pub fn chat_anchor_pos( + opt: &Options, + prompt: &[i32], + user_token_id: i32, + assistant_token_id: i32, +) -> i32 { + if user_token_id < 0 || assistant_token_id < 0 { + return -1; + } + let mut last_user = -1; + for (i, &token) in prompt.iter().enumerate() { + if token == assistant_token_id { + break; + } + if token == user_token_id { + last_user = i as i32; + } + } + if last_user >= opt.min_tokens { + last_user + } else { + -1 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn store_len_trims_to_align() { + let opt = Options::default(); + assert_eq!(store_len(&opt, 4096 + 40), 4096); + assert_eq!(store_len(&opt, 100), 100); + } + + #[test] + fn continued_target_needs_aligned_interval() { + let opt = Options::default(); + assert_eq!(continued_store_target(&opt, 0, 8192), 0); + assert_eq!(continued_store_target(&opt, 0, 10240), 10240); + assert_eq!(continued_store_target(&opt, 10240, 10240), 0); + } + + #[test] + fn continued_step_maps_default_10000_interval_to_10240() { + let opt = Options::default(); + assert_eq!(DEFAULT_CONTINUED_INTERVAL_TOKENS, 10000); + assert_eq!( + opt.continued_interval_tokens, + DEFAULT_CONTINUED_INTERVAL_TOKENS + ); + assert_eq!(continued_step(&opt), 10240); + } + + #[test] + fn bank_checkpoint_due_when_committed_at_10240_aligned_frontier() { + let opt = Options::default(); + assert!(bank_checkpoint_due(&opt, 10240, 0)); + } + + #[test] + fn bank_checkpoint_not_due_when_committed_is_raw_10000_interval() { + let opt = Options::default(); + assert!(!bank_checkpoint_due(&opt, 10000, 0)); + } + + #[test] + fn bank_checkpoint_not_due_when_committed_is_9999() { + let opt = Options::default(); + assert!(!bank_checkpoint_due(&opt, 9999, 0)); + } + + #[test] + fn continued_step_disables_unrepresentable_aligned_interval() { + let opt = Options { + min_tokens: 1, + continued_interval_tokens: i32::MAX, + boundary_align_tokens: 2048, + ..Options::default() + }; + assert_eq!(continued_step(&opt), 0); + assert_eq!(continued_store_target(&opt, 0, 2048), 0); + } + + #[test] + fn budget_adds_one_percent() { + assert_eq!(budget_required(100), Some(101)); + assert_eq!(budget_required(101), Some(103)); + } + + #[test] + fn equal_length_incoming_does_not_discount_continued() { + let text = b"same prefix"; + let sha = text_sha_hex(text); + let entry = ScoreEntry { + sha: &sha, + quant_bits: 2, + model_id: 0, + reason: Reason::Continued, + tokens: 512, + hits: 0, + ctx_size: 2048, + created_at: 1000, + last_used: 1000, + text_bytes: text.len() as u64, + file_size: 4096, + }; + let incoming = EvictionContext { + text, + model_id: 0, + quant_bits: 2, + ctx_size: 2048, + reject_different_quant: true, + }; + + assert_eq!( + eviction_score(&entry, 1000, Some(&incoming)), + eviction_score(&entry, 1000, None) + ); + } +} diff --git a/crates/ds4-kv/src/sha1.rs b/crates/ds4-kv/src/sha1.rs new file mode 100644 index 0000000000000000000000000000000000000000..c8746da4557035b0168121e280e7faf845f11e21 --- /dev/null +++ b/crates/ds4-kv/src/sha1.rs @@ -0,0 +1,140 @@ +//! Byte-identical port of the SHA-1 in `ds4_kvstore.c`. + +pub fn sha1(bytes: &[u8]) -> [u8; 20] { + let mut c = Ctx::new(); + c.update(bytes); + c.finish() +} + +pub fn sha1_hex(bytes: &[u8]) -> String { + hex20(&sha1(bytes)) +} + +struct Ctx { + h: [u32; 5], + bytes: u64, + block: [u8; 64], + used: usize, +} + +impl Ctx { + fn new() -> Self { + Self { + h: [ + 0x6745_2301, + 0xefcd_ab89, + 0x98ba_dcfe, + 0x1032_5476, + 0xc3d2_e1f0, + ], + bytes: 0, + block: [0; 64], + used: 0, + } + } + + fn update(&mut self, mut input: &[u8]) { + self.bytes += input.len() as u64; + while !input.is_empty() { + let n = (64 - self.used).min(input.len()); + self.block[self.used..self.used + n].copy_from_slice(&input[..n]); + self.used += n; + input = &input[n..]; + if self.used == 64 { + self.transform(); + self.used = 0; + } + } + } + + fn finish(&mut self) -> [u8; 20] { + let bits = self.bytes * 8; + self.update(&[0x80]); + let zero = [0u8; 1]; + while self.used != 56 { + self.update(&zero); + } + let mut len = [0u8; 8]; + for i in 0..8 { + len[7 - i] = (bits >> (8 * i)) as u8; + } + self.update(&len); + let mut out = [0u8; 20]; + for i in 0..5 { + out[i * 4] = (self.h[i] >> 24) as u8; + out[i * 4 + 1] = (self.h[i] >> 16) as u8; + out[i * 4 + 2] = (self.h[i] >> 8) as u8; + out[i * 4 + 3] = self.h[i] as u8; + } + out + } + + fn transform(&mut self) { + let mut w = [0u32; 80]; + for i in 0..16 { + w[i] = (u32::from(self.block[i * 4]) << 24) + | (u32::from(self.block[i * 4 + 1]) << 16) + | (u32::from(self.block[i * 4 + 2]) << 8) + | u32::from(self.block[i * 4 + 3]); + } + for i in 16..80 { + w[i] = rol32(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1); + } + + let mut a = self.h[0]; + let mut b = self.h[1]; + let mut cc = self.h[2]; + let mut d = self.h[3]; + let mut e = self.h[4]; + for i in 0..80 { + let (f, k) = if i < 20 { + ((b & cc) | ((!b) & d), 0x5a82_7999) + } else if i < 40 { + (b ^ cc ^ d, 0x6ed9_eba1) + } else if i < 60 { + ((b & cc) | (b & d) | (cc & d), 0x8f1b_bcdc) + } else { + (b ^ cc ^ d, 0xca62_c1d6) + }; + let tmp = rol32(a, 5) + .wrapping_add(f) + .wrapping_add(e) + .wrapping_add(k) + .wrapping_add(w[i]); + e = d; + d = cc; + cc = rol32(b, 30); + b = a; + a = tmp; + } + self.h[0] = self.h[0].wrapping_add(a); + self.h[1] = self.h[1].wrapping_add(b); + self.h[2] = self.h[2].wrapping_add(cc); + self.h[3] = self.h[3].wrapping_add(d); + self.h[4] = self.h[4].wrapping_add(e); + } +} + +fn rol32(v: u32, n: i32) -> u32 { + v.rotate_left(n as u32) +} + +fn hex20(digest: &[u8; 20]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(40); + for &b in digest { + out.push(HEX[(b >> 4) as usize] as char); + out.push(HEX[(b & 15) as usize] as char); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sha1_abc_matches_fips() { + assert_eq!(sha1_hex(b"abc"), "a9993e364706816aba3e25717850c26c9cd0d89d"); + } +} diff --git a/crates/ds4-kv/src/store.rs b/crates/ds4-kv/src/store.rs new file mode 100644 index 0000000000000000000000000000000000000000..2f9f9ceb8687384384126104039dfa9256a68b75 --- /dev/null +++ b/crates/ds4-kv/src/store.rs @@ -0,0 +1,2406 @@ +//! Directory-backed KVC catalog: refresh, prefix lookup, LCP, evict. + +use crate::format::{ + fill_header, is_automatic_exact_replay, is_bank_replay_v1, path_for_sha, read_envelope, + read_header_text, read_metadata, read_path, read_text_prefix, sha_hex_name, stage_stream, + text_sha_hex, write_path, Envelope, FormatError, Header, Reason, Record, EXT_IMAGE_PIXELS_V2, + EXT_TOOL_MAP, +}; +use crate::policy::{ + eviction_score, file_size_bytes, file_size_fits, EvictionContext, Options, ScoreEntry, + DEFAULT_MB, +}; +use std::fs; +use std::io::{self, Read as _, Seek as _, SeekFrom, Write as _}; +#[cfg(unix)] +use std::os::fd::AsRawFd as _; +#[cfg(unix)] +use std::os::unix::fs::{DirBuilderExt as _, OpenOptionsExt as _}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +static PAYLOAD_TEMP_SEQ: AtomicU64 = AtomicU64::new(0); + +/// Whether a record must leave something to prefill. The bank lane admits +/// only records it can extend; the serial lane may replay an exact one. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Suffix { + Optional, + Required, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum FileAccess { + ReadOnly, + ReadWrite, +} + +/// What a search would do with the best record it can see for a key. The +/// store answers what it holds; the lane turns that into a miss reason. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PrefixAnswer { + /// Nothing is keyed by this prompt. + None, + /// A record the search takes. Only the caller's own budget can refuse + /// it now, and the reuse contract has no reason for that. + Usable, + /// A record ruled out by model, quantization or context. + Mismatch, + /// A record shallower than the store's current minimum — written + /// before the minimum was raised, and skipped by every search since. + Shallow, +} + +#[derive(Clone, Debug)] +pub struct Entry { + pub sha: String, + pub path: PathBuf, + pub header: Header, + pub file_size: u64, + identity: Option<[u8; 32]>, +} + +/// A file the catalog cannot use: its payload no longer holds what its +/// header describes. The name still says which text it was keyed by. +#[derive(Clone, Debug)] +struct Damaged { + sha: String, + path: PathBuf, + reason: Reason, + ext_flags: u8, + text_bytes: u32, +} + +#[derive(Debug)] +pub struct Store { + pub dir: PathBuf, + pub budget_bytes: u64, + pub reject_different_quant: bool, + pub opt: Options, + pub continued_last_store_tokens: i32, + entries: Vec, + damaged: Vec, + identity: Option<[u8; 32]>, +} + +#[derive(Debug)] +pub struct PayloadTemp { + path: PathBuf, +} + +/// Holds the validated inode while a native loader reopens its descriptor path. +/// An atomic replacement at the catalog path cannot swap the loaded artifact. +#[derive(Debug)] +pub struct PayloadGuard { + _file: fs::File, + path: PathBuf, +} + +impl PayloadGuard { + pub fn path(&self) -> &Path { + &self.path + } +} + +impl PayloadTemp { + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for PayloadTemp { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +impl Store { + pub fn open( + dir: impl AsRef, + budget_mb: u64, + reject_different_quant: bool, + opt: Options, + ) -> io::Result { + let dir = dir.as_ref().to_path_buf(); + #[cfg(unix)] + { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true).mode(0o700).create(&dir)?; + } + #[cfg(not(unix))] + fs::create_dir_all(&dir)?; + let budget_mb = if budget_mb == 0 { + DEFAULT_MB + } else { + budget_mb + }; + let mut store = Self { + dir, + budget_bytes: budget_mb * 1024 * 1024, + reject_different_quant, + opt, + continued_last_store_tokens: 0, + entries: Vec::new(), + damaged: Vec::new(), + identity: None, + }; + store.evict(0, None); + Ok(store) + } + + /// Require this artifact/runtime identity for all subsequent reads and + /// writes. Legacy entries remain in the shared budget, but cannot be reused. + pub fn bind_identity(&mut self, identity: [u8; 32]) { + self.identity = Some(identity); + self.continued_last_store_tokens = 0; + self.refresh(); + } + + fn matches_identity(&self, flags: u8, identity: Option<[u8; 32]>) -> bool { + match self.identity { + Some(expected) => flags & crate::identity::FLAG != 0 && identity == Some(expected), + None => flags & crate::identity::FLAG == 0, + } + } + + fn envelope_identity(&self, path: &Path, e: &Envelope) -> bool { + self.matches_identity( + e.header.ext_flags, + crate::identity::read(path, e.file_size, e.trailer_bytes, e.header.ext_flags), + ) + } + + fn check_identity(&self, path: &Path, e: &Envelope) -> io::Result<()> { + if self.envelope_identity(path, e) { + return Ok(()); + } + Err(io::Error::new( + io::ErrorKind::InvalidData, + "KVC artifact/runtime identity mismatch", + )) + } + + fn scoped_trailer(&self, header: &mut Header, trailer: &[u8]) -> Vec { + let mut out = trailer.to_vec(); + if let Some(identity) = self.identity { + header.ext_flags |= crate::identity::FLAG; + crate::identity::append(&mut out, &identity); + } + out + } + + fn visible_envelope(&self, mut envelope: Envelope) -> Envelope { + // Callers validate protocol extensions independently. Identity has + // already been checked; it is neither a tool map nor payload content. + if self.identity.is_some() { + envelope.header.ext_flags &= !crate::identity::FLAG; + envelope.trailer_bytes -= crate::identity::BYTES as u64; + } + envelope + } + + pub fn payload_temp(&self) -> io::Result { + loop { + let seq = PAYLOAD_TEMP_SEQ.fetch_add(1, Ordering::Relaxed); + let path = self + .dir + .join(format!(".payload.{}.{}", std::process::id(), seq)); + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + match options.open(&path) { + Ok(_) => return Ok(PayloadTemp { path }), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + } + } + + pub fn refresh(&mut self) { + self.entries.clear(); + self.damaged.clear(); + let Ok(rd) = fs::read_dir(&self.dir) else { + return; + }; + for ent in rd.flatten() { + let name = ent.file_name(); + let Some(name) = name.to_str() else { continue }; + let Some(sha) = sha_hex_name(name) else { + continue; + }; + let path = ent.path(); + let Ok(metadata) = read_metadata(&path) else { + // No search can use it, but it can still be the reason a + // prompt keyed by its text finds nothing. + if let Ok((header, _)) = read_header_text(&path, 0) { + self.damaged.push(Damaged { + sha, + path, + reason: header.reason, + ext_flags: header.ext_flags, + text_bytes: header.text_bytes, + }); + } + continue; + }; + let identity = crate::identity::read( + &path, + metadata.file_size, + metadata.trailer_bytes, + metadata.header.ext_flags, + ); + self.entries.push(Entry { + sha, + path, + header: metadata.header, + file_size: metadata.file_size, + identity, + }); + } + } + + pub fn entries(&self) -> &[Entry] { + &self.entries + } + + pub fn write(&mut self, mut record: Record) -> io::Result { + record.trailer = self.scoped_trailer(&mut record.header, &record.trailer); + if !file_size_fits( + self.budget_bytes, + record.text.len() as u64, + record.payload.len() as u64, + record.trailer.len() as u64, + ) { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + "KVC file exceeds budget", + )); + } + record.header.text_bytes = record.text.len() as u32; + record.header.payload_bytes = record.payload.len() as u64; + let sha = text_sha_hex(&record.text); + let path = path_for_sha(&self.dir, &sha); + let incoming = EvictionContext { + text: &record.text, + model_id: record.header.model_id, + quant_bits: record.header.quant_bits, + ctx_size: record.header.ctx_size, + reject_different_quant: self.reject_different_quant, + }; + let extra = 48 + + 4 + + record.text.len() as u64 + + record.payload.len() as u64 + + record.trailer.len() as u64; + self.evict(extra, Some(&incoming)); + write_path(&path, &record).map_err(|e| io::Error::other(e))?; + self.refresh(); + Ok(path) + } + + /// Return an already-compatible KVC entry before the caller stages its + /// opaque native payload. A nonempty trailer follows the writer fast path. + pub fn reuse_compatible( + &mut self, + mut header: Header, + text: &[u8], + trailer: &[u8], + ) -> io::Result> { + if !trailer.is_empty() { + header.ext_flags |= EXT_TOOL_MAP; + } + let sha = text_sha_hex(text); + let path = path_for_sha(&self.dir, &sha); + let Ok(existing) = read_envelope(&path) else { + return Ok(None); + }; + let compatible = self.envelope_identity(&path, &existing) + && existing.header.model_id == header.model_id + && (!self.reject_different_quant || existing.header.quant_bits == header.quant_bits) + && existing.header.ctx_size <= header.ctx_size + && is_automatic_exact_replay(existing.header.reason, existing.header.ext_flags) + && is_automatic_exact_replay(header.reason, header.ext_flags) + && existing.text == text + && text_sha_hex(&existing.text) == sha; + if !compatible { + return Ok(None); + } + let visible = self.visible_envelope(existing.clone()); + if trailer.is_empty() { + // Revalidate header and footer through one descriptor on the + // no-write fast path too; catalog paths may be atomically replaced. + self.payload_file(&path, &visible, FileAccess::ReadOnly)?; + } else { + let trailer = self.scoped_trailer(&mut header, trailer); + let mut file = self.payload_file(&path, &visible, FileAccess::ReadWrite)?; + rewrite_compatible_trailer(&mut file, &existing, header.ext_flags, &trailer)?; + } + Ok(Some(path)) + } + + /// Store an opaque native payload without buffering it in Rust memory. + /// A nonempty trailer is the frozen server tool-map suffix. + pub fn write_payload_file( + &mut self, + mut header: Header, + text: &[u8], + payload_path: &Path, + trailer: &[u8], + ) -> io::Result { + if text.len() > u32::MAX as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "KVC text exceeds u32 length", + )); + } + if !trailer.is_empty() { + header.ext_flags |= EXT_TOOL_MAP; + } + if let Some(path) = self.reuse_compatible(header.clone(), text, trailer)? { + return Ok(path); + } + let trailer = self.scoped_trailer(&mut header, trailer); + let sha = text_sha_hex(text); + let path = path_for_sha(&self.dir, &sha); + if payload_path == path { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "KVC payload source aliases destination", + )); + } + let mut payload = fs::File::open(payload_path)?; + let payload_bytes = payload.metadata()?.len(); + let extra = file_size_bytes(text.len() as u64, payload_bytes, trailer.len() as u64) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "KVC file size overflows") + })?; + if !file_size_fits( + self.budget_bytes, + text.len() as u64, + payload_bytes, + trailer.len() as u64, + ) { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + "KVC file exceeds budget", + )); + } + header.text_bytes = text.len() as u32; + header.payload_bytes = payload_bytes; + let incoming = EvictionContext { + text, + model_id: header.model_id, + quant_bits: header.quant_bits, + ctx_size: header.ctx_size, + reject_different_quant: self.reject_different_quant, + }; + let staged = stage_stream(&path, &header, text, &mut payload, payload_bytes, &trailer) + .map_err(format_io_error)?; + self.evict_excluding(extra, Some(&incoming), Some(&path)); + if let Err(error) = fs::rename(&staged, &path) { + let _ = fs::remove_file(staged); + return Err(error); + } + self.refresh(); + Ok(path) + } + + pub fn read(&self, path: &Path) -> io::Result { + let envelope = read_envelope(path).map_err(format_io_error)?; + self.check_identity(path, &envelope)?; + let mut record = read_path(path).map_err(io::Error::other)?; + self.check_trailer(record.header.ext_flags, &record.trailer)?; + if self.identity.is_some() { + record + .trailer + .truncate(record.trailer.len().saturating_sub(crate::identity::BYTES)); + record.header.ext_flags &= !crate::identity::FLAG; + } + Ok(record) + } + + pub fn open_payload(&self, path: &Path, expected: &Envelope) -> io::Result { + let file = self.payload_file(path, expected, FileAccess::ReadOnly)?; + #[cfg(target_os = "linux")] + let held_path = PathBuf::from(format!("/proc/self/fd/{}", file.as_raw_fd())); + #[cfg(all(unix, not(target_os = "linux")))] + let held_path = PathBuf::from(format!("/dev/fd/{}", file.as_raw_fd())); + #[cfg(not(unix))] + let held_path = { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "held KVC payloads need Unix descriptors", + )); + }; + Ok(PayloadGuard { + _file: file, + path: held_path, + }) + } + + fn payload_file( + &self, + path: &Path, + expected: &Envelope, + access: FileAccess, + ) -> io::Result { + let mut file = fs::OpenOptions::new() + .read(true) + .write(access == FileAccess::ReadWrite) + .open(path)?; + let mut raw = [0; crate::format::FIXED_HEADER]; + file.read_exact(&mut raw)?; + let mut text_len = [0; 4]; + file.read_exact(&mut text_len)?; + let header = crate::format::parse_header(&raw, u32::from_le_bytes(text_len)) + .map_err(format_io_error)?; + let payload_offset = + (crate::format::FIXED_HEADER + 4) as u64 + u64::from(header.text_bytes); + let file_size = file.metadata()?.len(); + let payload_end = payload_offset + .checked_add(header.payload_bytes) + .filter(|end| *end <= file_size) + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "truncated KVC payload"))?; + let trailer_bytes = file_size - payload_end; + let mut footer = [0; crate::identity::BYTES]; + let identity = if header.ext_flags & crate::identity::FLAG != 0 + && trailer_bytes >= footer.len() as u64 + { + file.seek(SeekFrom::Start(file_size - footer.len() as u64))?; + file.read_exact(&mut footer)?; + crate::identity::decode(header.ext_flags, &footer) + } else { + None + }; + let wanted = &expected.header; + if !self.matches_identity(header.ext_flags, identity) + || payload_offset != expected.payload_offset + || header.payload_bytes != wanted.payload_bytes + || header.model_id != wanted.model_id + || header.quant_bits != wanted.quant_bits + || header.ctx_size != wanted.ctx_size + || header.tokens != wanted.tokens + || header.reason != wanted.reason + || header.ext_flags & !crate::identity::FLAG != wanted.ext_flags + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "KVC payload identity changed", + )); + } + // Match the selected text on this same descriptor, in bounded chunks. + file.seek(SeekFrom::Start((crate::format::FIXED_HEADER + 4) as u64))?; + let mut buffer = [0; 4096]; + for bytes in expected.text.chunks(buffer.len()) { + file.read_exact(&mut buffer[..bytes.len()])?; + if &buffer[..bytes.len()] != bytes { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "KVC payload text changed", + )); + } + } + Ok(file) + } + + /// Read tool metadata with the same required identity as payload candidates. + /// The identity footer is never exposed as a tool-map record. + pub fn read_trailer(&self, path: &Path, max_bytes: u64) -> io::Result<(Header, Vec)> { + let (mut header, mut trailer) = + crate::format::read_trailer(path, max_bytes).map_err(format_io_error)?; + self.check_trailer(header.ext_flags, &trailer)?; + if self.identity.is_some() { + trailer.truncate(trailer.len() - crate::identity::BYTES); + header.ext_flags &= !crate::identity::FLAG; + } + Ok((header, trailer)) + } + + fn check_trailer(&self, flags: u8, trailer: &[u8]) -> io::Result<()> { + if self.matches_identity(flags, crate::identity::decode(flags, trailer)) { + return Ok(()); + } + Err(io::Error::new( + io::ErrorKind::InvalidData, + "KVC artifact/runtime identity mismatch", + )) + } + + pub fn find_text_prefix( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + ) -> Option { + self.find_prefix(prompt, model_id, quant_bits, ctx_size, false) + } + + /// Reopen the selected file and repeat the C loader's header, text, and + /// filename-hash checks before a native payload restore consumes it. + pub fn text_prefix_candidate( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + ) -> io::Result> { + self.prefix_candidate(prompt, model_id, quant_bits, ctx_size, false) + } + + pub fn bank_text_prefix_candidate( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + ) -> io::Result> { + self.prefix_candidate(prompt, model_id, quant_bits, ctx_size, true) + } + + pub fn bank_text_prefix_candidate_identity( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + identity_flags: u8, + ) -> io::Result> { + self.prefix_candidate_identity( + prompt, + model_id, + quant_bits, + ctx_size, + true, + EXT_IMAGE_PIXELS_V2, + identity_flags, + ) + } + + pub fn bank_text_lcp_candidate( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + min_lcp: usize, + ) -> io::Result> { + self.bank_text_lcp_candidate_with_identity( + prompt, model_id, quant_bits, ctx_size, min_lcp, 0, 0, + ) + } + + pub fn bank_text_lcp_candidate_identity( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + min_lcp: usize, + identity_flags: u8, + ) -> io::Result> { + self.bank_text_lcp_candidate_with_identity( + prompt, + model_id, + quant_bits, + ctx_size, + min_lcp, + EXT_IMAGE_PIXELS_V2, + identity_flags, + ) + } + + fn bank_text_lcp_candidate_with_identity( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + min_lcp: usize, + identity_mask: u8, + identity_flags: u8, + ) -> io::Result> { + let Some((index, _)) = self.find_text_lcp_with_identity( + prompt, + model_id, + quant_bits, + ctx_size, + min_lcp, + identity_mask, + identity_flags, + ) else { + return Ok(None); + }; + let entry = self.entries[index].clone(); + let envelope = read_envelope(&entry.path).map_err(format_io_error)?; + let header = &envelope.header; + let unchanged = header.model_id == entry.header.model_id + && header.quant_bits == entry.header.quant_bits + && header.ctx_size == entry.header.ctx_size + && header.tokens == entry.header.tokens + && header.text_bytes == entry.header.text_bytes; + let lcp = prompt + .iter() + .zip(&envelope.text) + .take_while(|(left, right)| left == right) + .count(); + // Corruption past the shared prefix still leaves the scan's answer + // standing, so check the record against its own name here too. + if !unchanged || text_sha_hex(&envelope.text) != entry.sha { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "KVC record does not match its name", + )); + } + + if !self.envelope_identity(&entry.path, &envelope) + || !is_automatic_exact_replay(header.reason, header.ext_flags) + || header.model_id != model_id + || (self.reject_different_quant && header.quant_bits != quant_bits) + || header.ctx_size > ctx_size + || header.ext_flags & identity_mask != identity_flags & identity_mask + || lcp < min_lcp + || 8 * (lcp as u64) < u64::from(header.text_bytes) + { + return Ok(None); + } + Ok(Some((entry.path, self.visible_envelope(envelope), lcp))) + } + + fn prefix_candidate( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + require_suffix: bool, + ) -> io::Result> { + self.prefix_candidate_identity(prompt, model_id, quant_bits, ctx_size, require_suffix, 0, 0) + } + + fn prefix_candidate_identity( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + require_suffix: bool, + identity_mask: u8, + identity_flags: u8, + ) -> io::Result> { + let index = if require_suffix { + self.find_prefix_with_identity( + prompt, + model_id, + quant_bits, + ctx_size, + true, + identity_mask, + identity_flags, + ) + } else { + self.find_prefix_with_identity( + prompt, + model_id, + quant_bits, + ctx_size, + false, + identity_mask, + identity_flags, + ) + }; + let Some(index) = index else { + return Ok(None); + }; + let entry = self.entries[index].clone(); + let envelope = read_envelope(&entry.path).map_err(format_io_error)?; + let header = &envelope.header; + let unchanged = header.model_id == entry.header.model_id + && header.quant_bits == entry.header.quant_bits + && header.ctx_size == entry.header.ctx_size + && header.tokens == entry.header.tokens + && header.text_bytes == entry.header.text_bytes; + // The record no longer describes itself: its text does not hash to + // its own name, it is not this prompt's prefix after all, or the + // header moved under the read. The candidate was selected and then + // refused by its own contents, which is not an absence. + if !unchanged + || text_sha_hex(&envelope.text) != entry.sha + || !prompt.starts_with(&envelope.text) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "KVC record does not match its name", + )); + } + + // Sound, but not usable in this shape: nothing left to prefill, or + // an identity this search does not take. + let bank_without_logits = is_bank_replay_v1(header.reason, header.ext_flags) + && envelope.text.len() == prompt.len(); + let missing_suffix = require_suffix && envelope.text.len() >= prompt.len(); + if !self.envelope_identity(&entry.path, &envelope) + || !is_automatic_exact_replay(header.reason, header.ext_flags) + || header.model_id != model_id + || header.ext_flags & identity_mask != identity_flags & identity_mask + || bank_without_logits + || missing_suffix + { + return Ok(None); + } + Ok(Some((entry.path, self.visible_envelope(envelope)))) + } + + pub fn touch_hit(&mut self, path: &Path) -> io::Result<()> { + let envelope = read_envelope(path).map_err(format_io_error)?; + self.check_identity(path, &envelope)?; + let mut file = self.payload_file( + path, + &self.visible_envelope(envelope.clone()), + FileAccess::ReadWrite, + )?; + let header = envelope.header; + let raw = fill_header( + header.model_id, + header.quant_bits, + header.reason, + header.ext_flags, + header.tokens, + header.hits.wrapping_add(1), + header.ctx_size, + header.created_at, + now_secs(), + header.payload_bytes, + ); + file.seek(SeekFrom::Start(0))?; + file.write_all(&raw)?; + file.flush()?; + self.refresh(); + Ok(()) + } + + pub fn discard(&mut self, path: &Path) -> io::Result<()> { + self.discard_inner(path, true) + } + + pub fn discard_bank(&mut self, path: &Path) -> io::Result<()> { + self.discard_inner(path, false) + } + + fn discard_inner(&mut self, path: &Path, reset_continued: bool) -> io::Result<()> { + if !self.entries.iter().any(|entry| entry.path == path) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "KVC discard path is not a catalog entry", + )); + } + if reset_continued { + self.continued_last_store_tokens = 0; + } + match fs::remove_file(path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + self.refresh(); + Ok(()) + } + + pub fn find_bank_text_prefix( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + ) -> Option { + self.find_prefix(prompt, model_id, quant_bits, ctx_size, true) + } + + fn find_prefix( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + require_suffix: bool, + ) -> Option { + self.find_prefix_with_identity(prompt, model_id, quant_bits, ctx_size, require_suffix, 0, 0) + } + + fn find_prefix_with_identity( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + require_suffix: bool, + identity_mask: u8, + identity_flags: u8, + ) -> Option { + self.refresh(); + let prompt_bytes = prompt.len(); + let mut best: Option = None; + for (i, e) in self.entries.iter().enumerate() { + if !self.matches_identity(e.header.ext_flags, e.identity) + || !is_automatic_exact_replay(e.header.reason, e.header.ext_flags) + { + continue; + } + if e.header.ext_flags & identity_mask != identity_flags & identity_mask { + continue; + } + if e.header.text_bytes as usize > prompt_bytes { + continue; + } + if (require_suffix || is_bank_replay_v1(e.header.reason, e.header.ext_flags)) + && e.header.text_bytes as usize == prompt_bytes + { + continue; + } + if (e.header.tokens as i32) < self.opt.min_tokens { + continue; + } + if e.header.model_id != model_id { + continue; + } + if ctx_size < e.header.ctx_size { + continue; + } + if self.reject_different_quant && e.header.quant_bits != quant_bits { + continue; + } + if let Some(b) = best { + let be = &self.entries[b]; + if (e.header.text_bytes as usize) < be.header.text_bytes as usize { + continue; + } + if e.header.text_bytes == be.header.text_bytes + && e.header.tokens <= be.header.tokens + { + continue; + } + } + let sha = text_sha_hex(&prompt[..e.header.text_bytes as usize]); + if sha == e.sha { + best = Some(i); + } + } + best + } + + /// What the serial search would do with this prompt: it keys on the + /// rendered text and may replay a record of exactly that length. + pub fn prefix_answer( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + ) -> PrefixAnswer { + self.prefix_answer_masked( + prompt, + model_id, + quant_bits, + ctx_size, + Suffix::Optional, + 0, + 0, + ) + } + + /// What the bank search would do: it keys an image request by its + /// media-marked cache text and admits only records it can extend, so + /// asking about the rendered prompt would see nothing at all. + pub fn bank_prefix_answer( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + identity_flags: u8, + ) -> PrefixAnswer { + self.prefix_answer_masked( + prompt, + model_id, + quant_bits, + ctx_size, + Suffix::Required, + EXT_IMAGE_PIXELS_V2, + identity_flags, + ) + } + + fn prefix_answer_masked( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + suffix: Suffix, + identity_mask: u8, + identity_flags: u8, + ) -> PrefixAnswer { + self.refresh(); + let reject_quant = self.reject_different_quant; + let min_tokens = self.opt.min_tokens; + let mut answer = PrefixAnswer::None; + + for e in &self.entries { + if !is_automatic_exact_replay(e.header.reason, e.header.ext_flags) + || e.header.text_bytes as usize > prompt.len() + { + continue; + } + let exact_length = e.header.text_bytes as usize == prompt.len(); + if suffix == Suffix::Required && exact_length { + // Nothing left to prefill from it in this shape. + continue; + } + if text_sha_hex(&prompt[..e.header.text_bytes as usize]) != e.sha { + continue; + } + + // A bank snapshot carries no logits, so an exact replay cannot + // use one: the record is there and its layout refuses. The media + // flag is identity in the same way — a record keyed by this text + // without it is a mismatch, not one about another conversation. + let compatible = !(exact_length + && is_bank_replay_v1(e.header.reason, e.header.ext_flags)) + && self.matches_identity(e.header.ext_flags, e.identity) + && e.header.model_id == model_id + && ctx_size >= e.header.ctx_size + && (!reject_quant || e.header.quant_bits == quant_bits) + && e.header.ext_flags & identity_mask == identity_flags & identity_mask; + if compatible && (e.header.tokens as i32) >= min_tokens { + return PrefixAnswer::Usable; + } + let refused = if compatible { + PrefixAnswer::Shallow + } else { + PrefixAnswer::Mismatch + }; + answer = weaker_answer(answer, refused); + } + + // A record whose payload was truncated away is not in the catalog, + // but its name still says it was keyed by this text. It is why the + // prompt finds nothing, and that is a mismatch. + if self.damaged_prefix(prompt, suffix) { + answer = weaker_answer(answer, PrefixAnswer::Mismatch); + } + + answer + } + + /// A file the catalog dropped that still shares this prompt's required + /// prefix. Its payload is gone; the text in front of it need not be. + fn damaged_lcp(&self, prompt: &[u8], min_lcp: usize) -> bool { + self.damaged.iter().any(|d| { + if !is_automatic_exact_replay(d.reason, d.ext_flags) + || (d.text_bytes as usize) < min_lcp + || u64::from(d.text_bytes) > 8 * prompt.len() as u64 + { + return false; + } + let want = (d.text_bytes as usize).min(prompt.len()); + let Ok((_, text)) = read_header_text(&d.path, want) else { + return false; + }; + let lcp = text + .iter() + .zip(prompt) + .take_while(|(stored, asked)| stored == asked) + .count(); + lcp >= min_lcp && 8 * (lcp as u64) >= u64::from(d.text_bytes) + }) + } + + /// A file the catalog dropped, keyed by a prefix of this prompt. + fn damaged_prefix(&self, prompt: &[u8], suffix: Suffix) -> bool { + self.damaged.iter().any(|d| { + let text_bytes = d.text_bytes as usize; + if !is_automatic_exact_replay(d.reason, d.ext_flags) + || text_bytes > prompt.len() + || (suffix == Suffix::Required && text_bytes == prompt.len()) + { + return false; + } + text_sha_hex(&prompt[..text_bytes]) == d.sha + }) + } + + /// The LCP form: an edited prompt diverges from the record it shares a + /// prefix with, so the exact question cannot see it at all. `min_lcp` + /// is the partial minimum the bank search admits from. + pub fn bank_lcp_answer( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + min_lcp: usize, + identity_flags: u8, + ) -> PrefixAnswer { + if min_lcp == 0 || prompt.len() < min_lcp { + return PrefixAnswer::None; + } + self.refresh(); + let reject_quant = self.reject_different_quant; + let min_tokens = self.opt.min_tokens; + let mut answer = PrefixAnswer::None; + + for e in &self.entries { + if !is_automatic_exact_replay(e.header.reason, e.header.ext_flags) + || (e.header.text_bytes as usize) < min_lcp + || u64::from(e.header.text_bytes) > 8 * prompt.len() as u64 + { + continue; + } + let want = (e.header.text_bytes as usize).min(prompt.len()); + let Ok((metadata, text)) = read_text_prefix(&e.path, want) else { + continue; + }; + if metadata.header.text_bytes != e.header.text_bytes { + continue; + } + let lcp = text + .iter() + .zip(prompt) + .take_while(|(stored, asked)| stored == asked) + .count(); + if lcp < min_lcp || 8 * (lcp as u64) < u64::from(e.header.text_bytes) { + continue; + } + + let compatible = self.matches_identity(e.header.ext_flags, e.identity) + && e.header.model_id == model_id + && ctx_size >= e.header.ctx_size + && (!reject_quant || e.header.quant_bits == quant_bits) + && e.header.ext_flags & EXT_IMAGE_PIXELS_V2 == identity_flags & EXT_IMAGE_PIXELS_V2; + if compatible && (e.header.tokens as i32) >= min_tokens { + return PrefixAnswer::Usable; + } + let refused = if compatible { + PrefixAnswer::Shallow + } else { + PrefixAnswer::Mismatch + }; + answer = weaker_answer(answer, refused); + } + + // An edited prompt diverges from the record, so the damaged list has + // to be read through the same shared prefix, not by name. A record + // no search would replay from is not a refusal either. + if self.damaged_lcp(prompt, min_lcp) { + answer = weaker_answer(answer, PrefixAnswer::Mismatch); + } + + answer + } + + pub fn find_text_lcp( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + min_lcp: usize, + ) -> Option<(usize, usize)> { + self.find_text_lcp_with_identity(prompt, model_id, quant_bits, ctx_size, min_lcp, 0, 0) + } + + fn find_text_lcp_with_identity( + &mut self, + prompt: &[u8], + model_id: u8, + quant_bits: u8, + ctx_size: u32, + min_lcp: usize, + identity_mask: u8, + identity_flags: u8, + ) -> Option<(usize, usize)> { + if min_lcp == 0 || prompt.len() < min_lcp { + return None; + } + self.refresh(); + let mut best: Option<(usize, usize, u64)> = None; + for (i, e) in self.entries.iter().enumerate() { + if !self.matches_identity(e.header.ext_flags, e.identity) + || !is_automatic_exact_replay(e.header.reason, e.header.ext_flags) + { + continue; + } + if e.header.ext_flags & identity_mask != identity_flags & identity_mask { + continue; + } + if (e.header.tokens as i32) < self.opt.min_tokens { + continue; + } + if e.header.model_id != model_id || ctx_size < e.header.ctx_size { + continue; + } + if self.reject_different_quant && e.header.quant_bits != quant_bits { + continue; + } + if (e.header.text_bytes as usize) < min_lcp { + continue; + } + if u64::from(e.header.text_bytes) > 8 * prompt.len() as u64 { + continue; + } + let want = (e.header.text_bytes as usize).min(prompt.len()); + let Ok((metadata, text)) = read_text_prefix(&e.path, want) else { + continue; + }; + if metadata.header.text_bytes != e.header.text_bytes { + continue; + } + let mut lcp = 0; + while lcp < text.len() && text[lcp] == prompt[lcp] { + lcp += 1; + } + if lcp < min_lcp { + continue; + } + if 8 * (lcp as u64) < u64::from(e.header.text_bytes) { + continue; + } + if let Some((_, best_lcp, best_text)) = best { + if lcp < best_lcp { + continue; + } + if lcp == best_lcp && u64::from(e.header.text_bytes) >= best_text { + continue; + } + } + best = Some((i, lcp, u64::from(e.header.text_bytes))); + } + best.map(|(i, lcp, _)| (i, lcp)) + } + + pub fn evict(&mut self, extra_bytes: u64, incoming: Option<&EvictionContext<'_>>) { + self.evict_excluding(extra_bytes, incoming, None); + } + + fn evict_excluding( + &mut self, + extra_bytes: u64, + incoming: Option<&EvictionContext<'_>>, + protected: Option<&Path>, + ) { + if self.budget_bytes == 0 || extra_bytes > self.budget_bytes { + return; + } + self.refresh(); + let now = now_secs(); + let target = self.budget_bytes - extra_bytes; + loop { + let total: u64 = self + .entries + .iter() + .filter(|e| protected.is_none_or(|path| path != e.path)) + .map(|e| e.file_size) + .sum(); + if total <= target || self.entries.is_empty() { + break; + } + let mut victim: Option = None; + let mut victim_score = 0.0; + for i in 0..self.entries.len() { + if protected.is_some_and(|path| path == self.entries[i].path) { + continue; + } + let score = self.score_at(i, now, incoming); + if victim.is_none() + || score < victim_score + || (score == victim_score + && self.entries[i].header.last_used + < self.entries[victim.unwrap()].header.last_used) + { + victim = Some(i); + victim_score = score; + } + } + let Some(victim) = victim else { break }; + let path = self.entries[victim].path.clone(); + let unlinked = fs::remove_file(path).is_ok(); + self.entries.remove(victim); + if !unlinked { + break; + } + } + } + + fn score_at(&self, i: usize, now: u64, incoming: Option<&EvictionContext<'_>>) -> f64 { + let e = &self.entries[i]; + eviction_score( + &ScoreEntry { + sha: &e.sha, + quant_bits: e.header.quant_bits, + model_id: e.header.model_id, + reason: e.header.reason, + tokens: e.header.tokens, + hits: e.header.hits, + ctx_size: e.header.ctx_size, + created_at: e.header.created_at, + last_used: e.header.last_used, + text_bytes: u64::from(e.header.text_bytes), + file_size: e.file_size, + }, + now, + incoming, + ) + } +} + +fn rewrite_compatible_trailer( + file: &mut fs::File, + existing: &Envelope, + incoming_ext_flags: u8, + trailer: &[u8], +) -> io::Result<()> { + // The C fast path is O(trailer); copying a multi-GiB payload here would + // turn a compatible save into a full checkpoint rewrite. + let payload_end = existing + .payload_offset + .checked_add(existing.header.payload_bytes) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "KVC payload end overflows"))?; + file.seek(SeekFrom::Start(payload_end))?; + file.set_len(payload_end)?; + file.write_all(trailer)?; + file.flush()?; + + let mut header = existing.header.clone(); + header.ext_flags |= incoming_ext_flags & EXT_TOOL_MAP; + header.last_used = now_secs(); + let raw = fill_header( + header.model_id, + header.quant_bits, + header.reason, + header.ext_flags, + header.tokens, + header.hits, + header.ctx_size, + header.created_at, + header.last_used, + header.payload_bytes, + ); + file.seek(SeekFrom::Start(0))?; + file.write_all(&raw)?; + file.flush() +} + +/// Fold in a record the search would not take. A mismatch outranks one +/// that is only too shallow, the way the lanes report the two, and both +/// outrank having seen nothing. +fn weaker_answer(seen: PrefixAnswer, refused: PrefixAnswer) -> PrefixAnswer { + if refused == PrefixAnswer::Mismatch || seen == PrefixAnswer::None { + return refused; + } + seen +} + +fn format_io_error(error: FormatError) -> io::Error { + match error { + FormatError::Io(error) => error, + FormatError::Truncated => { + io::Error::new(io::ErrorKind::UnexpectedEof, "truncated KVC payload") + } + other => io::Error::new(io::ErrorKind::InvalidData, other), + } +} + +fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::{ + fill_header, le_put32, read_metadata, read_text_prefix, Header, Reason, FIXED_HEADER, + }; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt as _; + + fn rec(text: &[u8], tokens: u32) -> Record { + Record { + header: Header { + quant_bits: 2, + reason: Reason::Cold, + ext_flags: 0, + model_id: 0, + tokens, + hits: 0, + ctx_size: 2048, + created_at: 1, + last_used: 1, + payload_bytes: 1, + text_bytes: text.len() as u32, + }, + text: text.to_vec(), + payload: vec![0xAA], + trailer: Vec::new(), + } + } + + #[test] + fn identity_required_for_reuse() { + let dir = std::env::temp_dir().join(format!("ds4-kv-identity-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let options = Options { + min_tokens: 1, + ..Options::default() + }; + let mut store = Store::open(&dir, 16, false, options).unwrap(); + let row = rec(b"same prompt", 8); + let path = store.write(row.clone()).unwrap(); + store.bind_identity([1; 32]); + assert!(store + .text_prefix_candidate(b"same prompt plus", 0, 2, 2048) + .unwrap() + .is_none()); + assert_eq!( + store.prefix_answer(b"same prompt plus", 0, 2, 2048), + PrefixAnswer::Mismatch + ); + assert!(store.read(&path).is_err()); + assert!(store + .reuse_compatible(row.header.clone(), &row.text, &[]) + .unwrap() + .is_none()); + store.write(row.clone()).unwrap(); + assert!(store + .text_prefix_candidate(b"same prompt plus", 0, 2, 2048) + .unwrap() + .is_some()); + assert!(store + .bank_text_lcp_candidate(b"same prompt edit", 0, 2, 2048, 4) + .unwrap() + .is_some()); + assert_eq!(store.read(&path).unwrap(), row); + let mut other = Store::open(&dir, 16, false, options).unwrap(); + other.bind_identity([2; 32]); + assert!(other + .text_prefix_candidate(b"same prompt plus", 0, 2, 2048) + .unwrap() + .is_none()); + assert!(other + .bank_text_lcp_candidate(b"same prompt edit", 0, 2, 2048, 4) + .unwrap() + .is_none()); + assert!(other + .reuse_compatible(row.header.clone(), &row.text, &[]) + .unwrap() + .is_none()); + assert!(other.read(&path).is_err()); + assert!(other.touch_hit(&path).is_err()); + let (_, envelope) = store + .text_prefix_candidate(b"same prompt plus", 0, 2, 2048) + .unwrap() + .unwrap(); + let held = store.open_payload(&path, &envelope).unwrap(); + let mut replacement = row.clone(); + replacement.payload = vec![0xBB]; + other.write(replacement).unwrap(); + assert_eq!(read_path(held.path()).unwrap().payload, row.payload); + assert!(store.open_payload(&path, &envelope).is_err()); + assert!(store.read(&path).is_err()); + assert_eq!(other.entries().len(), 1); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn identity_keeps_tool_trailer() { + let dir = std::env::temp_dir().join(format!("ds4-kv-id-tools-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open( + &dir, + 16, + false, + Options { + min_tokens: 1, + ..Options::default() + }, + ) + .unwrap(); + store.bind_identity([3; 32]); + let mut row = rec(b"tool prompt", 8); + let tools = vec![crate::ExtensionRecord { + id: b"tool-1".to_vec(), + body: b"call()".to_vec(), + }]; + row.header.ext_flags = EXT_TOOL_MAP; + row.trailer = crate::encode_ktm(&tools); + let path = store.write(row.clone()).unwrap(); + assert_eq!(store.read(&path).unwrap(), row); + let raw = read_path(&path).unwrap(); + assert_eq!(crate::decode_ktm(&raw.trailer), tools); + assert!(store + .reuse_compatible(row.header.clone(), &row.text, &row.trailer) + .unwrap() + .is_some()); + let loaded = store.read(&path).unwrap(); + assert_eq!(loaded.trailer, row.trailer); + assert_eq!(loaded.payload, row.payload); + assert_eq!(store.read_trailer(&path, 4096).unwrap().1, row.trailer); + store.bind_identity([4; 32]); + assert!(store.read_trailer(&path, 4096).is_err()); + store.bind_identity([3; 32]); + let bytes = fs::read(&path).unwrap(); + fs::write(&path, &bytes[..bytes.len() - 1]).unwrap(); + assert!(store + .text_prefix_candidate(b"tool prompt suffix", 0, 2, 2048) + .unwrap() + .is_none()); + assert!(store.read(&path).is_err()); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn identities_share_one_budget() { + let dir = std::env::temp_dir().join(format!("ds4-kv-id-budget-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open( + &dir, + 16, + false, + Options { + min_tokens: 1, + ..Options::default() + }, + ) + .unwrap(); + store.bind_identity([1; 32]); + let first = store.write(rec(b"old artifact", 8)).unwrap(); + let bytes = fs::metadata(&first).unwrap().len(); + store.budget_bytes = bytes + 16; // Includes the policy's one-percent reserve. + store.bind_identity([2; 32]); + let second = store.write(rec(b"new artifact", 8)).unwrap(); + assert!(!first.exists() && second.exists()); + assert_eq!(store.entries().len(), 1); + assert_eq!(fs::metadata(second).unwrap().len(), bytes); + fs::remove_dir_all(dir).unwrap(); + } + + #[cfg(unix)] + #[test] + fn new_nested_store_directories_are_private() { + let base = std::env::temp_dir().join(format!("ds4-kv-private-dir-{}", std::process::id())); + let dir = base.join("nested/store"); + let _ = fs::remove_dir_all(&base); + + let _store = Store::open(&dir, 16, false, Options::default()).unwrap(); + for path in [&base, &base.join("nested"), &dir] { + assert_eq!(fs::metadata(path).unwrap().permissions().mode() & 0o077, 0); + } + + let _ = fs::remove_dir_all(&base); + } + + #[cfg(unix)] + #[test] + fn existing_store_directory_mode_is_unchanged() { + let dir = std::env::temp_dir().join(format!("ds4-kv-existing-dir-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).unwrap(); + + let _store = Store::open(&dir, 16, false, Options::default()).unwrap(); + assert_eq!( + fs::metadata(&dir).unwrap().permissions().mode() & 0o777, + 0o755 + ); + + let _ = fs::remove_dir_all(&dir); + } + + #[cfg(unix)] + #[test] + fn payload_temp_is_private_and_drop_removes_it() { + let dir = std::env::temp_dir().join(format!("ds4-kv-payload-temp-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let store = Store::open(&dir, 16, false, Options::default()).unwrap(); + + let temp = store.payload_temp().unwrap(); + let path = temp.path().to_path_buf(); + assert!(path.exists()); + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + + drop(temp); + assert!(!path.exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn payload_temps_are_distinct_and_drop_independently() { + let dir = std::env::temp_dir().join(format!("ds4-kv-payload-pair-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let store = Store::open(&dir, 16, false, Options::default()).unwrap(); + + let first = store.payload_temp().unwrap(); + let second = store.payload_temp().unwrap(); + let first_path = first.path().to_path_buf(); + let second_path = second.path().to_path_buf(); + assert_ne!(first_path, second_path); + assert!(first_path.exists()); + assert!(second_path.exists()); + + drop(first); + assert!(!first_path.exists()); + assert!(second_path.exists()); + drop(second); + assert!(!second_path.exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn lcp_refuses_a_moved_record() { + let dir = std::env::temp_dir().join(format!("ds4-kv-lcp-candidate-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut record = rec(b"shared opening/stored tail", 512); + record.header.reason = Reason::BankShutdown; + record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1; + let path = store.write(record).unwrap(); + let edited = b"shared opening/edited tail"; + + let (got, _, lcp) = store + .bank_text_lcp_candidate(edited, 0, 2, 8192, 8) + .unwrap() + .unwrap(); + assert_eq!(got, path); + assert_eq!(lcp, 15); + + // Corrupt the text past the shared prefix: the scan still picks the + // record, and only the full-text check can see that it moved. + let mut file = fs::OpenOptions::new().write(true).open(&path).unwrap(); + file.seek(SeekFrom::Start((FIXED_HEADER + 4 + 20) as u64)) + .unwrap(); + file.write_all(b"X").unwrap(); + file.flush().unwrap(); + assert_eq!( + store + .bank_text_lcp_candidate(edited, 0, 2, 8192, 8) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidData + ); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn a_cut_record_keeps_its_key() { + let dir = std::env::temp_dir().join(format!("ds4-kv-truncated-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let path = store.write(rec(b"shared prefix", 512)).unwrap(); + + // Cut the payload away: the header and text stay, so the catalog + // drops the record while its name still says what it was keyed by. + fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(FIXED_HEADER as u64 + 4 + 13) + .unwrap(); + + assert_eq!( + store.prefix_answer(b"shared prefix and suffix", 0, 2, 2048), + PrefixAnswer::Mismatch + ); + assert_eq!( + store.prefix_answer(b"another conversation", 0, 2, 2048), + PrefixAnswer::None + ); + + // A record no search replays from is not a refusal: it was never a + // candidate, damaged or not. + let mut session = rec(b"agent opening", 512); + session.header.reason = Reason::AgentSession; + let session_path = store.write(session).unwrap(); + fs::OpenOptions::new() + .write(true) + .open(&session_path) + .unwrap() + .set_len(FIXED_HEADER as u64 + 4 + 13) + .unwrap(); + assert_eq!( + store.prefix_answer(b"agent opening and more", 0, 2, 2048), + PrefixAnswer::None + ); + assert_eq!( + store.bank_lcp_answer(b"agent openinG edited", 0, 2, 2048, 8, 0), + PrefixAnswer::None + ); + + // An edited prompt diverges from the record, so only the shared + // prefix can find it — and the text in front of the payload is + // still there to be read. + assert_eq!( + store.bank_lcp_answer(b"shared prefiX edited", 0, 2, 2048, 8, 0), + PrefixAnswer::Mismatch + ); + assert_eq!( + store.bank_lcp_answer(b"another opening turn", 0, 2, 2048, 8, 0), + PrefixAnswer::None + ); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn an_exact_bank_answers_layout() { + let dir = std::env::temp_dir().join(format!("ds4-kv-exact-bank-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut record = rec(b"exact conversation", 512); + record.header.reason = Reason::BankShutdown; + record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1; + store.write(record).unwrap(); + + // A bank snapshot has no logits to replay from, so an exact serial + // prompt is refused by the record's layout, not by its absence. + assert_eq!( + store.prefix_answer(b"exact conversation", 0, 2, 2048), + PrefixAnswer::Mismatch + ); + // With a suffix to prefill, the same record is what the search takes. + assert_eq!( + store.prefix_answer(b"exact conversation plus a turn", 0, 2, 2048), + PrefixAnswer::Usable + ); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn a_thin_record_answers_shallow() { + let dir = std::env::temp_dir().join(format!("ds4-kv-shallow-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + // Written while the minimum was lower; every search skips it now. + store.write(rec(b"shared opening", 8)).unwrap(); + + assert_eq!( + store.prefix_answer(b"shared opening and more", 0, 2, 2048), + PrefixAnswer::Shallow + ); + assert_eq!( + store.prefix_answer(b"another conversation", 0, 2, 2048), + PrefixAnswer::None + ); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn an_image_key_owns_its_answer() { + let dir = std::env::temp_dir().join(format!("ds4-kv-image-key-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, true, Options::default()).unwrap(); + let mut record = rec(b"chat\xffDS4IMG2 turn", 512); + record.header.reason = Reason::BankCheckpoint; + record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1 | EXT_IMAGE_PIXELS_V2; + store.write(record).unwrap(); + let key = b"chat\xffDS4IMG2 turn and one more"; + + // The identity the record was stored with: the search takes it. + assert_eq!( + store.bank_prefix_answer(key, 0, 2, 2048, EXT_IMAGE_PIXELS_V2), + PrefixAnswer::Usable + ); + // A different quantization rules it out, so the miss is a mismatch. + assert_eq!( + store.bank_prefix_answer(key, 0, 4, 2048, EXT_IMAGE_PIXELS_V2), + PrefixAnswer::Mismatch + ); + // The rendered prompt is not the key it was stored under, so the + // text-keyed search sees nothing. Asked under this key without the + // media flag, the record is there and its layout rules it out. + assert_eq!( + store.prefix_answer(b"chat turn and one more", 0, 4, 2048), + PrefixAnswer::None + ); + assert_eq!( + store.bank_prefix_answer(key, 0, 2, 2048, 0), + PrefixAnswer::Mismatch + ); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn an_edit_mismatches_by_lcp() { + let dir = std::env::temp_dir().join(format!("ds4-kv-lcp-identity-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + store + .write(rec(b"shared opening turn/ORIGINAL", 512)) + .unwrap(); + let edited = b"shared opening turn/EDITED CONTINUATION"; + + // The stored text diverges from the prompt, so the exact-prefix + // question cannot see the record at all. + assert_eq!( + store.bank_prefix_answer(edited, 1, 2, 2048, 0), + PrefixAnswer::None + ); + // Through the LCP the partial search would have taken it, and only + // the model it was written for rules it out. + assert_eq!( + store.bank_lcp_answer(edited, 1, 2, 2048, 8, 0), + PrefixAnswer::Mismatch + ); + // The identity it was stored with is not a mismatch. + assert_eq!( + store.bank_lcp_answer(edited, 0, 2, 2048, 8, 0), + PrefixAnswer::Usable + ); + // Neither is a prompt that shares nothing with it. + assert_eq!( + store.bank_lcp_answer(b"a different opening", 1, 2, 2048, 8, 0), + PrefixAnswer::None + ); + + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn discard_removes_existing_candidate_and_refreshes_index() { + let dir = std::env::temp_dir().join(format!("ds4-kv-discard-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let discarded = store.write(rec(b"discarded", 512)).unwrap(); + let retained = store.write(rec(b"retained", 1024)).unwrap(); + store.continued_last_store_tokens = 512; + assert_eq!(store.entries().len(), 2); + + store.discard(&discarded).unwrap(); + + assert!(!discarded.exists()); + assert!(retained.exists()); + assert_eq!(store.entries().len(), 1); + assert_eq!(store.entries()[0].path, retained); + assert_eq!(store.continued_last_store_tokens, 0); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn bank_discard_preserves_the_serial_checkpoint_marker() { + let dir = std::env::temp_dir().join(format!("ds4-kv-bank-discard-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let discarded = store.write(rec(b"bank-corrupt", 512)).unwrap(); + store.continued_last_store_tokens = 777; + + store.discard_bank(&discarded).unwrap(); + + assert!(!discarded.exists()); + assert!(store.entries().is_empty()); + assert_eq!(store.continued_last_store_tokens, 777); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn discard_missing_candidate_is_ok_and_refreshes_index() { + let dir = + std::env::temp_dir().join(format!("ds4-kv-discard-missing-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let missing = store.write(rec(b"missing", 512)).unwrap(); + store.continued_last_store_tokens = 512; + fs::remove_file(&missing).unwrap(); + assert_eq!(store.entries().len(), 1); + + store.discard(&missing).unwrap(); + + assert!(store.entries().is_empty()); + assert_eq!(store.continued_last_store_tokens, 0); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn discard_rejects_paths_outside_the_catalog() { + let dir = std::env::temp_dir().join(format!("ds4-kv-discard-scope-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let outside = dir.with_extension("outside"); + fs::write(&outside, b"keep").unwrap(); + + let error = store.discard(&outside).unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(fs::read(&outside).unwrap(), b"keep"); + fs::remove_file(outside).unwrap(); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn prefix_prefers_longest_matching_text() { + let dir = std::env::temp_dir().join(format!("ds4-kv-prefix-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + store.write(rec(b"hello", 512)).unwrap(); + store.write(rec(b"hello world", 512)).unwrap(); + let idx = store.find_text_prefix(b"hello world!", 0, 2, 2048).unwrap(); + assert_eq!(store.entries()[idx].header.text_bytes, 11); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn bank_lookup_requires_suffix() { + let dir = std::env::temp_dir().join(format!("ds4-kv-bank-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut r = rec(b"hello", 512); + r.header.reason = Reason::BankEvict; + r.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1; + store.write(r).unwrap(); + assert!(store.find_bank_text_prefix(b"hello", 0, 2, 2048).is_none()); + assert!(store.find_bank_text_prefix(b"hello!", 0, 2, 2048).is_some()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn bank_identity_lookup_separates_image_and_text_records() { + let dir = std::env::temp_dir().join(format!("ds4-kv-bank-identity-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut plain = rec(b"plain identity", 512); + plain.header.reason = Reason::BankCheckpoint; + plain.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1; + store.write(plain).unwrap(); + let mut image = rec(b"image identity", 512); + image.header.reason = Reason::BankCheckpoint; + image.header.ext_flags = + crate::format::EXT_BANK_REPLAY_V1 | crate::format::EXT_IMAGE_PIXELS_V2; + store.write(image).unwrap(); + + assert!(store + .bank_text_prefix_candidate_identity( + b"plain identity tail", + 0, + 2, + 2048, + EXT_IMAGE_PIXELS_V2, + ) + .unwrap() + .is_none()); + assert!(store + .bank_text_prefix_candidate_identity(b"image identity tail", 0, 2, 2048, 0) + .unwrap() + .is_none()); + assert!(store + .bank_text_prefix_candidate_identity( + b"image identity tail", + 0, + 2, + 2048, + EXT_IMAGE_PIXELS_V2, + ) + .unwrap() + .is_some()); + assert!( + store + .bank_text_lcp_candidate_identity( + b"image identitX", + 0, + 2, + 2048, + 8, + EXT_IMAGE_PIXELS_V2, + ) + .unwrap() + .is_some() + ); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn lcp_candidate_reopens_and_revalidates_the_selected_record() { + let dir = std::env::temp_dir().join(format!("ds4-kv-lcp-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut record = rec(b"shared alpha", 512); + record.header.reason = Reason::BankCheckpoint; + record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1; + let path = store.write(record).unwrap(); + + let (candidate, envelope, lcp) = store + .bank_text_lcp_candidate(b"shared beta", 0, 2, 2048, 6) + .unwrap() + .unwrap(); + assert_eq!(candidate, path); + assert_eq!(envelope.text, b"shared alpha"); + assert_eq!(lcp, b"shared ".len()); + + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .unwrap(); + file.seek(SeekFrom::Start((FIXED_HEADER + 4) as u64)) + .unwrap(); + file.write_all(b"X").unwrap(); + file.flush().unwrap(); + assert!(store + .bank_text_lcp_candidate(b"shared beta", 0, 2, 2048, 6) + .unwrap() + .is_none()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn indexes_sparse_payload_without_reading_it() { + const PAYLOAD_BYTES: u64 = 4 * 1024 * 1024 * 1024; + const TRAILER_BYTES: u64 = 17; + + let dir = std::env::temp_dir().join(format!("ds4-kv-sparse-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let text = [b'x'; 64]; + let prompt = [b'x'; 8]; + let path = path_for_sha(&dir, &text_sha_hex(&text)); + let header = fill_header(0, 2, Reason::Cold, 0, 512, 0, 2048, 1, 1, PAYLOAD_BYTES); + let mut text_len = [0u8; 4]; + le_put32(&mut text_len, text.len() as u32); + let mut file = fs::File::create(&path).unwrap(); + file.write_all(&header).unwrap(); + file.write_all(&text_len).unwrap(); + file.write_all(&text).unwrap(); + let payload_offset = (FIXED_HEADER + 4 + text.len()) as u64; + file.set_len(payload_offset + PAYLOAD_BYTES + TRAILER_BYTES) + .unwrap(); + drop(file); + + let metadata = read_metadata(&path).unwrap(); + assert_eq!(metadata.header.text_bytes, text.len() as u32); + let (_, prefix) = read_text_prefix(&path, prompt.len()).unwrap(); + assert_eq!(prefix, prompt); + + let mut store = Store::open(&dir, 8192, false, Options::default()).unwrap(); + assert_eq!(store.entries().len(), 1); + assert_eq!( + store.entries()[0].file_size, + payload_offset + PAYLOAD_BYTES + TRAILER_BYTES + ); + let (_, lcp) = store.find_text_lcp(&prompt, 0, 2, 2048, 5).unwrap(); + assert_eq!(lcp, prompt.len()); + + let _ = fs::remove_dir_all(&dir); + } + + #[cfg(unix)] + #[test] + fn eviction_stops_after_unlink_failure() { + let dir = std::env::temp_dir().join(format!("ds4-kv-evict-unlink-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + store.write(rec(b"first", 512)).unwrap(); + store.write(rec(b"second", 1024)).unwrap(); + + let original_permissions = fs::metadata(&dir).unwrap().permissions(); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).unwrap(); + store.budget_bytes = 1; + store.evict(0, None); + let disk_files = fs::read_dir(&dir).unwrap().count(); + let remaining_tokens: Vec<_> = store.entries().iter().map(|e| e.header.tokens).collect(); + fs::set_permissions(&dir, original_permissions).unwrap(); + let _ = fs::remove_dir_all(&dir); + + assert_eq!(disk_files, 2); + assert_eq!(remaining_tokens, vec![1024]); + } + + #[test] + fn compatible_empty_trailer_reuses_existing_bytes() { + let dir = std::env::temp_dir().join(format!("ds4-kv-reuse-empty-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut existing = rec(b"same prompt", 512); + existing.header.hits = 7; + existing.trailer = b"old trailer".to_vec(); + let path = store.write(existing).unwrap(); + let sibling = store.write(rec(b"sibling", 1024)).unwrap(); + let before = fs::read(&path).unwrap(); + + let mut incoming = rec(b"same prompt", 2048).header; + incoming.quant_bits = 4; + incoming.reason = Reason::Continued; + incoming.ext_flags = crate::format::EXT_RESPONSES_VISIBLE; + incoming.ctx_size = 4096; + incoming.created_at = 99; + incoming.last_used = 99; + store.budget_bytes = 1; + + let got = store + .write_payload_file(incoming, b"same prompt", &dir.join("missing-payload"), b"") + .unwrap(); + assert_eq!(got, path); + assert_eq!(fs::read(&path).unwrap(), before); + assert!(sibling.exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn reuse_compatible_empty_trailer_returns_existing_without_staging() { + let dir = + std::env::temp_dir().join(format!("ds4-kv-preflight-empty-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut existing = rec(b"same prompt", 512); + existing.header.hits = 7; + existing.trailer = b"old trailer".to_vec(); + let path = store.write(existing).unwrap(); + let sibling = store.write(rec(b"sibling", 1024)).unwrap(); + let before = fs::read(&path).unwrap(); + let sibling_before = fs::read(&sibling).unwrap(); + + let mut incoming = rec(b"same prompt", 2048).header; + incoming.quant_bits = 4; + incoming.reason = Reason::Continued; + incoming.ext_flags = crate::format::EXT_RESPONSES_VISIBLE; + incoming.ctx_size = 4096; + incoming.created_at = 99; + incoming.last_used = 99; + store.budget_bytes = 1; + + let got = store + .reuse_compatible(incoming, b"same prompt", b"") + .unwrap(); + + assert_eq!(got, Some(path.clone())); + assert_eq!(fs::read(&path).unwrap(), before); + assert_eq!(fs::read(&sibling).unwrap(), sibling_before); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 2); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn reuse_compatible_incompatible_entry_returns_none_without_mutation() { + let dir = std::env::temp_dir().join(format!( + "ds4-kv-preflight-incompatible-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let path = store.write(rec(b"same prompt", 512)).unwrap(); + let sibling = store.write(rec(b"sibling", 1024)).unwrap(); + let before = fs::read(&path).unwrap(); + let sibling_before = fs::read(&sibling).unwrap(); + let mut incoming = rec(b"same prompt", 512).header; + incoming.model_id = 1; + store.budget_bytes = 1; + + let got = store + .reuse_compatible(incoming, b"same prompt", b"") + .unwrap(); + + assert_eq!(got, None); + assert_eq!(fs::read(&path).unwrap(), before); + assert_eq!(fs::read(&sibling).unwrap(), sibling_before); + assert_eq!(store.entries().len(), 2); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn reuse_compatible_nonempty_trailer_matches_writer_fast_path() { + let dir = + std::env::temp_dir().join(format!("ds4-kv-preflight-trailer-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut existing = rec(b"same prompt", 512); + existing.header.hits = 7; + existing.header.created_at = 42; + existing.header.last_used = 43; + existing.header.ext_flags = crate::format::EXT_RESPONSES_VISIBLE; + existing.payload = b"opaque payload".to_vec(); + existing.trailer = b"old trailer".to_vec(); + let path = store.write(existing).unwrap(); + let sibling = store.write(rec(b"sibling", 1024)).unwrap(); + let before = read_path(&path).unwrap(); + + let mut incoming = rec(b"same prompt", 2048).header; + incoming.reason = Reason::Continued; + incoming.ext_flags = crate::format::EXT_THINKING_VISIBLE; + incoming.ctx_size = 4096; + incoming.created_at = 99; + incoming.last_used = 99; + store.budget_bytes = 1; + let started = now_secs(); + + let got = store + .reuse_compatible(incoming, b"same prompt", b"new tool map") + .unwrap(); + let after = read_path(&path).unwrap(); + + assert_eq!(got, Some(path)); + assert_eq!(after.text, before.text); + assert_eq!(after.payload, before.payload); + assert_eq!(after.trailer, b"new tool map"); + assert_eq!(after.header.model_id, before.header.model_id); + assert_eq!(after.header.quant_bits, before.header.quant_bits); + assert_eq!(after.header.reason, before.header.reason); + assert_eq!(after.header.tokens, before.header.tokens); + assert_eq!(after.header.hits, before.header.hits); + assert_eq!(after.header.ctx_size, before.header.ctx_size); + assert_eq!(after.header.created_at, before.header.created_at); + assert_eq!(after.header.payload_bytes, before.header.payload_bytes); + assert_eq!(after.header.text_bytes, before.header.text_bytes); + assert_eq!( + after.header.ext_flags, + before.header.ext_flags | crate::format::EXT_TOOL_MAP + ); + assert!(after.header.last_used >= started); + assert!(sibling.exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn compatible_trailer_rewrite_preserves_payload_and_header() { + let dir = std::env::temp_dir().join(format!("ds4-kv-reuse-trailer-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut existing = rec(b"same prompt", 512); + existing.header.hits = 7; + existing.header.created_at = 42; + existing.header.last_used = 43; + existing.header.ext_flags = crate::format::EXT_RESPONSES_VISIBLE; + existing.payload = b"opaque payload".to_vec(); + existing.trailer = b"old trailer".to_vec(); + let path = store.write(existing).unwrap(); + let sibling = store.write(rec(b"sibling", 1024)).unwrap(); + let before = read_path(&path).unwrap(); + + let mut incoming = rec(b"same prompt", 2048).header; + incoming.reason = Reason::Continued; + incoming.ext_flags = crate::format::EXT_THINKING_VISIBLE; + incoming.ctx_size = 4096; + incoming.created_at = 99; + incoming.last_used = 99; + store.budget_bytes = 1; + let started = now_secs(); + + store + .write_payload_file( + incoming, + b"same prompt", + &dir.join("missing-payload"), + b"new tool map", + ) + .unwrap(); + let got = read_path(&path).unwrap(); + assert_eq!(got.text, before.text); + assert_eq!(got.payload, before.payload); + assert_eq!(got.trailer, b"new tool map"); + assert_eq!(got.header.model_id, before.header.model_id); + assert_eq!(got.header.quant_bits, before.header.quant_bits); + assert_eq!(got.header.reason, before.header.reason); + assert_eq!(got.header.tokens, before.header.tokens); + assert_eq!(got.header.hits, before.header.hits); + assert_eq!(got.header.ctx_size, before.header.ctx_size); + assert_eq!(got.header.created_at, before.header.created_at); + assert_eq!(got.header.payload_bytes, before.header.payload_bytes); + assert_eq!(got.header.text_bytes, before.header.text_bytes); + assert_eq!( + got.header.ext_flags, + before.header.ext_flags | crate::format::EXT_TOOL_MAP + ); + assert!(got.header.last_used >= started); + assert!(sibling.exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn payload_destination_alias_is_rejected_without_data_loss() { + let dir = std::env::temp_dir().join(format!("ds4-kv-alias-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let path = store.write(rec(b"same prompt", 512)).unwrap(); + let before = fs::read(&path).unwrap(); + let mut incompatible = rec(b"same prompt", 512).header; + incompatible.model_id = 1; + + let error = store + .write_payload_file(incompatible, b"same prompt", &path, b"") + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(fs::read(&path).unwrap(), before); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn stream_truncation_maps_to_unexpected_eof() { + assert_eq!( + format_io_error(FormatError::Truncated).kind(), + io::ErrorKind::UnexpectedEof + ); + } + + #[test] + fn failed_incompatible_stage_preserves_existing_entries() { + let dir = std::env::temp_dir().join(format!("ds4-kv-stage-fail-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let target = store.write(rec(b"same prompt", 8192)).unwrap(); + let sibling = store.write(rec(b"sibling", 512)).unwrap(); + let target_before = fs::read(&target).unwrap(); + let sibling_before = fs::read(&sibling).unwrap(); + let mut incoming = rec(b"same prompt", 1024).header; + incoming.model_id = 1; + + let error = store + .write_payload_file(incoming, b"same prompt", &dir.join("missing-payload"), b"") + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + assert_eq!(fs::read(&target).unwrap(), target_before); + assert_eq!(fs::read(&sibling).unwrap(), sibling_before); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn incompatible_replacement_credits_target_and_keeps_sibling() { + let dir = std::env::temp_dir().join(format!("ds4-kv-replace-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let text = b"same prompt"; + let target = store.write(rec(text, 8192)).unwrap(); + let sibling = store.write(rec(b"sibling", 512)).unwrap(); + let payload = b"new payload"; + let payload_path = dir.join("incoming.payload"); + fs::write(&payload_path, payload).unwrap(); + let incoming_bytes = file_size_bytes(text.len() as u64, payload.len() as u64, 0).unwrap(); + store.budget_bytes = fs::metadata(&sibling).unwrap().len() + incoming_bytes; + let mut incoming = rec(text, 1024).header; + incoming.model_id = 1; + + let got = store + .write_payload_file(incoming, text, &payload_path, b"") + .unwrap(); + let replaced = read_path(&target).unwrap(); + assert_eq!(got, target); + assert_eq!(replaced.header.model_id, 1); + assert_eq!(replaced.payload, payload); + assert!(sibling.exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn candidate_reopens_and_validates_text() { + let dir = std::env::temp_dir().join(format!("ds4-kv-candidate-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let path = store.write(rec(b"shared prefix", 512)).unwrap(); + + let (got_path, envelope) = store + .text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192) + .unwrap() + .unwrap(); + assert_eq!(got_path, path); + assert_eq!(envelope.text, b"shared prefix"); + + let mut file = fs::OpenOptions::new().write(true).open(&path).unwrap(); + file.seek(SeekFrom::Start((crate::format::FIXED_HEADER + 4) as u64)) + .unwrap(); + file.write_all(b"X").unwrap(); + file.flush().unwrap(); + // The text no longer hashes to its own name: the record is refused + // by its contents, which is not the same as not being there. + assert_eq!( + store + .text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidData + ); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn bank_candidate_requires_a_valid_strict_suffix() { + let dir = + std::env::temp_dir().join(format!("ds4-kv-bank-candidate-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut record = rec(b"shared prefix", 512); + record.header.reason = Reason::BankShutdown; + record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1; + let path = store.write(record).unwrap(); + let mut serial = rec(b"serial exact", 512); + serial.header.reason = Reason::Shutdown; + let serial_path = store.write(serial).unwrap(); + + assert!(store + .bank_text_prefix_candidate(b"shared prefix", 0, 2, 8192) + .unwrap() + .is_none()); + assert!(store + .bank_text_prefix_candidate(b"serial exact", 0, 2, 8192) + .unwrap() + .is_none()); + assert_eq!( + store + .bank_text_prefix_candidate(b"serial exact suffix", 0, 2, 8192) + .unwrap() + .unwrap() + .0, + serial_path + ); + let (got_path, envelope) = store + .bank_text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192) + .unwrap() + .unwrap(); + assert_eq!(got_path, path); + assert_eq!(envelope.text, b"shared prefix"); + + let mut file = fs::OpenOptions::new().write(true).open(&path).unwrap(); + file.seek(SeekFrom::Start((crate::format::FIXED_HEADER + 4) as u64)) + .unwrap(); + file.write_all(b"X").unwrap(); + file.flush().unwrap(); + assert_eq!( + store + .bank_text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192) + .unwrap_err() + .kind(), + io::ErrorKind::InvalidData + ); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn touch_hit_preserves_payload_and_trailer() { + let dir = std::env::temp_dir().join(format!("ds4-kv-touch-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, false, Options::default()).unwrap(); + let mut record = rec(b"shared prefix", 512); + record.header.hits = 7; + record.header.created_at = 42; + record.header.last_used = 43; + record.payload = b"opaque payload".to_vec(); + record.trailer = b"trailer".to_vec(); + let path = store.write(record).unwrap(); + let before = read_path(&path).unwrap(); + let started = now_secs(); + + store.touch_hit(&path).unwrap(); + let after = read_path(&path).unwrap(); + assert_eq!(after.header.hits, before.header.hits + 1); + assert_eq!(after.header.created_at, before.header.created_at); + assert!(after.header.last_used >= started); + assert_eq!(after.text, before.text); + assert_eq!(after.payload, before.payload); + assert_eq!(after.trailer, before.trailer); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/crates/ds4-kv/tests/bank_thinking.rs b/crates/ds4-kv/tests/bank_thinking.rs new file mode 100644 index 0000000000000000000000000000000000000000..f2012f2ce6227c0939d4fa18f96f40cd718bf432 --- /dev/null +++ b/crates/ds4-kv/tests/bank_thinking.rs @@ -0,0 +1,250 @@ +//! Bank thinking bit + one KTM extension record. +//! +//! Given / When / Then against the C KVC envelope: thinking is +//! `EXT_THINKING_VISIBLE`, the only post-DSV4 magic trailer is `KTM\x01`. + +use ds4_kv::{ + decode_file, encode_file, read_path, write_path, BankThinkingExtensions, ExtensionRecord, + Header, Reason, Record, EXT_BANK_REPLAY_V1, EXT_THINKING_VISIBLE, EXT_TOOL_MAP, +}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn sample_bank() -> Record { + Record { + header: Header { + quant_bits: 2, + reason: Reason::BankCheckpoint, + ext_flags: EXT_BANK_REPLAY_V1, + model_id: 3, + tokens: 64, + hits: 1, + ctx_size: 4096, + created_at: 1_700_000_000, + last_used: 1_700_000_050, + payload_bytes: 6, + text_bytes: 11, + }, + text: b"hello world".to_vec(), + payload: b"PAYLOD".to_vec(), + trailer: Vec::new(), + } +} + +fn tmp(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("ds4-kv-bank-think-{}-{}", std::process::id(), name)); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir.join("bank.kv") +} + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_KV_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/kv_c_oracle") +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn unhex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() +} + +fn c_write(path: &Path, rec: &Record) { + let status = Command::new(oracle()) + .args([ + "write", + "--path", + path.to_str().unwrap(), + "--text-hex", + &hex(&rec.text), + "--payload-hex", + &hex(&rec.payload), + "--model-id", + &rec.header.model_id.to_string(), + "--quant", + &rec.header.quant_bits.to_string(), + "--reason", + &(rec.header.reason as u8).to_string(), + "--ext", + &rec.header.ext_flags.to_string(), + "--tokens", + &rec.header.tokens.to_string(), + "--hits", + &rec.header.hits.to_string(), + "--ctx", + &rec.header.ctx_size.to_string(), + "--created", + &rec.header.created_at.to_string(), + "--used", + &rec.header.last_used.to_string(), + "--trailer-hex", + &hex(&rec.trailer), + ]) + .status() + .unwrap(); + assert!(status.success(), "C write failed"); +} + +fn c_read(path: &Path) -> Record { + let out = Command::new(oracle()) + .args(["read", path.to_str().unwrap()]) + .output() + .unwrap(); + assert!( + out.status.success(), + "C read failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let text = String::from_utf8(out.stdout).unwrap(); + let mut rec = sample_bank(); + for line in text.lines() { + let Some((k, v)) = line.split_once('=') else { + continue; + }; + match k { + "model_id" => rec.header.model_id = v.parse().unwrap(), + "quant_bits" => rec.header.quant_bits = v.parse().unwrap(), + "reason" => rec.header.reason = Reason::from_u8(v.parse().unwrap()), + "ext_flags" => rec.header.ext_flags = v.parse().unwrap(), + "tokens" => rec.header.tokens = v.parse().unwrap(), + "hits" => rec.header.hits = v.parse().unwrap(), + "ctx_size" => rec.header.ctx_size = v.parse().unwrap(), + "created_at" => rec.header.created_at = v.parse().unwrap(), + "last_used" => rec.header.last_used = v.parse().unwrap(), + "payload_bytes" => rec.header.payload_bytes = v.parse().unwrap(), + "text_bytes" => rec.header.text_bytes = v.parse().unwrap(), + "text_hex" => rec.text = unhex(v), + "payload_hex" => rec.payload = unhex(v), + "trailer_hex" => rec.trailer = unhex(v), + _ => {} + } + } + rec +} + +fn one_extension() -> BankThinkingExtensions { + BankThinkingExtensions { + thinking_visible: true, + records: vec![ExtensionRecord { + id: b"call_abc".to_vec(), + body: b"function_calls>x".to_vec(), + }], + } +} + +#[test] +fn bank_save_load_thinking_bit_and_one_extension_record() { + // Given: a bank checkpoint with thinking-visible text and one KTM record + let ext = one_extension(); + let mut rec = sample_bank(); + rec.persist_bank_extensions(&ext); + + // When: the envelope is written and read back + let path = tmp("roundtrip"); + write_path(&path, &rec).unwrap(); + let got = read_path(&path).unwrap(); + let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer); + + // Then: thinking bit + BANK_REPLAY_V1 + TOOL_MAP and the one record survive + assert_eq!( + got.header.ext_flags, + EXT_BANK_REPLAY_V1 | EXT_THINKING_VISIBLE | EXT_TOOL_MAP + ); + assert!(restored.thinking_visible); + assert_eq!(restored.records, ext.records); + assert_eq!(&got.trailer[..4], b"KTM\x01"); + assert_eq!(encode_file(&rec)[..3], *b"KVC"); + + let _ = fs::remove_dir_all(path.parent().unwrap()); +} + +#[test] +fn missing_extension_section_is_ignored_like_c() { + // Given: a bank file with replay-v1 only (no thinking bit, no trailer) + let rec = sample_bank(); + let bytes = encode_file(&rec); + let got = decode_file(&bytes).unwrap(); + + // When: restore runs on a missing section + let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer); + + // Then: C-like ignore — thinking off, zero records, file still valid + assert_eq!(got.header.ext_flags, EXT_BANK_REPLAY_V1); + assert!(!restored.thinking_visible); + assert!(restored.records.is_empty()); +} + +#[test] +fn invalid_ktm_magic_is_ignored_like_c() { + // Given: a trailer that is not KTM\x01 + let restored = BankThinkingExtensions::restore(EXT_TOOL_MAP, b"XXX\x01\x01\0\0\0"); + + // When/Then: C kv_tool_map_load_from_pos returns 0 + assert!(!restored.thinking_visible); + assert!(restored.records.is_empty()); +} + +#[test] +fn rust_bank_thinking_extension_is_c_readable() { + let oracle = oracle(); + assert!( + oracle.exists(), + "build the C oracle first: make tests/parity/kv_c_oracle (missing {})", + oracle.display() + ); + + let ext = one_extension(); + let mut rec = sample_bank(); + rec.persist_bank_extensions(&ext); + rec.header.text_bytes = rec.text.len() as u32; + rec.header.payload_bytes = rec.payload.len() as u64; + + let path = tmp("rust-c"); + write_path(&path, &rec).unwrap(); + let got = c_read(&path); + let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer); + + assert_eq!(got.header.ext_flags, rec.header.ext_flags); + assert_eq!(got.trailer, rec.trailer); + assert!(restored.thinking_visible); + assert_eq!(restored.records, ext.records); + + let _ = fs::remove_dir_all(path.parent().unwrap()); +} + +#[test] +fn c_bank_thinking_extension_loads_in_rust() { + let oracle = oracle(); + assert!( + oracle.exists(), + "build the C oracle first: make tests/parity/kv_c_oracle (missing {})", + oracle.display() + ); + + let ext = one_extension(); + let mut rec = sample_bank(); + rec.persist_bank_extensions(&ext); + rec.header.text_bytes = rec.text.len() as u32; + rec.header.payload_bytes = rec.payload.len() as u64; + + let path = tmp("c-rust"); + c_write(&path, &rec); + let got = decode_file(&fs::read(&path).unwrap()).unwrap(); + let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer); + + assert_eq!(got.header.ext_flags, rec.header.ext_flags); + assert_eq!(got.trailer, rec.trailer); + assert!(restored.thinking_visible); + assert_eq!(restored.records, ext.records); + + let _ = fs::remove_dir_all(path.parent().unwrap()); +} diff --git a/crates/ds4-kv/tests/four_way.rs b/crates/ds4-kv/tests/four_way.rs new file mode 100644 index 0000000000000000000000000000000000000000..c7454ad2e6ee166287534b5857f6aa5c1460d7a3 --- /dev/null +++ b/crates/ds4-kv/tests/four_way.rs @@ -0,0 +1,375 @@ +//! Four-way KVC matrix: C save/load × Rust save/load. + +use ds4_kv::{ + chat_anchor_pos, continued_store_target, decode_file, encode_file, eviction_score, read_path, + read_trailer, store_len, write_path, Header, Options, Reason, Record, ScoreEntry, Store, + EXT_TOOL_MAP, +}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_KV_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/kv_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/kv_c_oracle (missing {})", + p.display() + ); + p +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn unhex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() +} + +fn sample() -> Record { + Record { + header: Header { + quant_bits: 2, + reason: Reason::Cold, + ext_flags: 0, + model_id: 3, + tokens: 512, + hits: 7, + ctx_size: 4096, + created_at: 1_700_000_000, + last_used: 1_700_000_050, + payload_bytes: 6, + text_bytes: 11, + }, + text: b"hello world".to_vec(), + payload: b"PAYLOD".to_vec(), + trailer: Vec::new(), + } +} + +fn c_write(path: &Path, rec: &Record) { + let status = Command::new(require_oracle()) + .args([ + "write", + "--path", + path.to_str().unwrap(), + "--text-hex", + &hex(&rec.text), + "--payload-hex", + &hex(&rec.payload), + "--model-id", + &rec.header.model_id.to_string(), + "--quant", + &rec.header.quant_bits.to_string(), + "--reason", + &(rec.header.reason as u8).to_string(), + "--ext", + &rec.header.ext_flags.to_string(), + "--tokens", + &rec.header.tokens.to_string(), + "--hits", + &rec.header.hits.to_string(), + "--ctx", + &rec.header.ctx_size.to_string(), + "--created", + &rec.header.created_at.to_string(), + "--used", + &rec.header.last_used.to_string(), + "--trailer-hex", + &hex(&rec.trailer), + ]) + .status() + .unwrap(); + assert!(status.success(), "C write failed"); +} + +fn c_read(path: &Path) -> Record { + let out = Command::new(require_oracle()) + .args(["read", path.to_str().unwrap()]) + .output() + .unwrap(); + assert!( + out.status.success(), + "C read failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let text = String::from_utf8(out.stdout).unwrap(); + let mut rec = sample(); + for line in text.lines() { + let Some((k, v)) = line.split_once('=') else { + continue; + }; + match k { + "model_id" => rec.header.model_id = v.parse().unwrap(), + "quant_bits" => rec.header.quant_bits = v.parse().unwrap(), + "reason" => rec.header.reason = Reason::from_u8(v.parse().unwrap()), + "ext_flags" => rec.header.ext_flags = v.parse().unwrap(), + "tokens" => rec.header.tokens = v.parse().unwrap(), + "hits" => rec.header.hits = v.parse().unwrap(), + "ctx_size" => rec.header.ctx_size = v.parse().unwrap(), + "created_at" => rec.header.created_at = v.parse().unwrap(), + "last_used" => rec.header.last_used = v.parse().unwrap(), + "payload_bytes" => rec.header.payload_bytes = v.parse().unwrap(), + "text_bytes" => rec.header.text_bytes = v.parse().unwrap(), + "text_hex" => rec.text = unhex(v), + "payload_hex" => rec.payload = unhex(v), + "trailer_hex" => rec.trailer = unhex(v), + _ => {} + } + } + rec +} + +fn tmp(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("ds4-kv-4way-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + dir.join(name) +} + +fn same_envelope(a: &Record, b: &Record) { + assert_eq!(a.header.model_id, b.header.model_id); + assert_eq!(a.header.quant_bits, b.header.quant_bits); + assert_eq!(a.header.reason, b.header.reason); + assert_eq!(a.header.ext_flags, b.header.ext_flags); + assert_eq!(a.header.tokens, b.header.tokens); + assert_eq!(a.header.hits, b.header.hits); + assert_eq!(a.header.ctx_size, b.header.ctx_size); + assert_eq!(a.header.created_at, b.header.created_at); + assert_eq!(a.header.last_used, b.header.last_used); + assert_eq!(a.header.payload_bytes, b.header.payload_bytes); + assert_eq!(a.header.text_bytes, b.header.text_bytes); + assert_eq!(a.text, b.text); + assert_eq!(a.payload, b.payload); + assert_eq!(a.trailer, b.trailer); +} + +#[test] +fn rust_save_rust_load() { + let rec = sample(); + let path = tmp("rr.kv"); + write_path(&path, &rec).unwrap(); + let got = read_path(&path).unwrap(); + same_envelope(&rec, &got); + assert_eq!(encode_file(&rec), fs::read(&path).unwrap()); +} + +#[test] +fn c_save_c_load() { + let rec = sample(); + let path = tmp("cc.kv"); + c_write(&path, &rec); + let got = c_read(&path); + same_envelope(&rec, &got); +} + +#[test] +fn rust_save_c_load() { + let rec = sample(); + let path = tmp("rc.kv"); + write_path(&path, &rec).unwrap(); + let got = c_read(&path); + same_envelope(&rec, &got); +} + +#[test] +fn c_save_rust_load() { + let rec = sample(); + let path = tmp("cr.kv"); + c_write(&path, &rec); + let got = decode_file(&fs::read(&path).unwrap()).unwrap(); + same_envelope(&rec, &got); +} + +#[test] +fn c_save_rust_trailer_only_load() { + let mut rec = sample(); + rec.header.ext_flags = EXT_TOOL_MAP; + rec.trailer = b"KTM\x01\x01\0\0\0\xff".to_vec(); + let path = tmp("cr-trailer.kv"); + c_write(&path, &rec); + + let (header, trailer) = read_trailer(&path, rec.trailer.len() as u64).unwrap(); + assert_eq!(header.ext_flags, EXT_TOOL_MAP); + assert_eq!(trailer, rec.trailer); +} + +#[test] +fn rust_stream_payload_with_trailer_loads_in_c() { + let base = tmp("stream-rust-c"); + let store_dir = base.join("store"); + let _ = fs::remove_dir_all(&base); + fs::create_dir_all(&base).unwrap(); + + let mut rec = sample(); + rec.payload = b"DSV4 opaque native payload\0\xff".to_vec(); + rec.trailer = b"KVT1 tool-map trailer".to_vec(); + rec.header.ext_flags = ds4_kv::EXT_TOOL_MAP; + rec.header.text_bytes = rec.text.len() as u32; + rec.header.payload_bytes = rec.payload.len() as u64; + let payload_path = ds4_kv::path_for_sha(&store_dir, &ds4_kv::text_sha_hex(&rec.text)) + .with_extension(format!("kv.tmp.{}", std::process::id())); + fs::create_dir_all(&store_dir).unwrap(); + fs::write(&payload_path, &rec.payload).unwrap(); + + let mut store = Store::open(&store_dir, 16, false, Options::default()).unwrap(); + let path = store + .write_payload_file(rec.header.clone(), &rec.text, &payload_path, &rec.trailer) + .unwrap(); + assert_eq!(fs::read(&payload_path).unwrap(), rec.payload); + let got = c_read(&path); + same_envelope(&rec, &got); + + let _ = fs::remove_dir_all(&base); +} + +#[test] +fn sha1_matches_c() { + let text = b"hello world"; + let out = Command::new(require_oracle()) + .args(["sha1", "--text-hex", &hex(text)]) + .output() + .unwrap(); + assert!(out.status.success()); + let c = String::from_utf8(out.stdout).unwrap(); + assert_eq!(c.trim(), ds4_kv::text_sha_hex(text)); +} + +#[test] +fn store_len_matches_c() { + for tokens in [100, 512, 2048, 4096 + 40, 20000] { + let out = Command::new(require_oracle()) + .args(["store-len", "--tokens", &tokens.to_string()]) + .output() + .unwrap(); + assert!(out.status.success()); + let c: i32 = String::from_utf8(out.stdout) + .unwrap() + .trim() + .parse() + .unwrap(); + assert_eq!(store_len(&Options::default(), tokens), c, "tokens={tokens}"); + } +} + +#[test] +fn continued_target_matches_c() { + for (name, min, interval, align, last, live, expected) in [ + ("default", 512, 10000, 2048, 0, 10240, 10240), + ("disabled", 512, 0, 2048, 0, 10240, 0), + ("custom-align", 512, 10000, 4096, 0, 12288, 12288), + ("below-min", 20000, 10000, 2048, 0, 10240, 0), + ("prior-marker", 512, 10000, 2048, 10240, 10240, 0), + ("nonmultiple", 512, 10000, 2048, 0, 10239, 0), + ("serial-align1", 512, 6800, 1, 0, 6800, 6800), + ] { + let out = Command::new(require_oracle()) + .args([ + "continued-target", + "--min", + &min.to_string(), + "--interval", + &interval.to_string(), + "--align", + &align.to_string(), + "--last", + &last.to_string(), + "--live", + &live.to_string(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "C continued-target failed for {name}"); + let c: i32 = String::from_utf8(out.stdout) + .unwrap() + .trim() + .parse() + .unwrap(); + assert_eq!(c, expected, "C case={name}"); + let opt = Options { + min_tokens: min, + continued_interval_tokens: interval, + boundary_align_tokens: align, + ..Options::default() + }; + assert_eq!(continued_store_target(&opt, last, live), c, "case={name}"); + } +} + +#[test] +fn chat_anchor_matches_c() { + let mut tokens = vec![7; 600]; + tokens[512] = 99; + tokens[530] = 99; + tokens[550] = 100; + tokens[560] = 99; + let mut args = vec!["chat-anchor".to_string(), "99".into(), "100".into()]; + args.extend(tokens.iter().map(i32::to_string)); + let out = Command::new(require_oracle()).args(&args).output().unwrap(); + assert!(out.status.success()); + let c: i32 = String::from_utf8(out.stdout) + .unwrap() + .trim() + .parse() + .unwrap(); + assert_eq!(chat_anchor_pos(&Options::default(), &tokens, 99, 100), c); + assert_eq!(c, 530); +} + +#[test] +fn eviction_score_matches_c() { + let out = Command::new(require_oracle()) + .args([ + "score", + "--hits", + "4", + "--tokens", + "512", + "--file-size", + "4096", + "--reason", + "1", + "--created", + "1000", + "--used", + "1000", + "--now", + "1000", + ]) + .output() + .unwrap(); + assert!(out.status.success()); + let c: f64 = String::from_utf8(out.stdout) + .unwrap() + .trim() + .parse() + .unwrap(); + let rust = eviction_score( + &ScoreEntry { + sha: "x", + quant_bits: 2, + model_id: 0, + reason: Reason::Cold, + tokens: 512, + hits: 4, + ctx_size: 2048, + created_at: 1000, + last_used: 1000, + text_bytes: 11, + file_size: 4096, + }, + 1000, + None, + ); + assert!((rust - c).abs() <= 1e-12, "rust={rust} c={c}"); +} diff --git a/crates/ds4-perf-gpu/Cargo.toml b/crates/ds4-perf-gpu/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..0843a3ffb70f4b99159a8fa6e4454bec57d76b74 --- /dev/null +++ b/crates/ds4-perf-gpu/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "ds4-perf-gpu" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "Isolated CUDA calibration and CUPTI collection for ds4-perf" + +[lib] +crate-type = ["rlib", "cdylib"] + +[[bin]] +name = "ds4-perf-gpu" +path = "src/main.rs" +required-features = ["cuda"] + +[features] +default = [] +cuda = ["dep:cudarc", "dep:libc"] + +[dependencies] +ds4-perf = { path = "../ds4-perf" } +serde_json = "1" +cudarc = { version = "0.19.9", optional = true, default-features = false, features = ["std", "driver", "nvrtc", "cupti", "dynamic-loading", "cuda-13030"] } +libc = { version = "0.2", optional = true } diff --git a/crates/ds4-perf-gpu/src/calibration.rs b/crates/ds4-perf-gpu/src/calibration.rs new file mode 100644 index 0000000000000000000000000000000000000000..2dbd7a669a8033fc1dc0f6af0f526d5597b77920 --- /dev/null +++ b/crates/ds4-perf-gpu/src/calibration.rs @@ -0,0 +1,176 @@ +use cudarc::{ + driver::{CudaContext, LaunchConfig, PushKernelArg}, + nvrtc::{compile_ptx_with_opts, CompileOptions}, +}; +use ds4_perf::machine::Calibration; +use std::time::Instant; + +const WARMUPS: u32 = 2; +const REPEATS: u32 = 7; +const FMA_ITERATIONS: i32 = 4096; +const LAUNCHES: u32 = 256; +const SOURCE: &str = r#" +extern "C" __global__ void copy_words(float *out, const float *input, int n) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) out[i] = input[i]; +} +extern "C" __global__ void fma_envelope(float *out, int iterations) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + float a=0.1f,b=0.2f,c=0.3f,d=0.4f,e=0.5f,f=0.6f,g=0.7f,h=0.8f; + for (int j=0; j Result { + measure(ordinal).map_err(|e| e.to_string()) +} + +fn measure(ordinal: usize) -> Result> { + let gpu = crate::device::inspect(ordinal)?; + let ctx = CudaContext::new(ordinal)?; + let stream = ctx.default_stream(); + // Larger than L2; bounded to 512 MiB total so this is not a memory stress test. + let bytes = (64 * 1024 * 1024usize).max(gpu.l2_bytes.max(0) as usize * 2); + if bytes > 256 * 1024 * 1024 || gpu.free_memory_bytes < (bytes * 4) as u64 { + return Err("insufficient bounded calibration headroom".into()); + } + let n = (bytes / 4) as i32; + let source = stream.clone_htod(&vec![1.0f32; n as usize])?; + let mut target = stream.alloc_zeros::(n as usize)?; + let workers = gpu.multiprocessors as u32 * 256 * 8; + let mut compute = stream.alloc_zeros::(workers as usize)?; + let ptx = compile_ptx_with_opts( + SOURCE, + CompileOptions { + options: vec![format!( + "--gpu-architecture=compute_{}{}", + gpu.compute_major, gpu.compute_minor + )], + ..Default::default() + }, + )?; + let ptx_sha256 = ds4_perf::artifact::hash_bytes(ptx.to_src().as_bytes()); + let module = ctx.load_module(ptx)?; + let copy = module.load_function("copy_words")?; + let fma = module.load_function("fma_envelope")?; + let empty = module.load_function("empty_launch")?; + let mut copy_gb_s = Vec::new(); + let mut fp32_gflop_s = Vec::new(); + let mut launch_us = Vec::new(); + let mut copy_times = Vec::new(); + let mut compute_times = Vec::new(); + let mut launch_times = Vec::new(); + for repetition in 0..WARMUPS + REPEATS { + let start = + stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?; + // All pointers are cudarc-owned slices on this stream; n bounds the kernel. + unsafe { + stream + .launch_builder(©) + .arg(&mut target) + .arg(&source) + .arg(&n) + .launch(LaunchConfig::for_num_elems(n as u32))?; + } + let end = + stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?; + end.synchronize()?; + let copy_ms = start.elapsed_ms(&end)? as f64; + + let start = + stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?; + // Exactly workers threads write an allocation of workers f32 elements. + unsafe { + stream + .launch_builder(&fma) + .arg(&mut compute) + .arg(&FMA_ITERATIONS) + .launch(LaunchConfig { + grid_dim: (workers / 256, 1, 1), + block_dim: (256, 1, 1), + shared_mem_bytes: 0, + })?; + } + let end = + stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?; + end.synchronize()?; + let compute_ms = start.elapsed_ms(&end)? as f64; + + let host_start = Instant::now(); + for _ in 0..LAUNCHES { + // This kernel has no arguments or memory accesses. + unsafe { + stream.launch_builder(&empty).launch(LaunchConfig { + grid_dim: (1, 1, 1), + block_dim: (1, 1, 1), + shared_mem_bytes: 0, + })?; + } + } + stream.synchronize()?; + let launch_ms = host_start.elapsed().as_secs_f64() * 1000.0; + let latency = launch_ms * 1000.0 / LAUNCHES as f64; + if repetition >= WARMUPS { + copy_gb_s.push(2.0 * bytes as f64 / (copy_ms * 1e6)); + fp32_gflop_s.push(workers as f64 * FMA_ITERATIONS as f64 * 16.0 / (compute_ms * 1e6)); + launch_us.push(latency); + copy_times.push(copy_ms); + compute_times.push(compute_ms); + launch_times.push(launch_ms); + } + } + if stream.clone_dtoh(&target)?.iter().any(|v| *v != 1.0) { + return Err("copy calibration validation failed".into()); + } + let compute_result = stream.clone_dtoh(&compute)?; + if !valid_compute(&compute_result) { + return Err("FMA calibration validation failed".into()); + } + if [©_gb_s, &fp32_gflop_s, &launch_us] + .iter() + .any(|values| values.iter().any(|v| !v.is_finite() || *v <= 0.0)) + { + return Err("invalid calibration timing".into()); + } + Ok(Calibration { ptx_sha256, method:"cuda-event-copy-fp32-fma-v1".into(),source_sha256:ds4_perf::artifact::hash_bytes(SOURCE.as_bytes()), + copy_ms:copy_times,compute_ms:compute_times,launch_batch_ms:launch_times, + compute_threads:workers,compute_block_threads:256,fma_iterations:FMA_ITERATIONS as u32,launch_count:LAUNCHES, + gpu, transfer_bytes: bytes as u64, warmups: WARMUPS, repeats: REPEATS, copy_gb_s, fp32_gflop_s, launch_us, + notes: vec!["Copy counts read plus write bytes, uses CUDA event time, and exceeds L2 capacity.".into(), + "FP32 measures eight independent SIMT FMA chains; it is not Tensor Core peak throughput.".into(), + "Launch latency is host wall time for 256 empty kernels plus completion synchronization, per kernel.".into(), + "Observed rates depend on current clocks, temperature and other processes; this is a measured envelope, not a hardware guarantee.".into()] }) +} + +fn valid_compute(values: &[f32]) -> bool { + let mut expected = [0.1f32, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]; + for _ in 0..FMA_ITERATIONS { + for value in &mut expected { + *value = value.mul_add(0.9999, 0.0001); + } + } + let expected: f32 = expected.iter().sum(); + !values.is_empty() + && values + .iter() + .all(|v| v.is_finite() && (*v - expected).abs() <= 0.00002) +} + +#[cfg(test)] +mod tests { + #[test] + fn rejects_short_fma_work() { + let mut one_step = [0.1f32, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]; + for value in &mut one_step { + *value = value.mul_add(0.9999, 0.0001); + } + assert!(!super::valid_compute(&[one_step.iter().sum()])); + } +} diff --git a/crates/ds4-perf-gpu/src/collector.rs b/crates/ds4-perf-gpu/src/collector.rs new file mode 100644 index 0000000000000000000000000000000000000000..07e8f8482583b6e7c1aa275be204c5200afc42a0 --- /dev/null +++ b/crates/ds4-perf-gpu/src/collector.rs @@ -0,0 +1,288 @@ +// CUPTI injection follows NVIDIA's cupti_trace_injection sample. NVTX injection +// points directly to libcupti; all application annotations remain NVIDIA nvtx. +use cudarc::{ + cupti::{ + result::activity, + sys::{self, CUpti_ActivityKind::*}, + }, + driver, +}; +use ds4_perf::trace::Event; +use std::{ + alloc::{alloc, dealloc, Layout}, + ffi::CStr, + fs::OpenOptions, + io::{BufWriter, Write}, + sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, OnceLock, + }, +}; + +const BUFFER_BYTES: usize = 8 * 1024 * 1024; +const BUFFER_ALIGN: usize = 8; +static OUTPUT: OnceLock>> = OnceLock::new(); +static INITIALIZED: OnceLock> = OnceLock::new(); +static ERRORS: AtomicU64 = AtomicU64::new(0); +static DROPPED: AtomicU64 = AtomicU64::new(0); + +fn emit(event: Event) { + let Some(output) = OUTPUT.get() else { + return; + }; + let Ok(mut output) = output.lock() else { + ERRORS.fetch_add(1, Ordering::Relaxed); + return; + }; + if serde_json::to_writer(&mut *output, &event).is_err() || output.write_all(b"\n").is_err() { + ERRORS.fetch_add(1, Ordering::Relaxed); + } +} + +fn error(message: String) { + ERRORS.fetch_add(1, Ordering::Relaxed); + emit(Event::Error { message }); +} + +// These callbacks are called by CUPTI with its documented output pointers. +extern "C" fn request_buffer(buffer: *mut *mut u8, size: *mut usize, records: *mut usize) { + let layout = Layout::from_size_align(BUFFER_BYTES, BUFFER_ALIGN).expect("constant layout"); + unsafe { + *buffer = alloc(layout); + *size = if (*buffer).is_null() { 0 } else { BUFFER_BYTES }; + *records = 0; + if (*buffer).is_null() { + ERRORS.fetch_add(1, Ordering::Relaxed); + } + } +} + +extern "C" fn complete_buffer( + context: driver::sys::CUcontext, + stream: u32, + buffer: *mut u8, + _size: usize, + valid: usize, +) { + // No panic may cross CUPTI's C boundary. Record failures for the parent. + let result = std::panic::catch_unwind(|| unsafe { consume(buffer, valid) }); + match result { + Ok(Ok(())) => {} + Ok(Err(message)) => error(message), + Err(_) => error("activity callback panicked".into()), + } + let mut dropped = 0; + // CUPTI owns context lifetime throughout this callback. + match unsafe { activity::get_num_dropped_records(context, stream, &mut dropped) } { + Ok(()) => { + DROPPED.fetch_add(dropped as u64, Ordering::Relaxed); + } + Err(e) => error(e.to_string()), + } + if !buffer.is_null() { + // This is exactly the allocation made by request_buffer. + unsafe { + dealloc( + buffer, + Layout::from_size_align(BUFFER_BYTES, BUFFER_ALIGN).expect("constant layout"), + ); + } + } +} + +unsafe fn string(pointer: *const std::ffi::c_char) -> String { + if pointer.is_null() { + return String::new(); + } + // CUPTI guarantees activity strings remain valid until this callback returns. + unsafe { CStr::from_ptr(pointer) } + .to_string_lossy() + .into_owned() +} + +unsafe fn consume(buffer: *mut u8, valid: usize) -> Result<(), String> { + if valid == 0 { + return Ok(()); + } + if buffer.is_null() || valid > BUFFER_BYTES { + return Err("invalid CUPTI buffer length".into()); + } + let mut record = std::ptr::null_mut(); + loop { + match unsafe { activity::get_next_record(buffer, valid, &mut record) } { + Ok(()) => {} + Err(cudarc::cupti::result::CuptiError( + sys::CUptiResult::CUPTI_ERROR_MAX_LIMIT_REACHED, + )) => return Ok(()), + Err(error) => return Err(error.to_string()), + } + if record.is_null() { + return Err("null activity record".into()); + } + // Runtime API version is checked before enable. Cast only the documented + // activity kind to its CUDA 13.3 record; copy values before buffer release. + match unsafe { (*record).kind } { + CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL => { + let k = unsafe { &*record.cast::() }; + if [k.gridX, k.gridY, k.gridZ, k.blockX, k.blockY, k.blockZ] + .iter() + .any(|v| *v < 0) + { + return Err("negative launch dimension".into()); + } + emit(Event::Kernel { + name: unsafe { string(k.name) }, + start: k.start, + end: k.end, + device: k.deviceId, + context: k.contextId, + stream: k.streamId, + correlation: k.correlationId, + grid: [k.gridX as u32, k.gridY as u32, k.gridZ as u32], + block: [k.blockX as u32, k.blockY as u32, k.blockZ as u32], + registers: k.registersPerThread as u32, + shared_bytes: k.staticSharedMemory.max(0) as u64 + + k.dynamicSharedMemory.max(0) as u64, + graph_id: k.graphId, + }); + } + CUPTI_ACTIVITY_KIND_MARKER => { + let m = unsafe { &*record.cast::() }; + if !matches!( + m.objectKind, + sys::CUpti_ActivityObjectKind::CUPTI_ACTIVITY_OBJECT_THREAD + | sys::CUpti_ActivityObjectKind::CUPTI_ACTIVITY_OBJECT_PROCESS + ) { + continue; + } + let id = unsafe { m.objectId.pt }; + emit(Event::Marker { + name: unsafe { string(m.name) }, + domain: unsafe { string(m.domain) }, + timestamp: m.timestamp, + id: m.id, + flags: m.flags as u32, + pid: id.processId, + tid: id.threadId, + }); + } + CUPTI_ACTIVITY_KIND_MEMCPY => { + let m = unsafe { &*record.cast::() }; + emit(Event::Memop { + operation: "memcpy".into(), + start: m.start, + end: m.end, + bytes: m.bytes, + device: m.deviceId, + context: m.contextId, + stream: m.streamId, + }); + } + CUPTI_ACTIVITY_KIND_MEMSET => { + let m = unsafe { &*record.cast::() }; + emit(Event::Memop { + operation: "memset".into(), + start: m.start, + end: m.end, + bytes: m.bytes, + device: m.deviceId, + context: m.contextId, + stream: m.streamId, + }); + } + _ => {} + } + } +} + +extern "C" fn finish() { + let _ = std::panic::catch_unwind(|| { + if let Err(e) = + activity::flush_all(sys::CUpti_ActivityFlag::CUPTI_ACTIVITY_FLAG_FLUSH_FORCED as u32) + { + error(e.to_string()); + } + emit(Event::End { + dropped: DROPPED.load(Ordering::Relaxed), + errors: ERRORS.load(Ordering::Relaxed), + }); + if let Some(output) = OUTPUT.get() { + if let Ok(mut output) = output.lock() { + let _ = output.flush(); + let _ = output.get_ref().sync_all(); + } + } + }); +} + +fn initialize() -> Result<(), String> { + let path = + std::env::var_os("DS4_PERF_CUPTI_OUTPUT").ok_or("DS4_PERF_CUPTI_OUTPUT is required")?; + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(path) + .map_err(|e| e.to_string())?; + OUTPUT + .set(Mutex::new(BufWriter::new(file))) + .map_err(|_| "collector already initialized")?; + let mut version = 0; + // cudarc supplies the binding; the output pointer is a live u32 local. + unsafe { sys::cuptiGetVersion(&mut version) } + .result() + .map_err(|e| e.to_string())?; + emit(Event::Start { + schema_version: 1, + api_version: version, + pid: std::process::id(), + }); + if version != sys::CUPTI_API_VERSION { + return Err(format!( + "CUPTI record ABI mismatch: runtime {version}, bindings {}", + sys::CUPTI_API_VERSION + )); + } + activity::register_callbacks(Some(request_buffer), Some(complete_buffer)) + .map_err(|e| e.to_string())?; + for kind in [ + CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL, + CUPTI_ACTIVITY_KIND_MEMCPY, + CUPTI_ACTIVITY_KIND_MEMSET, + CUPTI_ACTIVITY_KIND_MARKER, + ] { + activity::enable(kind).map_err(|e| e.to_string())?; + } + // Linux atexit is the documented CUPTI injection sample flush point. + if unsafe { libc::atexit(finish) } != 0 { + return Err("cannot register CUPTI exit flush".into()); + } + Ok(()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn InitializeInjection() -> i32 { + let result = std::panic::catch_unwind(|| INITIALIZED.get_or_init(initialize)); + match result { + Ok(Ok(())) => 1, + Ok(Err(message)) => { + error(message.clone()); + flush_output(); + 0 + } + Err(_) => { + error("CUPTI initialization panicked".into()); + flush_output(); + 0 + } + } +} + +fn flush_output() { + if let Some(output) = OUTPUT.get() { + if let Ok(mut output) = output.lock() { + if output.flush().is_err() || output.get_ref().sync_all().is_err() { + eprintln!("ds4-perf CUPTI: could not persist collector output"); + } + } + } +} diff --git a/crates/ds4-perf-gpu/src/device.rs b/crates/ds4-perf-gpu/src/device.rs new file mode 100644 index 0000000000000000000000000000000000000000..19727a01624947a8881237338d7b2b24d9b5e991 --- /dev/null +++ b/crates/ds4-perf-gpu/src/device.rs @@ -0,0 +1,51 @@ +use cudarc::driver::{sys::CUdevice_attribute::*, CudaContext}; +use ds4_perf::machine::Gpu; + +pub fn inspect(ordinal: usize) -> Result { + let ctx = CudaContext::new(ordinal).map_err(|e| e.to_string())?; + let attr = |key| ctx.attribute(key).map_err(|e| e.to_string()); + let (free, total) = ctx.mem_get_info().map_err(|e| e.to_string())?; + let uuid = ctx.uuid().map_err(|e| e.to_string())?; + let mut driver_version = 0; + // CUDA writes one integer to this live local, through cudarc's bindings. + unsafe { cudarc::driver::sys::cuDriverGetVersion(&mut driver_version) } + .result() + .map_err(|e| e.to_string())?; + let mut unavailable = std::collections::BTreeMap::new(); + let mut optional = |name: &str, key| match attr(key) { + Ok(value) if value > 0 => Some(value), + other => { + unavailable.insert(name.into(), format!("{other:?}")); + None + } + }; + let memory_bus_bits = optional( + "memory_bus_bits", + CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH, + ); + let memory_clock_khz = optional("memory_clock_khz", CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE); + let clock_khz = optional("clock_khz", CU_DEVICE_ATTRIBUTE_CLOCK_RATE); + Ok(Gpu { + ordinal, + name: ctx.name().map_err(|e| e.to_string())?, + uuid: uuid.bytes.iter().map(|b| format!("{:02x}", *b)).collect(), + compute_major: attr(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR)?, + compute_minor: attr(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR)?, + driver_version, + total_memory_bytes: total as u64, + free_memory_bytes: free as u64, + multiprocessors: attr(CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)?, + warp_size: attr(CU_DEVICE_ATTRIBUTE_WARP_SIZE)?, + max_threads_per_block: attr(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK)?, + max_threads_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR)?, + max_blocks_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_BLOCKS_PER_MULTIPROCESSOR)?, + registers_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_MULTIPROCESSOR)?, + shared_bytes_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR)?, + shared_bytes_per_block: attr(CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN)?, + l2_bytes: attr(CU_DEVICE_ATTRIBUTE_L2_CACHE_SIZE)?, + memory_bus_bits, + memory_clock_khz, + clock_khz, + unavailable, + }) +} diff --git a/crates/ds4-perf-gpu/src/lib.rs b/crates/ds4-perf-gpu/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..24943c07dba0c07a60c2a2c78217b8ccdcca70af --- /dev/null +++ b/crates/ds4-perf-gpu/src/lib.rs @@ -0,0 +1,11 @@ +// This helper owns profiler CUDA calls; it is never an inference dependency. +#![cfg_attr(not(feature = "cuda"), forbid(unsafe_code))] + +#[cfg(feature = "cuda")] +pub mod device; + +#[cfg(feature = "cuda")] +pub mod calibration; + +#[cfg(feature = "cuda")] +mod collector; diff --git a/crates/ds4-perf-gpu/src/main.rs b/crates/ds4-perf-gpu/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..f839e5c9950cdb3ce3e0a110cae4bbc8c954a029 --- /dev/null +++ b/crates/ds4-perf-gpu/src/main.rs @@ -0,0 +1,23 @@ +fn main() { + let args: Vec<_> = std::env::args().skip(1).collect(); + let result = match args.as_slice() { + [command, ordinal] if command == "inspect" => ordinal + .parse::() + .map_err(|e| e.to_string()) + .and_then(ds4_perf_gpu::device::inspect) + .and_then(|gpu| serde_json::to_string(&gpu).map_err(|e| e.to_string())), + [command, ordinal] if command == "calibrate" => ordinal + .parse::() + .map_err(|e| e.to_string()) + .and_then(ds4_perf_gpu::calibration::run) + .and_then(|calibration| serde_json::to_string(&calibration).map_err(|e| e.to_string())), + _ => Err("usage: ds4-perf-gpu inspect|calibrate DEVICE".into()), + }; + match result { + Ok(text) => println!("{text}"), + Err(error) => { + eprintln!("ds4-perf-gpu: {error}"); + std::process::exit(1); + } + } +} diff --git a/crates/ds4-perf-gpu/tests/live.rs b/crates/ds4-perf-gpu/tests/live.rs new file mode 100644 index 0000000000000000000000000000000000000000..d5b080712e4a82e00652dfc16ebe10d5e8dc5854 --- /dev/null +++ b/crates/ds4-perf-gpu/tests/live.rs @@ -0,0 +1,14 @@ +#![cfg(feature = "cuda")] + +#[test] +#[ignore = "requires an idle CUDA device and NVRTC"] +fn measured_envelope() { + let result = ds4_perf_gpu::calibration::run(0).unwrap(); + for samples in [&result.copy_gb_s, &result.fp32_gflop_s, &result.launch_us] { + assert_eq!(samples.len(), result.repeats as usize); + assert!(samples + .iter() + .all(|value| value.is_finite() && *value > 0.0)); + } + assert!(result.transfer_bytes > result.gpu.l2_bytes as u64); +} diff --git a/crates/ds4-perf/Cargo.toml b/crates/ds4-perf/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..cb167d9d62bc64fee32a5d050a73dfba136878a3 --- /dev/null +++ b/crates/ds4-perf/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "ds4-perf" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "Model-free performance orchestration and Nsight diagnostics for ds4" + +[dependencies] +clap = { version = "4.6", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +# Evidence summaries must retain the exact f64 samples used for decisions. +serde_json = { version = "1", features = ["float_roundtrip"] } +sha2 = { version = "0.11", default-features = false } +ctrlc = { version = "3.5", features = ["termination"] } + +[target.'cfg(unix)'.dependencies] +nix = { version = "0.31", default-features = false, features = ["signal", "process"] } diff --git a/crates/ds4-perf/src/artifact.rs b/crates/ds4-perf/src/artifact.rs new file mode 100644 index 0000000000000000000000000000000000000000..b8796ee185f07d7762855f9ab49ecc3ccf8286ac --- /dev/null +++ b/crates/ds4-perf/src/artifact.rs @@ -0,0 +1,564 @@ +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + io::{Read, Write}, + path::Path, + time::SystemTime, +}; + +/// Each payload evolves independently. Unknown versions fail closed. +pub trait Payload { + const KIND: &'static str; + const SCHEMA: u32 = 1; + fn validate(&self) -> Result<(), String>; +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Reference { + pub path: String, + pub sha256: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Artifact { + pub schema_version: u32, + pub kind: String, + pub producer_version: String, + pub created_unix: u64, + pub complete: bool, + pub inputs: Vec, + pub warnings: Vec, + pub data: Option, +} + +impl Artifact { + pub fn new(data: T) -> Self { + Self { + schema_version: T::SCHEMA, + kind: T::KIND.into(), + producer_version: env!("CARGO_PKG_VERSION").into(), + created_unix: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + complete: true, + inputs: Vec::new(), + warnings: Vec::new(), + data: Some(data), + } + } + pub fn failed(message: String) -> Self { + Self { + schema_version: T::SCHEMA, + kind: T::KIND.into(), + producer_version: env!("CARGO_PKG_VERSION").into(), + created_unix: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + complete: false, + inputs: Vec::new(), + warnings: vec![message], + data: None, + } + } + pub fn require(&self) -> Result<&T, String> { + if !self.complete { + return Err(format!("incomplete {} evidence", T::KIND)); + } + self.data + .as_ref() + .ok_or_else(|| format!("missing {} payload", T::KIND)) + } +} + +fn hex(bytes: impl IntoIterator) -> String { + bytes + .into_iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +pub fn hash_bytes(bytes: &[u8]) -> String { + hex(Sha256::digest(bytes)) +} + +pub fn hash(path: &Path) -> Result { + let mut file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + hash_file(&mut file) +} + +fn hash_file(file: &mut fs::File) -> Result { + let mut state = Sha256::new(); + let mut buffer = [0u8; 64 * 1024]; + loop { + let count = file.read(&mut buffer).map_err(|e| e.to_string())?; + if count == 0 { + break; + } + state.update(&buffer[..count]); + } + Ok(hex(state.finalize())) +} + +#[derive(Clone, PartialEq, Eq)] +struct FileStamp { + size: u64, + modified: SystemTime, + #[cfg(unix)] + identity: [u64; 6], +} + +impl FileStamp { + fn read(metadata: &fs::Metadata) -> Result { + #[cfg(unix)] + use std::os::unix::fs::MetadataExt; + if !metadata.is_file() { + return Err("evidence input must be a regular file".into()); + } + Ok(Self { + size: metadata.len(), + modified: metadata.modified().map_err(|e| e.to_string())?, + #[cfg(unix)] + identity: [ + metadata.dev(), + metadata.ino(), + metadata.mtime() as u64, + metadata.mtime_nsec() as u64, + metadata.ctime() as u64, + metadata.ctime_nsec() as u64, + ], + }) + } +} + +/// Share full-byte verification only within one operation. Reuse never trusts +/// an expected hash alone: replacement or metadata drift invalidates the read. +#[derive(Default)] +pub struct Verification { + files: std::collections::BTreeMap, + aliases: std::collections::BTreeMap, + #[cfg(test)] + full_reads: usize, +} + +impl Verification { + pub fn verify(&mut self, path: &Path, expected: &str) -> Result<(), String> { + let alias = std::env::current_dir() + .map_err(|e| e.to_string())? + .join(path); + let canonical = path.canonicalize().map_err(|e| e.to_string())?; + if self + .aliases + .get(&alias) + .is_some_and(|previous| previous != &canonical) + { + return Err(format!("evidence path changed: {}", path.display())); + } + let mut file = fs::File::open(&canonical).map_err(|e| e.to_string())?; + let before = FileStamp::read(&file.metadata().map_err(|e| e.to_string())?)?; + let digest = match self.files.get(&canonical) { + Some((stamp, digest)) => { + if *stamp != before { + return Err(format!( + "evidence changed during verification: {}", + path.display() + )); + } + digest.clone() + } + None => { + #[cfg(test)] + { + self.full_reads += 1; + } + hash_file(&mut file)? + } + }; + if digest != expected { + return Err(format!("evidence changed: {}", path.display())); + } + let after = FileStamp::read(&file.metadata().map_err(|e| e.to_string())?)?; + let current = FileStamp::read(&fs::metadata(&canonical).map_err(|e| e.to_string())?)?; + if before != after + || before != current + || path.canonicalize().ok().as_ref() != Some(&canonical) + { + return Err(format!( + "evidence changed during verification: {}", + path.display() + )); + } + self.aliases.insert(alias, canonical.clone()); + self.files.insert(canonical, (before, digest)); + Ok(()) + } + + pub fn finish(&self) -> Result<(), String> { + for (alias, target) in &self.aliases { + if alias.canonicalize().ok().as_ref() != Some(target) { + return Err(format!("evidence path changed: {}", alias.display())); + } + } + for (path, (before, _)) in &self.files { + let current = FileStamp::read(&fs::metadata(path).map_err(|e| e.to_string())?)?; + if *before != current { + return Err(format!( + "evidence changed during verification: {}", + path.display() + )); + } + } + Ok(()) + } +} + +pub fn reference(path: &Path, directory: &Path) -> Result { + let canonical = path.canonicalize().map_err(|e| e.to_string())?; + let base = directory.canonicalize().map_err(|e| e.to_string())?; + let relative = canonical.strip_prefix(base).unwrap_or(&canonical); + Ok(Reference { + path: relative.to_string_lossy().into(), + sha256: hash(&canonical)?, + }) +} + +pub fn save(path: &Path, value: &Artifact) -> Result<(), String> { + if value.kind != T::KIND || value.schema_version != T::SCHEMA { + return Err("artifact type/schema mismatch".into()); + } + if let Some(data) = &value.data { + data.validate()?; + } + if value.complete && value.data.is_none() { + return Err("complete artifact has no data".into()); + } + let mut bytes = serde_json::to_vec_pretty(value).map_err(|e| e.to_string())?; + bytes.push(b'\n'); + let nonce = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let temporary = path.with_extension(format!("tmp-{}-{nonce}", std::process::id())); + let result = (|| { + let mut file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + // Atomic publication with no overwrite, even if another writer wins. + fs::hard_link(&temporary, path) + })(); + let _ = fs::remove_file(temporary); + result.map_err(|e| format!("{}: {e}", path.display())) +} + +pub fn load(path: &Path) -> Result, String> { + let mut checked = Verification::default(); + let value = load_verified(path, &mut checked)?; + checked.finish()?; + Ok(value) +} + +pub fn load_verified( + path: &Path, + checked: &mut Verification, +) -> Result, String> { + let file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + const MAX_JSON: u64 = 64 * 1024 * 1024; + if file.metadata().map_err(|e| e.to_string())?.len() > MAX_JSON { + return Err("artifact exceeds 64 MiB".into()); + } + let mut bytes = Vec::new(); + file.take(MAX_JSON + 1) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if bytes.len() as u64 > MAX_JSON { + return Err("artifact exceeds 64 MiB".into()); + } + checked.verify(path, &hash_bytes(&bytes))?; + let value: Artifact = + serde_json::from_slice(&bytes).map_err(|e| format!("{}: {e}", path.display()))?; + if value.schema_version != T::SCHEMA || value.kind != T::KIND { + return Err(format!( + "{}: expected {} schema {}", + path.display(), + T::KIND, + T::SCHEMA + )); + } + if let Some(data) = &value.data { + data.validate()?; + } + if value.complete { + value.require()?; + } + verify_refs( + path, + &value.inputs, + checked, + &mut std::collections::BTreeSet::new(), + 0, + )?; + Ok(value) +} + +fn verify_refs( + path: &Path, + refs: &[Reference], + checked: &mut Verification, + traversed: &mut std::collections::BTreeSet, + depth: usize, +) -> Result<(), String> { + if depth > 32 { + return Err("evidence reference depth exceeds 32".into()); + } + for input in refs { + let alias = path.parent().unwrap_or(Path::new(".")).join(&input.path); + let target = alias.canonicalize().map_err(|e| e.to_string())?; + checked.verify(&alias, &input.sha256)?; + if !traversed.insert(target.clone()) { + continue; + } + if target.extension().is_none_or(|e| e != "json") { + continue; + } + let file = fs::File::open(&target).map_err(|e| e.to_string())?; + if file.metadata().map_err(|e| e.to_string())?.len() > 64 * 1024 * 1024 { + return Err("JSON evidence exceeds 64 MiB".into()); + } + let value: serde_json::Value = serde_json::from_reader(file).map_err(|e| e.to_string())?; + if value.get("kind").is_none() || value.get("schema_version").is_none() { + continue; + } + let refs: Vec = serde_json::from_value( + value + .get("inputs") + .ok_or("nested artifact inputs missing")? + .clone(), + ) + .map_err(|e| e.to_string())?; + verify_refs(&target, &refs, checked, traversed, depth + 1)?; + } + Ok(()) +} + +pub fn directory(path: &Path) -> Result<(), String> { + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + fs::create_dir(path).map_err(|e| { + format!( + "cannot create new evidence directory {}: {e}", + path.display() + ) + }) +} + +/// Import an inspected machine/calibration bundle with its relative raw inputs. +/// External references require re-inspection into a self-contained directory. +pub fn bundle( + path: &Path, + out: &Path, +) -> Result { + load::(path)?; + let source = path.canonicalize().map_err(|e| e.to_string())?; + let root = source.parent().ok_or("bundle has no parent")?; + directory(out)?; + let mut visited = std::collections::BTreeSet::new(); + let mut pending = vec![source.clone()]; + let mut total = 0u64; + while let Some(path) = pending.pop() { + let path = path.canonicalize().map_err(|e| e.to_string())?; + if !visited.insert(path.clone()) { + continue; + } + let relative = path + .strip_prefix(root) + .map_err(|_| "bundle has external raw inputs; create a self-contained inspection")?; + total = total + .checked_add(fs::metadata(&path).map_err(|e| e.to_string())?.len()) + .ok_or("bundle size overflow")?; + if total > 256 * 1024 * 1024 { + return Err("machine/calibration bundle exceeds 256 MiB".into()); + } + let destination = out.join(relative); + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + let mut source_file = fs::File::open(&path).map_err(|e| e.to_string())?; + let mut copy = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&destination) + .map_err(|e| e.to_string())?; + std::io::copy(&mut source_file, &mut copy).map_err(|e| e.to_string())?; + if path.extension().is_none_or(|e| e != "json") { + continue; + } + let value: serde_json::Value = + serde_json::from_reader(fs::File::open(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if value.get("kind").is_none() || value.get("schema_version").is_none() { + continue; + } + let inputs: Vec = + serde_json::from_value(value.get("inputs").ok_or("bundle inputs missing")?.clone()) + .map_err(|e| e.to_string())?; + pending.extend( + inputs + .into_iter() + .map(|r| path.parent().unwrap().join(r.path)), + ); + } + let imported = out.join(source.file_name().unwrap()); + load::(&imported)?; + Ok(imported) +} + +#[cfg(test)] +mod tests { + use super::*; + #[derive(Debug, Serialize, Deserialize)] + struct Example { + value: u32, + } + impl Payload for Example { + const KIND: &'static str = "example"; + fn validate(&self) -> Result<(), String> { + Ok(()) + } + } + #[test] + fn verification_rejects_drift_and_conflicting_digest() { + let root = std::env::temp_dir().join(format!("ds4-verified-{}", std::process::id())); + directory(&root).unwrap(); + let path = root.join("input"); + fs::write(&path, b"first").unwrap(); + let expected = hash(&path).unwrap(); + let mut checked = Verification::default(); + checked.verify(&path, &expected).unwrap(); + checked.verify(&path, &expected).unwrap(); + assert_eq!(checked.full_reads, 1); + assert!(checked.verify(&path, &hash_bytes(b"other")).is_err()); + fs::write(&path, b"other").unwrap(); + assert!(checked.verify(&path, &expected).is_err()); + assert!(checked.finish().is_err()); + + let mut replaced = Verification::default(); + replaced.verify(&path, &hash_bytes(b"other")).unwrap(); + let replacement = root.join("replacement"); + fs::write(&replacement, b"other").unwrap(); + fs::rename(replacement, &path).unwrap(); + assert!( + replaced.finish().is_err(), + "same bytes on another inode must fail" + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn operation_reuses_shared_inputs_across_artifacts() { + let root = std::env::temp_dir().join(format!("ds4-shared-{}", std::process::id())); + directory(&root).unwrap(); + let input = root.join("weights"); + fs::write(&input, b"same input").unwrap(); + for index in 0..2 { + let mut evidence = Artifact::new(Example { value: index }); + evidence.inputs.push(reference(&input, &root).unwrap()); + save(&root.join(format!("{index}.json")), &evidence).unwrap(); + } + let mut checked = Verification::default(); + for index in 0..2 { + load_verified::(&root.join(format!("{index}.json")), &mut checked).unwrap(); + } + assert_eq!(checked.full_reads, 3, "two envelopes and one shared input"); + checked.finish().unwrap(); + let envelope = root.join("0.json"); + let bytes = fs::read(&envelope).unwrap(); + fs::write(&envelope, [bytes.as_slice(), b"\n"].concat()).unwrap(); + assert!( + checked.finish().is_err(), + "root envelope must remain pinned" + ); + fs::write(&input, b"new input!").unwrap(); + assert!(load_verified::(&root.join("1.json"), &mut checked).is_err()); + assert!(checked.finish().is_err()); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn verification_rejects_retargeted_alias() { + let root = std::env::temp_dir().join(format!("ds4-alias-{}", std::process::id())); + directory(&root).unwrap(); + fs::write(root.join("a"), b"same").unwrap(); + fs::write(root.join("b"), b"same").unwrap(); + let alias = root.join("alias"); + std::os::unix::fs::symlink(root.join("a"), &alias).unwrap(); + let mut checked = Verification::default(); + checked.verify(&alias, &hash_bytes(b"same")).unwrap(); + fs::remove_file(&alias).unwrap(); + std::os::unix::fs::symlink(root.join("b"), &alias).unwrap(); + assert!(checked.finish().is_err()); + assert!(checked.verify(&alias, &hash_bytes(b"same")).is_err()); + fs::remove_dir_all(root).unwrap(); + } + #[test] + fn transitive_hashes() { + let root = std::env::temp_dir().join(format!("ds4-tree-{}", std::process::id())); + directory(&root).unwrap(); + fs::write(root.join("raw"), "original").unwrap(); + let mut first = Artifact::new(Example { value: 1 }); + first + .inputs + .push(reference(&root.join("raw"), &root).unwrap()); + save(&root.join("first.json"), &first).unwrap(); + let mut second = Artifact::new(Example { value: 2 }); + second + .inputs + .push(reference(&root.join("first.json"), &root).unwrap()); + save(&root.join("second.json"), &second).unwrap(); + fs::write(root.join("raw"), "altered").unwrap(); + let rejected = load::(&root.join("second.json")).is_err(); + fs::remove_dir_all(root).unwrap(); + assert!(rejected, "nested evidence tampering must fail"); + } + + #[test] + fn moved_evidence_and_tampering() { + let root = std::env::temp_dir().join(format!("ds4-evidence-{}", std::process::id())); + let source = root.join("source"); + directory(&source).unwrap(); + fs::write(source.join("raw"), "original").unwrap(); + let mut artifact = Artifact::new(Example { value: 7 }); + artifact + .inputs + .push(reference(&source.join("raw"), &source).unwrap()); + save(&source.join("example.json"), &artifact).unwrap(); + assert!(save( + &source.join("example.json"), + &Artifact::new(Example { value: 99 }) + ) + .is_err()); + let moved = root.join("moved"); + fs::rename(&source, &moved).unwrap(); + assert_eq!( + load::(&moved.join("example.json")) + .unwrap() + .require() + .unwrap() + .value, + 7 + ); + fs::write(moved.join("raw"), "changed").unwrap(); + assert!(load::(&moved.join("example.json")) + .unwrap_err() + .contains("evidence changed")); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/crates/ds4-perf/src/bench.rs b/crates/ds4-perf/src/bench.rs new file mode 100644 index 0000000000000000000000000000000000000000..d1c42c217139ed3bc27c5670b4433eda23bc701e --- /dev/null +++ b/crates/ds4-perf/src/bench.rs @@ -0,0 +1,84 @@ +use crate::csv; +use std::io::BufRead; + +pub const HEADER: &str = + "ctx_tokens,prefill_tokens,prefill_tps,gen_tokens,gen_tps,first_token_sec,kvcache_bytes"; + +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Row { + pub ctx: u64, + pub prefill_tokens: u64, + pub prefill_tps: f64, + pub gen_tokens: u64, + pub gen_tps: f64, + pub first_token_sec: f64, + pub kvcache_bytes: u64, +} + +impl Row { + pub fn csv(&self) -> String { + format!( + "{},{},{},{},{},{},{}\n", + self.ctx, + self.prefill_tokens, + self.prefill_tps, + self.gen_tokens, + self.gen_tps, + self.first_token_sec, + self.kvcache_bytes + ) + } +} + +pub fn parse(reader: impl BufRead) -> Result, String> { + let mut header = Vec::new(); + let mut rows = Vec::new(); + csv::records(reader, |row| { + if csv::column(&row, &["prefill_tps"]).is_some() { + header = row; + return Ok(()); + } + if header.is_empty() || row.iter().all(|v| v.trim().is_empty()) { + return Ok(()); + } + let field = |name| { + csv::value(&row, &header, &[name]) + .ok_or_else(|| format!("missing ds4-bench column: {name}")) + }; + let integer = |name| { + field(name)? + .parse::() + .map_err(|_| format!("invalid {name}")) + }; + rows.push(Row { + ctx: integer("ctx_tokens")?, + prefill_tokens: integer("prefill_tokens")?, + prefill_tps: csv::number(field("prefill_tps")?)?, + gen_tokens: integer("gen_tokens")?, + gen_tps: csv::number(field("gen_tps")?)?, + first_token_sec: csv::number(field("first_token_sec")?)?, + kvcache_bytes: integer("kvcache_bytes")?, + }); + Ok(()) + })?; + Ok(rows) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bench_named_columns() { + let input = include_str!("../tests/fixtures/bench.csv"); + let rows = parse(input.as_bytes()).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].prefill_tps, 1502.8); + assert_eq!(rows[0].gen_tps, 24.3); + assert_eq!(rows[0].first_token_sec, 0.0512); + assert_eq!(rows[0].kvcache_bytes, 4294967296); + assert_eq!(rows[0].ctx, 8192); + assert!(parse(b"opaque output\n".as_slice()).unwrap().is_empty()); + } +} diff --git a/crates/ds4-perf/src/cli.rs b/crates/ds4-perf/src/cli.rs new file mode 100644 index 0000000000000000000000000000000000000000..f011808b1b3fb4179596e4d7a739d5c17a7731df --- /dev/null +++ b/crates/ds4-perf/src/cli.rs @@ -0,0 +1,214 @@ +use clap::{Args, Parser, Subcommand, ValueEnum}; +use std::{ffi::OsString, path::PathBuf}; + +#[derive(Parser, Debug)] +#[command( + version, + about = "Reproducible CUDA performance evidence and experiments" +)] +pub struct Cli { + #[command(subcommand)] + pub command: Command, +} + +#[derive(Subcommand, Debug)] +pub enum Command { + /// Inspect machine capabilities; optionally measure the device envelope. + Inspect(Inspect), + /// Compatibility entry point for the v0.1.0 capability report. + #[command(hide = true)] + Doctor { + #[arg(long, default_value = "./ds4-bench")] + bench: String, + }, + /// Profile a fresh benchmark and identify execution-path bottlenecks. + Scout(Scout), + /// Compare matched unprofiled experiments and correctness evidence. + Compare(Compare), + /// Choose experiments from evidence; --auto executes a bounded loop. + Optimize(Optimize), + /// Measure ordered HTTP conversation workloads and check their contracts. + Serving(Serving), + /// Propose one-control experiments from the server's capability report. + ServingControls(ServingControls), + /// Select a profile only from complete, repeated serving contracts. + ServingProfile(ServingProfile), + /// Check or execute a pinned serving profile; additional options are forbidden. + ApplyProfile(ApplyProfile), +} + +#[derive(Args, Debug)] +pub struct ServingControls { + #[arg(long)] + pub plan: PathBuf, + #[arg(long)] + pub out: PathBuf, +} + +#[derive(Args, Debug)] +pub struct ServingProfile { + #[arg(long)] + pub plan: PathBuf, + #[arg(long)] + pub out: PathBuf, +} + +#[derive(Args, Debug)] +pub struct ApplyProfile { + #[arg(long)] + pub profile: PathBuf, + #[arg(long)] + pub out: PathBuf, + #[arg(long)] + pub execute: bool, +} + +#[derive(Args, Debug)] +pub struct Serving { + #[command(flatten)] + pub budget: Budget, + /// Local server origin, e.g. http://127.0.0.1:8002. + #[arg(long)] + pub url: String, + #[arg(long)] + pub workload: PathBuf, + #[arg(long)] + pub out: PathBuf, + /// Replay the ordered workload; KV is deliberately preserved between repeats. + #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..=100))] + pub repeats: u32, + /// Pin the local server executable, start time and command line. + #[arg(long)] + pub server_pid: Option, + /// Physical NVIDIA device ordinal used for clock/temperature snapshots. + #[arg(long, default_value_t = 0)] + pub device: usize, +} + +#[derive(Args, Debug)] +pub struct Inspect { + #[arg(long)] + pub out: PathBuf, + #[arg(long)] + pub calibrate: bool, + #[arg(long, default_value_t = 0)] + pub device: usize, + #[arg(long)] + pub gpu_helper: Option, + #[arg(long)] + pub bench: Option, +} + +#[derive(Clone, Copy, Debug, ValueEnum)] +pub enum Collector { + Nsys, + Cupti, +} + +#[derive(Args, Clone, Debug)] +pub struct Scout { + #[arg(skip)] + pub exact_environment: bool, + #[command(flatten)] + pub budget: Budget, + #[arg(long)] + pub out: PathBuf, + #[arg(long, value_enum, default_value = "nsys")] + pub collector: Collector, + #[arg(long)] + pub fit: bool, + /// Request ds4-bench token and full-vocabulary frontier proof files. + #[arg(long)] + pub proof: bool, + /// CUPTI injection library (default: beside ds4-perf-gpu). + #[arg(long, requires = "collector")] + pub cupti_library: Option, + /// NVIDIA libcupti.so used for NVTX injection and activity collection. + #[arg(long)] + pub cupti_sdk: Option, + #[arg(long)] + pub ncu: bool, + #[arg(long)] + pub machine: Option, + #[arg(long, requires = "fit")] + pub calibration: Option, + #[arg(long)] + pub gpu_helper: Option, + #[arg(long, default_value_t = 0)] + pub device: usize, + #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..=100))] + pub repeats: u32, + /// Explicit workload/cache contract used when comparing runs. + #[arg(long)] + pub workload: Option, + #[arg(long, default_value = "inherited", value_parser = ["inherited", "warmup-then-fresh"])] + pub cache_policy: String, + /// Allowlisted runtime control, passed directly to the child process. + #[arg(long = "env")] + pub environment: Vec, + #[arg(last = true, required = true)] + pub command: Vec, +} + +#[derive(Args, Debug)] +pub struct Compare { + #[arg(long)] + pub baseline: PathBuf, + #[arg(long)] + pub candidate: PathBuf, + #[arg(long)] + pub out: PathBuf, + /// Exit unsuccessfully on regression or insufficient comparison evidence. + #[arg(long)] + pub regression: bool, + #[arg(long, default_value_t = 3.0)] + pub max_slowdown_percent: f64, + #[arg(long, default_value_t = 0.0001)] + pub logit_atol: f64, + #[arg(long, default_value_t = 0.0001)] + pub logit_rtol: f64, + /// Relaxed contract for summation-order changes: when positive, each + /// frontier must stay within this relative RMS of the baseline at 1024 + /// tokens (the bound grows with log2 of the context) and keep its argmax; + /// greedy sequences may diverge. Zero keeps the exact contract. + #[arg(long, default_value_t = 0.0)] + pub logit_rel_rms: f64, +} + +#[derive(Args, Debug)] +pub struct Optimize { + #[command(flatten)] + pub budget: Budget, + #[arg(long)] + pub scout: PathBuf, + #[arg(long)] + pub out: PathBuf, + /// Run evidence-selected experiments and compare each candidate. + #[arg(long = "auto")] + pub automatic: bool, + #[arg(long)] + pub plan: Option, + #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(u32).range(1..=32))] + pub rounds: u32, + #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(u32).range(3..=100))] + pub repeats: u32, +} + +#[derive(Args, Clone, Debug, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Budget { + /// Deadline for each benchmark/profiler process, including descendants. + #[arg(long,default_value_t=1800,value_parser=clap::value_parser!(u64).range(1..=86400))] + pub timeout_seconds: u64, + /// Total raw output budget per scout; auto also bounds the campaign. + #[arg(long,default_value_t=2048,value_parser=clap::value_parser!(u64).range(1..=65536))] + pub max_output_mib: u64, +} +impl Budget { + pub fn limits(&self) -> crate::process::Limits { + crate::process::Limits { + timeout: std::time::Duration::from_secs(self.timeout_seconds), + bytes: self.max_output_mib.saturating_mul(1024 * 1024), + } + } +} diff --git a/crates/ds4-perf/src/compare.rs b/crates/ds4-perf/src/compare.rs new file mode 100644 index 0000000000000000000000000000000000000000..cb4a9487c57bf9fe1e87f4efe2ada72b56401baa --- /dev/null +++ b/crates/ds4-perf/src/compare.rs @@ -0,0 +1,695 @@ +use crate::{ + cli, + experiment::{self, Scout}, + fit, +}; +use ds4_perf::artifact::{self, Artifact, Payload, Reference}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, fs, path::Path}; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Delta { + pub metric: String, + pub ctx: u64, + pub unit: String, + pub baseline: Vec, + pub candidate: Vec, + pub baseline_median: f64, + pub candidate_median: f64, + pub median_percent: f64, + pub lower_percent: f64, + pub upper_percent: f64, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LogitError { + pub max_abs: f64, + pub max_scaled: f64, + pub bad: u64, + pub checked: u64, + /// Largest relative RMS of a candidate frontier against its reference + /// (sqrt(sum diff^2 / sum ref^2)); the relaxed contract's statistic. + #[serde(default)] + pub rel_rms: f64, + /// Largest `rel_rms / rel_rms_bound(ctx)`; above 1 the relaxed contract + /// is violated. + #[serde(default)] + pub rel_rms_scaled: f64, +} + +/// Reference context of the relaxed contract: `--logit-rel-rms` is the bound +/// at this width and grows with log2(ctx) from it. +const REL_RMS_REFERENCE_CTX: f64 = 1024.0; + +/// Context-dependent relative RMS bound. Reordering noise accumulates with +/// the attended length but the model's amplification saturates, and a +/// one-ulp perturbation of Inkling MQ85GB measured 0.063 at 16 tokens and +/// 0.105 at 515 (log-linear), so the bound grows logarithmically: at 2K it +/// is 1.1x the reference, at 8K 1.3x, at 64K 1.6x, never linear in ctx. +fn rel_rms_bound(reference: f64, ctx: u64) -> f64 { + reference * (ctx.max(2) as f64).log2() / REL_RMS_REFERENCE_CTX.log2() +} + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Verdict { + Pass, + Improved, + Regressed, + Inconclusive, + Incorrect, + Incomparable, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Comparison { + pub verdict: Verdict, + pub metrics: Vec, + pub correctness: LogitError, + /// Number of proof comparisons with a greedy sequence or argmax mismatch. + pub token_mismatches: u64, + /// Number of proof comparisons whose frontier argmax differs (a subset of + /// `token_mismatches`). + #[serde(default)] + pub argmax_mismatches: u64, + pub max_slowdown_percent: f64, + pub logit_atol: f64, + pub logit_rtol: f64, + /// Relaxed contract when positive: a frontier is correct when its relative + /// RMS stays within `rel_rms_bound(ctx)` (this value at 1024 tokens, + /// growing with log2(ctx)) and its argmax matches; greedy sequences may + /// diverge. Zero keeps the per-logit atol/rtol and sequence contract. + #[serde(default)] + pub logit_rel_rms: f64, + pub changes: BTreeMap, + pub reasons: Vec, + pub method: String, +} + +impl Payload for Comparison { + const KIND: &'static str = "compare"; + fn validate(&self) -> Result<(), String> { + if self.method != "sample-extrema-envelope-v1" { + return Err("unsupported comparison method".into()); + } + if [ + self.max_slowdown_percent, + self.logit_atol, + self.logit_rtol, + self.logit_rel_rms, + self.correctness.max_abs, + self.correctness.max_scaled, + self.correctness.rel_rms, + self.correctness.rel_rms_scaled, + ] + .into_iter() + .any(|v| !v.is_finite() || v < 0.0) + { + return Err("invalid comparison policy or result".into()); + } + if self.verdict == Verdict::Incomparable { + if self.reasons.is_empty() || !self.metrics.is_empty() { + return Err("incomparable result lacks a reason".into()); + } + return Ok(()); + } + if self.metrics.is_empty() + || self.correctness.checked == 0 + || self.correctness.bad > self.correctness.checked + || self.argmax_mismatches > self.token_mismatches + { + return Err("conclusive comparison lacks timing/correctness evidence".into()); + } + let expected = verdict( + &self.metrics, + &self.correctness, + self.token_mismatches, + self.argmax_mismatches, + self.max_slowdown_percent, + self.logit_rel_rms, + ); + if self.verdict != expected { + return Err("comparison verdict disagrees with evidence".into()); + } + for d in &self.metrics { + if [ + d.baseline_median, + d.candidate_median, + d.median_percent, + d.lower_percent, + d.upper_percent, + ] + .into_iter() + .any(|v| !v.is_finite()) + || d.baseline + .iter() + .chain(&d.candidate) + .any(|v| !v.is_finite() || *v <= 0.0) + { + return Err("invalid comparison samples".into()); + } + let computed = delta(&d.metric, d.ctx, &d.baseline, &d.candidate)?; + if d.baseline.len() < 3 + || d.baseline.len() != d.candidate.len() + || d.unit != computed.unit + || [ + d.baseline_median, + d.candidate_median, + d.median_percent, + d.lower_percent, + d.upper_percent, + ] != [ + computed.baseline_median, + computed.candidate_median, + computed.median_percent, + computed.lower_percent, + computed.upper_percent, + ] + { + return Err("comparison summary disagrees with raw samples".into()); + } + } + Ok(()) + } +} + +fn delta(metric: &str, ctx: u64, a: &[f64], b: &[f64]) -> Result { + if a.is_empty() || b.is_empty() || a.iter().chain(b).any(|v| !v.is_finite() || *v <= 0.0) { + return Err("nonpositive or missing timing samples".into()); + } + let baseline_median = fit::median(a); + let candidate_median = fit::median(b); + let extrema = |v: &[f64]| { + ( + v.iter().copied().fold(f64::INFINITY, f64::min), + v.iter().copied().fold(0.0, f64::max), + ) + }; + let (amin, amax) = extrema(a); + let (bmin, bmax) = extrema(b); + Ok(Delta { + metric: metric.into(), + ctx, + unit: if metric == "first_token" { + "seconds" + } else { + "seconds/token" + } + .into(), + baseline: a.into(), + candidate: b.into(), + baseline_median, + candidate_median, + median_percent: 100.0 * (candidate_median / baseline_median - 1.0), + lower_percent: 100.0 * (bmin / amax - 1.0), + upper_percent: 100.0 * (bmax / amin - 1.0), + }) +} + +fn logit_error(a: &[f64], b: &[f64], atol: f64, rtol: f64) -> Result { + if a.is_empty() || a.len() != b.len() || a.iter().chain(b).any(|v| !v.is_finite()) { + return Err("invalid full-vocabulary logits".into()); + } + let mut error = LogitError::default(); + let (mut diff2, mut ref2) = (0.0f64, 0.0f64); + for (a, b) in a.iter().zip(b) { + let difference = (a - b).abs(); + let tolerance = atol + rtol * a.abs(); + error.max_abs = error.max_abs.max(difference); + // A zero tolerance is valid for an exact parity contract. + error.max_scaled = error + .max_scaled + .max((difference / tolerance.max(f64::MIN_POSITIVE)).min(f64::MAX)); + error.bad += u64::from(difference > tolerance); + error.checked += 1; + diff2 += difference * difference; + ref2 += a * a; + } + error.rel_rms = (diff2 / ref2.max(f64::MIN_POSITIVE)).sqrt().min(f64::MAX); + Ok(error) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Frontier { + source: String, + model: String, + backend: String, + quality: bool, + quant_bits: u64, + prompt_tokens: u64, + frontier_tokens: u64, + prefill_tokens: u64, + ctx: u64, + vocab: usize, + argmax_id: usize, + argmax_logit: f64, + logits: Vec, +} + +fn proof_file( + root: &Path, + reference: &Reference, +) -> Result { + let path = root.join(&reference.path); + if artifact::hash(&path)? != reference.sha256 { + return Err("correctness evidence changed".into()); + } + let file = fs::File::open(&path).map_err(|e| e.to_string())?; + if file.metadata().map_err(|e| e.to_string())?.len() > 64 * 1024 * 1024 { + return Err("proof file exceeds 64 MiB".into()); + } + serde_json::from_reader(file).map_err(|e| format!("{}: {e}", path.display())) +} + +type FrontierProofs = BTreeMap)>>; + +fn proofs(root: &Path, scout: &Scout) -> Result { + let mut proofs: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for sample in &scout.samples { + for row in &sample.rows { + let name = format!("frontier_{:06}.logits.json", row.ctx); + let frontier: Frontier = proof_file( + root, + sample + .proof_files + .get(&name) + .ok_or("missing full-vocabulary frontier; use scout --proof")?, + )?; + let name = format!("tokens-{}.json", row.ctx); + let tokens: Vec = proof_file( + root, + sample + .proof_files + .get(&name) + .ok_or("missing greedy-token evidence; use scout --proof")?, + )?; + if frontier.source != "ds4-bench" + || frontier.backend != "cuda" + || frontier.model.is_empty() + || frontier.vocab == 0 + || frontier.logits.len() != frontier.vocab + || frontier.argmax_id >= frontier.vocab + || frontier.frontier_tokens != row.ctx + || frontier.prompt_tokens != row.ctx + || frontier.prefill_tokens != row.prefill_tokens + || frontier.ctx <= row.ctx + || tokens.len() as u64 != row.gen_tokens + || tokens + .iter() + .any(|t| *t < 0 || *t as usize >= frontier.vocab) + || frontier.logits.iter().any(|v| !v.is_finite()) + || !frontier.argmax_logit.is_finite() + || frontier.logits[frontier.argmax_id] != frontier.argmax_logit + || frontier.logits.iter().any(|v| *v > frontier.argmax_logit) + { + return Err("frontier/token proof disagrees with benchmark row".into()); + } + if scout.workload.is_some() + && scout.model_argument.as_deref() != Some(Path::new(&frontier.model)) + { + return Err( + "proof model differs from the captured and hashed workload argument".into(), + ); + } + proofs.entry(row.ctx).or_default().push((frontier, tokens)); + } + } + Ok(proofs) +} + +pub fn validate_proofs(root: &Path, scout: &Scout) -> Result<(), String> { + if scout.samples.len() != scout.requested_repeats as usize + || scout.samples.iter().any(|s| s.rows.is_empty()) + { + return Err("missing samples for requested proof".into()); + } + proofs(root, scout).map(|_| ()) +} + +fn identity(a: &Scout, b: &Scout) -> Result, String> { + if !a.process_scope.verified + || !b.process_scope.verified + || a.process_scope.expected_owners != b.process_scope.expected_owners + { + return Err("GPU process snapshots do not match the intended owners for both runs".into()); + } + if !a.unreviewed_environment.is_empty() || !b.unreviewed_environment.is_empty() { + return Err("unreviewed DS4/CUDA/LD environment controls prevent matched comparison; see scout.json".into()); + } + let (Some(ga), Some(gb)) = (&a.gpu, &b.gpu) else { + return Err("comparison requires verified CUDA device identity".into()); + }; + if !ga.same_device(gb) { + return Err("device or driver differs".into()); + } + if a.command[1..] != b.command[1..] || a.cwd != b.cwd { + return Err("benchmark arguments or working directory differ".into()); + } + if a.cache_policy != "warmup-then-fresh" || a.cache_policy != b.cache_policy { + return Err("comparison requires matching explicit --cache-policy".into()); + } + let (Some(wa), Some(wb)) = (&a.workload, &b.workload) else { + return Err( + "comparison requires --workload with hashed model/prompt and cache contract".into(), + ); + }; + let hashes = |w: &experiment::Workload| { + w.files + .iter() + .map(|(k, v)| (k.clone(), v.sha256.clone())) + .collect::>() + }; + if wa.name != wb.name + || wa.family != wb.family + || hashes(wa) != hashes(wb) + || wa.shape != wb.shape + || wa.cache_state != wb.cache_state + { + return Err("workload model/prompt/shape/cache identity differs".into()); + } + let mut changes = BTreeMap::new(); + if a.executable_sha256 != b.executable_sha256 { + changes.insert( + "executable_sha256".into(), + format!("{} -> {}", a.executable_sha256, b.executable_sha256), + ); + } + let keys: std::collections::BTreeSet<_> = + a.environment.keys().chain(b.environment.keys()).collect(); + for key in keys { + if a.environment.get(key) == b.environment.get(key) { + continue; + } + if !crate::knobs::tunable(key) { + return Err(format!("non-tuning runtime/memory policy changed: {key}")); + } + for value in [a.environment.get(key), b.environment.get(key)] + .into_iter() + .flatten() + { + crate::knobs::validate(key, value, &wa.family)?; + } + changes.insert( + key.clone(), + format!( + "{:?} -> {:?}", + a.environment.get(key), + b.environment.get(key) + ), + ); + } + Ok(changes) +} + +fn analyze( + a: &Scout, + ar: &Path, + b: &Scout, + br: &Path, + args: &cli::Compare, +) -> Result { + let changes = identity(a, b)?; + if a.requested_repeats != b.requested_repeats + || a.samples.len() != a.requested_repeats as usize + || b.samples.len() != b.requested_repeats as usize + { + return Err("comparison requires equal complete requested sample counts".into()); + } + if a.samples.len() < 3 || b.samples.len() < 3 { + return Err("comparison requires at least three fresh unprofiled samples per side".into()); + } + type TimingPair = (Vec, Vec); + let mut timing: BTreeMap<(String, u64), TimingPair> = BTreeMap::new(); + let shape = |s: &experiment::Sample| { + s.rows + .iter() + .map(|r| (r.ctx, r.prefill_tokens, r.gen_tokens, r.kvcache_bytes)) + .collect::>() + }; + let expected = shape(&a.samples[0]); + if expected.is_empty() + || expected + .iter() + .map(|r| r.0) + .collect::>() + .len() + != expected.len() + { + return Err("empty or duplicate benchmark frontiers".into()); + } + for (candidate, scout) in [(false, a), (true, b)] { + for sample in &scout.samples { + if shape(sample) != expected { + return Err("token counts/frontiers/KV allocation differ across samples".into()); + } + for row in &sample.rows { + for (name, value) in [ + ("prefill", 1.0 / row.prefill_tps), + ("decode", 1.0 / row.gen_tps), + ("first_token", row.first_token_sec), + ] { + let entry = timing.entry((name.into(), row.ctx)).or_default(); + if candidate { + entry.1.push(value); + } else { + entry.0.push(value); + } + } + } + } + } + let baseline = proofs(ar, a)?; + let candidate = proofs(br, b)?; + let mut correctness = LogitError::default(); + let mut token_mismatches = 0; + let mut argmax_mismatches = 0; + for (ctx, base) in &baseline { + let (reference, tokens) = &base[0]; + let candidates = candidate.get(ctx).ok_or("missing candidate frontier")?; + for (frontier, ids) in base.iter().chain(candidates) { + if frontier.source != reference.source + || frontier.backend != reference.backend + || frontier.model != reference.model + || frontier.quality != reference.quality + || frontier.quant_bits != reference.quant_bits + || frontier.ctx != reference.ctx + { + return Err("quality/quantization/context proof metadata differs".into()); + } + token_mismatches += + u64::from(ids != tokens || frontier.argmax_id != reference.argmax_id); + argmax_mismatches += u64::from(frontier.argmax_id != reference.argmax_id); + let error = logit_error( + &reference.logits, + &frontier.logits, + args.logit_atol, + args.logit_rtol, + )?; + correctness.max_abs = correctness.max_abs.max(error.max_abs); + correctness.max_scaled = correctness.max_scaled.max(error.max_scaled); + correctness.bad += error.bad; + correctness.checked += error.checked; + correctness.rel_rms = correctness.rel_rms.max(error.rel_rms); + if args.logit_rel_rms > 0.0 { + let scaled = error.rel_rms / rel_rms_bound(args.logit_rel_rms, *ctx); + correctness.rel_rms_scaled = correctness.rel_rms_scaled.max(scaled.min(f64::MAX)); + } + } + } + let metrics = timing + .into_iter() + .map(|((name, ctx), (a, b))| delta(&name, ctx, &a, &b)) + .collect::, _>>()?; + let verdict = verdict( + &metrics, + &correctness, + token_mismatches, + argmax_mismatches, + args.max_slowdown_percent, + args.logit_rel_rms, + ); + Ok(Comparison { + verdict, + metrics, + correctness, + token_mismatches, + argmax_mismatches, + max_slowdown_percent: args.max_slowdown_percent, + logit_atol: args.logit_atol, + logit_rtol: args.logit_rtol, + logit_rel_rms: args.logit_rel_rms, + changes, + reasons: Vec::new(), + method: "sample-extrema-envelope-v1".into(), + }) +} + +fn verdict( + metrics: &[Delta], + correctness: &LogitError, + mismatches: u64, + argmax_mismatches: u64, + tolerance: f64, + logit_rel_rms: f64, +) -> Verdict { + // The relaxed contract accepts summation-order changes that a chaotic + // model amplifies into different greedy continuations: it bounds the + // frontier's relative RMS per context and keeps its argmax instead of + // every logit and the whole sequence. + let incorrect = if logit_rel_rms > 0.0 { + correctness.rel_rms_scaled > 1.0 || argmax_mismatches > 0 + } else { + correctness.bad > 0 || mismatches > 0 + }; + if incorrect { + return Verdict::Incorrect; + } + if metrics.iter().any(|d| d.lower_percent > tolerance) { + return Verdict::Regressed; + } + if metrics.iter().any(|d| d.upper_percent > tolerance) { + return Verdict::Inconclusive; + } + if metrics + .iter() + .any(|d| d.metric != "first_token" && d.upper_percent < -1.0) + { + return Verdict::Improved; + } + Verdict::Pass +} + +pub fn run(args: &cli::Compare) -> Result<(), String> { + if [ + args.max_slowdown_percent, + args.logit_atol, + args.logit_rtol, + args.logit_rel_rms, + ] + .into_iter() + .any(|v| !v.is_finite() || v < 0.0) + { + return Err("comparison tolerances must be finite and nonnegative".into()); + } + artifact::directory(&args.out)?; + let result = (|| { + let a = experiment::load(&args.baseline)?; + let b = experiment::load(&args.candidate)?; + analyze( + a.require()?, + args.baseline.parent().unwrap_or(Path::new(".")), + b.require()?, + args.candidate.parent().unwrap_or(Path::new(".")), + args, + ) + })(); + let data = match result { + Ok(data) => data, + Err(error) => Comparison { + verdict: Verdict::Incomparable, + metrics: Vec::new(), + correctness: LogitError::default(), + token_mismatches: 0, + argmax_mismatches: 0, + max_slowdown_percent: args.max_slowdown_percent, + logit_atol: args.logit_atol, + logit_rtol: args.logit_rtol, + logit_rel_rms: args.logit_rel_rms, + changes: BTreeMap::new(), + reasons: vec![error], + method: "sample-extrema-envelope-v1".into(), + }, + }; + let passed = matches!(data.verdict, Verdict::Pass | Verdict::Improved); + let mut artifact = Artifact::new(data); + artifact.complete = artifact + .data + .as_ref() + .is_some_and(|d| d.verdict != Verdict::Incomparable); + for path in [&args.baseline, &args.candidate] { + if path.is_file() { + artifact.inputs.push(artifact::reference(path, &args.out)?); + } + } + artifact::save(&args.out.join("compare.json"), &artifact)?; + println!( + "compare: {:?}; {}", + artifact.data.as_ref().unwrap().verdict, + args.out.join("compare.json").display() + ); + if args.regression && !passed { + return Err("regression gate did not pass; see compare.json".into()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn preserves_live_roundtrip() { + // These measured Qwen ratios exposed a lossy JSON parse on reload. + let recorded: Comparison = + serde_json::from_str(include_str!("../tests/fixtures/compare-live.json")).unwrap(); + recorded.validate().unwrap(); + let encoded = serde_json::to_vec(&recorded).unwrap(); + let mut loaded: Comparison = serde_json::from_slice(&encoded).unwrap(); + loaded.validate().unwrap(); + loaded.metrics[0].median_percent += 0.01; + assert!(loaded.validate().is_err()); + } + + #[test] + fn separates_regression_noise() { + let stable = delta("prefill", 2048, &[1.0, 1.01, 0.99], &[1.1, 1.11, 1.09]).unwrap(); + assert!(stable.lower_percent > 3.0); + let noise = delta("prefill", 2048, &[0.9, 1.0, 1.1], &[0.9, 1.03, 1.2]).unwrap(); + assert!(noise.lower_percent < 3.0 && noise.upper_percent > 3.0); + assert!(delta("prefill", 2048, &[1.0], &[0.0]).is_err()); + assert!(delta("prefill", 2048, &[1.0], &[f64::NAN]).is_err()); + } + #[test] + fn checks_every_logit() { + let a = vec![0.0, 1.0, -2.0, 0.001]; + assert!(logit_error(&a, &a, 1e-4, 1e-4).unwrap().bad == 0); + let mut b = a.clone(); + b[3] = 0.01; + assert_eq!(logit_error(&a, &b, 1e-4, 1e-4).unwrap().bad, 1); + assert!(logit_error(&a, &[f64::NAN], 1e-4, 1e-4).is_err()); + } + #[test] + fn relaxed_contract_bounds_rms_and_argmax() { + let a = vec![0.0, 3.0, -4.0]; + let b = vec![0.3, 3.0, -4.0]; + let mut error = logit_error(&a, &b, 1e-4, 1e-4).unwrap(); + assert_eq!(error.bad, 1); + assert!((error.rel_rms - 0.06).abs() < 1e-12); + // The bound is the reference at 1024 tokens and grows with log2(ctx). + assert!((rel_rms_bound(0.1, 1024) - 0.1).abs() < 1e-12); + assert!((rel_rms_bound(0.1, 8192) - 0.13).abs() < 1e-12); + assert!(rel_rms_bound(0.1, 65536) < 0.17); + let stable = delta("prefill", 2048, &[1.0, 1.01, 0.99], &[0.9, 0.91, 0.89]).unwrap(); + let metrics = [stable]; + // Strict: any bad logit or sequence mismatch is incorrect. + assert_eq!( + verdict(&metrics, &error, 1, 0, 3.0, 0.0), + Verdict::Incorrect + ); + // Relaxed: the sequence may diverge while the scaled RMS and argmax hold. + error.rel_rms_scaled = error.rel_rms / rel_rms_bound(0.1, 2048); + assert_eq!(verdict(&metrics, &error, 1, 0, 3.0, 0.1), Verdict::Improved); + assert_eq!( + verdict(&metrics, &error, 1, 1, 3.0, 0.1), + Verdict::Incorrect + ); + error.rel_rms_scaled = error.rel_rms / rel_rms_bound(0.05, 2048); + assert_eq!( + verdict(&metrics, &error, 1, 0, 3.0, 0.05), + Verdict::Incorrect + ); + } +} diff --git a/crates/ds4-perf/src/csv.rs b/crates/ds4-perf/src/csv.rs new file mode 100644 index 0000000000000000000000000000000000000000..74cebab56fe5f1aecc60f6b599421883f706e962 --- /dev/null +++ b/crates/ds4-perf/src/csv.rs @@ -0,0 +1,155 @@ +use std::io::{self, BufRead}; + +// Read records incrementally: GPU traces can contain millions of launches. +// Quotes, escaped quotes and embedded newlines are legal in Nsight names. +pub fn records( + reader: impl BufRead, + mut visit: impl FnMut(Vec) -> Result<(), String>, +) -> Result<(), String> { + let mut field = Vec::new(); + let mut row = Vec::new(); + let mut quoted = false; + let mut closed = false; + for byte in reader.bytes() { + let byte = byte.map_err(|e| e.to_string())?; + if quoted { + if byte == b'"' { + quoted = false; + closed = true; + } else { + field.push(byte); + } + continue; + } + match byte { + b'"' if closed => { + field.push(b'"'); + quoted = true; + closed = false; + } + b'"' if field.is_empty() => quoted = true, + b',' | b'\n' => { + row.push(take_field(&mut field)?); + closed = false; + if byte == b'\n' { + visit(std::mem::take(&mut row))?; + } + } + b'\r' => {} + b' ' | b'\t' if closed => {} + _ if closed => return Err("characters after closing CSV quote".into()), + _ => field.push(byte), + } + } + if quoted { + return Err("unterminated CSV quote".into()); + } + if !field.is_empty() || !row.is_empty() || closed { + row.push(take_field(&mut field)?); + visit(row)?; + } + Ok(()) +} + +fn take_field(bytes: &mut Vec) -> Result { + String::from_utf8(std::mem::take(bytes)).map_err(|e| format!("CSV is not UTF-8: {e}")) +} + +pub fn field(value: &str) -> String { + if value.contains([',', '"', '\r', '\n']) { + format!("\"{}\"", value.replace('"', "\"\"")) + } else { + value.into() + } +} + +pub fn key(value: &str) -> String { + value + .trim() + .trim_start_matches('\u{feff}') + .to_ascii_lowercase() +} + +pub fn column(header: &[String], names: &[&str]) -> Option { + header.iter().position(|s| names.contains(&key(s).as_str())) +} + +pub fn value<'a>(row: &'a [String], header: &[String], names: &[&str]) -> Option<&'a str> { + row.get(column(header, names)?) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) +} + +pub fn number(value: &str) -> Result { + value + .trim() + .parse::() + .ok() + .filter(|v| v.is_finite() && *v >= 0.0) + .ok_or_else(|| format!("invalid nonnegative number: {value}")) +} + +pub fn time(row: &[String], header: &[String], name: &str) -> Option { + for (unit, factor) in [ + ("ns", 1.0), + ("us", 1_000.0), + ("ms", 1_000_000.0), + ("s", 1_000_000_000.0), + ] { + let col = format!("{} ({unit})", name.to_ascii_lowercase()); + if let Some(value) = value(row, header, &[&col]) { + return number(value) + .ok() + .map(|v| v * factor) + .filter(|v| v.is_finite()); + } + } + None +} + +pub fn open(path: &std::path::Path) -> io::Result> { + std::fs::File::open(path).map(io::BufReader::new) +} + +pub fn table( + reader: impl BufRead, + marker: &str, + mut visit: impl FnMut(&[String], &[String]) -> Result<(), String>, +) -> Result<(), String> { + let mut header = Vec::new(); + records(reader, |row| { + if column(&row, &[marker]).is_some() { + header = row; + return Ok(()); + } + if header.is_empty() || row.iter().all(|s| s.trim().is_empty()) { + return Ok(()); + } + if row.len() != header.len() { + return Err("unexpected CSV record width after header".into()); + } + visit(&header, &row) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quoted_templates_and_newlines() { + let input = "Name,Count\r\n\"foo>\",2\r\n\"line\n\"\"two\"\"\",3"; + let mut rows = Vec::new(); + records(input.as_bytes(), |row| { + rows.push(row); + Ok(()) + }) + .unwrap(); + assert_eq!(rows[1], ["foo>", "2"]); + assert_eq!(rows[2], ["line\n\"two\"", "3"]); + assert!(records(b"\"bad".as_slice(), |_| Ok(())).is_err()); + for invalid in ["NaN", "inf", "-1"] { + assert!(number(invalid).is_err()); + } + } +} diff --git a/crates/ds4-perf/src/cupti.rs b/crates/ds4-perf/src/cupti.rs new file mode 100644 index 0000000000000000000000000000000000000000..3a1c15466374dcd9877d3f8b6177e7dd7c9f0d80 --- /dev/null +++ b/crates/ds4-perf/src/cupti.rs @@ -0,0 +1,397 @@ +use crate::{ + csv, + nsys::{Evidence, Kernel, Phase}, + runner, +}; +use ds4_perf::trace::Event; +use std::{ + collections::BTreeMap, + fs::File, + io::{BufRead, BufReader}, + path::Path, +}; + +pub fn capture( + out: &Path, + args: &crate::cli::Scout, + controls: &BTreeMap, +) -> Result { + let helper = crate::inspect::helper(args.gpu_helper.as_deref()); + let library = args + .cupti_library + .clone() + .or_else(|| helper.map(|p| p.with_file_name("libds4_perf_gpu.so"))) + .ok_or("CUPTI collector unavailable; build make ds4-perf-gpu or pass --cupti-library")? + .canonicalize() + .map_err(|e| format!("CUPTI collector library: {e}"))?; + let sdk = args + .cupti_sdk + .clone() + .or_else(|| { + runner::resolve("nvcc".as_ref()).and_then(|p| { + p.parent()? + .parent() + .map(|p| p.join("extras/CUPTI/lib64/libcupti.so")) + }) + }) + .ok_or("NVIDIA libcupti.so not found; pass --cupti-sdk")? + .canonicalize() + .map_err(|e| format!("NVIDIA CUPTI library: {e}"))?; + let before = ds4_perf::artifact::hash(&library)?; + let config = crate::experiment::Replay { + helper: None, + calibration: None, + collector: Some(crate::experiment::InputFile { + path: library.clone(), + sha256: before.clone(), + }), + sdk: Some(crate::experiment::InputFile { + path: sdk.clone(), + sha256: ds4_perf::artifact::hash(&sdk)?, + }), + }; + std::fs::write( + out.join("collector-config.json"), + serde_json::to_vec_pretty(&config).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + + runner::write( + &out.join("collector.txt"), + &format!( + "collector={}\ncollector_sha256={}\nsdk={}\nsdk_sha256={}\n", + library.display(), + before, + sdk.display(), + ds4_perf::artifact::hash(&sdk)? + ), + )?; + let mut env = controls.clone(); + env.insert( + "CUDA_INJECTION64_PATH".into(), + library.clone().into_os_string(), + ); + env.insert("NVTX_INJECTION64_PATH".into(), sdk.clone().into_os_string()); + env.insert( + "DS4_PERF_CUPTI_OUTPUT".into(), + out.join("activity.jsonl").into_os_string(), + ); + let mut paths = vec![sdk + .parent() + .ok_or("CUPTI library has no parent")? + .to_path_buf()]; + if let Some(inherited) = controls.get(std::ffi::OsStr::new("LD_LIBRARY_PATH")) { + paths.extend(std::env::split_paths(inherited)); + } + env.insert( + "LD_LIBRARY_PATH".into(), + std::env::join_paths(paths).map_err(|e| e.to_string())?, + ); + eprintln!("ds4-perf: fresh CUPTI-instrumented process"); + runner::run_limited(&args.command, out, "profile", &env, args.budget.limits())?; + for input in [&config.collector, &config.sdk].into_iter().flatten() { + if ds4_perf::artifact::hash(&input.path)? != input.sha256 { + return Err("CUPTI collector or SDK changed during capture".into()); + } + } + collect(out) +} + +// Normalize owned activity records only after checking the collector footer. +// Subtract the integer epoch before conversion: CUPTI timestamps exceed f64's +// exact integer range, whereas the duration of a bounded capture does not. +pub fn collect(out: &Path) -> Result { + let input = BufReader::new(File::open(out.join("activity.jsonl")).map_err(|e| e.to_string())?); + let (ranges, gpu, mut evidence) = normalize(input)?; + runner::write(&out.join("nsys-ranges.csv"), &ranges)?; + runner::write(&out.join("nsys-gpu-trace.csv"), &gpu)?; + crate::timeline::load(out, &mut evidence.phases)?; + if evidence.phases.is_empty() { + evidence + .warnings + .push("CUDA-only analysis: expected NVTX phases absent".into()); + } + Ok(evidence) +} + +fn normalize(input: impl BufRead) -> Result<(String, String, Evidence), String> { + let mut records = Vec::new(); + let mut started = false; + let mut ended = false; + for line in input.lines() { + let event: Event = + serde_json::from_str(&line.map_err(|e| e.to_string())?).map_err(|e| e.to_string())?; + if ended { + return Err("CUPTI records after footer".into()); + } + match &event { + Event::Start { + schema_version: 1, + api_version: 130301, + .. + } if !started => started = true, + Event::Start { .. } => return Err("unsupported or duplicate CUPTI header".into()), + _ if !started => return Err("CUPTI header missing".into()), + Event::End { + dropped: 0, + errors: 0, + } => ended = true, + Event::End { dropped, errors } => { + return Err(format!( + "CUPTI incomplete: {dropped} dropped, {errors} errors" + )) + } + Event::Error { message } => return Err(format!("CUPTI: {message}")), + _ => records.push(event), + } + } + if !started || !ended { + return Err("CUPTI capture has no successful footer".into()); + } + let origin = records + .iter() + .filter_map(|e| match e { + Event::Kernel { start, .. } | Event::Memop { start, .. } => Some(*start), + Event::Marker { timestamp, .. } => Some(*timestamp), + _ => None, + }) + .min() + .ok_or("empty CUPTI trace")?; + let mut open = BTreeMap::new(); + let mut windows = Vec::new(); + let mut ranges = "Name,Start (ns),Duration (ns),PID,TID\n".to_string(); + let mut markers: Vec<_> = records + .iter() + .filter(|e| matches!(e, Event::Marker { .. })) + .collect(); + markers.sort_by_key(|e| match e { + Event::Marker { timestamp, .. } => *timestamp, + _ => 0, + }); + for record in markers { + let Event::Marker { + name, + domain, + timestamp, + id, + flags, + pid, + tid, + } = record + else { + continue; + }; + let key = (*pid, *id); + if flags & 2 != 0 + && domain.is_empty() + && crate::nsys::phase_name(name).is_some() + && open.insert(key, (name.clone(), *timestamp, *tid)).is_some() + { + return Err("duplicate CUPTI range start".into()); + } + if flags & 4 != 0 { + if let Some((name, start, start_tid)) = open.remove(&key) { + if *timestamp <= start || *tid != start_tid { + return Err("invalid CUPTI local range".into()); + } + ranges.push_str(&format!( + "{},{},{},{},{}\n", + csv::field(&name), + start - origin, + timestamp - start, + pid, + tid + )); + windows.push((name, start, *timestamp)); + } + } + } + if !open.is_empty() { + return Err("unterminated CUPTI phase range".into()); + } + let mut gpu = "Name,Start (ns),Duration (ns),Device,Ctx,Strm,Bytes (B),GrdX,GrdY,GrdZ,BlkX,BlkY,BlkZ,Reg/Trd,StcSMem (B),DymSMem (B)\n".to_string(); + let mut grouped: BTreeMap<(String, String), Kernel> = BTreeMap::new(); + for record in records { + match record { + Event::Kernel { + name, + start, + end, + device, + context, + stream, + grid, + block, + registers, + shared_bytes, + .. + } => { + if end <= start { + return Err("invalid CUPTI kernel duration".into()); + } + gpu.push_str(&format!( + "{},{},{},{},{},{},,{},{},{},{},{},{},{},{},0\n", + csv::field(&name), + start - origin, + end - start, + device, + context, + stream, + grid[0], + grid[1], + grid[2], + block[0], + block[1], + block[2], + registers, + shared_bytes + )); + let phase = windows + .iter() + .find(|(_, a, b)| start >= *a && end <= *b) + .map(|w| w.0.as_str()) + .unwrap_or(""); + for scope in [Some(""), (!phase.is_empty()).then_some(phase)] + .into_iter() + .flatten() + { + let k = grouped + .entry((scope.into(), name.clone())) + .or_insert_with(|| Kernel { + name: name.clone(), + total_ns: 0.0, + count: Some(0), + }); + k.total_ns += (end - start) as f64; + k.count = k.count.and_then(|n| n.checked_add(1)); + } + } + Event::Memop { + operation, + start, + end, + bytes, + device, + context, + stream, + } => { + if end <= start { + return Err("invalid CUPTI memory duration".into()); + } + gpu.push_str(&format!( + "{},{},{},{},{},{},{},,,,,,,,,\n", + csv::field(&operation), + start - origin, + end - start, + device, + context, + stream, + bytes + )); + } + _ => {} + } + } + let mut evidence = Evidence::default(); + evidence.warnings.push("CUPTI phases use GPU execution contained in the Rust NVTX time window; this is not CUDA API correlation attribution".into()); + for ((phase, _), kernel) in grouped { + if phase.is_empty() { + evidence.global.push(kernel); + continue; + } + evidence + .phases + .entry(phase.clone()) + .or_insert_with(|| Phase { + name: phase, + ..Default::default() + }) + .kernels + .push(kernel); + } + for kernels in std::iter::once(&mut evidence.global) + .chain(evidence.phases.values_mut().map(|p| &mut p.kernels)) + { + kernels.sort_by(|a, b| b.total_ns.total_cmp(&a.total_ns)); + } + Ok((ranges, gpu, evidence)) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn rejects_incomplete_activity() { + for text in [ + "", + "{\"kind\":\"start\",\"schema_version\":1,\"api_version\":130301,\"pid\":1}\n", + "{\"kind\":\"end\",\"dropped\":0,\"errors\":0}\n", + ] { + assert!(normalize(text.as_bytes()).is_err()); + } + } + #[test] + fn preserves_epoch_precision() { + let epoch = 1_788_000_000_000_000_000; + let events = [ + Event::Start { + schema_version: 1, + api_version: 130301, + pid: 1, + }, + Event::Marker { + name: "ds4.prefill".into(), + domain: String::new(), + timestamp: epoch, + id: 2, + flags: 2, + pid: 1, + tid: 3, + }, + Event::Kernel { + name: "copy".into(), + start: epoch + 13, + end: epoch + 113, + device: 0, + context: 1, + stream: 1, + correlation: 1, + grid: [2, 1, 1], + block: [32, 1, 1], + registers: 16, + shared_bytes: 0, + graph_id: 0, + }, + Event::Marker { + name: String::new(), + domain: String::new(), + timestamp: epoch + 201, + id: 2, + flags: 4, + pid: 1, + tid: 3, + }, + Event::End { + dropped: 0, + errors: 0, + }, + ]; + let text = events + .iter() + .map(|e| serde_json::to_string(e).unwrap() + "\n") + .collect::(); + let (ranges, gpu, e) = normalize(text.as_bytes()).unwrap(); + assert!(ranges.contains("ds4.prefill,0,201,1,3")); + assert!(gpu.contains("copy,13,100,0,1,1")); + assert_eq!(e.phases["ds4.prefill"].kernels[0].total_ns, 100.0); + let mut reordered: Vec<_> = events.iter().collect(); + reordered.swap(1, 3); + let unordered = reordered + .iter() + .map(|e| serde_json::to_string(e).unwrap() + "\n") + .collect::(); + let (paired, _, _) = normalize(unordered.as_bytes()).unwrap(); + assert_eq!(paired, ranges); + assert!(normalize(text.replace("\"dropped\":0", "\"dropped\":1").as_bytes()).is_err()); + } +} diff --git a/crates/ds4-perf/src/doctor.rs b/crates/ds4-perf/src/doctor.rs new file mode 100644 index 0000000000000000000000000000000000000000..c91001d9165de9c43c5c79427d7f0b941cf84b3c --- /dev/null +++ b/crates/ds4-perf/src/doctor.rs @@ -0,0 +1,381 @@ +use crate::runner::{self, Capture, Probe}; +use std::collections::BTreeMap; + +#[derive(Default, Debug)] +pub struct Capabilities { + pub profile_help: String, + pub reports: String, + pub sync_rule: bool, + pub ncu_help: String, +} + +impl Capabilities { + pub fn report(&self, name: &str) -> Option { + let base = name.split(':').next()?; + let line = self.reports.lines().find(|line| { + line.trim_start() + .strip_prefix(base) + .is_some_and(|rest| rest.starts_with([' ', '['])) + })?; + let mut selected = base.to_string(); + for option in name.split(':').skip(1) { + if line.contains(&format!(":{option}")) { + selected.push(':'); + selected.push_str(option); + } + } + Some(selected) + } + + pub fn can_scout(&self) -> bool { + self.profile_help.contains("cuda") + && self.profile_help.contains("nvtx") + && (self.report("nvtx_kern_sum").is_some() + || self.report("cuda_gpu_kern_sum").is_some()) + } + + pub fn graph_nodes(&self) -> bool { + self.profile_help.contains("--cuda-graph-trace") && self.profile_help.contains("'node'") + } +} + +pub struct Doctor { + pub facts: BTreeMap, + pub caps: Capabilities, + pub warnings: Vec, + pub probes: Vec<(String, Capture)>, +} + +pub fn inspect(probe: &mut impl Probe, benchmark: Option<&str>) -> Doctor { + let mut probes = Vec::new(); + let mut run = |program: &str, args: &[&str]| { + let result = probe.capture(program, args); + probes.push((format!("{program} {}", args.join(" ")), result.clone())); + result + }; + let mut facts = BTreeMap::new(); + facts.insert("os".into(), std::env::consts::OS.into()); + facts.insert("arch".into(), std::env::consts::ARCH.into()); + facts.insert( + "git_sha".into(), + summary(&run("git", &["rev-parse", "HEAD"])), + ); + let dirty = run( + "git", + &["status", "--porcelain", "--untracked-files=normal"], + ); + facts.insert( + "git_dirty".into(), + if dirty.ok { + (!dirty.out.trim().is_empty()).to_string() + } else { + "unknown".into() + }, + ); + facts.insert("hostname".into(), summary(&run("hostname", &[]))); + let smi = run("nvidia-smi", &[]); + facts.insert( + "nvidia-smi".into(), + if smi.ok { "OK" } else { "unavailable" }.into(), + ); + let gpu = run( + "nvidia-smi", + &[ + "--query-gpu=name,compute_cap,driver_version", + "--format=csv,noheader", + ], + ); + let mut gpu_rows = Vec::new(); + if gpu.ok { + let _ = crate::csv::records(gpu.out.as_bytes(), |row| { + gpu_rows.push(row); + Ok(()) + }); + } + // Older drivers may not expose compute_cap; retain the name/driver anyway. + if gpu_rows.is_empty() { + let gpu = run( + "nvidia-smi", + &["--query-gpu=name,driver_version", "--format=csv,noheader"], + ); + if gpu.ok { + let _ = crate::csv::records(gpu.out.as_bytes(), |mut row| { + if row.len() == 2 { + row.insert(1, "unknown".into()); + } + gpu_rows.push(row); + Ok(()) + }); + } + } + for (name, index) in [("gpu", 0), ("compute", 1), ("driver", 2)] { + let values: Vec<_> = gpu_rows + .iter() + .filter_map(|row| row.get(index)) + .map(|s| s.trim().to_string()) + .collect(); + facts.insert( + name.into(), + if values.is_empty() { + "unknown".into() + } else { + values.join("; ") + }, + ); + } + facts.insert("cuda".into(), summary(&run("nvcc", &["--version"]))); + facts.insert( + "cuda_driver_api".into(), + smi.out + .lines() + .find(|line| line.contains("CUDA") && line.contains("Version")) + .unwrap_or("unknown") + .trim() + .into(), + ); + facts.insert("nsys".into(), summary(&run("nsys", &["--version"]))); + let profile = run("nsys", &["profile", "--help"]); + let reports = run("nsys", &["stats", "--help-reports"]); + let rules = run("nsys", &["analyze", "--help-rules"]); + facts.insert("ncu".into(), summary(&run("ncu", &["--version"]))); + let ncu = run("ncu", &["--help"]); + let bench = benchmark.map(|program| run(program, &["--help"])); + facts.insert( + "benchmark".into(), + benchmark.unwrap_or("supplied scout command").into(), + ); + facts.insert("NVTX".into(), "unknown (not probed)".into()); + + let caps = Capabilities { + profile_help: if profile.ok { + profile.combined() + } else { + String::new() + }, + // Nsight 2026.1 exits 1 after successfully listing reports/rules. + // Accept the documented help body, not arbitrary failed-command text. + reports: if reports.ok + || reports + .combined() + .contains("The following built-in reports are available:") + { + reports.combined() + } else { + String::new() + }, + sync_rule: (rules.ok + || rules + .combined() + .contains("The following built-in reports are available:")) + && rules + .combined() + .lines() + .any(|l| l.trim_start().starts_with("cuda_api_sync[")), + ncu_help: if ncu.ok { + ncu.combined() + } else { + String::new() + }, + }; + let mut warnings = Vec::new(); + if !smi.ok { + warnings.push("nvidia-smi unavailable; GPU/driver readiness unknown".into()); + } + if !caps.can_scout() { + warnings + .push("nsys lacks CUDA/NVTX collection or kernel reports; scout unavailable".into()); + } + if !caps.graph_nodes() { + warnings.push( + "CUDA graph-node tracing unavailable; captured decode may lack kernel detail".into(), + ); + } + for report in [ + "nvtx_gpu_proj_sum", + "nvtx_kern_sum", + "nvtx_pushpop_trace", + "cuda_gpu_trace", + "cuda_gpu_mem_time_sum", + ] { + if caps.report(report).is_none() { + warnings.push(format!( + "{report} unavailable; related scout evidence will be UNKNOWN" + )); + } + } + if !caps.sync_rule && caps.report("cuda_api_sync").is_none() { + warnings.push("cuda_api_sync unavailable; scout omits synchronization analysis".into()); + } + if caps.ncu_help.is_empty() { + warnings.push("ncu unavailable; scout works, kernel deep-dive unavailable".into()); + } + let mut doctor = Doctor { + facts, + caps, + warnings, + probes, + }; + if let Some(bench) = bench { + if bench.ok { + doctor.benchmark(&bench.combined(), "NVTX: "); + } else { + doctor + .warnings + .push("target benchmark --help failed; NVTX capability unknown".into()); + } + } + doctor +} + +fn summary(result: &Capture) -> String { + if !result.ok { + return "unavailable".into(); + } + let text = result.combined(); + let lines: Vec<_> = text + .lines() + .map(str::trim) + .filter(|l| !l.is_empty()) + .collect(); + lines.join(" | ") +} + +impl Doctor { + // Only the named target's help or the actual baseline's stderr supplies this fact. + pub fn benchmark(&mut self, output: &str, prefix: &str) { + let value = output.lines().find_map(|line| line.strip_prefix(prefix)); + let capability = match value { + Some("available") => "available (ds4.prefill / ds4.decode)", + Some(s) if s.starts_with("unavailable") => { + "unavailable (build with make ds4-bench-perf)" + } + _ => "unknown (benchmark has no capability marker)", + }; + self.facts.insert("NVTX".into(), capability.into()); + if value != Some("available") { + self.warnings.push(format!( + "benchmark NVTX {capability}; phase diagnosis requires ds4 ranges" + )); + } + } + + pub fn render(&self) -> String { + let mut out = String::from("ds4-perf doctor\n\n"); + for name in [ + "git_sha", + "git_dirty", + "hostname", + "os", + "arch", + "gpu", + "compute", + "driver", + "cuda", + "cuda_driver_api", + "nsys", + "ncu", + "benchmark", + "NVTX", + ] { + out.push_str(&format!("{name:<16} {}\n", self.facts[name])); + } + out.push_str(&format!( + "status {}\n", + if self.warnings.is_empty() { + "READY" + } else { + "DEGRADED" + } + )); + for warning in &self.warnings { + out.push_str(&format!("reason/impact {warning}\n")); + } + out + } + + pub fn save_probes(&self, out: &std::path::Path) -> Result<(), String> { + for (i, (command, result)) in self.probes.iter().enumerate() { + runner::write( + &out.join(format!("probe-{i:02}.txt")), + &format!( + "command: {command}\nstatus: {}\nstdout:\n{}\nstderr:\n{}", + result.ok, result.out, result.err + ), + )?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Fake; + impl Probe for Fake { + fn capture(&mut self, program: &str, args: &[&str]) -> Capture { + let out = match (program, args) { + ("nsys", ["profile", "--help"]) => "--trace cuda nvtx --cuda-graph-trace 'node' --sample --cpuctxsw", + ("nsys", ["stats", "--help-reports"]) => " cuda_gpu_kern_sum[:nvtx-name][:base|:mangled] -- kernels\n nvtx_kern_sum[:base|:mangled] -- phases", + ("nvidia-smi", [_, _]) => "NVIDIA GB10, 12.1, 610.43.02", + ("./ds4-bench", _) => "help\nNVTX: available\n", + ("ncu", _) => return Capture::failed("not found"), + _ => "version", + }; + Capture { + ok: true, + out: out.into(), + err: String::new(), + } + } + } + + #[test] + fn missing_ncu_keeps_scout() { + let d = inspect(&mut Fake, Some("./ds4-bench")); + assert!(d.caps.can_scout()); + assert!(d.caps.graph_nodes()); + assert_eq!(d.facts["gpu"], "NVIDIA GB10"); + assert_eq!( + d.caps.report("cuda_gpu_kern_sum:nvtx-name:base").as_deref(), + Some("cuda_gpu_kern_sum:nvtx-name:base") + ); + assert!(d.caps.report("nvtx_gpu_proj_sum").is_none()); + assert!(d.render().contains("DEGRADED")); + assert!(d + .render() + .contains("scout works, kernel deep-dive unavailable")); + } + + #[test] + fn older_report_drops_options() { + let caps = Capabilities { + reports: "cuda_gpu_kern_sum -- summary\n".into(), + ..Default::default() + }; + assert_eq!( + caps.report("cuda_gpu_kern_sum:nvtx-name:base").as_deref(), + Some("cuda_gpu_kern_sum") + ); + assert!(!caps.can_scout()); + } + + #[test] + fn help_exit_one_is_supported() { + struct HelpExitOne; + impl Probe for HelpExitOne { + fn capture(&mut self, program: &str, args: &[&str]) -> Capture { + let mut result = Fake.capture(program, args); + if program == "nsys" && args == ["stats", "--help-reports"] { + result.ok = false; + result.out = format!( + "The following built-in reports are available:\n{}", + result.out + ); + } + result + } + } + assert!(inspect(&mut HelpExitOne, None).caps.can_scout()); + } +} diff --git a/crates/ds4-perf/src/experiment.rs b/crates/ds4-perf/src/experiment.rs new file mode 100644 index 0000000000000000000000000000000000000000..29c5c0d3b95941dade91cd17c439a6f255844fc9 --- /dev/null +++ b/crates/ds4-perf/src/experiment.rs @@ -0,0 +1,629 @@ +use crate::{bench, cli, csv, inspect, nsys, runner}; +use ds4_perf::{ + artifact::{self, Artifact, Payload, Reference}, + machine::{Gpu, Machine}, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeMap, + ffi::OsString, + fs, + io::Read, + path::{Path, PathBuf}, +}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct InputFile { + pub path: PathBuf, + pub sha256: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Workload { + pub protocol: String, + pub name: String, + pub family: String, + pub files: BTreeMap, + pub shape: BTreeMap, + pub cache_state: String, +} + +impl Workload { + pub fn verify(&self) -> Result<(), String> { + if self.name.is_empty() + || self.family.is_empty() + || self.cache_state.is_empty() + || !self.files.contains_key("model") + || !self.files.contains_key("prompt") + { + return Err( + "workload requires name, family, cache_state and hashed model/prompt files".into(), + ); + } + for (name, input) in &self.files { + if artifact::hash(&input.path)? != input.sha256 { + return Err(format!("workload input changed: {name}")); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Sample { + pub rows: Vec, + pub proof_files: BTreeMap, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scout { + pub process_scope: ProcessScope, + pub unreviewed_environment: Vec, + pub model_argument: Option, + pub replay: Replay, + pub budget: cli::Budget, + pub requested_repeats: u32, + pub requested_proof: bool, + pub collector: String, + pub gpu: Option, + pub command: Vec>, + pub cwd: PathBuf, + pub executable: PathBuf, + pub executable_sha256: String, + pub environment: BTreeMap, + pub workload: Option, + pub cache_policy: String, + pub samples: Vec, + pub evidence: nsys::Evidence, + pub timing_source: String, +} + +impl Payload for Scout { + const KIND: &'static str = "scout"; + fn validate(&self) -> Result<(), String> { + if self.process_scope.verified + && (self.process_scope.before != self.process_scope.expected_owners + || self.process_scope.after != self.process_scope.expected_owners) + { + return Err("GPU process snapshot verification disagrees with recorded PIDs".into()); + } + if let Some(gpu) = &self.gpu { + gpu.validate()?; + } + if !["nsys", "cupti"].contains(&self.collector.as_str()) + || self.command.is_empty() + || self.executable_sha256.len() != 64 + || self.timing_source != "fresh-unprofiled-processes" + { + return Err("invalid scout identity or timing source".into()); + } + for sample in &self.samples { + for row in &sample.rows { + if row.ctx == 0 + || row.prefill_tokens == 0 + || row.gen_tokens == 0 + || [row.prefill_tps, row.gen_tps, row.first_token_sec] + .into_iter() + .any(|v| !v.is_finite() || v <= 0.0) + { + return Err("invalid benchmark sample".into()); + } + } + } + for p in self.evidence.phases.values() { + if [p.wall_ns, p.projected_ns, p.busy_ns, p.mem_ns, p.gap_ns] + .into_iter() + .flatten() + .any(|v| !v.is_finite() || v < 0.0) + || p.kernels + .iter() + .any(|k| !k.total_ns.is_finite() || k.total_ns < 0.0) + { + return Err("invalid phase evidence".into()); + } + } + Ok(()) + } +} + +/// Bind normalized samples to the exact hashed, unprofiled process outputs. +pub fn load(path: &Path) -> Result, String> { + let value: Artifact = artifact::load(path)?; + let scout = value.require()?; + let root = path.parent().unwrap_or(Path::new(".")); + const MAX_TIMING_BYTES: u64 = 64 * 1024 * 1024; + for (index, sample) in scout.samples.iter().enumerate() { + let name = if index == 0 { + "bench.stdout".into() + } else { + format!("bench-{index:02}.stdout") + }; + let reference = value + .inputs + .iter() + .find(|r| r.path == name) + .ok_or_else(|| format!("unreferenced raw benchmark: {name}"))?; + let mut bytes = Vec::new(); + fs::File::open(root.join(&name)) + .map_err(|e| e.to_string())? + .take(MAX_TIMING_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if bytes.len() as u64 > MAX_TIMING_BYTES { + return Err("raw benchmark exceeds 64 MiB".into()); + } + // Hash and parse the same bytes, so a later file change cannot replace + // the data used to validate this sample. + if artifact::hash_bytes(&bytes) != reference.sha256 { + return Err(format!("raw benchmark changed: {name}")); + } + if bench::parse(bytes.as_slice())? != sample.rows { + return Err(format!( + "sample summary disagrees with raw benchmark: {name}" + )); + } + } + Ok(value) +} + +pub struct Prepared { + collector: Option, + limits: crate::process::Limits, + calibration: Option, + pub machine: Artifact, + pub environment: BTreeMap, + workload: Option, + executable: PathBuf, + executable_sha256: String, +} + +impl Prepared { + pub fn run(&self, command: &[OsString], out: &Path, name: &str) -> Result<(), String> { + let mut command = command.to_vec(); + if command.first().is_some_and(|v| v == "nsys") { + command[0] = self + .collector + .as_ref() + .ok_or("nsys executable unavailable")? + .path + .clone() + .into(); + } + runner::run_limited(&command, out, name, &self.environment, self.limits) + } + pub fn pin(&self, args: &cli::Scout) -> cli::Scout { + let mut args = args.clone(); + args.command[0] = self.executable.clone().into_os_string(); + args.calibration = self.calibration.clone(); + args + } +} + +pub fn prepare(out: &Path, args: &cli::Scout) -> Result { + let environment = if args.exact_environment { + runner::replay_controls(&args.environment)? + } else { + runner::controls(&args.environment)? + }; + runner::device_environment(&environment)?; + let workload: Option = args + .workload + .as_ref() + .map(|p| { + let bytes = fs::read(p).map_err(|e| e.to_string())?; + let mut workload: Workload = + serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + for input in workload.files.values_mut() { + input.path = p + .parent() + .unwrap_or(Path::new(".")) + .join(&input.path) + .canonicalize() + .map_err(|e| e.to_string())?; + } + workload.verify_scope(&args.command, &environment)?; + workload.verify()?; + Ok::<_, String>(workload) + }) + .transpose()?; + let executable = runner::resolve(&args.command[0]).ok_or("benchmark executable not found")?; + let executable_sha256 = artifact::hash(&executable)?; + artifact::directory(&out.join("machine"))?; + let machine = inspect::collect( + &out.join("machine"), + args.device, + args.gpu_helper.as_deref(), + None, + )?; + if let Some(expected) = &args.machine { + let previous: Artifact = artifact::load(expected)?; + let a = previous + .require()? + .gpu + .as_ref() + .ok_or("input machine has no GPU")?; + let b = machine + .require()? + .gpu + .as_ref() + .ok_or("current machine has no GPU")?; + if !a.same_device(b) { + return Err("selected machine differs from current CUDA device/driver".into()); + } + artifact::bundle::(expected, &out.join("machine-input"))?; + } + if args.proof + && args + .command + .iter() + .any(|a| a == "--dump-frontier-logits-dir") + { + return Err("--proof owns --dump-frontier-logits-dir; remove the benchmark flag".into()); + } + let calibration = args + .calibration + .as_ref() + .map(|path| { + artifact::bundle::(path, &out.join("calibration")) + }) + .transpose()?; + let collector = if matches!(args.collector, cli::Collector::Nsys) { + runner::resolve(std::ffi::OsStr::new("nsys")) + .map(|path| { + Ok::<_, String>(InputFile { + sha256: artifact::hash(&path)?, + path, + }) + }) + .transpose()? + } else { + None + }; + if collector.is_some() { + fs::write( + out.join("collector-config.json"), + serde_json::to_vec_pretty(&Replay { + helper: None, + collector: collector.clone(), + sdk: None, + calibration: None, + }) + .map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + } + Ok(Prepared { + collector, + calibration, + limits: args.budget.limits(), + machine, + environment, + workload, + executable, + executable_sha256, + }) +} + +pub fn bench_command(args: &cli::Scout, out: &Path, name: &str) -> Vec { + let mut command = args.command.clone(); + if args.proof { + command.push("--dump-frontier-logits-dir".into()); + command.push(out.join(format!("{name}-proof")).into_os_string()); + } + command +} + +pub fn sample(out: &Path, name: &str) -> Result { + let rows = + bench::parse(csv::open(&out.join(format!("{name}.stdout"))).map_err(|e| e.to_string())?)?; + let mut proof_files = BTreeMap::new(); + let proof = out.join(format!("{name}-proof")); + if proof.is_dir() { + for entry in fs::read_dir(&proof).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + if entry.file_type().map_err(|e| e.to_string())?.is_file() { + proof_files.insert( + entry.file_name().to_string_lossy().into(), + artifact::reference(&entry.path(), out)?, + ); + } + } + } + Ok(Sample { rows, proof_files }) +} + +// Every local raw file is referenced directly so consumers can verify a moved +// evidence directory without trusting only an intermediate manifest hash. +pub fn raw_refs(root: &Path, dir: &Path) -> Result, String> { + let mut paths = fs::read_dir(dir) + .map_err(|e| e.to_string())? + .collect::, _>>() + .map_err(|e| e.to_string())?; + paths.sort_by_key(|e| e.file_name()); + let mut refs = Vec::new(); + for entry in paths { + let kind = entry.file_type().map_err(|e| e.to_string())?; + if kind.is_dir() { + refs.extend(raw_refs(root, &entry.path())?); + } + if kind.is_file() { + refs.push(artifact::reference(&entry.path(), root)?); + } + } + Ok(refs) +} + +pub fn finish( + out: &Path, + args: &cli::Scout, + prepared: Prepared, + result: &Result, +) -> Result<(), String> { + let mut samples = Vec::new(); + let mut warnings = Vec::new(); + for index in 0..args.repeats { + let name = if index == 0 { + "bench".into() + } else { + format!("bench-{index:02}") + }; + match sample(out, &name) { + Ok(s) => samples.push(s), + Err(e) => warnings.push(e), + } + } + let verified = (|| { + if artifact::hash(&prepared.executable)? != prepared.executable_sha256 { + return Err("benchmark binary changed during scout".into()); + } + if let Some(workload) = &prepared.workload { + workload.verify()?; + } + if let Some(input) = &prepared.collector { + if artifact::hash(&input.path)? != input.sha256 { + return Err("nsys executable changed during scout".into()); + } + } + crate::process::check_bytes(out, args.budget.limits())?; + Ok::<_, String>(()) + })(); + if let Err(e) = &verified { + warnings.push(e.clone()); + } + if let Err(e) = result { + warnings.push(e.clone()); + } + #[cfg(unix)] + let command = { + use std::os::unix::ffi::OsStrExt; + args.command.iter().map(|a| a.as_bytes().to_vec()).collect() + }; + #[cfg(not(unix))] + let command = args + .command + .iter() + .map(|a| a.to_string_lossy().as_bytes().to_vec()) + .collect(); + let mut replay = Replay { + helper: None, + collector: None, + sdk: None, + calibration: None, + }; + if let Some(machine) = &prepared.machine.data { + if let Some(path) = machine.facts.get("gpu_helper_path") { + replay.helper = Some(InputFile { + path: path.into(), + sha256: machine + .facts + .get("gpu_helper_sha256") + .cloned() + .ok_or("helper hash missing")?, + }); + } + } + let collector = out.join("collector-config.json"); + if collector.exists() { + let collected: Replay = + serde_json::from_slice(&fs::read(collector).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + replay.collector = collected.collector; + replay.sdk = collected.sdk; + } + replay.calibration = prepared + .calibration + .as_ref() + .map(|path| artifact::reference(path, out)) + .transpose()?; + let mut scout = Artifact::new(Scout { + process_scope: process_scope(out, &prepared.environment), + unreviewed_environment: runner::unreviewed_env(), + model_argument: crate::workload::model_argument(&args.command).ok(), + replay, + budget: args.budget.clone(), + requested_repeats: args.repeats, + requested_proof: args.proof, + collector: match args.collector { + cli::Collector::Nsys => "nsys", + cli::Collector::Cupti => "cupti", + } + .into(), + gpu: prepared.machine.data.as_ref().and_then(|m| m.gpu.clone()), + command, + cwd: std::env::current_dir().map_err(|e| e.to_string())?, + executable: prepared.executable, + executable_sha256: prepared.executable_sha256, + environment: prepared + .environment + .iter() + .map(|(k, v)| (k.to_string_lossy().into(), v.to_string_lossy().into())) + .collect(), + workload: prepared.workload, + cache_policy: args.cache_policy.clone(), + samples, + evidence: result.as_ref().cloned().unwrap_or_default(), + timing_source: "fresh-unprofiled-processes".into(), + }); + let data = scout.data.as_ref().unwrap(); + let samples_complete = data.samples.len() == args.repeats as usize + && data.samples.iter().all(|s| !s.rows.is_empty()); + if !samples_complete { + warnings.push("requested benchmark samples are missing or empty".into()); + } + let proof = if args.proof { + crate::compare::validate_proofs(out, data) + } else { + Ok(()) + }; + if let Err(e) = &proof { + warnings.push(e.clone()); + } + let scope_ok = data.process_scope.verified; + if !scope_ok { + warnings.push( + "GPU/host process snapshots are unavailable or differ from intended owners".into(), + ); + } + scout.complete = + result.is_ok() && verified.is_ok() && samples_complete && proof.is_ok() && scope_ok; + scout.warnings = warnings; + scout.inputs = raw_refs(out, out)?; + // Publish immutable collection evidence first; final scout includes all + // requested analysis and its output accounting, without a reference cycle. + artifact::save(&out.join("collection.json"), &scout)?; + let mut advanced: Result<(), String> = (|| { + verified?; + proof?; + if !scope_ok { + return Err("unverified process snapshots; see scout.json".into()); + } + if !samples_complete && args.repeats > 1 { + return Err("incomplete repeated benchmark samples; see scout.json".into()); + } + if result.is_err() { + return Ok(()); + } + if args.fit { + let shape = scout + .data + .as_ref() + .and_then(|s| s.workload.as_ref()) + .map(|w| w.shape.clone()) + .unwrap_or_default(); + crate::fit::run(out, &prepared.machine, args.calibration.as_deref(), shape)?; + } + if args.ncu { + crate::ncu::run( + out, + &args.command, + &prepared.environment, + &scout, + args.budget.limits(), + )?; + } + Ok(()) + })(); + scout.inputs = raw_refs(out, out)?; + // Reserve the terminal status and the final JSON before claiming complete. + let final_bytes = serde_json::to_vec_pretty(&scout) + .map_err(|e| e.to_string())? + .len() as u64 + + 1 + + b"COMPLETE\n".len() as u64; + if advanced.is_ok() + && crate::process::bytes(out)?.saturating_add(final_bytes) > args.budget.limits().bytes + { + advanced = Err("profiler output budget exceeded at publication".into()); + } + if let Err(error) = &advanced { + scout.complete = false; + scout.warnings.push(error.clone()); + } + artifact::save(&out.join("scout.json"), &scout)?; + advanced +} + +pub fn failed(out: &Path, message: String) -> Result<(), String> { + let mut scout = Artifact::::failed(message); + scout.inputs = raw_refs(out, out)?; + artifact::save(&out.join("scout.json"), &scout) +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Replay { + pub helper: Option, + pub collector: Option, + pub sdk: Option, + pub calibration: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProcessScope { + pub expected_owners: Vec, + pub before: Vec, + pub after: Vec, + pub verified: bool, +} + +fn process_scope(out: &Path, env: &BTreeMap) -> ProcessScope { + let mut expected = Vec::new(); + let manifest = env.get(std::ffi::OsStr::new("DS4_CUDA_WEIGHT_IPC_MANIFEST")); + if let Some(path) = manifest { + if let Ok(text) = fs::read_to_string(path) { + expected = text + .lines() + .filter_map(|line| { + line.strip_prefix("owner ") + .and_then(|v| v.split_whitespace().next()) + .and_then(|v| v.parse().ok()) + }) + .collect(); + } + } + expected.sort_unstable(); + let read = |label| -> Result, String> { + let text = fs::read_to_string(out.join(format!("processes-{label}.txt"))) + .map_err(|e| e.to_string())?; + let text = text + .strip_prefix("gpu_query_ok=true\n") + .ok_or("GPU process query failed")?; + let (text, ps) = text + .split_once("ps_ok=") + .ok_or("process snapshot missing")?; + if !ps.starts_with("true\n") { + return Err("host process query failed".into()); + } + let mut pids = Vec::new(); + csv::records(text.as_bytes(), |row| { + if row.iter().all(|v| v.trim().is_empty()) { + return Ok(()); + } + pids.push( + row[0] + .trim() + .parse::() + .map_err(|_| "unrecognized GPU process record")?, + ); + Ok(()) + })?; + pids.sort_unstable(); + Ok(pids) + }; + let before = read("before"); + let after = read("after"); + let verified = before.as_ref().is_ok_and(|p| *p == expected) + && after.as_ref().is_ok_and(|p| *p == expected) + && (manifest.is_none() || !expected.is_empty()); + ProcessScope { + expected_owners: expected, + before: before.unwrap_or_default(), + after: after.unwrap_or_default(), + verified, + } +} diff --git a/crates/ds4-perf/src/fit.rs b/crates/ds4-perf/src/fit.rs new file mode 100644 index 0000000000000000000000000000000000000000..8b61bfb6b092249f0ae0f32d1e388b04d9d4cb95 --- /dev/null +++ b/crates/ds4-perf/src/fit.rs @@ -0,0 +1,369 @@ +use crate::{csv, nsys}; +use ds4_perf::{ + artifact::{self, Artifact, Payload}, + machine::{Calibration, Gpu, Machine}, +}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, path::Path}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Launch { + pub phase: String, + pub kernel: String, + pub grid: [u64; 3], + pub block: [u64; 3], + pub registers_per_thread: u64, + pub shared_bytes: u64, + pub instances: u64, + pub total_ns: f64, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Bound { + pub operand_shape: Option>, + pub operand_shape_source: String, + pub launch: Launch, + pub resident_blocks_upper: u64, + pub occupancy_upper: f64, + pub waves_lower: u64, + pub last_wave_fill: f64, + pub block_limits: BTreeMap, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Fit { + pub calibration_status: String, + pub workload_shape_status: String, + pub bounds: Vec, + pub unknown_launches: u64, + pub observed_copy_gb_s: Option, + pub observed_fp32_gflop_s: Option, + pub observed_launch_us: Option, + pub shape: BTreeMap, + pub limits: Vec, +} + +impl Payload for Fit { + const KIND: &'static str = "fit"; + fn validate(&self) -> Result<(), String> { + for b in &self.bounds { + if b.resident_blocks_upper == 0 + || b.waves_lower == 0 + || !b.occupancy_upper.is_finite() + || !(0.0..=1.0).contains(&b.occupancy_upper) + || !b.last_wave_fill.is_finite() + || !(0.0..=1.0).contains(&b.last_wave_fill) + || !b.launch.total_ns.is_finite() + || b.launch.total_ns <= 0.0 + { + return Err("invalid geometry bound".into()); + } + } + for v in [ + self.observed_copy_gb_s, + self.observed_fp32_gflop_s, + self.observed_launch_us, + ] + .into_iter() + .flatten() + { + if !v.is_finite() || v <= 0.0 { + return Err("invalid measured envelope".into()); + } + } + Ok(()) + } +} + +fn product(values: [u64; 3]) -> Result { + values.into_iter().try_fold(1u64, |a, b| { + a.checked_mul(b) + .filter(|n| *n > 0) + .ok_or("invalid launch dimensions".into()) + }) +} + +fn bound(gpu: &Gpu, launch: Launch) -> Result { + gpu.validate()?; + let threads = product(launch.block)?; + let blocks = product(launch.grid)?; + if threads > gpu.max_threads_per_block as u64 || launch.registers_per_thread == 0 { + return Err("invalid block size or unknown register count".into()); + } + let warp_threads = threads.div_ceil(gpu.warp_size as u64) * gpu.warp_size as u64; + let registers = warp_threads + .checked_mul(launch.registers_per_thread) + .ok_or("register demand overflow")?; + let mut limits = BTreeMap::from([ + ("block_slots".into(), gpu.max_blocks_per_sm as u64), + ( + "threads".into(), + gpu.max_threads_per_sm as u64 / warp_threads, + ), + ( + "registers".into(), + gpu.registers_per_sm.max(0) as u64 / registers, + ), + ]); + if let Some(blocks) = (gpu.shared_bytes_per_sm.max(0) as u64).checked_div(launch.shared_bytes) { + limits.insert("shared_memory".into(), blocks); + } + let resident = *limits.values().min().ok_or("resource limits unavailable")?; + if resident == 0 { + return Err("launch exceeds reported SM resources".into()); + } + let capacity = resident + .checked_mul(gpu.multiprocessors as u64) + .ok_or("grid capacity overflow")?; + let waves = blocks.div_ceil(capacity); + Ok(Bound { + operand_shape: None, + operand_shape_source: + "unavailable: activity records expose launch geometry, not operand dimensions".into(), + launch, + resident_blocks_upper: resident, + occupancy_upper: resident as f64 * warp_threads as f64 / gpu.max_threads_per_sm as f64, + waves_lower: waves, + last_wave_fill: (1 + (blocks - 1) % capacity) as f64 / capacity as f64, + block_limits: limits, + }) +} + +fn byte_value(header: &[String], row: &[String], name: &str) -> Option { + // Resource bounds require integer bytes. Rounded MB/KB reports cannot + // support an occupancy upper bound; scout requests csv:noconv. + csv::value(row, header, &[&format!("{name} (b)")])? + .parse() + .ok() +} + +fn launches(out: &Path) -> Result<(Vec, u64), String> { + let mut ranges = Vec::new(); + csv::table( + csv::open(&out.join("nsys-ranges.csv")).map_err(|e| e.to_string())?, + "name", + |header, row| { + let Some(name) = csv::value(row, header, &["name"]).and_then(nsys::phase_name) else { + return Ok(()); + }; + let start = csv::time(row, header, "Start").ok_or("range Start missing")?; + let duration = csv::time(row, header, "Duration").ok_or("range Duration missing")?; + ranges.push((name, start, start + duration)); + Ok(()) + }, + )?; + type LaunchKey = (String, String, [u64; 3], [u64; 3], u64, u64); + let mut groups: BTreeMap = BTreeMap::new(); + let mut unknown = 0; + csv::table( + csv::open(&out.join("nsys-gpu-trace.csv")).map_err(|e| e.to_string())?, + "name", + |header, row| { + let name = csv::value(row, header, &["name"]).ok_or("kernel Name missing")?; + if header + .iter() + .position(|s| csv::key(s).starts_with("bytes (")) + .and_then(|i| row.get(i)) + .is_some_and(|s| !s.trim().is_empty()) + || name.starts_with("[CUDA ") + { + return Ok(()); + } + let start = csv::time(row, header, "Start").ok_or("kernel Start missing")?; + let duration = csv::time(row, header, "Duration").ok_or("kernel Duration missing")?; + let Some((phase, _, _)) = ranges + .iter() + .find(|(_, a, b)| start >= *a && start + duration <= *b) + else { + return Ok(()); + }; + let integer = + |name| csv::value(row, header, &[name]).and_then(|v| v.parse::().ok()); + let dimensions: Option> = + ["grdx", "grdy", "grdz", "blkx", "blky", "blkz", "reg/trd"] + .into_iter() + .map(integer) + .collect(); + let shared = byte_value(header, row, "stcsmem") + .zip(byte_value(header, row, "dymsmem")) + .and_then(|(a, b)| a.checked_add(b)); + let (Some(d), Some(shared)) = (dimensions, shared) else { + unknown += 1; + return Ok(()); + }; + let grid = [d[0], d[1], d[2]]; + let block = [d[3], d[4], d[5]]; + let key = ( + phase.to_string(), + name.to_string(), + grid, + block, + d[6], + shared, + ); + let launch = groups.entry(key).or_insert_with(|| Launch { + phase: phase.to_string(), + kernel: name.to_string(), + grid, + block, + registers_per_thread: d[6], + shared_bytes: shared, + instances: 0, + total_ns: 0.0, + }); + launch.instances += 1; + launch.total_ns += duration; + Ok(()) + }, + )?; + Ok((groups.into_values().collect(), unknown)) +} + +pub fn median(values: &[f64]) -> f64 { + let mut values = values.to_vec(); + values.sort_by(f64::total_cmp); + let middle = values.len() / 2; + if values.len().is_multiple_of(2) { + (values[middle - 1] + values[middle]) / 2.0 + } else { + values[middle] + } +} + +pub fn run( + out: &Path, + machine: &Artifact, + calibration: Option<&Path>, + shape: BTreeMap, +) -> Result<(), String> { + let result: Result = (|| { + let gpu = machine + .require()? + .gpu + .as_ref() + .ok_or("fit requires CUDA device properties")?; + let calibration = calibration.map(artifact::load::).transpose()?; + let envelope = calibration.as_ref().map(|c| c.require()).transpose()?; + if envelope.is_some_and(|c| !c.gpu.same_device(gpu)) { + return Err("calibration device/driver differs from scout".into()); + } + let (launches, mut unknown) = launches(out)?; + let mut bounds = Vec::new(); + let mut limits = vec!["Resource occupancy is an upper bound: allocation granularity, carveout, clusters and barriers are not modeled. Waves/fill use that bound, not observed occupancy.".into(),"Operand shapes are declared workload metadata. Kernel names do not establish M/N/K, tensor-core FLOPs, or memory traffic.".into(),"Calibration is an observed FP32 SIMT/copy/host-launch envelope, not a tensor-core peak or a workload roofline.".into()]; + for launch in launches { + let count = launch.instances; + match bound(gpu, launch) { + Ok(b) => bounds.push(b), + Err(e) => { + unknown += count; + limits.push(e); + } + } + } + bounds.sort_by(|a, b| b.launch.total_ns.total_cmp(&a.launch.total_ns)); + Ok(Fit { + calibration_status: if envelope.is_some() { + "measured" + } else { + "missing" + } + .into(), + workload_shape_status: if shape.is_empty() { + "missing" + } else { + "declared" + } + .into(), + bounds, + unknown_launches: unknown, + observed_copy_gb_s: envelope.map(|c| median(&c.copy_gb_s)), + observed_fp32_gflop_s: envelope.map(|c| median(&c.fp32_gflop_s)), + observed_launch_us: envelope.map(|c| median(&c.launch_us)), + shape, + limits, + }) + })(); + let mut artifact = match result { + Ok(data) => { + let mut a = Artifact::new(data); + a.complete = a.data.as_ref().is_some_and(|d| { + !d.bounds.is_empty() + && d.unknown_launches == 0 + && d.calibration_status == "measured" + && d.workload_shape_status == "declared" + }); + a + } + Err(ref error) => Artifact::::failed(error.clone()), + }; + for path in [ + Some(out.join("collection.json")), + Some(out.join("machine/machine.json")), + calibration.map(Path::to_path_buf), + ] + .into_iter() + .flatten() + { + if path.is_file() { + artifact.inputs.push(artifact::reference(&path, out)?); + } + } + artifact::save(&out.join("fit.json"), &artifact)?; + if !artifact.complete { + return Err( + "fit incomplete; see fit.json for missing geometry, calibration or workload metadata" + .into(), + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + fn gpu() -> Gpu { + serde_json::from_value(serde_json::json!({ + "ordinal":0,"name":"fixture","uuid":"test","compute_major":12,"compute_minor":1,"driver_version":13030, + "total_memory_bytes":1024,"free_memory_bytes":512,"multiprocessors":2,"warp_size":32, + "max_threads_per_block":1024,"max_threads_per_sm":1536,"max_blocks_per_sm":24, + "registers_per_sm":65536,"shared_bytes_per_sm":102400,"shared_bytes_per_block":101376,"l2_bytes":1, + "memory_bus_bits":null,"memory_clock_khz":null,"clock_khz":null,"unavailable":{} + })).unwrap() + } + fn launch() -> Launch { + Launch { + phase: "ds4.prefill".into(), + kernel: "fixture".into(), + grid: [49, 1, 1], + block: [33, 1, 1], + registers_per_thread: 16, + shared_bytes: 0, + instances: 1, + total_ns: 100.0, + } + } + #[test] + fn rounds_warps_and_grid_tail() { + let b = bound(&gpu(), launch()).unwrap(); + assert_eq!(b.resident_blocks_upper, 24); + assert_eq!(b.occupancy_upper, 1.0); + assert_eq!(b.waves_lower, 2); + assert_eq!(b.last_wave_fill, 1.0 / 48.0); + } + #[test] + fn limits_shared_and_unknowns() { + let mut l = launch(); + l.shared_bytes = 51200; + assert_eq!(bound(&gpu(), l.clone()).unwrap().resident_blocks_upper, 2); + l.registers_per_thread = 0; + assert!(bound(&gpu(), l).is_err()); + assert!(product([u64::MAX, 2, 1]).is_err()); + assert_eq!( + byte_value(&["StcSMem (MB)".into()], &["0.01".into()], "stcsmem"), + None + ); + } +} diff --git a/crates/ds4-perf/src/inspect.rs b/crates/ds4-perf/src/inspect.rs new file mode 100644 index 0000000000000000000000000000000000000000..dd4fc876e11d1500eabff3304492e75f441a9b4f --- /dev/null +++ b/crates/ds4-perf/src/inspect.rs @@ -0,0 +1,185 @@ +use crate::{ + doctor, + runner::{self, Probe}, +}; +use ds4_perf::{ + artifact::{self, Artifact, Payload}, + machine::{Calibration, Gpu, Machine}, +}; +use std::{ + ffi::OsString, + path::{Path, PathBuf}, +}; + +pub fn helper(explicit: Option<&Path>) -> Option { + if let Some(path) = explicit { + return path.canonicalize().ok(); + } + let sibling = std::env::current_exe().ok()?.with_file_name("ds4-perf-gpu"); + if sibling.is_file() { + return Some(sibling); + } + runner::resolve("ds4-perf-gpu".as_ref()) +} + +fn raw_inputs(artifact: &mut Artifact, out: &Path, prefix: &str) -> Result<(), String> { + for suffix in ["command.txt", "stdout", "stderr", "status.txt"] { + let path = out.join(format!("{prefix}.{suffix}")); + if path.is_file() { + artifact.inputs.push(artifact::reference(&path, out)?); + } + } + Ok(()) +} + +pub fn collect( + out: &Path, + device: usize, + gpu_helper: Option<&Path>, + bench: Option<&str>, +) -> Result, String> { + let mut system = runner::System; + let d = doctor::inspect(&mut system, bench); + d.save_probes(out)?; + runner::write(&out.join("doctor.txt"), &d.render())?; + let processes = system.capture( + "nvidia-smi", + &[ + "--query-compute-apps=pid,process_name", + "--format=csv,noheader", + ], + ); + let mut warnings = d.warnings; + let mut facts = d.facts; + if let Ok(exe) = std::env::current_exe() { + facts.insert("ds4_perf_sha256".into(), artifact::hash(&exe)?); + } + let gpu = match helper(gpu_helper) { + Some(path) => { + let before = artifact::hash(&path)?; + facts.insert("gpu_helper_path".into(), path.to_string_lossy().into()); + facts.insert("gpu_helper_sha256".into(), before.clone()); + let command = [ + path.clone().into_os_string(), + "inspect".into(), + device.to_string().into(), + ]; + match runner::run(&command, out, "gpu-inspect").and_then(|_| { + if before != artifact::hash(&path)? { + return Err("GPU helper changed during inspection".into()); + } + let bytes = + std::fs::read(out.join("gpu-inspect.stdout")).map_err(|e| e.to_string())?; + let gpu: Gpu = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + gpu.validate()?; + if gpu.ordinal != device { + return Err("helper inspected another device".into()); + } + Ok(gpu) + }) { + Ok(gpu) => Some(gpu), + Err(error) => { + warnings.push(format!("CUDA device properties unavailable: {error}")); + None + } + } + } + None => { + warnings.push( + "CUDA helper unavailable; build make ds4-perf-gpu or pass --gpu-helper".into(), + ); + None + } + }; + if !processes.ok { + warnings.push("GPU process state unknown".into()); + } + let complete = gpu.is_some() && processes.ok; + let mut machine = Artifact::new(Machine { + facts, + gpu, + gpu_processes: processes.combined(), + process_query_ok: processes.ok, + }); + machine.complete = complete; + machine.warnings = warnings; + raw_inputs(&mut machine, out, "gpu-inspect")?; + machine + .inputs + .push(artifact::reference(&out.join("doctor.txt"), out)?); + artifact::save(&out.join("machine.json"), &machine)?; + Ok(machine) +} + +pub fn run( + out: &Path, + device: usize, + calibrate: bool, + gpu_helper: Option<&Path>, + bench: Option<&str>, +) -> Result<(), String> { + artifact::directory(out)?; + let machine = collect(out, device, gpu_helper, bench)?; + println!( + "machine: {} ({})", + out.join("machine.json").display(), + if machine.complete { + "device properties measured" + } else { + "partial; see warnings" + } + ); + if !calibrate { + return Ok(()); + } + let result = calibrate_device(out, device, gpu_helper, &machine); + let mut calibration = match &result { + Ok(data) => Artifact::new(data.clone()), + Err(error) => Artifact::::failed(error.clone()), + }; + calibration + .inputs + .push(artifact::reference(&out.join("machine.json"), out)?); + raw_inputs(&mut calibration, out, "calibration")?; + artifact::save(&out.join("calibration.json"), &calibration)?; + result?; + println!("calibration: {}", out.join("calibration.json").display()); + Ok(()) +} + +fn calibrate_device( + out: &Path, + device: usize, + gpu_helper: Option<&Path>, + machine: &Artifact, +) -> Result { + let inspected = machine.require()?; + let helper = + helper(gpu_helper).ok_or("calibration requires ds4-perf-gpu; build make ds4-perf-gpu")?; + let hash = artifact::hash(&helper)?; + if inspected.facts.get("gpu_helper_sha256") != Some(&hash) { + return Err("GPU helper changed since inspection".into()); + } + let command: Vec = vec![ + helper.clone().into_os_string(), + "calibrate".into(), + device.to_string().into(), + ]; + runner::run(&command, out, "calibration")?; + if artifact::hash(&helper)? != hash { + return Err("GPU helper changed during calibration".into()); + } + let data: Calibration = serde_json::from_slice( + &std::fs::read(out.join("calibration.stdout")).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + data.validate()?; + if !inspected + .gpu + .as_ref() + .is_some_and(|gpu| gpu.same_device(&data.gpu) && gpu.ordinal == data.gpu.ordinal) + { + return Err("calibration device differs from machine inspection".into()); + } + Ok(data) +} diff --git a/crates/ds4-perf/src/knobs.rs b/crates/ds4-perf/src/knobs.rs new file mode 100644 index 0000000000000000000000000000000000000000..272a7a263d9881a6c3d329ec83b8a9f6190cb357 --- /dev/null +++ b/crates/ds4-perf/src/knobs.rs @@ -0,0 +1,432 @@ +// Only controls whose values are consumed by the native execution path may +// vary inside a matched experiment. Memory/residency identity stays fixed. +pub fn tunable(key: &str) -> bool { + matches!( + key, + "DS4_QWEN_PREFILL_CHUNK" + | "DS4_IQUEST_PREFILL_CHUNK" + | "DS4_QWEN_PLE_WORKERS" + | "DS4_DOTS3_PREFILL_CHUNK" + | "DS4_INKLING_NO_LINEAR" + | "DS4_INKLING_NO_MOE_BATCH" + | "DS4_INKLING_NO_Q8_BATCH" + | "DS4_INKLING_NO_MOE_TILE" + | "DS4_INKLING_NO_LINEAR_TILE" + | "DS4_INKLING_NO_LOGIT_TILE" + | "DS4_INKLING_NO_LINEAR_PANEL" + | "DS4_INKLING_NO_ATTN_GROUP" + | "DS4_INKLING_NO_Q8_TILE" + | "DS4_INKLING_NO_SHARED_Q8" + | "DS4_INKLING_NO_SHARED_TILE" + | "DS4_INKLING_NO_SHARED_DOWN_TILE" + | "DS4_INKLING_NO_Q4_TILE" + | "DS4_INKLING_NO_Q8_ROUTED_TILE" + | "DS4_INKLING_NO_IQ2_ALIGNED" + | "DS4_INKLING_NO_IQ2_XS_ALIGNED" + | "DS4_INKLING_NO_SHARED_SOA" + | "DS4_INKLING_NO_IQ2_LEAN" + | "DS4_INKLING_NO_Q3_TILE" + | "DS4_INKLING_NO_ATTN_TRANSPOSE" + | "DS4_INKLING_NO_SHARED_COLUMN" + | "DS4_INKLING_NO_ATTN_PAIR" + | "DS4_INKLING_NO_Q4_LEAN" + | "DS4_INKLING_NO_SHARED_PIPE" + | "DS4_INKLING_NO_IQ2_SLAB" + | "DS4_INKLING_ATTN_HMMA" + | "DS4_MIMO2_NO_PREFILL_HMMA" + | "DS4_MIMO2_NO_PREFILL_ASYNC" + | "DS4_MIMO2_NO_SWA_HMMA" + | "DS4_MIMO2_SWA_DECODE" + | "DS4_MIMO2_SWA_VEC" + | "DS4_MIMO2_ROUTER_WARP" + | "DS4_MIMO2_DFLASH_CPU" + | "DS4_MIMO2_SWIGLU_Q8" + | "DS4_MIMO2_SUM_RESIDUAL" + | "DS4_MIMO2_ATTN_RESIDUAL" + | "DS4_MIMO2_GATEUP_BOUNDED" + | "DS4_MIMO2_INPUT_Q8_COMPACT" + | "DS4_MIMO2_DOWN_PIPE64" + | "DS4_NAIVE_DECODE_SCORES" + | "DS4_NAIVE_SWA_PREFILL_SCORES" + | "DS4_NAIVE_DSA_DECODE_TILE" + | "DS4_NAIVE_DSA_DIRECT" + | "DS4_NAIVE_SWIGLU_Q8" + | "DS4_NAIVE_INDEX_PACK" + | "DS4_NAIVE_INDEX_U2" + | "DS4_NAIVE_SWA_DECODE_UNIT" + | "DS4_NAIVE_SWA_RING_WALK" + | "DS4_NAIVE_ROUTER_WARP" + | "DS4_NAIVE_SUM_ADD" + | "DS4_INKLING_PREFILL_CHUNK" + | "DS4_CUDA_SOLAR_GQA_CHUNK" + | "DS4_FATTN_HMMA_LDSM" + | "DS4_SOLAR_FATTN_GQA2" + | "DS4_SOLAR_FATTN_WS" + | "DS4_STEP37_PREFILL_CHUNK" + | "DS4_STEP37_NO_SWA_HMMA" + | "DS4_EXAONE_PREFILL_GQA" + | "DS4_LING3VL_PREFILL_CHUNK" + | "DS4_LING3VL_NO_BF16_VEC" + | "DS4_LING3VL_NO_MLA_TILE" + | "DS4_LING3VL_MLA_TILE" + | "DS4_LING3VL_NO_MLA_HMMA" + | "DS4_LING3VL_NO_BF16_REUSE" + | "DS4_LING3VL_NO_BF16_PAIR" + | "DS4_LING3VL_NO_GEMV_XREG" + | "DS4_LING3VL_NO_F32_VEC" + | "DS4_LING3VL_NO_MOE_FUSE" + | "DS4_MMQ_Q5_PAIR" + | "DS4_MMQ_VEC_SANITIZE" + | "DS4_CUDA_LAYER_GRAPHS" + ) +} + +pub fn validate(key: &str, value: &str, family: &str) -> Result<(), String> { + let n = value + .parse::() + .map_err(|_| format!("{key}: expected a positive integer"))?; + let family = family.to_ascii_lowercase(); + let valid = match key { + "DS4_QWEN_PREFILL_CHUNK" => family.starts_with("qwen") && (1..=16384).contains(&n), + "DS4_IQUEST_PREFILL_CHUNK" => family == "iquest-q1" && (1..=8192).contains(&n), + "DS4_QWEN_PLE_WORKERS" => family.starts_with("qwen") && (1..=64).contains(&n), + "DS4_DOTS3_PREFILL_CHUNK" => family.starts_with("dots") && (1..=8192).contains(&n), + // Native diagnostic switches test presence; "0" would still disable. + "DS4_INKLING_NO_LINEAR" + | "DS4_INKLING_NO_MOE_BATCH" + | "DS4_INKLING_NO_Q8_BATCH" + | "DS4_INKLING_NO_MOE_TILE" + | "DS4_INKLING_NO_LINEAR_TILE" + | "DS4_INKLING_NO_LOGIT_TILE" + | "DS4_INKLING_NO_LINEAR_PANEL" + | "DS4_INKLING_NO_ATTN_GROUP" + | "DS4_INKLING_NO_Q8_TILE" + | "DS4_INKLING_NO_SHARED_Q8" + | "DS4_INKLING_NO_SHARED_TILE" + | "DS4_INKLING_NO_SHARED_DOWN_TILE" + | "DS4_INKLING_NO_Q4_TILE" + | "DS4_INKLING_NO_Q8_ROUTED_TILE" + | "DS4_INKLING_NO_IQ2_ALIGNED" + | "DS4_INKLING_NO_IQ2_XS_ALIGNED" + | "DS4_INKLING_NO_SHARED_SOA" + | "DS4_INKLING_NO_IQ2_LEAN" + | "DS4_INKLING_NO_Q3_TILE" + | "DS4_INKLING_NO_ATTN_TRANSPOSE" + | "DS4_INKLING_NO_SHARED_COLUMN" + | "DS4_INKLING_NO_ATTN_PAIR" + | "DS4_INKLING_NO_Q4_LEAN" + | "DS4_INKLING_NO_SHARED_PIPE" + | "DS4_INKLING_NO_IQ2_SLAB" + | "DS4_INKLING_ATTN_HMMA" => family == "inkling" && value == "1", + "DS4_MIMO2_NO_PREFILL_HMMA" | "DS4_MIMO2_NO_PREFILL_ASYNC" | "DS4_MIMO2_NO_SWA_HMMA" => { + family == "mimo2" && value == "1" + } + "DS4_MIMO2_SWA_DECODE" | "DS4_MIMO2_SWA_VEC" | "DS4_MIMO2_ROUTER_WARP" => { + family == "mimo2" && matches!(value, "0" | "1") + } + "DS4_MIMO2_DFLASH_CPU" => family == "mimo2" && value == "1", + "DS4_MIMO2_SWIGLU_Q8" => family == "mimo2" && matches!(value, "0" | "1"), + "DS4_MIMO2_SUM_RESIDUAL" + | "DS4_MIMO2_ATTN_RESIDUAL" + | "DS4_MIMO2_GATEUP_BOUNDED" + | "DS4_MIMO2_INPUT_Q8_COMPACT" + | "DS4_MIMO2_DOWN_PIPE64" => family == "mimo2" && matches!(value, "0" | "1"), + "DS4_NAIVE_DECODE_SCORES" + | "DS4_NAIVE_SWA_PREFILL_SCORES" + | "DS4_NAIVE_DSA_DECODE_TILE" + | "DS4_NAIVE_DSA_DIRECT" + | "DS4_NAIVE_SWIGLU_Q8" + | "DS4_NAIVE_INDEX_PACK" + | "DS4_NAIVE_INDEX_U2" + | "DS4_NAIVE_SWA_DECODE_UNIT" + | "DS4_NAIVE_SWA_RING_WALK" + | "DS4_NAIVE_ROUTER_WARP" + | "DS4_NAIVE_SUM_ADD" => family == "naive_n05_flash" && matches!(value, "0" | "1"), + "DS4_INKLING_PREFILL_CHUNK" => family == "inkling" && (1..=8192).contains(&n), + "DS4_CUDA_SOLAR_GQA_CHUNK" => { + family.starts_with("solar") && [64, 128, 256, 512, 1024, 2048].contains(&n) + } + "DS4_FATTN_HMMA_LDSM" | "DS4_SOLAR_FATTN_GQA2" | "DS4_SOLAR_FATTN_WS" => { + family == "solar-open2" && matches!(value, "0" | "1") + } + "DS4_STEP37_PREFILL_CHUNK" => family.starts_with("step") && (1..=4096).contains(&n), + "DS4_LING3VL_PREFILL_CHUNK" => { + (family.starts_with("ling") || family == "bailingmoe3") && (1..=4096).contains(&n) + } + "DS4_LING3VL_NO_BF16_VEC" + | "DS4_LING3VL_NO_MLA_TILE" + | "DS4_LING3VL_MLA_TILE" + | "DS4_LING3VL_NO_MLA_HMMA" + | "DS4_LING3VL_NO_BF16_REUSE" + | "DS4_LING3VL_NO_BF16_PAIR" + | "DS4_LING3VL_NO_GEMV_XREG" + | "DS4_LING3VL_NO_F32_VEC" + | "DS4_LING3VL_NO_MOE_FUSE" + | "DS4_MMQ_VEC_SANITIZE" => { + (family.starts_with("ling") || family == "bailingmoe3") && value == "1" + } + "DS4_MMQ_Q5_PAIR" => { + (family.starts_with("ling") || family == "bailingmoe3") && value == "0" + } + "DS4_CUDA_LAYER_GRAPHS" => { + (family.starts_with("ling") || family == "bailingmoe3") && matches!(value, "0" | "1") + } + "DS4_STEP37_NO_SWA_HMMA" => family.starts_with("step") && value == "1", + "DS4_EXAONE_PREFILL_GQA" => { + (family.starts_with("step") || family.starts_with("exaone") || family.starts_with("k2")) + && value == "0" + } + _ => false, + }; + if !valid { + return Err(format!( + "unsupported experiment control/value for {family}: {key}={value}" + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + fn iquest_chunk_is_scoped_and_bounded() { + assert!(super::tunable("DS4_IQUEST_PREFILL_CHUNK")); + assert!(super::validate("DS4_IQUEST_PREFILL_CHUNK", "8192", "IQuest-Q1").is_ok()); + assert!(super::validate("DS4_IQUEST_PREFILL_CHUNK", "0", "iquest-q1").is_err()); + assert!(super::validate("DS4_IQUEST_PREFILL_CHUNK", "8193", "iquest-q1").is_err()); + assert!(super::validate("DS4_IQUEST_PREFILL_CHUNK", "128", "qwen4exp").is_err()); + } + use super::*; + + #[test] + fn naive_attention_controls() { + for key in [ + "DS4_NAIVE_DECODE_SCORES", + "DS4_NAIVE_SWA_PREFILL_SCORES", + "DS4_NAIVE_DSA_DECODE_TILE", + "DS4_NAIVE_DSA_DIRECT", + "DS4_NAIVE_SWIGLU_Q8", + "DS4_NAIVE_INDEX_PACK", + "DS4_NAIVE_INDEX_U2", + "DS4_NAIVE_SWA_DECODE_UNIT", + "DS4_NAIVE_SWA_RING_WALK", + "DS4_NAIVE_ROUTER_WARP", + "DS4_NAIVE_SUM_ADD", + ] { + assert!(tunable(key)); + for value in ["0", "1"] { + assert!(validate(key, value, "naive_n05_flash").is_ok()); + } + for value in ["2", "01", "true", ""] { + assert!(validate(key, value, "naive_n05_flash").is_err()); + } + assert!(validate(key, "1", "mimo2").is_err()); + } + } + + #[test] + fn solar_attention_controls() { + for key in [ + "DS4_FATTN_HMMA_LDSM", + "DS4_SOLAR_FATTN_GQA2", + "DS4_SOLAR_FATTN_WS", + ] { + assert!(tunable(key)); + for value in ["0", "1"] { + assert!(validate(key, value, "solar-open2").is_ok()); + } + for value in ["2", "01", "true", ""] { + assert!(validate(key, value, "solar-open2").is_err()); + } + assert!(validate(key, "1", "inkling").is_err()); + } + assert!(!tunable("DS4_METAL_PREFILL_CHUNK")); + assert!(!tunable("DS4_SOLAR_KV_FORMAT")); + } + + #[test] + fn inkling_controls() { + for key in [ + "DS4_INKLING_NO_LINEAR", + "DS4_INKLING_NO_MOE_BATCH", + "DS4_INKLING_NO_Q8_BATCH", + "DS4_INKLING_NO_MOE_TILE", + "DS4_INKLING_NO_LINEAR_TILE", + "DS4_INKLING_NO_LOGIT_TILE", + "DS4_INKLING_NO_LINEAR_PANEL", + "DS4_INKLING_NO_ATTN_GROUP", + "DS4_INKLING_NO_Q8_TILE", + "DS4_INKLING_NO_SHARED_Q8", + "DS4_INKLING_NO_SHARED_TILE", + "DS4_INKLING_NO_SHARED_DOWN_TILE", + "DS4_INKLING_NO_Q4_TILE", + "DS4_INKLING_NO_Q8_ROUTED_TILE", + "DS4_INKLING_NO_IQ2_ALIGNED", + "DS4_INKLING_NO_IQ2_XS_ALIGNED", + "DS4_INKLING_NO_SHARED_SOA", + "DS4_INKLING_NO_IQ2_LEAN", + "DS4_INKLING_NO_Q3_TILE", + "DS4_INKLING_NO_ATTN_TRANSPOSE", + "DS4_INKLING_NO_SHARED_COLUMN", + "DS4_INKLING_NO_ATTN_PAIR", + "DS4_INKLING_NO_Q4_LEAN", + "DS4_INKLING_NO_SHARED_PIPE", + "DS4_INKLING_NO_IQ2_SLAB", + "DS4_INKLING_ATTN_HMMA", + ] { + assert!(tunable(key)); + assert!(validate(key, "1", "Inkling").is_ok()); + for value in ["0", "2", "01", "true", ""] { + assert!(validate(key, value, "inkling").is_err()); + } + for family in ["qwen", "inkling-other", ""] { + assert!(validate(key, "1", family).is_err()); + } + } + assert!(tunable("DS4_INKLING_PREFILL_CHUNK")); + for value in ["1", "512", "1024", "2048", "2049", "8192"] { + assert!(validate("DS4_INKLING_PREFILL_CHUNK", value, "inkling").is_ok()); + } + for value in ["0", "8193", "x"] { + assert!(validate("DS4_INKLING_PREFILL_CHUNK", value, "inkling").is_err()); + } + assert!(validate("DS4_INKLING_PREFILL_CHUNK", "512", "qwen").is_err()); + assert!(!tunable("DS4_INKLING_UNKNOWN")); + } + + #[test] + fn mimo2_prefill_hmma_kill() { + assert!(tunable("DS4_MIMO2_NO_PREFILL_HMMA")); + assert!(validate("DS4_MIMO2_NO_PREFILL_HMMA", "1", "mimo2").is_ok()); + assert!(validate("DS4_MIMO2_NO_PREFILL_HMMA", "1", "MiMo2").is_ok()); + for value in ["0", "2", "01", "true", ""] { + assert!(validate("DS4_MIMO2_NO_PREFILL_HMMA", value, "mimo2").is_err()); + } + assert!(validate("DS4_MIMO2_NO_PREFILL_HMMA", "1", "inkling").is_err()); + assert!(tunable("DS4_MIMO2_NO_PREFILL_ASYNC")); + assert!(validate("DS4_MIMO2_NO_PREFILL_ASYNC", "1", "mimo2").is_ok()); + assert!(validate("DS4_MIMO2_NO_PREFILL_ASYNC", "0", "mimo2").is_err()); + assert!(tunable("DS4_MIMO2_NO_SWA_HMMA")); + assert!(validate("DS4_MIMO2_NO_SWA_HMMA", "1", "mimo2").is_ok()); + assert!(validate("DS4_MIMO2_NO_SWA_HMMA", "0", "mimo2").is_err()); + } + + #[test] + fn mimo2_decode_controls() { + for key in [ + "DS4_MIMO2_SWA_DECODE", + "DS4_MIMO2_SWA_VEC", + "DS4_MIMO2_ROUTER_WARP", + ] { + assert!(tunable(key)); + for value in ["0", "1"] { + assert!(validate(key, value, "mimo2").is_ok()); + } + for value in ["2", "01", "true", ""] { + assert!(validate(key, value, "mimo2").is_err()); + } + assert!(validate(key, "1", "qwen").is_err()); + } + } + + #[test] + fn mimo2_swiglu_control() { + let key = "DS4_MIMO2_SWIGLU_Q8"; + assert!(tunable(key)); + assert!(validate(key, "0", "mimo2").is_ok()); + assert!(validate(key, "1", "mimo2").is_ok()); + assert!(validate(key, "2", "mimo2").is_err()); + assert!(validate(key, "1", "qwen").is_err()); + } + + #[test] + fn mimo2_fusion_controls() { + for key in [ + "DS4_MIMO2_SUM_RESIDUAL", + "DS4_MIMO2_ATTN_RESIDUAL", + "DS4_MIMO2_GATEUP_BOUNDED", + "DS4_MIMO2_INPUT_Q8_COMPACT", + "DS4_MIMO2_DOWN_PIPE64", + ] { + assert!(tunable(key), "{key} is missing from matched comparisons"); + for value in ["0", "1"] { + assert!(validate(key, value, "mimo2").is_ok()); + assert!(validate(key, value, "MiMo2").is_ok()); + } + for value in ["2", "01", "true", ""] { + assert!(validate(key, value, "mimo2").is_err()); + } + assert!(validate(key, "1", "qwen").is_err()); + } + } + + #[test] + fn mimo2_dflash_control() { + let key = "DS4_MIMO2_DFLASH_CPU"; + assert!(tunable(key)); + assert!(validate(key, "1", "mimo2").is_ok()); + assert!(validate(key, "0", "mimo2").is_err()); + assert!(validate(key, "1", "qwen").is_err()); + } + + #[test] + fn step37_controls() { + assert!(tunable("DS4_STEP37_PREFILL_CHUNK")); + assert!(tunable("DS4_STEP37_NO_SWA_HMMA")); + assert!(tunable("DS4_EXAONE_PREFILL_GQA")); + assert!(validate("DS4_EXAONE_PREFILL_GQA", "0", "step37").is_ok()); + assert!(validate("DS4_EXAONE_PREFILL_GQA", "0", "k2-horizon").is_ok()); + assert!(validate("DS4_EXAONE_PREFILL_GQA", "0", "k2").is_ok()); + assert!(validate("DS4_EXAONE_PREFILL_GQA", "1", "step37").is_err()); + assert!(validate("DS4_EXAONE_PREFILL_GQA", "0", "qwen").is_err()); + for value in ["1", "512", "1024", "2048", "4096"] { + assert!(validate("DS4_STEP37_PREFILL_CHUNK", value, "step37").is_ok()); + } + for value in ["0", "4097", "x"] { + assert!(validate("DS4_STEP37_PREFILL_CHUNK", value, "step37").is_err()); + } + assert!(validate("DS4_STEP37_PREFILL_CHUNK", "512", "qwen").is_err()); + assert!(validate("DS4_STEP37_NO_SWA_HMMA", "1", "step37").is_ok()); + assert!(validate("DS4_STEP37_NO_SWA_HMMA", "1", "qwen").is_err()); + for value in ["0", "2", ""] { + assert!(validate("DS4_STEP37_NO_SWA_HMMA", value, "step37").is_err()); + } + } + + #[test] + fn ling3vl_controls() { + assert!(tunable("DS4_LING3VL_PREFILL_CHUNK")); + for value in ["1", "512", "1024", "2048", "4096"] { + assert!(validate("DS4_LING3VL_PREFILL_CHUNK", value, "ling3vl").is_ok()); + assert!(validate("DS4_LING3VL_PREFILL_CHUNK", value, "ling").is_ok()); + } + for value in ["0", "4097", "x"] { + assert!(validate("DS4_LING3VL_PREFILL_CHUNK", value, "ling3vl").is_err()); + } + assert!(validate("DS4_LING3VL_PREFILL_CHUNK", "2048", "qwen").is_err()); + for key in [ + "DS4_LING3VL_NO_BF16_VEC", + "DS4_LING3VL_NO_MLA_TILE", + "DS4_LING3VL_MLA_TILE", + "DS4_LING3VL_NO_MLA_HMMA", + "DS4_LING3VL_NO_BF16_REUSE", + "DS4_LING3VL_NO_BF16_PAIR", + "DS4_LING3VL_NO_GEMV_XREG", + "DS4_LING3VL_NO_F32_VEC", + "DS4_LING3VL_NO_MOE_FUSE", + "DS4_MMQ_VEC_SANITIZE", + ] { + assert!(tunable(key)); + assert!(validate(key, "1", "ling3vl").is_ok()); + assert!(validate(key, "0", "ling3vl").is_err()); + assert!(validate(key, "1", "qwen").is_err()); + } + assert!(tunable("DS4_MMQ_Q5_PAIR")); + assert!(validate("DS4_MMQ_Q5_PAIR", "0", "ling3vl").is_ok()); + assert!(validate("DS4_MMQ_Q5_PAIR", "1", "ling3vl").is_err()); + assert!(tunable("DS4_CUDA_LAYER_GRAPHS")); + assert!(validate("DS4_CUDA_LAYER_GRAPHS", "0", "ling3vl").is_ok()); + assert!(validate("DS4_CUDA_LAYER_GRAPHS", "1", "ling3vl").is_ok()); + assert!(validate("DS4_CUDA_LAYER_GRAPHS", "0", "qwen").is_err()); + } +} diff --git a/crates/ds4-perf/src/lib.rs b/crates/ds4-perf/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..e933daf22ec720d13e84939fb076a98674e7d714 --- /dev/null +++ b/crates/ds4-perf/src/lib.rs @@ -0,0 +1,5 @@ +#![forbid(unsafe_code)] + +pub mod artifact; +pub mod machine; +pub mod trace; diff --git a/crates/ds4-perf/src/machine.rs b/crates/ds4-perf/src/machine.rs new file mode 100644 index 0000000000000000000000000000000000000000..a32af7408515c103c2671f434ab41ffd305ff9bc --- /dev/null +++ b/crates/ds4-perf/src/machine.rs @@ -0,0 +1,146 @@ +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Gpu { + pub ordinal: usize, + pub name: String, + pub uuid: String, + pub compute_major: i32, + pub compute_minor: i32, + pub driver_version: i32, + pub total_memory_bytes: u64, + pub free_memory_bytes: u64, + pub multiprocessors: i32, + pub warp_size: i32, + pub max_threads_per_block: i32, + pub max_threads_per_sm: i32, + pub max_blocks_per_sm: i32, + pub registers_per_sm: i32, + pub shared_bytes_per_sm: i32, + pub shared_bytes_per_block: i32, + pub l2_bytes: i32, + pub memory_bus_bits: Option, + pub memory_clock_khz: Option, + pub clock_khz: Option, + pub unavailable: BTreeMap, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Machine { + pub facts: BTreeMap, + pub gpu: Option, + pub gpu_processes: String, + pub process_query_ok: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Calibration { + pub gpu: Gpu, + pub transfer_bytes: u64, + pub warmups: u32, + pub repeats: u32, + pub copy_gb_s: Vec, + pub fp32_gflop_s: Vec, + pub launch_us: Vec, + pub method: String, + pub source_sha256: String, + pub ptx_sha256: String, + pub copy_ms: Vec, + pub compute_ms: Vec, + pub launch_batch_ms: Vec, + pub compute_threads: u32, + pub compute_block_threads: u32, + pub fma_iterations: u32, + pub launch_count: u32, + pub notes: Vec, +} + +impl crate::artifact::Payload for Machine { + const KIND: &'static str = "machine"; + fn validate(&self) -> Result<(), String> { + if let Some(gpu) = &self.gpu { + gpu.validate()?; + } + Ok(()) + } +} + +impl Gpu { + pub fn validate(&self) -> Result<(), String> { + if self.uuid.is_empty() + || self.name.is_empty() + || self.multiprocessors <= 0 + || self.warp_size <= 0 + || self.max_threads_per_sm <= 0 + || self.max_threads_per_block <= 0 + || self.max_blocks_per_sm <= 0 + || self.total_memory_bytes == 0 + || self.free_memory_bytes > self.total_memory_bytes + { + return Err("invalid CUDA device properties".into()); + } + Ok(()) + } + /// Free memory is a dynamic observation, excluded from comparison identity. + pub fn same_device(&self, other: &Self) -> bool { + self.uuid == other.uuid + && self.driver_version == other.driver_version + && self.compute_major == other.compute_major + && self.compute_minor == other.compute_minor + } +} + +impl crate::artifact::Payload for Calibration { + const KIND: &'static str = "calibration"; + fn validate(&self) -> Result<(), String> { + self.gpu.validate()?; + if self.method != "cuda-event-copy-fp32-fma-v1" + || self.source_sha256.len() != 64 + || self.ptx_sha256.len() != 64 + || self.repeats == 0 + || self.repeats > 100 + || self.transfer_bytes <= self.gpu.l2_bytes.max(0) as u64 + || self.compute_threads == 0 + || self.compute_block_threads == 0 + || self.fma_iterations == 0 + || self.launch_count == 0 + { + return Err("invalid calibration method or workload".into()); + } + for values in [ + &self.copy_gb_s, + &self.fp32_gflop_s, + &self.launch_us, + &self.copy_ms, + &self.compute_ms, + &self.launch_batch_ms, + ] { + if values.len() != self.repeats as usize + || values.iter().any(|v| !v.is_finite() || *v <= 0.0) + { + return Err("invalid calibration samples".into()); + } + } + for i in 0..self.repeats as usize { + let calculated = [ + 2.0 * self.transfer_bytes as f64 / (self.copy_ms[i] * 1e6), + self.compute_threads as f64 * self.fma_iterations as f64 * 16.0 + / (self.compute_ms[i] * 1e6), + self.launch_batch_ms[i] * 1000.0 / self.launch_count as f64, + ]; + let reported = [self.copy_gb_s[i], self.fp32_gflop_s[i], self.launch_us[i]]; + if calculated + .iter() + .zip(reported) + .any(|(a, b)| (a - b).abs() > a.abs() * 1e-9) + { + return Err("calibration rates disagree with raw timings".into()); + } + } + Ok(()) + } +} diff --git a/crates/ds4-perf/src/main.rs b/crates/ds4-perf/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..250c7523f317e7b1af717d769be5d45154b9bd22 --- /dev/null +++ b/crates/ds4-perf/src/main.rs @@ -0,0 +1,58 @@ +#![forbid(unsafe_code)] + +mod bench; +mod cli; +mod compare; +mod csv; +mod cupti; +mod doctor; +mod experiment; +mod fit; +mod inspect; +mod knobs; +mod ncu; +mod nsys; +mod optimize; +mod process; +mod report; +mod runner; +mod serving; +mod serving_http; +mod timeline; +mod workload; + +use clap::Parser; + +fn main() { + if let Err(error) = process::signals() { + eprintln!("ds4-perf: signal handler: {error}"); + std::process::exit(1); + } + let result = match cli::Cli::parse().command { + cli::Command::Inspect(args) => inspect::run( + &args.out, + args.device, + args.calibrate, + args.gpu_helper.as_deref(), + args.bench.as_deref(), + ), + cli::Command::Doctor { bench } => { + print!( + "{}", + doctor::inspect(&mut runner::System, Some(&bench)).render() + ); + Ok(()) + } + cli::Command::Scout(args) => runner::scout(&args), + cli::Command::Compare(args) => compare::run(&args), + cli::Command::Optimize(args) => optimize::run(&args), + cli::Command::Serving(args) => serving::run(&args), + cli::Command::ServingControls(args) => serving::controls(&args), + cli::Command::ServingProfile(args) => serving::profile(&args), + cli::Command::ApplyProfile(args) => serving::apply(&args), + }; + if let Err(error) = result { + eprintln!("ds4-perf: {error}"); + std::process::exit(1); + } +} diff --git a/crates/ds4-perf/src/ncu.rs b/crates/ds4-perf/src/ncu.rs new file mode 100644 index 0000000000000000000000000000000000000000..3c8fb2dd68b1ecb5df7a3f8d2cb9deab2408ab09 --- /dev/null +++ b/crates/ds4-perf/src/ncu.rs @@ -0,0 +1,350 @@ +use crate::{csv, experiment, nsys, runner}; +use ds4_perf::artifact::{self, Artifact, Payload}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, ffi::OsString, path::Path}; + +const METRICS: &[&str] = &[ + "gpu__time_duration.sum", + "sm__throughput.avg.pct_of_peak_sustained_elapsed", + "dram__throughput.avg.pct_of_peak_sustained_elapsed", + "sm__warps_active.avg.pct_of_peak_sustained_active", + "launch__registers_per_thread", + "launch__grid_size", + "launch__block_size", + "launch__waves_per_multiprocessor", +]; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Metric { + pub name: String, + pub unit: String, + pub value: Option, + pub unavailable: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Target { + pub selection: String, + pub launch: Option, + pub phase: String, + pub kernel: String, + pub metrics: Vec, + pub captured: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Counters { + pub targets: Vec, + pub limits: Vec, +} + +impl Payload for Counters { + const KIND: &'static str = "ncu"; + fn validate(&self) -> Result<(), String> { + for target in &self.targets { + if !nsys::PHASES.contains(&target.phase.as_str()) + || (target.captured && target.launch.is_none()) + { + return Err("invalid NCU phase".into()); + } + for m in &target.metrics { + if m.value.is_some_and(|v| !v.is_finite() || v < 0.0) + || m.value.is_some() == m.unavailable.is_some() + { + return Err("invalid NCU metric status".into()); + } + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LaunchIdentity { + pub id: String, + pub kernel: String, + pub process: String, + pub device: String, + pub grid: String, + pub block: String, +} + +fn identity(input: impl std::io::BufRead, expected: &str) -> Result { + let mut identity = None; + csv::table(input, "metric name", |header, row| { + let get = |name| { + csv::value(row, header, &[name]) + .map(str::to_string) + .ok_or_else(|| format!("NCU launch identity lacks {name}")) + }; + let current = LaunchIdentity { + id: get("id")?, + kernel: get("kernel name")?, + process: get("process id")?, + device: get("device")?, + grid: get("grid size")?, + block: get("block size")?, + }; + if current.kernel != expected + || identity + .as_ref() + .is_some_and(|previous| *previous != current) + { + return Err("NCU launch identity differs from filtered target".into()); + } + identity = Some(current); + Ok(()) + })?; + identity.ok_or("NCU captured no identified launch".into()) +} + +fn metrics(input: impl std::io::BufRead) -> Result, String> { + let mut result = BTreeMap::new(); + let mut launches = std::collections::BTreeSet::new(); + csv::table(input, "metric name", |header, row| { + let name = csv::value(row, header, &["metric name"]).ok_or("NCU metric name missing")?; + let unit = csv::value(row, header, &["metric unit"]).unwrap_or(""); + let raw = csv::value(row, header, &["metric value"]).unwrap_or("missing"); + let value = csv::number(&raw.replace(',', "")).ok(); + if let Some(id) = csv::value(row, header, &["id"]) { + launches.insert(id.to_string()); + } + if result + .insert( + name.to_string(), + Metric { + name: name.into(), + unit: unit.into(), + value, + unavailable: value.is_none().then(|| raw.into()), + }, + ) + .is_some() + { + return Err("duplicate NCU metric; expected one bounded launch".into()); + } + Ok(()) + })?; + if launches.len() != 1 { + return Err("NCU needs exactly one launch ID".into()); + } + for name in METRICS { + result.entry((*name).into()).or_insert_with(|| Metric { + name: (*name).into(), + unit: String::new(), + value: None, + unavailable: Some("not reported by installed NCU/device".into()), + }); + } + Ok(result.into_values().collect()) +} + +pub fn run( + out: &Path, + command: &[OsString], + env: &BTreeMap, + scout: &Artifact, + limits: crate::process::Limits, +) -> Result<(), String> { + let dir = out.join("ncu"); + artifact::directory(&dir)?; + let mut warnings = Vec::new(); + let mut targets = Vec::new(); + let result = (|| { + runner::run_limited(&["ncu".into(), "--help".into()], &dir, "help", env, limits)?; + let help = std::fs::read_to_string(dir.join("help.stdout")).map_err(|e| e.to_string())?; + runner::run_limited( + &[ + "ncu".into(), + "--query-metrics".into(), + "--query-metrics-mode".into(), + "all".into(), + ], + &dir, + "query", + env, + limits, + )?; + let query = std::fs::read_to_string(dir.join("query.stdout")).map_err(|e| e.to_string())?; + let supported: Vec<_> = METRICS + .iter() + .filter(|m| query.contains(**m)) + .copied() + .collect(); + if !supported.contains(&"gpu__time_duration.sum") { + return Err("NCU cannot measure kernel duration on this device".into()); + } + for phase in nsys::PHASES { + let Some(kernel) = scout + .require()? + .evidence + .phases + .get(phase) + .and_then(|p| p.kernels.first()) + else { + warnings.push(format!("{phase}: no measured kernel to target")); + continue; + }; + let label = phase.replace('.', "-"); + let mut capture = crate::report::bounded_ncu(&help, phase, &kernel.name, &[]) + .ok_or("NCU lacks bounded NVTX/kernel filtering")?; + for required in [ + "--target-processes", + "--print-kernel-base", + "--rename-kernels", + "--print-units", + ] { + if !help.contains(required) { + return Err(format!( + "NCU cannot verify bounded process/kernel identity: lacks {required}" + )); + } + } + let base = if kernel.name.starts_with("_Z") { + "mangled" + } else if kernel.name.contains(['<', '(']) { + "demangled" + } else { + "function" + }; + capture.extend([ + "--target-processes".into(), + "application-only".into(), + "--print-kernel-base".into(), + base.into(), + "--rename-kernels".into(), + "0".into(), + "--csv".into(), + "--page".into(), + "details".into(), + "--print-units".into(), + "base".into(), + "--metrics".into(), + supported.join(",").into(), + "--export".into(), + dir.join(&label).into_os_string(), + ]); + capture.extend_from_slice(command); + let mut captured = runner::run_limited(&capture, &dir, &label, env, limits).is_ok(); + if !dir.join(format!("{label}.ncu-rep")).is_file() { + captured = false; + } + let normalized = metrics( + csv::open(&dir.join(format!("{label}.stdout"))).map_err(|e| e.to_string())?, + )?; + let launch = identity( + csv::open(&dir.join(format!("{label}.stdout"))).map_err(|e| e.to_string())?, + &kernel.name, + ); + if let Err(error) = &launch { + warnings.push(format!("{phase}: {error}")); + captured = false; + } + captured &= normalized + .iter() + .any(|m| m.name == "gpu__time_duration.sum" && m.value.is_some_and(|v| v > 0.0)); + if !captured { + warnings.push(format!( + "{phase}: NCU capture incomplete; see raw stdout/stderr" + )); + } + targets.push(Target { + selection:"first matching instance of the phase's top aggregate kernel; not necessarily its dominant geometry".into(), + launch:launch.ok(), + phase: phase.into(), + kernel: kernel.name.clone(), + metrics: normalized, + captured, + }); + } + if targets.is_empty() { + return Err("NCU requires a measured prefill/decode target".into()); + } + Ok::<_, String>(()) + })(); + if let Err(e) = &result { + warnings.push(e.clone()); + } + let complete = result.is_ok() + && !targets.is_empty() + && targets.iter().all(|t| t.captured) + && warnings.is_empty(); + let mut artifact = Artifact::new(Counters {targets,limits:vec!["One matching launch per phase with application replay and strict all-attribute matching. Replay/cache controls affect counters; NCU throughput is excluded from experiment speed comparisons.".into(),"Missing counters remain unknown. High utilization alone does not establish a causal bottleneck.".into()]}); + artifact.complete = complete; + artifact.warnings = warnings; + artifact.inputs = experiment::raw_refs(out, &dir)?; + for name in ["collection.json", "fit.json"] { + let path = out.join(name); + if path.exists() { + artifact.inputs.push(artifact::reference(&path, out)?); + } + } + artifact::save(&out.join("ncu.json"), &artifact)?; + if !complete { + return Err(format!( + "NCU incomplete; see {}", + out.join("ncu.json").display() + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn parses_live_details_export() { + let bytes = include_bytes!("../tests/fixtures/ncu-details.csv"); + let launch = identity(&bytes[..], "ds4_moe_worklist_mmq_kernel").unwrap(); + assert_eq!(launch.grid, "(48, 1, 1)"); + assert_eq!(launch.block, "(32, 8, 1)"); + let values = metrics(&bytes[..]).unwrap(); + let duration = values + .iter() + .find(|m| m.name == "gpu__time_duration.sum") + .unwrap(); + assert_eq!(duration.unit, "ns"); + assert_eq!(duration.value, Some(3101888.0)); + assert!(values + .iter() + .find(|m| m.name.starts_with("dram__")) + .unwrap() + .value + .is_none()); + } + + #[test] + fn requires_launch_identity() { + assert!(metrics( + b"Metric Name,Metric Unit,Metric Value\ngpu__time_duration.sum,nsecond,100\n" + .as_slice() + ) + .is_err()); + } + #[test] + fn counter_units_and_missing() { + let m = metrics(b"ID,Metric Name,Metric Unit,Metric Value\n0,gpu__time_duration.sum,nsecond,\"1,234.5\"\n0,launch__block_size,thread,NA\n".as_slice()).unwrap(); + assert_eq!( + m.iter() + .find(|m| m.name == "gpu__time_duration.sum") + .unwrap() + .value, + Some(1234.5) + ); + assert_eq!( + m.iter() + .find(|m| m.name == "launch__block_size") + .unwrap() + .value, + None + ); + assert!( + metrics(b"ID,Metric Name,Metric Unit,Metric Value\n0,x,,1\n1,x,,2\n".as_slice()) + .is_err() + ); + } +} diff --git a/crates/ds4-perf/src/nsys.rs b/crates/ds4-perf/src/nsys.rs new file mode 100644 index 0000000000000000000000000000000000000000..806455590e79d8a162f86636b641fbec9e5f4d76 --- /dev/null +++ b/crates/ds4-perf/src/nsys.rs @@ -0,0 +1,315 @@ +use crate::{csv, doctor::Capabilities}; +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::io::BufRead; +use std::path::Path; + +pub const PHASES: [&str; 2] = ["ds4.prefill", "ds4.decode"]; + +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Kernel { + pub name: String, + pub total_ns: f64, + pub count: Option, +} + +#[derive(Clone, Default, Debug, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Phase { + pub name: String, + pub wall_ns: Option, + pub projected_ns: Option, + pub busy_ns: Option, + pub mem_ns: Option, + pub gap_ns: Option, + pub kernels: Vec, +} + +#[derive(Clone, Default, Debug, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Evidence { + pub phases: BTreeMap, + pub global: Vec, + pub warnings: Vec, +} + +pub fn phase_name(name: &str) -> Option<&'static str> { + let name = name.strip_prefix(':').unwrap_or(name); + PHASES.into_iter().find(|p| *p == name) +} + +pub fn profile_command(caps: &Capabilities, out: &Path, bench: &[OsString]) -> Vec { + let mut args: Vec = ["nsys", "profile", "--trace=cuda,nvtx"] + .map(Into::into) + .into(); + for option in ["--sample", "--cpuctxsw"] { + if caps.profile_help.contains(option) { + args.push(format!("{option}=none").into()); + } + } + if caps.graph_nodes() { + args.push("--cuda-graph-trace=node".into()); + } + args.push("--output".into()); + args.push(out.join("trace").into()); + args.extend_from_slice(bench); + args +} + +pub fn collect_prepared( + caps: &Capabilities, + out: &Path, + prepared: &crate::experiment::Prepared, +) -> Evidence { + collect_with(caps, out, |cmd, out, name| prepared.run(cmd, out, name)) +} + +fn collect_with( + caps: &Capabilities, + out: &Path, + mut run: impl FnMut(&[OsString], &Path, &str) -> Result<(), String>, +) -> Evidence { + let mut evidence = Evidence::default(); + let mut input = out.join("trace.nsys-rep"); + // Keep each stdout/stderr verbatim. In particular cuda_api_sync is an + // analyze rule in newer Nsight, not a stats report. + for (requested, file) in [ + ("nvtx_gpu_proj_sum", "nsys-nvtx"), + ("nvtx_kern_sum:base", "nsys-nvtx-kernels"), + // Keep global CUDA kernels even when NVTX_EVENTS is absent. + ("cuda_gpu_kern_sum:base", "nsys-kernels"), + ("cuda_gpu_mem_time_sum", "nsys-memory"), + ("nvtx_pushpop_trace", "nsys-ranges"), + ("cuda_gpu_trace:base", "nsys-gpu-trace"), + ("cuda_api_sync", "nsys-sync"), + ] { + let rule = requested == "cuda_api_sync" && caps.sync_rule; + let selected = if rule { + Some(requested.into()) + } else { + caps.report(requested) + }; + let Some(selected) = selected else { + evidence + .warnings + .push(format!("{requested}: unsupported by installed nsys")); + continue; + }; + let mut command: Vec = [ + "nsys", + if rule { "analyze" } else { "stats" }, + "--format", + "csv:noconv", + if rule { "--rule" } else { "--report" }, + ] + .map(Into::into) + .into(); + command.push(selected.into()); + command.push(input.clone().into()); + match run(&command, out, file) { + Ok(()) => { + // Reuse the completed export. Reopening the .nsys-rep can + // trigger Nsight's stale-export check between reports. + let sqlite = out.join("trace.sqlite"); + if sqlite.is_file() { + input = sqlite; + } + if let Err(err) = std::fs::copy( + out.join(format!("{file}.stdout")), + out.join(format!("{file}.csv")), + ) { + evidence.warnings.push(err.to_string()); + } + } + Err(err) => evidence.warnings.push(err), + } + } + load(out, evidence) +} + +fn load(out: &Path, mut evidence: Evidence) -> Evidence { + for file in ["nsys-nvtx-kernels", "nsys-kernels"] { + let Ok(reader) = csv::open(&out.join(format!("{file}.csv"))) else { + continue; + }; + match kernels(reader) { + Ok(groups) => { + for (name, kernels) in groups { + if name.is_empty() { + evidence.global = kernels; + continue; + } + let phase = evidence + .phases + .entry(name.clone()) + .or_insert_with(|| Phase { + name, + ..Default::default() + }); + // Prefer explicit NVTX association; never add the same report twice. + if phase.kernels.is_empty() { + phase.kernels = kernels; + } + } + } + Err(err) => evidence.warnings.push(format!("{file}: {err}")), + } + } + if let Ok(reader) = csv::open(&out.join("nsys-nvtx.csv")) { + if let Err(err) = projection(reader, &mut evidence.phases) { + evidence.warnings.push(format!("nsys-nvtx: {err}")); + } + } + match crate::timeline::load(out, &mut evidence.phases) { + Ok(()) => {} + Err(err) => evidence + .warnings + .push(format!("GPU busy coverage unknown: {err}")), + } + if evidence.phases.is_empty() { + evidence.warnings.push( + "CUDA-only analysis: no ds4.prefill / ds4.decode NVTX evidence; phase diagnosis UNKNOWN; build with make ds4-bench-perf".into(), + ); + } + evidence +} + +pub fn kernels(reader: impl BufRead) -> Result>, String> { + let mut grouped: BTreeMap<(String, String), Kernel> = BTreeMap::new(); + let mut header = Vec::new(); + csv::records(reader, |row| { + if csv::column(&row, &["kernel name", "name"]).is_some() + && row.iter().any(|s| csv::key(s).starts_with("total time (")) + { + header = row; + return Ok(()); + } + if header.is_empty() || row.iter().all(|s| s.trim().is_empty()) { + return Ok(()); + } + let name = + csv::value(&row, &header, &["kernel name", "name"]).ok_or("missing kernel name")?; + let (phase, name) = if let Some(range) = csv::value(&row, &header, &["nvtx range", "range"]) + { + (phase_name(range).unwrap_or(""), name) + } else if let Some((range, kernel)) = name.split_once('/') { + match phase_name(range) { + Some(phase) => (phase, kernel), + None => ("", name), + } + } else { + ("", name) + }; + let time = csv::time(&row, &header, "Total Time").ok_or("missing/invalid Total Time")?; + let count = csv::value(&row, &header, &["kern inst", "instances"]) + .map(|v| v.parse::()) + .transpose() + .map_err(|_| "invalid kernel count")?; + let entry = grouped + .entry((phase.into(), name.into())) + .or_insert_with(|| Kernel { + name: name.into(), + count: Some(0), + ..Default::default() + }); + entry.total_ns += time; + entry.count = entry.count.zip(count).and_then(|(a, b)| a.checked_add(b)); + Ok(()) + })?; + let mut groups: BTreeMap> = BTreeMap::new(); + for ((phase, _), kernel) in grouped { + groups.entry(phase).or_default().push(kernel); + } + for kernels in groups.values_mut() { + kernels.sort_by(|a, b| b.total_ns.total_cmp(&a.total_ns).then(a.name.cmp(&b.name))); + } + Ok(groups) +} + +fn projection(reader: impl BufRead, phases: &mut BTreeMap) -> Result<(), String> { + csv::table(reader, "range", |header, row| { + let Some(name) = csv::value(row, header, &["range"]).and_then(phase_name) else { + return Ok(()); + }; + let phase = phases.entry(name.into()).or_insert_with(|| Phase { + name: name.into(), + ..Default::default() + }); + if let Some(time) = csv::time(row, header, "Total Range Time") { + *phase.wall_ns.get_or_insert(0.0) += time; + } + if let Some(time) = csv::time(row, header, "Total Proj Time") { + *phase.projected_ns.get_or_insert(0.0) += time; + } + Ok(()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reports_reuse_export() { + let out = std::env::temp_dir().join(format!("ds4-nsys-export-{}", std::process::id())); + std::fs::create_dir(&out).unwrap(); + let caps = Capabilities { + reports: "nvtx_gpu_proj_sum -- projection\nnvtx_kern_sum[:base] -- kernels\ncuda_gpu_kern_sum[:base] -- kernels\n".into(), + ..Default::default() + }; + let mut completed = Vec::new(); + let evidence = collect_with(&caps, &out, |command, dir, name| { + let input = Path::new(command.last().unwrap()); + let sqlite = dir.join("trace.sqlite"); + // Reproduce Nsight rejecting a second .nsys-rep read as stale, + // even though the first report just completed its SQLite export. + if sqlite.exists() && input != sqlite { + return Err("Existing SQLite export found: older than input file".into()); + } + std::fs::write(sqlite, []).unwrap(); + let csv = if name == "nsys-nvtx-kernels" { + "Name,Total Time (ns)\n:ds4.decode/kernel,3000\n" + } else { + "Name,Total Time (ns)\nkernel,3000\n" + }; + std::fs::write(dir.join(format!("{name}.stdout")), csv).unwrap(); + completed.push(name.to_owned()); + Ok(()) + }); + std::fs::remove_dir_all(out).unwrap(); + assert_eq!( + completed, + ["nsys-nvtx", "nsys-nvtx-kernels", "nsys-kernels"] + ); + assert_eq!(evidence.phases["ds4.decode"].kernels[0].total_ns, 3000.0); + assert_eq!(evidence.global[0].name, "kernel"); + } + + #[test] + fn templates_repeats_and_phases() { + let groups = kernels(include_bytes!("../tests/fixtures/kernels.csv").as_slice()).unwrap(); + let prefill = &groups["ds4.prefill"]; + assert_eq!(prefill[0].name, "foo>"); + assert_eq!(prefill[0].total_ns, 3000.0); + assert_eq!(prefill[0].count, Some(3)); + assert_eq!(groups["ds4.decode"][0].name, "ordinary_kernel"); + } + + #[test] + fn older_and_empty_reports() { + let groups = kernels(b"Name,Total Time (us)\na/b,3\n".as_slice()).unwrap(); + assert_eq!(groups[""][0].name, "a/b"); + let groups = + kernels(b"Name,Total Time (us)\n:ds4.decode/ordinary_kernel,3\n".as_slice()).unwrap(); + assert_eq!(groups["ds4.decode"][0].count, None); + assert_eq!(groups["ds4.decode"][0].total_ns, 3000.0); + for text in [ + "", + "SKIPPED: no CUDA kernel data\n", + "UNKNOWN REPORT: nvtx_kern_sum\n", + ] { + assert!(kernels(text.as_bytes()).unwrap().is_empty()); + } + } +} diff --git a/crates/ds4-perf/src/optimize.rs b/crates/ds4-perf/src/optimize.rs new file mode 100644 index 0000000000000000000000000000000000000000..34c18c9f448a59ec5d734d022748e4c76f0c7938 --- /dev/null +++ b/crates/ds4-perf/src/optimize.rs @@ -0,0 +1,596 @@ +use crate::{ + cli, + compare::{self, Comparison, Verdict}, + experiment::{self, Scout, Workload}, + fit::Fit, + knobs, + ncu::Counters, + report, runner, +}; +use ds4_perf::artifact::{self, Artifact, Payload, Reference}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, +}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Candidate { + pub name: String, + pub environment: BTreeMap, + pub reason: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Plan { + candidates: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Trial { + pub candidate: Candidate, + pub accepted: bool, + pub comparisons: Vec, + pub verdicts: Vec, + pub failure: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Decision { + pub automatic: bool, + pub candidates: Vec, + pub trials: Vec, + pub selected: Option, + pub stop_reason: String, + pub evidence: Vec, + pub method: String, +} + +impl Payload for Decision { + const KIND: &'static str = "decision"; + fn validate(&self) -> Result<(), String> { + if self.method != "bracketed-controls-v1" || self.stop_reason.is_empty() { + return Err("invalid optimization decision".into()); + } + let mut accepted = BTreeMap::new(); + for trial in &self.trials { + if trial.accepted + && (trial.failure.is_some() + || trial.comparisons.len() != 2 + || trial.verdicts.len() != 2 + || trial.verdicts.iter().any(|v| *v != Verdict::Improved)) + { + return Err("accepted trial lacks bracketed comparison".into()); + } + if !trial.accepted && trial.failure.as_ref().is_none_or(|v| v.is_empty()) { + return Err("rejected trial lacks a recorded reason".into()); + } + if trial.accepted { + accepted.extend(trial.candidate.environment.clone()); + } + } + match &self.selected { + Some(selected) if selected.environment != accepted || accepted.is_empty() => { + return Err("selected controls lack accepted trial evidence".into()) + } + None if !accepted.is_empty() => { + return Err("accepted controls missing from selection".into()) + } + _ => {} + } + Ok(()) + } +} + +fn candidates( + scout: &Scout, + fit: Option<&Fit>, + counters: Option<&Counters>, +) -> (Vec, Vec) { + let Some(workload) = &scout.workload else { + return ( + Vec::new(), + vec!["Provide --workload before choosing runtime experiments".into()], + ); + }; + let family = workload.family.to_ascii_lowercase(); + let mut reasons = Vec::new(); + let mut host = false; + let mut wide = false; + let mut prefill = false; + for (name, phase) in &scout.evidence.phases { + let diagnosis = report::classify(phase); + reasons.push(format!( + "{name}: {diagnosis:?}; wall_ns={:?}, busy_ns={:?}", + phase.wall_ns, phase.busy_ns + )); + // Initial runtime proposals target prefill. Decode still contributes + // diagnostic context and must pass the later regression comparison. + if name == "ds4.prefill" { + prefill |= diagnosis != report::Diagnosis::Unknown; + host |= diagnosis == report::Diagnosis::HostIdle; + wide |= diagnosis == report::Diagnosis::Fragmentation; + } + } + if let Some(fit) = fit { + if let Some(bound) = fit.bounds.iter().find(|b| b.launch.phase == "ds4.prefill") { + reasons.push(format!( + "top prefill geometry: {} grid={:?} block={:?} waves_lower={} tail_fill={:.3}", + bound.launch.kernel, + bound.launch.grid, + bound.launch.block, + bound.waves_lower, + bound.last_wave_fill + )); + wide |= bound.waves_lower == 1 && bound.last_wave_fill < 0.5; + } + reasons.push(format!( + "measured envelope: copy_GB_s={:?}, fp32_GFLOP_s={:?}, launch_us={:?}", + fit.observed_copy_gb_s, fit.observed_fp32_gflop_s, fit.observed_launch_us + )); + } + if let Some(counters) = counters { + for target in &counters.targets { + for metric in &target.metrics { + if metric.name == "sm__warps_active.avg.pct_of_peak_sustained_active" { + reasons.push(format!( + "{} observed active warps: {:?} {}", + target.phase, metric.value, metric.unit + )); + if target.phase == "ds4.prefill" { + wide |= metric.value.is_some_and(|v| v < 30.0); + } + } + } + } + } + let mut result = Vec::new(); + if family.starts_with("qwen") && host { + let current = scout + .environment + .get("DS4_QWEN_PLE_WORKERS") + .and_then(|v| v.parse::().ok()) + .unwrap_or(32); + for workers in [(current * 2).min(64), (current / 2).max(1)] { + if workers == current { + continue; + } + result.push(Candidate {name:format!("ple-workers-{workers}"),environment:BTreeMap::from([("DS4_QWEN_PLE_WORKERS".into(),workers.to_string())]),reason:"Host GPU-free spans: test PLE worker parallelism with fixed cache allocation; not a claim that PLE caused the gap".into()}); + } + } + if !prefill { + reasons.push("No measured prefill phase for automatic prefill controls".into()); + return (result, reasons); + } + let control = if family.starts_with("qwen") { + Some(("DS4_QWEN_PREFILL_CHUNK", 256, 16384)) + } else if family.starts_with("dots") { + Some(("DS4_DOTS3_PREFILL_CHUNK", 4096, 8192)) + } else if family.starts_with("solar") + && scout + .environment + .get("DS4_CUDA_SOLAR_GQA_GROUPED") + .is_none_or(|v| v != "0") + { + Some(("DS4_CUDA_SOLAR_GQA_CHUNK", 64, 2048)) + } else if family.starts_with("step") { + Some(("DS4_STEP37_PREFILL_CHUNK", 2048, 4096)) + } else { + None + }; + if let Some((key, default, maximum)) = control { + let frontier = scout + .samples + .first() + .and_then(|s| s.rows.iter().map(|r| r.ctx).max()) + .unwrap_or(maximum) as u32; + let limit = (maximum as u32).min(frontier).max(1); + let current = scout + .environment + .get(key) + .and_then(|v| v.parse::().ok()) + .unwrap_or(default) + .min(limit); + let values = if wide { + [ + current.saturating_mul(2), + current / 2, + current.saturating_mul(4), + ] + } else { + [current / 2, current.saturating_mul(2), current / 4] + }; + for value in values.map(|v| v.clamp(1, limit)) { + if value == current + || knobs::validate(key, &value.to_string(), &family).is_err() + || result + .iter() + .any(|c| c.environment.get(key) == Some(&value.to_string())) + { + continue; + } + result.push(Candidate {name:format!("chunk-{value}"),environment:BTreeMap::from([(key.into(),value.to_string())]),reason:if wide {"Fragmentation/underfilled geometry: test work granularity before changing kernels"} else {"Test work decomposition at a smaller/larger chunk while preserving model, context and cache policy"}.into()}); + } + } + (result, reasons) +} + +fn validate_plan(plan: &[Candidate], workload: &Workload) -> Result<(), String> { + if plan.is_empty() || plan.len() > 32 { + return Err("experiment plan needs 1..32 candidates".into()); + } + let mut names = std::collections::BTreeSet::new(); + for candidate in plan { + if candidate.name.is_empty() + || !names.insert(&candidate.name) + || candidate.reason.is_empty() + || candidate.environment.len() != 1 + { + return Err( + "each candidate needs a unique name, reason and one controlled variable".into(), + ); + } + for (key, value) in &candidate.environment { + knobs::validate(key, value, &workload.family)?; + } + } + Ok(()) +} + +fn replay( + source: &Scout, + out: PathBuf, + workload: &Path, + repeats: u32, + overrides: &BTreeMap, + source_root: &Path, +) -> Result { + let mut environment = source.environment.clone(); + environment.extend(overrides.clone()); + #[cfg(unix)] + let command = { + use std::os::unix::ffi::OsStringExt; + source + .command + .iter() + .cloned() + .map(std::ffi::OsString::from_vec) + .collect() + }; + #[cfg(not(unix))] + let command = source + .command + .iter() + .map(|v| { + String::from_utf8(v.clone()) + .map(Into::into) + .map_err(|e| e.to_string()) + }) + .collect::>()?; + for input in [ + &source.replay.helper, + &source.replay.collector, + &source.replay.sdk, + ] + .into_iter() + .flatten() + { + if artifact::hash(&input.path)? != input.sha256 { + return Err("profiling helper/collector binary changed since source scout".into()); + } + } + if source.collector == "nsys" { + let collector = source + .replay + .collector + .as_ref() + .ok_or("source scout lacks nsys executable identity")?; + if runner::resolve(std::ffi::OsStr::new("nsys")).as_ref() != Some(&collector.path) { + return Err("nsys executable resolution changed since source scout".into()); + } + } + Ok(cli::Scout { + exact_environment: true, + budget: source.budget.clone(), + out, + collector: if source.collector == "cupti" { + cli::Collector::Cupti + } else { + cli::Collector::Nsys + }, + // A phase-only source can still run an explicit controlled experiment. + fit: source.replay.calibration.is_some(), + ncu: false, + proof: true, + machine: None, + calibration: source + .replay + .calibration + .as_ref() + .map(|r| source_root.join(&r.path)), + gpu_helper: source.replay.helper.as_ref().map(|v| v.path.clone()), + device: source + .gpu + .as_ref() + .ok_or("automatic experiments need verified CUDA properties")? + .ordinal, + cupti_library: source + .replay + .collector + .as_ref() + .filter(|_| source.collector == "cupti") + .map(|v| v.path.clone()), + cupti_sdk: source.replay.sdk.as_ref().map(|v| v.path.clone()), + repeats, + workload: Some(workload.to_path_buf()), + cache_policy: source.cache_policy.clone(), + environment: environment + .into_iter() + .map(|(k, v)| format!("{k}={v}")) + .collect(), + command, + }) +} + +fn compare_trial(base: &Path, candidate: &Path, out: &Path) -> Result { + compare::run(&cli::Compare { + baseline: base.join("scout.json"), + candidate: candidate.join("scout.json"), + out: out.into(), + regression: false, + max_slowdown_percent: 3.0, + logit_atol: 0.0001, + logit_rtol: 0.0001, + logit_rel_rms: 0.0, + })?; + let artifact: Artifact = artifact::load(&out.join("compare.json"))?; + let data = artifact.data.ok_or("comparison has no result")?; + Ok(data.verdict) +} + +fn automatic(args: &cli::Optimize, source: &Scout, decision: &mut Decision) -> Result<(), String> { + if !source.process_scope.verified { + return Err("automatic execution needs matching GPU process snapshots".into()); + } + if !source.unreviewed_environment.is_empty() { + return Err("automatic replay requires reviewed DS4/CUDA/LD environment controls".into()); + } + let workload = source + .workload + .as_ref() + .ok_or("--auto requires hashed --workload evidence")?; + if source.cache_policy != "warmup-then-fresh" { + return Err("--auto requires an explicit cache policy on the source scout".into()); + } + if std::env::current_dir().map_err(|e| e.to_string())? != source.cwd { + return Err("run --auto from the source scout working directory".into()); + } + if artifact::hash(&source.executable)? != source.executable_sha256 { + return Err("source benchmark executable changed".into()); + } + workload.verify()?; + validate_plan(&decision.candidates, workload)?; + let workload_path = args + .out + .join("workload.json") + .canonicalize() + .unwrap_or_else(|_| args.out.join("workload.json")); + fs::write( + &workload_path, + serde_json::to_vec_pretty(workload).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + let workload_path = workload_path.canonicalize().map_err(|e| e.to_string())?; + let source_root = args.scout.parent().unwrap_or(Path::new(".")); + let execute = |out: &Path, env: &BTreeMap| -> Result<(), String> { + let remaining = args + .budget + .limits() + .bytes + .saturating_sub(crate::process::bytes(&args.out)?) + / (1024 * 1024); + if remaining == 0 { + return Err("automatic campaign output budget reached".into()); + } + let mut replay = replay( + source, + out.to_path_buf(), + &workload_path, + args.repeats, + env, + source_root, + )?; + replay.budget = args.budget.clone(); + replay.budget.max_output_mib = remaining; + runner::scout(&replay)?; + crate::process::check_bytes(&args.out, args.budget.limits()) + }; + let mut control_env = BTreeMap::new(); + let mut before = args.out.join("control-00"); + execute(&before, &control_env)?; + for (index, candidate) in decision + .candidates + .iter() + .take(args.rounds as usize) + .enumerate() + { + let candidate_dir = args.out.join(format!("candidate-{:02}", index + 1)); + let mut env = control_env.clone(); + env.extend(candidate.environment.clone()); + let work = execute(&candidate_dir, &env); + let after = args.out.join(format!("control-{:02}", index + 1)); + // Re-measure the unchanged control after the candidate to expose drift. + let control = execute(&after, &control_env); + let mut trial = Trial { + candidate: candidate.clone(), + accepted: false, + comparisons: Vec::new(), + verdicts: Vec::new(), + failure: work + .as_ref() + .err() + .cloned() + .or_else(|| control.as_ref().err().cloned()), + }; + if work.is_ok() && control.is_ok() { + for (label, base) in [("before", &before), ("after", &after)] { + let comparison = args.out.join(format!("compare-{:02}-{label}", index + 1)); + match compare_trial(base, &candidate_dir, &comparison) { + Ok(verdict) => trial.verdicts.push(verdict), + Err(error) => { + trial.failure = Some(format!("{label} comparison: {error}")); + } + } + let path = comparison.join("compare.json"); + if path.is_file() { + match artifact::reference(&path, &args.out) { + Ok(reference) => trial.comparisons.push(reference), + Err(error) => { + trial.failure = Some(format!("{label} reference: {error}")); + } + } + } + } + trial.accepted = trial.failure.is_none() + && trial.comparisons.len() == 2 + && trial.verdicts.len() == 2 + && trial.verdicts.iter().all(|v| *v == Verdict::Improved); + if !trial.accepted && trial.failure.is_none() { + trial.failure = Some(format!("candidate not retained: {:?}", trial.verdicts)); + } + } + if trial.accepted { + control_env = env; + decision.selected = Some(Candidate {name:candidate.name.clone(),environment:control_env.clone(),reason:"Improved against both bracketing controls with full-vocabulary and greedy-token parity".into()}); + before = candidate_dir; + } else { + before = after; + } + decision.trials.push(trial); + if control.is_err() { + return Err("control run failed; optimization stopped".into()); + } + } + decision.stop_reason = if decision.candidates.len() > args.rounds as usize { + "round budget reached" + } else { + "candidate plan exhausted" + } + .into(); + Ok(()) +} + +pub fn run(args: &cli::Optimize) -> Result<(), String> { + artifact::directory(&args.out)?; + let mut decision = Decision { + automatic: args.automatic, + candidates: Vec::new(), + trials: Vec::new(), + selected: None, + stop_reason: "analysis complete".into(), + evidence: Vec::new(), + method: "bracketed-controls-v1".into(), + }; + let mut refs = Vec::new(); + let result = (|| { + let source = experiment::load(&args.scout)?; + refs.push(artifact::reference(&args.scout, &args.out)?); + let root = args.scout.parent().unwrap_or(Path::new(".")); + let fit = root.join("fit.json"); + let fit: Option> = if fit.exists() { + refs.push(artifact::reference(&fit, &args.out)?); + Some(artifact::load(&fit)?) + } else { + None + }; + let ncu = root.join("ncu.json"); + let ncu: Option> = if ncu.exists() { + refs.push(artifact::reference(&ncu, &args.out)?); + Some(artifact::load(&ncu)?) + } else { + None + }; + let scout = source.require()?; + let (suggested, reasons) = candidates( + scout, + fit.as_ref() + .filter(|a| a.complete) + .and_then(|a| a.data.as_ref()), + ncu.as_ref() + .filter(|a| a.complete) + .and_then(|a| a.data.as_ref()), + ); + decision.candidates = suggested; + decision.evidence = reasons; + if let Some(path) = &args.plan { + let plan: Plan = serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + validate_plan( + &plan.candidates, + scout + .workload + .as_ref() + .ok_or("--plan requires source workload identity")?, + )?; + decision.candidates = plan.candidates; + refs.push(artifact::reference(path, &args.out)?); + } + if args.automatic { + automatic(args, scout, &mut decision)?; + } + crate::process::check_bytes(&args.out, args.budget.limits()) + })(); + let mut result = result; + let mut artifact = Artifact::new(decision); + artifact.complete = result.is_ok(); + if let Err(e) = &result { + artifact.data.as_mut().unwrap().stop_reason = e.clone(); + artifact.warnings.push(e.clone()); + } + refs.extend(experiment::raw_refs(&args.out, &args.out)?); + artifact.inputs = refs; + let final_bytes = serde_json::to_vec_pretty(&artifact) + .map_err(|e| e.to_string())? + .len() as u64 + + 1; + if result.is_ok() + && crate::process::bytes(&args.out)?.saturating_add(final_bytes) + > args.budget.limits().bytes + { + let error = "automatic campaign output budget exceeded at publication".to_string(); + artifact.complete = false; + artifact.data.as_mut().unwrap().stop_reason = error.clone(); + artifact.warnings.push(error.clone()); + result = Err(error); + } + artifact::save(&args.out.join("decision.json"), &artifact)?; + println!("decision: {}", args.out.join("decision.json").display()); + result +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn rejects_uncontrolled_plan() { + let workload = Workload { + protocol: "ds4-bench-v1".into(), + name: "test".into(), + family: "qwen".into(), + files: BTreeMap::new(), + shape: BTreeMap::new(), + cache_state: "warm".into(), + }; + let candidate = Candidate { + name: "bad".into(), + reason: "test".into(), + environment: BTreeMap::from([("DS4_QWEN_PLE_CACHE_MB".into(), "512".into())]), + }; + assert!(validate_plan(&[candidate], &workload).is_err()); + } +} diff --git a/crates/ds4-perf/src/process.rs b/crates/ds4-perf/src/process.rs new file mode 100644 index 0000000000000000000000000000000000000000..f594f5777f31596acfa173fca7979f3f787b4073 --- /dev/null +++ b/crates/ds4-perf/src/process.rs @@ -0,0 +1,153 @@ +use std::{ + fs, + path::Path, + process::{Child, Command, ExitStatus}, + time::{Duration, Instant}, +}; + +static INTERRUPTED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +pub fn signals() -> Result<(), String> { + ctrlc::set_handler(|| INTERRUPTED.store(true, std::sync::atomic::Ordering::Relaxed)) + .map_err(|e| e.to_string()) +} + +#[derive(Clone, Copy, Debug)] +pub struct Limits { + pub timeout: Duration, + pub bytes: u64, +} + +impl Default for Limits { + fn default() -> Self { + Self { + timeout: Duration::from_secs(1800), + bytes: 2 * 1024 * 1024 * 1024, + } + } +} + +pub fn bytes(path: &Path) -> Result { + let mut total = 0u64; + for entry in fs::read_dir(path).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let metadata = entry.metadata().map_err(|e| e.to_string())?; + let size = if entry.file_type().map_err(|e| e.to_string())?.is_dir() { + bytes(&entry.path())? + } else { + metadata.len() + }; + total = total.checked_add(size).ok_or("output size overflow")?; + } + Ok(total) +} + +fn stop(child: &mut Child) { + #[cfg(unix)] + { + use nix::{ + sys::signal::{killpg, Signal}, + unistd::Pid, + }; + let group = Pid::from_raw(child.id() as i32); + let _ = killpg(group, Signal::SIGTERM); + std::thread::sleep(Duration::from_millis(200)); + let _ = killpg(group, Signal::SIGKILL); + } + #[cfg(not(unix))] + { + let _ = child.kill(); + } + let end = Instant::now() + Duration::from_secs(2); + while Instant::now() < end { + if child.try_wait().is_ok_and(|s| s.is_some()) { + break; + } + std::thread::sleep(Duration::from_millis(25)); + } +} + +pub fn run(command: &mut Command, out: &Path, limits: Limits) -> Result { + if INTERRUPTED.load(std::sync::atomic::Ordering::Relaxed) { + return Err("interrupted".into()); + } + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let mut child = command.spawn().map_err(|e| e.to_string())?; + let start = Instant::now(); + let result = (|| loop { + if INTERRUPTED.load(std::sync::atomic::Ordering::Relaxed) { + return Err("interrupted".into()); + } + if bytes(out)? > limits.bytes { + return Err("profiler output budget exceeded".into()); + } + if let Some(status) = child.try_wait().map_err(|e| e.to_string())? { + #[cfg(unix)] + if nix::sys::signal::killpg(nix::unistd::Pid::from_raw(child.id() as i32), None).is_ok() + { + return Err("process exited with live descendants in its group".into()); + } + return Ok(status); + } + if start.elapsed() >= limits.timeout { + return Err(format!( + "process {} exceeded {} second deadline", + child.id(), + limits.timeout.as_secs() + )); + } + std::thread::sleep(Duration::from_millis(25)); + })(); + if result.is_err() { + stop(&mut child); + } + result +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + #[test] + fn rejects_orphaned_descendants() { + let root = std::env::temp_dir().join(format!("ds4-orphan-{}", std::process::id())); + fs::create_dir(&root).unwrap(); + let mut command = Command::new("sh"); + command.args(["-c", "sleep 2 &"]); + let result = run(&mut command, &root, Limits::default()); + fs::remove_dir_all(root).unwrap(); + assert!(result.is_err_and(|e| e.contains("descendants"))); + } + + #[test] + fn terminates_hanging_group() { + let root = std::env::temp_dir().join(format!("ds4-deadline-{}", std::process::id())); + fs::create_dir(&root).unwrap(); + let mut command = Command::new("sh"); + command.args(["-c", "sleep 30 & wait"]); + let start = Instant::now(); + let error = run( + &mut command, + &root, + Limits { + timeout: Duration::from_millis(100), + bytes: 1024, + }, + ) + .unwrap_err(); + assert!(error.contains("deadline")); + assert!(start.elapsed() < Duration::from_secs(5)); + fs::remove_dir_all(root).unwrap(); + } +} + +/// Include generated reports and copies in the final output accounting. +pub fn check_bytes(out: &Path, limits: Limits) -> Result<(), String> { + if bytes(out)? > limits.bytes { + return Err("profiler output budget exceeded".into()); + } + Ok(()) +} diff --git a/crates/ds4-perf/src/report.rs b/crates/ds4-perf/src/report.rs new file mode 100644 index 0000000000000000000000000000000000000000..b08624f681dcb5d429ffbaabf12e9e90b873aaf7 --- /dev/null +++ b/crates/ds4-perf/src/report.rs @@ -0,0 +1,317 @@ +use crate::{ + bench, + doctor::Doctor, + nsys::{Evidence, Kernel, Phase}, + runner, +}; +use std::ffi::OsString; + +// Scout v1 heuristics: execution-path triage, never a hardware bottleneck proof. +const HOST_MAX_COVERAGE: f64 = 0.75; +const HOST_MIN_GAP_NS: f64 = 100_000.0; +const MEM_MIN_BUSY_SHARE: f64 = 0.20; +const SHORT_KERNEL_NS: f64 = 10_000.0; +const SHORT_MIN_LAUNCHES: u64 = 100; +const SHORT_MIN_TIME_SHARE: f64 = 0.35; +const DOMINANT_MIN_SHARE: f64 = 0.15; +const TOP_KERNEL_LIMIT: usize = 10; + +#[derive(Debug, PartialEq, Eq)] +pub enum Diagnosis { + HostIdle, + Memop, + Fragmentation, + Dominant, + Mixed, + Unknown, +} + +impl Diagnosis { + fn label(&self) -> &str { + match self { + Self::HostIdle => "HOST / IDLE", + Self::Memop => "GPU MEMOP", + Self::Fragmentation => "LAUNCH FRAGMENTATION", + Self::Dominant => "DOMINANT KERNEL", + Self::Mixed => "MIXED", + Self::Unknown => "UNKNOWN", + } + } + fn next(&self) -> &str { + match self { + Self::HostIdle => "inspect host-side PLE, routing, indexing, synchronization and GPU-free spans before NCU", + Self::Memop => "inspect phase-local memcpy/memset, repeated copies and materialization before NCU", + Self::Fragmentation => "inspect fusion, graph capture and repeated transforms before tuning tiles", + Self::Dominant => "inspect this kernel's dispatch/fallback path; then use targeted NCU if the path is expected", + Self::Mixed => "inspect repeated transforms, routing and kernel families across this phase", + Self::Unknown => "inspect raw reports and missing phase/trace evidence before choosing a target", + } + } +} + +fn ratio(a: Option, b: Option) -> Option { + a.zip(b) + .filter(|(a, b)| a.is_finite() && b.is_finite() && *a >= 0.0 && *b > 0.0) + .map(|(a, b)| a / b) +} + +fn kernel_total(phase: &Phase) -> f64 { + phase.kernels.iter().map(|k| k.total_ns).sum() +} + +fn short_work(phase: &Phase) -> (u64, f64) { + phase.kernels.iter().fold((0u64, 0.0), |(count, time), k| { + if k.count + .is_some_and(|n| n > 0 && k.total_ns / n as f64 <= SHORT_KERNEL_NS) + { + ( + count.saturating_add(k.count.unwrap_or(0)), + time + k.total_ns, + ) + } else { + (count, time) + } + }) +} + +pub fn classify(phase: &Phase) -> Diagnosis { + let Some(coverage) = ratio(phase.busy_ns, phase.wall_ns) else { + return Diagnosis::Unknown; + }; + if coverage > 1.0 || phase.mem_ns.is_none() || phase.gap_ns.is_none() { + return Diagnosis::Unknown; + } + if coverage < HOST_MAX_COVERAGE && phase.gap_ns.unwrap_or(0.0) >= HOST_MIN_GAP_NS { + return Diagnosis::HostIdle; + } + if ratio(phase.mem_ns, phase.busy_ns).is_some_and(|share| share >= MEM_MIN_BUSY_SHARE) { + return Diagnosis::Memop; + } + let total = kernel_total(phase); + if total <= 0.0 || !total.is_finite() { + return Diagnosis::Unknown; + } + let (short_count, short_time) = short_work(phase); + if short_count >= SHORT_MIN_LAUNCHES && short_time / total >= SHORT_MIN_TIME_SHARE { + return Diagnosis::Fragmentation; + } + if phase + .kernels + .iter() + .any(|k| k.total_ns / total >= DOMINANT_MIN_SHARE) + { + return Diagnosis::Dominant; + } + Diagnosis::Mixed +} + +fn seconds(value: Option) -> String { + value + .map(|v| format!("{:.6} s", v / 1e9)) + .unwrap_or_else(|| "UNKNOWN".into()) +} +fn percent(value: Option) -> String { + value + .map(|v| format!("{:.1} %", v * 100.0)) + .unwrap_or_else(|| "UNKNOWN".into()) +} + +pub fn render( + d: &Doctor, + command: &[OsString], + rows: &[bench::Row], + evidence: &Evidence, +) -> String { + let mut out = format!("DS4 PERFORMANCE SCOUT\n\nWORKLOAD\n command {}\n git {} (dirty={})\n gpu {}\n\nAPPLICATION THROUGHPUT (fresh, unprofiled process)\n", + runner::shell(command), d.facts["git_sha"], d.facts["git_dirty"], d.facts["gpu"]); + if rows.is_empty() { + out.push_str(" UNKNOWN: no recognized ds4-bench CSV on stdout\n"); + } + for row in rows { + out.push_str(&format!(" ctx {}: prefill {} tokens, {:.2} tok/s; decode {} tokens, {:.2} tok/s\n first_token_sec {:.4}; kvcache_bytes {}\n", row.ctx, row.prefill_tokens, row.prefill_tps, row.gen_tokens, row.gen_tps, row.first_token_sec, row.kvcache_bytes)); + } + out.push_str("\nEXECUTION-PATH STRUCTURE (separate profiled process)\n Profiled throughput is informational and is never compared with baseline TPS.\n Projection is a first-to-last GPU span, including gaps; it is not GPU busy time.\n Coverage unions traced GPU intervals clipped to CPU NVTX phases.\n Kernel percentages use summed kernel time within each phase; overlaps can double-count.\n"); + if rows.len() > 1 { + out.push_str(" Multiple benchmark rows: phase reports aggregate frontiers; use one frontier for attribution.\n"); + } + for name in crate::nsys::PHASES { + let Some(p) = evidence.phases.get(name) else { + out.push_str(&format!( + "\n{name}: UNKNOWN (no phase evidence; may be absent from workload)\n" + )); + continue; + }; + let coverage = ratio(p.busy_ns, p.wall_ns); + let mem_share = ratio(p.mem_ns, p.busy_ns); + out.push_str(&format!("\nNVTX GPU PROJECTION: {name}\n range time {}\n projected span {}\n GPU busy union {}\n GPU coverage {}\n largest GPU-free span inside phase {}\n GPU memop union {} ({} of busy)\n\n{} TOP KERNELS\n", seconds(p.wall_ns), seconds(p.projected_ns), seconds(p.busy_ns), percent(coverage), seconds(p.gap_ns), seconds(p.mem_ns), percent(mem_share), name.to_uppercase())); + top_kernels(&mut out, &p.kernels); + let diagnosis = classify(p); + let (short_count, short_time) = short_work(p); + out.push_str(&format!("\nDIAGNOSIS {}\nEVIDENCE\n GPU coverage {}; largest gap {}\n memop share {}; short-kernel launches {}, {} of kernel time (mean <= {:.0} us)\nNEXT\n {}\n", diagnosis.label(), percent(coverage), seconds(p.gap_ns), percent(mem_share), short_count, percent(ratio(Some(short_time), Some(kernel_total(p)))), SHORT_KERNEL_NS / 1000.0, diagnosis.next())); + if diagnosis == Diagnosis::Dominant { + if let Some(kernel) = p.kernels.first() { + out.push_str(&format!("\nNEXT NCU TARGET\n kernel: {}\n reason: {:.1} % of {name} summed kernel time\n", kernel.name, 100.0 * kernel.total_ns / kernel_total(p))); + match ncu_command(&d.caps.ncu_help, name, &kernel.name, command) { + Some(cmd) => out.push_str(&format!(" suggested: {}\n Profiles one matching launch in this NVTX phase. Review memory headroom and graph replay before running.\n With a resident weight owner, prefer the matching model-free probe (see the optimization playbook).\n", runner::shell(&cmd))), + None => out.push_str(" suggested: unavailable; installed NCU lacks verified bounded kernel/NVTX filtering\n"), + } + } + } + } + if !evidence.global.is_empty() { + out.push_str( + "\nGLOBAL CUDA KERNELS (all phases and setup; excluded from phase diagnosis)\n", + ); + top_kernels(&mut out, &evidence.global); + } + out.push_str(&format!("\nHEURISTICS v1\n host: coverage < {}%, gap >= {} us; memop: >= {}% busy;\n fragmentation: >= {} short launches, >= {}% kernel time, mean <= {} us;\n dominant: >= {}% summed phase kernel time. Missing structural evidence => UNKNOWN.\n", HOST_MAX_COVERAGE*100.0, HOST_MIN_GAP_NS/1000.0, MEM_MIN_BUSY_SHARE*100.0, SHORT_MIN_LAUNCHES, SHORT_MIN_TIME_SHARE*100.0, SHORT_KERNEL_NS/1000.0, DOMINANT_MIN_SHARE*100.0)); + out.push_str("\nLIMITS\n Nsight Systems does not establish memory-bound versus compute-bound behavior.\n A profiler result is not a correctness proof or a claimed performance improvement.\n Return every code change to same-workload, same-hour A/B and existing correctness gates.\n Cache warmth, clocks, binary/model/prompt identity and unrecorded environment still require review.\n"); + for warning in d.warnings.iter().chain(&evidence.warnings) { + out.push_str(&format!(" WARNING: {warning}\n")); + } + out +} + +fn top_kernels(out: &mut String, kernels: &[Kernel]) { + let total: f64 = kernels.iter().map(|k| k.total_ns).sum(); + for k in kernels.iter().take(TOP_KERNEL_LIMIT) { + out.push_str(&format!( + " {} {} launches={} {}\n", + percent(ratio(Some(k.total_ns), Some(total))), + seconds(Some(k.total_ns)), + k.count + .map(|n| n.to_string()) + .unwrap_or_else(|| "UNKNOWN".into()), + k.name + )); + } +} + +pub fn bounded_ncu( + help: &str, + phase: &str, + name: &str, + bench: &[OsString], +) -> Option> { + ncu_command(help, phase, name, bench) +} + +fn ncu_command(help: &str, phase: &str, name: &str, bench: &[OsString]) -> Option> { + if ![ + "--kernel-name-base", + "--kernel-name", + "--launch-count", + "--nvtx-include", + "--nvtx", + "--replay-mode", + "--app-replay-match", + "--app-replay-mode", + ] + .iter() + .all(|flag| help.contains(flag)) + { + return None; + } + let base = if name.starts_with("_Z") { + "mangled" + } else if name.contains(['<', '(']) { + "demangled" + } else { + "function" + }; + let mut regex = String::from("regex:^"); + for c in name.chars() { + if ".+*?()[]{}^$|\\".contains(c) { + regex.push('\\'); + } + regex.push(c); + } + regex.push('$'); + let mut command: Vec = [ + "ncu", + "--kernel-name-base", + base, + "--kernel-name", + ®ex, + "--launch-count", + "1", + "--nvtx", + "--nvtx-include", + &format!("{phase}/"), + // Avoid saving the full accessible model allocation for kernel replay. + "--replay-mode", + "application", + "--app-replay-match", + "all", + "--app-replay-mode", + "strict", + ] + .map(Into::into) + .into(); + command.extend_from_slice(bench); + Some(command) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn phase(shares: &[f64]) -> Phase { + Phase { + name: "ds4.prefill".into(), + wall_ns: Some(1e9), + busy_ns: Some(0.95e9), + mem_ns: Some(0.0), + gap_ns: Some(0.01e9), + kernels: shares + .iter() + .enumerate() + .map(|(i, s)| Kernel { + name: format!("kernel{i}"), + total_ns: s * 0.95e9, + count: Some(1), + }) + .collect(), + ..Default::default() + } + } + #[test] + fn classifier_priority() { + let mut p = phase(&[0.8, 0.2]); + assert_eq!(classify(&p), Diagnosis::Dominant); + p.busy_ns = Some(0.6e9); + assert_eq!(classify(&p), Diagnosis::HostIdle); + p.busy_ns = Some(0.95e9); + p.mem_ns = Some(0.3e9); + assert_eq!(classify(&p), Diagnosis::Memop); + p.mem_ns = Some(0.0); + p.kernels[0].count = Some(100_000); + assert_eq!(classify(&p), Diagnosis::Fragmentation); + assert_eq!(classify(&phase(&[0.1; 10])), Diagnosis::Mixed); + p.busy_ns = None; + assert_eq!(classify(&p), Diagnosis::Unknown); + assert_eq!(classify(&Phase::default()), Diagnosis::Unknown); + } + #[test] + fn ncu_is_phase_and_launch_limited() { + let help = "--kernel-name --kernel-name-base --launch-count --nvtx --nvtx-include --replay-mode --app-replay-match --app-replay-mode"; + let cmd = ncu_command( + help, + "ds4.decode", + "foo", + &["./ds4-bench".into(), "a'$(b)".into()], + ) + .unwrap(); + assert!(cmd.windows(2).any(|w| w == ["--launch-count", "1"])); + assert!(cmd + .windows(2) + .any(|w| w == ["--nvtx-include", "ds4.decode/"])); + assert!(cmd + .windows(2) + .any(|w| w == ["--replay-mode", "application"])); + assert!(cmd.windows(2).any(|w| w == ["--app-replay-match", "all"])); + assert!(cmd.windows(2).any(|w| w == ["--app-replay-mode", "strict"])); + assert!(!cmd.iter().any(|s| s == "full")); + assert!(ncu_command("", "ds4.decode", "foo", &[]).is_none()); + } +} diff --git a/crates/ds4-perf/src/runner.rs b/crates/ds4-perf/src/runner.rs new file mode 100644 index 0000000000000000000000000000000000000000..8b336a7dcc18a0d841e34e9a4ad46c0b465c3df8 --- /dev/null +++ b/crates/ds4-perf/src/runner.rs @@ -0,0 +1,794 @@ +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File}; +use std::path::Path; +use std::process::{Command, Stdio}; + +#[derive(Clone)] +pub struct Capture { + pub ok: bool, + pub out: String, + pub err: String, +} + +impl Capture { + pub fn failed(error: &str) -> Self { + Self { + ok: false, + out: String::new(), + err: error.into(), + } + } + pub fn combined(&self) -> String { + format!("{}\n{}", self.out, self.err) + } +} + +pub trait Probe { + fn capture(&mut self, program: &str, args: &[&str]) -> Capture; +} + +pub struct System; +impl Probe for System { + fn capture(&mut self, program: &str, args: &[&str]) -> Capture { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let out = std::env::temp_dir().join(format!("ds4-probe-{}-{nonce}", std::process::id())); + if let Err(error) = fs::create_dir(&out) { + return Capture::failed(&error.to_string()); + } + let mut command = vec![OsString::from(program)]; + command.extend(args.iter().map(OsString::from)); + let result = controls(&[]).and_then(|env| { + run_limited( + &command, + &out, + "probe", + &env, + crate::process::Limits { + timeout: std::time::Duration::from_secs(60), + bytes: 64 * 1024 * 1024, + }, + ) + }); + let stdout = fs::read(out.join("probe.stdout")).unwrap_or_default(); + let mut stderr = fs::read(out.join("probe.stderr")).unwrap_or_default(); + if let Err(error) = &result { + stderr.extend_from_slice(format!("\n{error}").as_bytes()); + } + let _ = fs::remove_dir_all(&out); + Capture { + ok: result.is_ok(), + out: String::from_utf8_lossy(&stdout).into(), + err: String::from_utf8_lossy(&stderr).into(), + } + } +} + +pub fn write(path: &Path, text: &str) -> Result<(), String> { + fs::write(path, text).map_err(|e| format!("{}: {e}", path.display())) +} + +// Quoting is for reproduction only: execution never invokes a shell. +pub fn shell(command: &[OsString]) -> String { + command + .iter() + .map(|s| quote(&s.to_string_lossy())) + .collect::>() + .join(" ") +} + +pub fn quote(s: &str) -> String { + format!("'{}'", s.replace('\'', "'\\''")) +} + +pub fn run(command: &[OsString], out: &Path, name: &str) -> Result<(), String> { + run_with(command, out, name, &controls(&[])?) +} + +pub fn run_with( + command: &[OsString], + out: &Path, + name: &str, + env: &std::collections::BTreeMap, +) -> Result<(), String> { + run_limited(command, out, name, env, crate::process::Limits::default()) +} + +pub fn run_limited( + command: &[OsString], + out: &Path, + name: &str, + env: &std::collections::BTreeMap, + limits: crate::process::Limits, +) -> Result<(), String> { + if command.is_empty() { + return Err("empty process command".into()); + } + write(&out.join(format!("{name}.command.txt")), &shell(command))?; + let stdout = File::create(out.join(format!("{name}.stdout"))).map_err(|e| e.to_string())?; + let stderr = File::create(out.join(format!("{name}.stderr"))).map_err(|e| e.to_string())?; + let start = std::time::Instant::now(); + let mut process = Command::new(&command[0]); + for key in PERF_ENV { + process.env_remove(key); + } + process + .args(&command[1..]) + .env_remove("CUDA_INJECTION64_PATH") + .env_remove("NVTX_INJECTION64_PATH") + .env_remove("DS4_PERF_CUPTI_OUTPUT") + .envs(env) + .stdin(Stdio::null()) + .stdout(stdout) + .stderr(stderr); + let status = crate::process::run(&mut process, out, limits); + let success = status.as_ref().is_ok_and(|s| s.success()); + write( + &out.join(format!("{name}.status.txt")), + &format!( + "status={status:?}\nwall_sec={}\ntimeout_sec={}\noutput_budget_bytes={}\n", + start.elapsed().as_secs_f64(), + limits.timeout.as_secs(), + limits.bytes + ), + )?; + if success { + Ok(()) + } else { + Err(format!( + "{name} failed ({status:?}); see {}/{}.stderr", + out.display(), + name + )) + } +} + +// Explicitly reviewed runtime controls only. Never record arbitrary DS4_* keys. +const PERF_ENV: &[&str] = &[ + "LD_LIBRARY_PATH", + "LD_PRELOAD", + "CUDA_CACHE_PATH", + "CUDA_HOME", + "CUBLAS_WORKSPACE_CONFIG", + "CUDA_VISIBLE_DEVICES", + "CUDA_DEVICE_ORDER", + "CUDA_MODULE_LOADING", + "CUDA_CACHE_DISABLE", + "CUDA_CACHE_MAXSIZE", + "CUDA_FORCE_PTX_JIT", + "CUDA_LAUNCH_BLOCKING", + "NVIDIA_VISIBLE_DEVICES", + "NVIDIA_DRIVER_CAPABILITIES", + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "DS4_WEIGHT_SERVER", + "DS4_CUDA_WEIGHT_IPC_MANIFEST", + "DS4_CUDA_WEIGHT_IPC_SCOPE", + "DS4_CUDA_WEIGHT_IPC_NO_DRAFTER", + "DS4_WEIGHT_RESIDENCY", + "DS4_WEIGHT_RESIDENCY_BASE", + "DS4_WEIGHT_RESIDENCY_MTP", + "DS4_WEIGHT_RESIDENCY_DRAFTER", + "DS4_CUDA_KEEP_MODEL_PAGES", + "DS4_CUDA_DIRECT_MODEL", + "DS4_CUDA_NO_FD_CACHE", + "DS4_CUDA_NO_HBM_CACHE", + "DS4_MEMGOV", + "DS4_MEMGOV_HEADROOM_MB", + "DS4_SESSION_LAZY_GRAPH", + "DS4_CUDA_CAPTURE", + "DS4_CUDA_NO_GRAPH", + "DS4_CUDA_FP8_KV", + "DS4_CUDA_FP4_INDEX", + "DS4_CUDA_MMQ", + "DS4_MTP_SPEC_DISABLE", + "DS4_GLM53_PREFILL_ROWS", + "DS4_GLM53_DSA_EXPANDED", + "DS4_GLM53_MTP", + "DS4_GLM53_LOW_ATTN", + "DS4_GLM53_POOL_WARP", + "DS4_GLM53_SHARED_Q8", + "DS4_GLM53_Q2_WORKLIST", + "DS4_QWEN_BATCH", + "DS4_QWEN_PREFILL_CHUNK", + "DS4_QWEN_PLE_CACHE_MB", + "DS4_QWEN_PLE_WORKERS", + "DS4_QWEN_PLE_DIR", + "DS4_PLE_CUDA_TILE_ROWS", + "DS4_PLE_NO_BATCH_ACQUIRE", + "DS4_QWEN_NO_SWIGLU_Q8_EMIT", + "DS4_QWEN_QSA_NO_FUSED", + "DS4_QWEN_PLE_NO_LOOKAHEAD", + "DS4_QWEN_MTP_SEQUENTIAL_VERIFY", + "DS4_SOLAR_KV_FORMAT", + "DS4_SOLAR_MOE_RESIDUAL", + "DS4_CUDA_SOLAR_GQA_GROUPED", + "DS4_CUDA_SOLAR_GQA_CHUNK", + // Solar graph width; DS4_CONT_PREFILL_CHUNK does not set it. + "DS4_METAL_PREFILL_CHUNK", + "DS4_FATTN_HMMA_LDSM", + "DS4_SOLAR_FATTN_GQA2", + "DS4_SOLAR_FATTN_WS", + "DS4_DOTS3_PREFILL_CHUNK", + "DS4_STEP37_PREFILL_CHUNK", + "DS4_LING3VL_PREFILL_CHUNK", + "DS4_LING3VL_NO_BF16_VEC", + "DS4_LING3VL_NO_MLA_TILE", + "DS4_LING3VL_MLA_TILE", + "DS4_LING3VL_NO_MLA_HMMA", + "DS4_LING3VL_NO_BF16_REUSE", + "DS4_LING3VL_NO_BF16_PAIR", + "DS4_LING3VL_NO_GEMV_XREG", + "DS4_LING3VL_NO_F32_VEC", + "DS4_LING3VL_NO_MOE_FUSE", + "DS4_CUDA_LAYER_GRAPHS", + "DS4_STEP37_NO_SWA_HMMA", + "DS4_EXAONE_PREFILL_GQA", + "DS4_INKLING_NO_LINEAR", + "DS4_INKLING_NO_MOE_BATCH", + "DS4_INKLING_NO_Q8_BATCH", + "DS4_INKLING_NO_MOE_TILE", + "DS4_INKLING_NO_LINEAR_TILE", + "DS4_INKLING_NO_LOGIT_TILE", + "DS4_INKLING_NO_LINEAR_PANEL", + "DS4_INKLING_NO_ATTN_GROUP", + "DS4_INKLING_NO_Q8_TILE", + "DS4_INKLING_NO_SHARED_Q8", + "DS4_INKLING_NO_SHARED_TILE", + "DS4_INKLING_NO_SHARED_DOWN_TILE", + "DS4_INKLING_NO_Q4_TILE", + "DS4_INKLING_NO_Q8_ROUTED_TILE", + "DS4_INKLING_NO_IQ2_ALIGNED", + "DS4_INKLING_NO_IQ2_XS_ALIGNED", + "DS4_INKLING_NO_SHARED_SOA", + "DS4_INKLING_NO_IQ2_LEAN", + "DS4_INKLING_NO_Q3_TILE", + "DS4_INKLING_NO_ATTN_TRANSPOSE", + "DS4_INKLING_NO_SHARED_COLUMN", + "DS4_INKLING_NO_ATTN_PAIR", + "DS4_INKLING_NO_Q4_LEAN", + "DS4_INKLING_NO_SHARED_PIPE", + "DS4_INKLING_NO_IQ2_SLAB", + "DS4_INKLING_ATTN_HMMA", + "DS4_MIMO2_NO_PREFILL_HMMA", + "DS4_MIMO2_NO_PREFILL_ASYNC", + "DS4_MIMO2_NO_SWA_HMMA", + "DS4_MIMO2_SWA_DECODE", + "DS4_MIMO2_SWA_VEC", + "DS4_MIMO2_ROUTER_WARP", + "DS4_MIMO2_DFLASH_CPU", + "DS4_MIMO2_SWIGLU_Q8", + "DS4_MIMO2_SUM_RESIDUAL", + "DS4_MIMO2_ATTN_RESIDUAL", + "DS4_MIMO2_GATEUP_BOUNDED", + "DS4_MIMO2_INPUT_Q8_COMPACT", + "DS4_MIMO2_DOWN_PIPE64", + "DS4_NAIVE_DECODE_SCORES", + "DS4_NAIVE_SWA_PREFILL_SCORES", + "DS4_NAIVE_DSA_DECODE_TILE", + "DS4_NAIVE_DSA_DIRECT", + "DS4_NAIVE_SWIGLU_Q8", + "DS4_NAIVE_INDEX_PACK", + "DS4_NAIVE_INDEX_U2", + "DS4_NAIVE_SWA_DECODE_UNIT", + "DS4_NAIVE_SWA_RING_WALK", + "DS4_NAIVE_ROUTER_WARP", + "DS4_NAIVE_SUM_ADD", + "DS4_INKLING_PREFILL_CHUNK", + "DS4_DOTS3_ATTN_NO_HMMA", + "DS4_DOTS3_ATTN_NO_SPLIT", + "DS4_DOTS3_VALUE_NO_HMMA", + "DS4_DOTS3_VALUE_NO_DECODE", + "DS4_DOTS3_ABSORB_NO_HMMA", + "DS4_DOTS3_NO_FUSED", + "DS4_QWEN_PREFILL_OPENING", + "DS4_QWEN_HC_NO_FUSED_MIX", + "DS4_CUDA_NO_NORM_Q8EMIT", + "DS4_MMQ_DENSE_D2R", + "DS4_MMQ_D2R_MAX_K", + "DS4_MMQ_D2R", + "DS4_MMQ_D2R_IQ2", + "DS4_MMQ_D2R_MIN_COLS", + "DS4_MMQ_NO_YIND", + "DS4_MMQ_PIPE", + "DS4_MMQ_PIPE_MAX_X", + "DS4_MMQ_IQ1M_PREFILL", + "DS4_MMQ_IQ1M_SLOT_LOOP", + "DS4_MMQ_IQ1M_WORKLIST", + "DS4_MMQ_IQ1_PAIR", + "DS4_MMQ_Q5_PAIR", + "DS4_MMQ_VEC_SANITIZE", + "DS4_MMQ_IQ2XS_WORKLIST", + "DS4_PLE_LATENCY_STATS", + "DS4_MMQ_WORKLIST", + "DS4_MMQ_IQ1S_WORKLIST", + "DS4_MMQ_IQ2XXS_WORKLIST", + "DS4_SERVER_CONTINUOUS", + "DS4_CONT_PREFILL_CHUNK", + "DS4_CONT_CAPTURE", + "DS4_BATCH_MAX_SEQ", + "DS4_BATCH_FIT_HEADROOM_MB", + "DS4_CONT_MTP_MODE", +]; + +pub fn reviewed_control(name: &str) -> bool { + PERF_ENV.contains(&name) +} + +pub fn controls( + overrides: &[String], +) -> Result, String> { + let mut env: std::collections::BTreeMap<_, _> = std::env::vars_os() + .filter(|(k, _)| k.to_str().is_some_and(|k| PERF_ENV.contains(&k))) + .collect(); + for assignment in overrides { + let (key, value) = assignment + .split_once('=') + .ok_or("--env requires NAME=VALUE")?; + if !PERF_ENV.contains(&key) { + return Err(format!("unreviewed performance environment control: {key}")); + } + env.insert(key.into(), value.into()); + } + Ok(env) +} + +pub fn environment(vars: impl IntoIterator) -> String { + let mut vars: Vec<_> = vars + .into_iter() + .filter(|(key, _)| key.to_str().is_some_and(|k| PERF_ENV.contains(&k))) + .collect(); + vars.sort(); + let mut out = String::from("# Strict allowlist; all inherited variables still reach both children.\n# This file is not a complete environment or a cache-state proof.\n"); + for (key, value) in vars { + out.push_str(&format!( + "{}={}\n", + key.to_string_lossy(), + quote(&value.to_string_lossy()) + )); + } + out +} + +pub fn resolve(program: &OsStr) -> Option { + let path = Path::new(program); + if path.components().count() > 1 { + return path.canonicalize().ok(); + } + std::env::split_paths(&std::env::var_os("PATH")?) + .map(|p| p.join(path)) + .find(|p| p.is_file()) + .and_then(|p| p.canonicalize().ok()) +} + +pub fn scout(args: &crate::cli::Scout) -> Result<(), String> { + if args.device != 0 { + return Err("ds4-bench uses visible CUDA device 0; select hardware with CUDA_VISIBLE_DEVICES before running ds4-perf".into()); + } + if matches!(args.collector, crate::cli::Collector::Nsys) + && (args.cupti_library.is_some() || args.cupti_sdk.is_some()) + { + return Err("CUPTI library options require --collector cupti".into()); + } + let out = &args.out; + if out.as_os_str().to_string_lossy().contains('%') { + return Err( + "output path must not contain Nsight filename substitution character '%'".into(), + ); + } + if let Some(parent) = out.parent().filter(|p| !p.as_os_str().is_empty()) { + fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + fs::create_dir(out) + .map_err(|e| format!("cannot create new output directory {}: {e}", out.display()))?; + let out = out.canonicalize().map_err(|e| e.to_string())?; + let result = match crate::experiment::prepare(&out, args) { + Ok(prepared) => { + let pinned = prepared.pin(args); + let work = scout_inner(&out, &pinned, &prepared); + let finish = crate::experiment::finish(&out, &pinned, prepared, &work); + work.map(|_| ()).and(finish) + } + Err(error) => { + crate::experiment::failed(&out, error.clone())?; + Err(error) + } + }; + let result = result.and_then(|()| crate::process::check_bytes(&out, args.budget.limits())); + write( + &out.join("status.txt"), + &match &result { + Ok(()) => "COMPLETE\n".into(), + Err(err) => format!("FAILED\n{err}\n"), + }, + )?; + result +} + +fn scout_inner( + out: &Path, + args: &crate::cli::Scout, + prepared: &crate::experiment::Prepared, +) -> Result { + let command = &args.command; + use crate::{bench, csv, doctor, nsys, report}; + eprintln!("ds4-perf: capability probes; artifacts {}", out.display()); + let mut system = System; + let mut d = doctor::inspect(&mut system, None); + d.facts.insert("benchmark".into(), shell(command)); + d.save_probes(out)?; + write(&out.join("doctor.txt"), &d.render())?; + write(&out.join("command.txt"), &format!("{}\n", shell(command)))?; + // Preserve even non-UTF-8 Unix argv byte-for-byte alongside the shell display. + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + let mut argv = Vec::new(); + for arg in command { + argv.extend_from_slice(arg.as_bytes()); + argv.push(0); + } + fs::write(out.join("command.argv"), argv).map_err(|e| e.to_string())?; + } + write( + &out.join("env.txt"), + &environment(prepared.environment.clone()), + )?; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| e.to_string())? + .as_secs(); + let mut manifest = format!( + "format=ds4-perf-v1\ntimestamp_unix={timestamp}\ncwd={}\nbenchmark_command={}\n", + std::env::current_dir() + .map_err(|e| e.to_string())? + .display(), + shell(command) + ); + if let Some(binary) = resolve(&command[0]) { + manifest.push_str(&format!("benchmark_executable={}\n", binary.display())); + let hash = Command::new("sha256sum").arg(&binary).output(); + if let Ok(hash) = hash { + if hash.status.success() { + manifest.push_str(&format!( + "benchmark_sha256={}\n", + String::from_utf8_lossy(&hash.stdout) + .split_whitespace() + .next() + .unwrap_or("unknown") + )); + } + } + } + manifest.push_str("throughput_source=bench.stdout (unprofiled)\nprofiled_throughput=informational only\ncache_policy=inherited; fresh process does not establish cold OS/owner caches\nenvironment_scope=strict allowlist, not a full comparison identity\n"); + save_manifest(out, &d, &manifest)?; + hygiene(&mut system, out, "before", &mut d.warnings)?; + let mut evidence = nsys::Evidence::default(); + let mut rows = Vec::new(); + let work = (|| -> Result<(), String> { + if matches!(args.collector, crate::cli::Collector::Nsys) && !d.caps.can_scout() { + return Err( + "nsys CUDA/NVTX collection or required kernel reports unavailable; see doctor.txt" + .into(), + ); + } + if args.cache_policy == "warmup-then-fresh" { + prepared.run(command, out, "warmup")?; + } + eprintln!("ds4-perf: fresh unprofiled baseline"); + prepared.run( + &crate::experiment::bench_command(args, out, "bench"), + out, + "bench", + )?; + let stderr = fs::read_to_string(out.join("bench.stderr")).map_err(|e| e.to_string())?; + d.benchmark(&stderr, "ds4-bench NVTX: "); + if d.facts["NVTX"].starts_with("unavailable") { + return Err("target benchmark lacks NVTX; build with make ds4-bench-perf and pass ./ds4-bench-perf to scout".into()); + } + match bench::parse(csv::open(&out.join("bench.stdout")).map_err(|e| e.to_string())?) { + Ok(parsed) => rows = parsed, + Err(err) => evidence + .warnings + .push(format!("baseline CSV unavailable: {err}")), + } + if rows.is_empty() { + evidence.warnings.push( + "baseline has no recognized CSV; emit ds4-bench CSV on stdout (omit --csv FILE)" + .into(), + ); + } + write( + &out.join("baseline.csv"), + &format!( + "{}\n{}", + bench::HEADER, + rows.iter().map(bench::Row::csv).collect::() + ), + )?; + for index in 1..args.repeats { + let name = format!("bench-{index:02}"); + if args.cache_policy == "warmup-then-fresh" { + prepared.run(command, out, &format!("warmup-{index:02}"))?; + } + eprintln!("ds4-perf: fresh unprofiled sample {index}"); + prepared.run( + &crate::experiment::bench_command(args, out, &name), + out, + &name, + )?; + } + if args.cache_policy == "warmup-then-fresh" { + prepared.run(command, out, "profile-warmup")?; + } + let warnings = std::mem::take(&mut evidence.warnings); + evidence = match args.collector { + crate::cli::Collector::Nsys => { + eprintln!("ds4-perf: fresh Nsight Systems process"); + prepared.run( + &nsys::profile_command(&d.caps, out, command), + out, + "profile", + )?; + if !out.join("trace.nsys-rep").is_file() { + return Err( + "nsys did not produce trace.nsys-rep; see profile.stdout/stderr".into(), + ); + } + nsys::collect_prepared(&d.caps, out, prepared) + } + crate::cli::Collector::Cupti => { + crate::cupti::capture(out, args, &prepared.environment)? + } + }; + evidence.warnings.extend(warnings); + Ok(()) + })(); + if let Err(err) = &work { + evidence.warnings.push(format!("INCOMPLETE SCOUT: {err}")); + } + hygiene(&mut system, out, "after", &mut d.warnings)?; + write(&out.join("doctor.txt"), &d.render())?; + save_manifest(out, &d, &manifest)?; + normalized(out, &evidence)?; + let report = report::render(&d, command, &rows, &evidence); + write(&out.join("report.txt"), &report)?; + print!("{report}"); + work.map(|_| evidence) +} + +fn save_manifest(out: &Path, d: &crate::doctor::Doctor, context: &str) -> Result<(), String> { + let mut manifest = context.to_string(); + for (name, value) in &d.facts { + manifest.push_str(&format!("{name}={}\n", value.replace('\n', "\\n"))); + } + write(&out.join("manifest.txt"), &manifest) +} + +fn hygiene( + probe: &mut impl Probe, + out: &Path, + label: &str, + warnings: &mut Vec, +) -> Result<(), String> { + let gpu = probe.capture( + "nvidia-smi", + &[ + "--query-compute-apps=pid,process_name", + "--format=csv,noheader", + ], + ); + let processes = probe.capture("ps", &["-eo", "pid=,comm="]); + write( + &out.join(format!("processes-{label}.txt")), + &format!( + "gpu_query_ok={}\n{}\n{}\nps_ok={}\n{}\n{}", + gpu.ok, gpu.out, gpu.err, processes.ok, processes.out, processes.err + ), + )?; + if !gpu.ok || !processes.ok { + warnings.push(format!( + "process hygiene {label}: could not establish process state; no processes were stopped" + )); + } else if !gpu.out.trim().is_empty() { + warnings.push(format!("process hygiene {label}: resident GPU processes exist (see processes-{label}.txt); check owner/worker contention before comparisons")); + } + Ok(()) +} + +fn normalized(out: &Path, evidence: &crate::nsys::Evidence) -> Result<(), String> { + let mut phases = + String::from("phase,range_ns,projected_ns,busy_union_ns,memop_union_ns,largest_gap_ns\n"); + let mut kernels = String::from("phase,kernel,total_ns,instances\n"); + for p in evidence.phases.values() { + let numbers = [p.wall_ns, p.projected_ns, p.busy_ns, p.mem_ns, p.gap_ns] + .map(|v| v.map(|v| v.to_string()).unwrap_or_default()); + phases.push_str(&format!("{},{}\n", p.name, numbers.join(","))); + for k in &p.kernels { + kernels.push_str(&format!( + "{},{},{},{}\n", + p.name, + crate::csv::field(&k.name), + k.total_ns, + k.count.map(|n| n.to_string()).unwrap_or_default() + )); + } + } + // Empty phase denotes whole-capture CUDA evidence, including setup. + for k in &evidence.global { + kernels.push_str(&format!( + ",{},{},{}\n", + crate::csv::field(&k.name), + k.total_ns, + k.count.map(|n| n.to_string()).unwrap_or_default() + )); + } + write(&out.join("normalized-phases.csv"), &phases)?; + write(&out.join("normalized-kernels.csv"), &kernels) +} + +pub fn replay_controls( + overrides: &[String], +) -> Result, String> { + let mut env = controls(overrides)?; + let keys: std::collections::BTreeSet<_> = overrides + .iter() + .filter_map(|v| v.split_once('=').map(|(k, _)| OsString::from(k))) + .collect(); + env.retain(|k, _| keys.contains(k)); + Ok(env) +} + +pub fn device_environment( + env: &std::collections::BTreeMap, +) -> Result<(), String> { + for key in [ + "CUDA_VISIBLE_DEVICES", + "CUDA_DEVICE_ORDER", + "LD_LIBRARY_PATH", + "LD_PRELOAD", + ] { + if env.get(std::ffi::OsStr::new(key)).cloned() != std::env::var_os(key) { + return Err(format!("set {key} on ds4-perf itself so inspection and benchmark use the same device/libraries")); + } + } + Ok(()) +} + +pub fn unreviewed_env() -> Vec { + std::env::vars_os() + .filter_map(|(k, _)| k.into_string().ok()) + .filter(|k| { + ["DS4_", "CUDA_", "LD_"] + .iter() + .any(|prefix| k.starts_with(prefix)) + && !PERF_ENV.contains(&k.as_str()) + && ![ + "CUDA_INJECTION64_PATH", + "NVTX_INJECTION64_PATH", + "DS4_PERF_CUPTI_OUTPUT", + ] + .contains(&k.as_str()) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn glm_kernel_controls() { + for key in [ + "DS4_GLM53_LOW_ATTN", + "DS4_GLM53_POOL_WARP", + "DS4_GLM53_SHARED_Q8", + "DS4_GLM53_Q2_WORKLIST", + ] { + for value in ["0", "1"] { + let env = controls(&[format!("{key}={value}")]).unwrap(); + assert_eq!(env.get(std::ffi::OsStr::new(key)), Some(&value.into())); + assert!(environment(env).contains(&format!("{key}='{value}'"))); + } + } + } + + #[test] + fn glm_ssd_controls() { + for (key, value) in [ + ("DS4_GLM53_PREFILL_ROWS", "128"), + ("DS4_GLM53_DSA_EXPANDED", "0"), + ("DS4_GLM53_MTP", "1"), + ] { + let env = controls(&[format!("{key}={value}")]).unwrap(); + assert_eq!(env.get(std::ffi::OsStr::new(key)), Some(&value.into())); + assert!(environment(env).contains(&format!("{key}='{value}'"))); + } + assert!(controls(&["DS4_GLM53_UNKNOWN=secret".into()]).is_err()); + assert!(!environment([("DS4_GLM53_UNKNOWN".into(), "secret".into())]).contains("secret")); + } + + #[test] + fn naive_attention_env() { + for key in [ + "DS4_NAIVE_DECODE_SCORES", + "DS4_NAIVE_SWA_PREFILL_SCORES", + "DS4_NAIVE_DSA_DECODE_TILE", + "DS4_NAIVE_DSA_DIRECT", + "DS4_NAIVE_SWIGLU_Q8", + "DS4_NAIVE_INDEX_PACK", + "DS4_NAIVE_INDEX_U2", + "DS4_NAIVE_SWA_DECODE_UNIT", + "DS4_NAIVE_SWA_RING_WALK", + "DS4_NAIVE_ROUTER_WARP", + "DS4_NAIVE_SUM_ADD", + ] { + for value in ["0", "1"] { + let env = controls(&[format!("{key}={value}")]).unwrap(); + assert_eq!(env.get(std::ffi::OsStr::new(key)), Some(&value.into())); + assert!(environment(env).contains(&format!("{key}='{value}'"))); + } + } + } + + #[test] + fn mimo2_fusion_env() { + for key in [ + "DS4_MIMO2_SUM_RESIDUAL", + "DS4_MIMO2_ATTN_RESIDUAL", + "DS4_MIMO2_GATEUP_BOUNDED", + "DS4_MIMO2_INPUT_Q8_COMPACT", + "DS4_MIMO2_DOWN_PIPE64", + ] { + for value in ["0", "1"] { + let env = controls(&[format!("{key}={value}")]).unwrap(); + assert_eq!(env.get(std::ffi::OsStr::new(key)), Some(&value.into())); + assert!(environment(env).contains(&format!("{key}='{value}'"))); + } + } + } + + #[test] + fn env_is_allowlisted() { + let out = environment( + [ + ("DS4_QWEN_PLE_CACHE_MB", "2048"), + ("DS4_QWEN_PLE_DIR", "/models/PLE-FP8"), + ("DS4_QWEN_PREFILL_OPENING", "1"), + ("DS4_MMQ_DENSE_D2R", "0"), + ("DS4_METAL_PREFILL_CHUNK", "4096"), + ("DS4_STEP37_PREFILL_CHUNK", "512"), + ("DS4_STEP37_NO_SWA_HMMA", "1"), + ("DS4_EXAONE_PREFILL_GQA", "0"), + ("DS4_FATTN_HMMA_LDSM", "0"), + ("DS4_SOLAR_FATTN_GQA2", "1"), + ("DS4_MIMO2_SWA_DECODE", "0"), + ("DS4_API_KEY", "secret"), + ("DS4_UNKNOWN", "secret"), + ("HF_TOKEN", "secret"), + ("CUDA_VISIBLE_DEVICES", "0"), + ] + .map(|(k, v)| (k.into(), v.into())), + ); + assert!(out.contains("DS4_QWEN_PLE_CACHE_MB='2048'")); + assert!(out.contains("DS4_QWEN_PLE_DIR='/models/PLE-FP8'")); + assert!(out.contains("DS4_QWEN_PREFILL_OPENING='1'")); + assert!(out.contains("DS4_MMQ_DENSE_D2R='0'")); + assert!(out.contains("DS4_METAL_PREFILL_CHUNK='4096'")); + assert!(out.contains("DS4_STEP37_PREFILL_CHUNK='512'")); + assert!(out.contains("DS4_STEP37_NO_SWA_HMMA='1'")); + assert!(out.contains("DS4_EXAONE_PREFILL_GQA='0'")); + assert!(out.contains("DS4_FATTN_HMMA_LDSM='0'")); + assert!(out.contains("DS4_SOLAR_FATTN_GQA2='1'")); + assert!(out.contains("DS4_MIMO2_SWA_DECODE='0'")); + assert!(controls(&[ + "DS4_MIMO2_SWA_DECODE=0".into(), + "DS4_MIMO2_SWA_VEC=0".into() + ]) + .is_ok()); + assert!(!out.contains("secret")); + assert_eq!(quote("a'$(x)`id`"), "'a'\\''$(x)`id`'"); + } +} diff --git a/crates/ds4-perf/src/serving.rs b/crates/ds4-perf/src/serving.rs new file mode 100644 index 0000000000000000000000000000000000000000..5bdd47da0597f08f94bc5a5db24bfb63e95b1a94 --- /dev/null +++ b/crates/ds4-perf/src/serving.rs @@ -0,0 +1,843 @@ +use crate::{ + cli, process, + serving_http::{Event, Http}, +}; +use ds4_perf::artifact::{self, Artifact, Payload}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{collections::BTreeSet, fs, path::Path}; + +mod context; +mod controls; +mod overlap; +mod profile; +mod restart; +mod review; +mod telemetry; +mod tools; + +pub fn controls(args: &cli::ServingControls) -> Result<(), String> { + controls::run(args) +} + +pub fn profile(args: &cli::ServingProfile) -> Result<(), String> { + profile::run(args) +} + +pub fn apply(args: &cli::ApplyProfile) -> Result<(), String> { + profile::apply(args) +} + +const PROTOCOL: &str = "ds4-serving-v1"; +const MAX_MANIFEST_BYTES: u64 = 16 * 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Workload { + protocol: String, + name: String, + family: String, + cases: Vec, + #[serde(default)] + overlaps: Vec, + #[serde(default)] + inputs: std::collections::BTreeMap, + #[serde(default)] + expected_clock_range_mhz: Option, + #[serde(default)] + restart_from: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Case { + name: String, + scenario: Scenario, + request: Value, + expect: Expect, + limits: Contract, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Scenario { + KvColdPrefill, + WarmAppend, + PartialBranch, + Media, + Mtp, + Tool, + RestartRestore, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Expect { + content: String, + finish_reason: String, + reuse_kind: String, + effective_lane: String, + speculation_active: bool, + fallback_reason: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + tool_calls: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + min_cached_tokens: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Contract { + ttft_ms: f64, + total_ms: f64, + min_host_available_bytes: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Observation { + name: String, + repeat: u32, + scenario: Scenario, + ttft_ms: f64, + first_content_ms: Option, + total_ms: f64, + content: String, + finish_reason: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + tool_calls: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + cached_tokens: Option, + host_available_before_bytes: u64, + host_available_after_bytes: u64, + #[serde(default)] + host_min_available_bytes: Option, + trace: Value, + failures: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Evidence { + protocol: String, + name: String, + family: String, + serving_plan: Value, + cases: Vec, + overlaps: Vec, + requested_repeats: u32, + identity: context::Identity, + gpu_samples: Vec, + #[serde(default)] + gpu_windows: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + restart_from: Option, + context_failures: Vec, + passed: bool, + timing_method: String, +} + +impl Payload for Evidence { + const KIND: &'static str = "serving"; + fn validate(&self) -> Result<(), String> { + if self.protocol != PROTOCOL + || (self.cases.is_empty() && self.overlaps.is_empty()) + || self.passed + != (self.cases.iter().all(|case| case.failures.is_empty()) + && self.overlaps.iter().all(overlap::Observation::passed) + && self.context_failures.is_empty()) + || self.cases.iter().any(|case| { + !case.ttft_ms.is_finite() + || case.ttft_ms <= 0.0 + || !case.total_ms.is_finite() + || case.total_ms < case.ttft_ms + }) + { + return Err("invalid serving evidence".into()); + } + Ok(()) + } +} + +impl Workload { + fn validate(&self) -> Result<(), String> { + if let Some(range) = &self.expected_clock_range_mhz { + range.validate()?; + } + if self.protocol != PROTOCOL + || self.name.is_empty() + || self.family.is_empty() + || (self.cases.is_empty() && self.overlaps.is_empty()) + || self.cases.len() > 128 + || self.overlaps.len() > 32 + { + return Err( + "serving workload needs protocol ds4-serving-v1, name, family and 1..128 cases" + .into(), + ); + } + let mut names = BTreeSet::new(); + if self.restart_from.is_some() + != self + .cases + .first() + .is_some_and(|case| matches!(case.scenario, Scenario::RestartRestore)) + { + return Err("restart_from requires restart_restore as the first case".into()); + } + for case in &self.cases { + if case.name.is_empty() + || !names.insert(&case.name) + || case.request.get("stream") != Some(&Value::Bool(true)) + || case + .request + .get("messages") + .and_then(Value::as_array) + .is_none_or(Vec::is_empty) + || case.request.get("n").is_some_and(|n| n.as_u64() != Some(1)) + || !tools::valid_finish( + &case.expect.content, + &case.expect.finish_reason, + &case.expect.tool_calls, + ) + || case + .expect + .tool_calls + .iter() + .any(|call| !call.declared(&case.request)) + || case.expect.min_cached_tokens.is_some_and(|n| { + n == 0 || case.request["stream_options"]["include_usage"] != true + }) + || !["cold", "exact", "partial", "fork"].contains(&case.expect.reuse_kind.as_str()) + || !["serial", "continuous", "static"] + .contains(&case.expect.effective_lane.as_str()) + || [case.limits.ttft_ms, case.limits.total_ms] + .iter() + .any(|v| !v.is_finite() || *v <= 0.0) + || case.limits.min_host_available_bytes == 0 + { + return Err(format!( + "{}: invalid request, expectation or limits", + case.name + )); + } + let valid = match case.scenario { + Scenario::KvColdPrefill => case.expect.reuse_kind == "cold", + Scenario::WarmAppend => { + matches!( + case.expect.reuse_kind.as_str(), + "exact" | "partial" | "fork" + ) + } + Scenario::PartialBranch => { + matches!(case.expect.reuse_kind.as_str(), "partial" | "fork") + } + Scenario::Mtp => case.expect.speculation_active, + Scenario::Tool => !case.expect.tool_calls.is_empty(), + Scenario::RestartRestore => { + self.restart_from.is_some() + && case.expect.min_cached_tokens.is_some() + && matches!( + case.expect.reuse_kind.as_str(), + "exact" | "partial" | "fork" + ) + } + Scenario::Media => case.request["messages"].as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message["content"].as_array().is_some_and(|parts| { + parts.iter().any(|part| { + matches!(part["type"].as_str(), Some("image_url" | "input_audio")) + }) + }) + }) + }), + }; + if !valid { + return Err(format!( + "{}: expectation contradicts the serving scenario", + case.name + )); + } + } + for overlap in &self.overlaps { + overlap.validate()?; + if !names.insert(overlap.name()) { + return Err("serving case names must be unique".into()); + } + } + Ok(()) + } +} + +fn host_available() -> Result { + let text = fs::read_to_string("/proc/meminfo") + .map_err(|e| format!("local host memory unavailable: {e}"))?; + text.lines() + .find_map(|line| { + line.strip_prefix("MemAvailable:") + .and_then(|v| v.split_whitespace().next()) + .and_then(|v| v.parse::().ok()) + }) + .and_then(|kb| kb.checked_mul(1024)) + .ok_or("MemAvailable missing".into()) +} + +fn plan(stats: &Value, family: &str) -> Result { + let plan = &stats["serving"]; + if plan["family"] != family + || !plan["requested"].is_object() + || !plan["effective"].is_object() + || !plan["qualified"].is_object() + || plan["issues"] + .as_array() + .is_none_or(|issues| issues.iter().any(|i| i["level"] == "error")) + { + return Err("server stats lack an error-free resolved plan for the workload family".into()); + } + if stats["queue_depth"].as_u64() != Some(0) + || stats["clients"].as_u64().is_none_or(|clients| clients > 1) + { + return Err("serving collection requires an idle server with no competing clients".into()); + } + Ok(plan.clone()) +} + +fn route_count(stats: &Value) -> Result { + stats["routes"] + .as_object() + .ok_or("server stats lack route counters")? + .values() + .try_fold(0u64, |sum, n| { + sum.checked_add(n.as_u64().ok_or("invalid route counter")?) + .ok_or("route counter overflow") + }) + .map_err(Into::into) +} + +fn post_open_quote(plan: &Value) -> Value { + let mut quote = plan["quote"].clone(); + if let Some(fields) = quote.as_object_mut() { + fields.remove("available"); + } + quote +} + +struct Completion { + ttft_ms: f64, + first_content_ms: Option, + total_ms: f64, + content: String, + finish_reason: String, + tool_calls: Vec, + cached_tokens: Option, +} + +fn summarize(events: &[Event]) -> Result { + let mut first = None; + let mut visible = None; + let mut done = None; + let mut content = String::new(); + let mut finished = None; + let mut tool_calls = tools::Calls::default(); + let mut cached_tokens = None; + let mut usage_seen = false; + for event in events { + if done.is_some() { + return Err("SSE data after DONE".into()); + } + if event.data == "[DONE]" { + done = Some(event.elapsed_ms); + continue; + } + let value: Value = + serde_json::from_str(&event.data).map_err(|e| format!("invalid SSE JSON: {e}"))?; + if value.get("error").is_some() { + return Err(format!("server streaming error: {}", value["error"])); + } + if let Some(usage) = value.get("usage").filter(|usage| !usage.is_null()) { + if usage_seen { + return Err("duplicate streaming usage".into()); + } + usage_seen = true; + cached_tokens = usage["prompt_tokens_details"]["cached_tokens"].as_u64(); + if cached_tokens.is_some_and(|cached| { + usage["prompt_tokens"] + .as_u64() + .is_none_or(|prompt| cached > prompt) + }) { + return Err("cached token usage exceeds prompt".into()); + } + } + let choices = value["choices"] + .as_array() + .ok_or("SSE event lacks choices")?; + if choices.len() > 1 { + return Err("serving collection supports one completion per request".into()); + } + let Some(choice) = choices.first() else { + continue; + }; + if finished.is_some() { + return Err("SSE choice after finish reason".into()); + } + if let Some(reason) = choice["finish_reason"].as_str() { + if !["stop", "length", "tool_calls"].contains(&reason) || finished.is_some() { + return Err("unsupported or duplicate SSE finish reason".into()); + } + finished = Some(reason.to_owned()); + } + let delta = &choice["delta"]; + if tool_calls.add(delta)? { + first.get_or_insert(event.elapsed_ms); + } + if let Some(text) = delta["content"].as_str().filter(|text| !text.is_empty()) { + visible.get_or_insert(event.elapsed_ms); + content.push_str(text); + } + if delta["content"] + .as_str() + .is_some_and(|text| !text.is_empty()) + || delta["reasoning_content"] + .as_str() + .is_some_and(|text| !text.is_empty()) + { + first.get_or_insert(event.elapsed_ms); + } + } + let finish_reason = finished.ok_or("SSE stream lacks a finish reason")?; + let tool_calls = tool_calls.finish()?; + let generated_calls = !tool_calls.is_empty(); + if generated_calls != (finish_reason == "tool_calls") { + return Err("tool call finish reason disagrees with generated calls".into()); + } + Ok(Completion { + ttft_ms: first.ok_or("SSE stream has no generated output")?, + first_content_ms: visible, + total_ms: done.ok_or("SSE stream lacks DONE")?, + content, + finish_reason, + tool_calls, + cached_tokens, + }) +} + +fn generated(event: &Event) -> bool { + serde_json::from_str::(&event.data).is_ok_and(|value| { + let delta = &value["choices"][0]["delta"]; + ["content", "reasoning_content"] + .into_iter() + .any(|key| delta[key].as_str().is_some_and(|text| !text.is_empty())) + || delta["tool_calls"].as_array().is_some_and(|calls| { + calls.iter().any(|call| { + ["name", "arguments"].iter().any(|key| { + call["function"][key] + .as_str() + .is_some_and(|text| !text.is_empty()) + }) + }) + }) + }) +} + +fn references(dir: &Path, root: &Path, out: &mut Vec) -> Result<(), String> { + let mut paths = fs::read_dir(dir) + .map_err(|e| e.to_string())? + .map(|entry| entry.map(|e| e.path()).map_err(|e| e.to_string())) + .collect::, _>>()?; + paths.sort(); + for path in paths { + if path.is_dir() { + references(&path, root, out)?; + } else { + out.push(artifact::reference(&path, root)?); + } + } + Ok(()) +} + +fn collect(args: &cli::Serving, workload: Workload) -> Result { + let http = Http::new(&args.url, &args.out, args.budget.limits())?; + let mut checked = artifact::Verification::default(); + let identity = context::Identity::capture(args, &workload, &mut checked)?; + let restart_from = + restart::prepare(&workload, &identity, &args.out, args.repeats, &mut checked)?; + checked.finish()?; + let mut gpu_samples = Vec::new(); + let mut gpu_windows = Vec::new(); + let mut context_failures = Vec::new(); + let mut observations = Vec::new(); + let mut overlaps = Vec::new(); + let mut resolved = None; + for repeat in 0..args.repeats { + let root = if args.repeats == 1 { + args.out.clone() + } else { + let root = args.out.join(format!("repeat-{repeat:03}")); + fs::create_dir(&root).map_err(|e| e.to_string())?; + root + }; + let gpu = context::gpu(args, &root, "gpu-before")?; + context_failures.extend(gpu.failures(workload.expected_clock_range_mhz.as_ref())); + gpu_samples.push(gpu); + let (_, window) = telemetry::observe(args, &root, || { + for (index, case) in workload.cases.iter().cloned().enumerate() { + let dir = root.join(format!("case-{index:03}")); + fs::create_dir(&dir).map_err(|e| e.to_string())?; + fs::write( + dir.join("request.json"), + serde_json::to_vec_pretty(&case.request).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + let before = http.stats(&dir, "before")?; + let current = plan(&before, &workload.family)?; + if matches!(case.scenario, Scenario::RestartRestore) { + restart::first_request( + &workload, + ¤t, + &before, + index, + repeat, + &mut checked, + )?; + } + if resolved + .as_ref() + .is_some_and(|previous| previous != ¤t) + { + return Err("resolved serving plan changed during collection".into()); + } + resolved = Some(current.clone()); + let available_before = host_available()?; + let (events, available_min) = context::measure(&dir, || http.stream(&dir))?; + let available_after = host_available()?; + let after = http.stats(&dir, "after")?; + if current != plan(&after, &workload.family)? { + return Err("resolved serving plan changed during request".into()); + } + if route_count(&after)?.checked_sub(route_count(&before)?) != Some(1) { + return Err( + "request trace cannot be attributed: route count did not advance by one" + .into(), + ); + } + let output = summarize(&events)?; + let trace = &after["last_request"]; + let mut failures = Vec::new(); + for (key, expected) in [ + ("reuse_kind", Value::String(case.expect.reuse_kind)), + ("effective_lane", Value::String(case.expect.effective_lane)), + ( + "speculation_active", + Value::Bool(case.expect.speculation_active), + ), + ( + "fallback_reason", + case.expect + .fallback_reason + .map(Value::String) + .unwrap_or(Value::Null), + ), + ] { + if trace.get(key) != Some(&expected) { + failures.push(format!( + "{key}: expected {expected}, observed {}", + trace[key] + )); + } + } + if output.content != case.expect.content { + failures.push("content differs from expected output".into()); + } + if output.finish_reason != case.expect.finish_reason { + failures.push("finish_reason differs from expected output".into()); + } + if output.tool_calls != case.expect.tool_calls { + failures.push("tool calls differ from expected names/JSON arguments".into()); + } + if case.expect.min_cached_tokens.is_some_and(|minimum| { + output.cached_tokens.is_none_or(|cached| cached < minimum) + }) { + failures.push("cached token usage below contract".into()); + } + if output.ttft_ms > case.limits.ttft_ms { + failures.push(format!( + "TTFT {:.3} ms exceeds {} ms", + output.ttft_ms, case.limits.ttft_ms + )); + } + if output.total_ms > case.limits.total_ms { + failures.push(format!( + "total {:.3} ms exceeds {} ms", + output.total_ms, case.limits.total_ms + )); + } + if available_min.min(available_before).min(available_after) + < case.limits.min_host_available_bytes + { + failures.push("sampled host available memory below contract".into()); + } + let observation = Observation { + name: case.name, + repeat, + scenario: case.scenario, + ttft_ms: output.ttft_ms, + first_content_ms: output.first_content_ms, + total_ms: output.total_ms, + content: output.content, + finish_reason: output.finish_reason, + tool_calls: output.tool_calls, + cached_tokens: output.cached_tokens, + host_available_before_bytes: available_before, + host_available_after_bytes: available_after, + host_min_available_bytes: Some(available_min), + trace: trace.clone(), + failures, + }; + fs::write( + dir.join("result.json"), + serde_json::to_vec_pretty(&observation).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + observations.push(observation); + process::check_bytes(&args.out, args.budget.limits())?; + } + for (index, overlap) in workload.overlaps.iter().enumerate() { + let dir = root.join(format!("overlap-{index:03}")); + let (observation, current) = overlap::collect( + &http, + overlap, + &dir, + &workload.family, + resolved.as_ref(), + repeat, + )?; + resolved = Some(current); + overlaps.push(observation); + process::check_bytes(&args.out, args.budget.limits())?; + } + Ok(()) + })?; + context_failures.extend(window.failures(workload.expected_clock_range_mhz.as_ref())); + gpu_windows.push(window); + let gpu = context::gpu(args, &root, "gpu-after")?; + context_failures.extend(gpu.failures(workload.expected_clock_range_mhz.as_ref())); + gpu_samples.push(gpu); + } + identity.verify(args)?; + let passed = observations.iter().all(|case| case.failures.is_empty()) + && overlaps.iter().all(overlap::Observation::passed) + && context_failures.is_empty(); + Ok(Evidence { protocol: PROTOCOL.into(), name: workload.name, family: workload.family, + serving_plan: resolved.ok_or("no serving plan collected")?, cases: observations, overlaps, requested_repeats: args.repeats, passed, + identity, gpu_samples, gpu_windows, restart_from, context_failures, + timing_method: "client dispatch through complete SSE event; curl --no-buffer; 1 ms file observer; includes curl startup; no profiler".into() }) +} + +pub fn run(args: &cli::Serving) -> Result<(), String> { + if fs::metadata(&args.workload) + .map_err(|e| e.to_string())? + .len() + > MAX_MANIFEST_BYTES + { + return Err("serving workload exceeds 16 MiB".into()); + } + let bytes = fs::read(&args.workload).map_err(|e| e.to_string())?; + let mut workload: Workload = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + workload.validate()?; + if let Some(path) = &mut workload.restart_from { + *path = args + .workload + .parent() + .unwrap_or(Path::new(".")) + .join(&*path) + .canonicalize() + .map_err(|e| e.to_string())?; + } + for input in workload.inputs.values_mut() { + input.path = args + .workload + .parent() + .unwrap_or(Path::new(".")) + .join(&input.path) + .canonicalize() + .map_err(|e| e.to_string())?; + } + fs::create_dir(&args.out).map_err(|e| format!("{}: {e}", args.out.display()))?; + fs::write(args.out.join("workload.json"), bytes).map_err(|e| e.to_string())?; + let result = collect(args, workload); + let (mut evidence, failure) = match result { + Ok(data) => { + let failure = (!data.passed).then(|| "serving workload contract failed".to_owned()); + (Artifact::new(data), failure) + } + Err(error) => (Artifact::::failed(error.clone()), Some(error)), + }; + references(&args.out, &args.out, &mut evidence.inputs)?; + if let Some(data) = &evidence.data { + data.identity.add_refs(&mut evidence.inputs); + if let Some(seed) = &data.restart_from { + evidence.inputs.push(seed.clone()); + } + } + artifact::save(&args.out.join("serving.json"), &evidence)?; + if let Some(error) = failure { + return Err(error); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn warm_append_uses_observed_reuse() { + for reuse in ["exact", "partial", "fork", "cold"] { + let workload: Workload = serde_json::from_value(serde_json::json!({ + "protocol":PROTOCOL,"name":"append","family":"qwen4exp", + "cases":[{ + "name":"append","scenario":"warm_append", + "request":{"stream":true,"stream_options":{"include_usage":true}, + "messages":[{"role":"user","content":"Continue"}]}, + "expect":{"content":"READY","finish_reason":"stop","reuse_kind":reuse, + "effective_lane":"continuous","speculation_active":true, + "fallback_reason":null,"min_cached_tokens":32}, + "limits":{"ttft_ms":1000,"total_ms":2000,"min_host_available_bytes":1} + }] + })) + .unwrap(); + assert_eq!(workload.validate().is_ok(), reuse != "cold", "{reuse}"); + } + } + + #[test] + fn disk_restore_can_place_a_fork() { + let workload: Workload = serde_json::from_value(serde_json::json!({ + "protocol":PROTOCOL,"name":"restore-fork","family":"qwen4exp", + "restart_from":"seed/serving.json","cases":[{ + "name":"restored","scenario":"restart_restore", + "request":{"stream":true,"stream_options":{"include_usage":true}, + "messages":[{"role":"user","content":"Continue"}]}, + "expect":{"content":"READY","finish_reason":"stop","reuse_kind":"fork", + "effective_lane":"continuous","speculation_active":true, + "fallback_reason":null,"min_cached_tokens":32}, + "limits":{"ttft_ms":1000,"total_ms":2000,"min_host_available_bytes":1} + }] + })) + .unwrap(); + assert!( + workload.validate().is_ok(), + "a restored full prefix may fork into a free bank" + ); + } + + fn event(elapsed_ms: f64, data: &str) -> Event { + Event { + elapsed_ms, + data: data.into(), + } + } + + #[test] + fn reasoning_and_visible_ttft() { + let events = [ + event(1.0, r#"{"choices":[{"delta":{"role":"assistant"}}]}"#), + event( + 12.0, + r#"{"choices":[{"delta":{"reasoning_content":"think"}}]}"#, + ), + event(23.0, r#"{"choices":[{"delta":{"content":"answer"}}]}"#), + event(25.0, r#"{"choices":[{"delta":{},"finish_reason":"stop"}]}"#), + event(30.0, "[DONE]"), + ]; + let output = summarize(&events).unwrap(); + assert_eq!(output.ttft_ms, 12.0); + assert_eq!(output.first_content_ms, Some(23.0)); + assert_eq!(output.total_ms, 30.0); + assert_eq!(output.content, "answer"); + assert_eq!(output.finish_reason, "stop"); + assert!(summarize(&events[..4]).is_err_and(|e| e.contains("DONE"))); + assert!(summarize(&events[..3]).is_err_and(|e| e.contains("finish"))); + } + + #[test] + fn incomplete_or_error_stream() { + for data in [ + r#"{"error":{"message":"failed"}}"#, + "not json", + r#"{"choices":[{"delta":{"tool_calls":[{}]}}]}"#, + ] { + assert!(summarize(&[event(1.0, data), event(2.0, "[DONE]")]).is_err()); + } + assert!(summarize(&[event(1.0, "[DONE]"), event(2.0, "[DONE]")]).is_err()); + } + + #[test] + fn finish_is_terminal_for_choice_deltas() { + for delta in [ + r#"{"content":"late"}"#, + r#"{"reasoning_content":"late"}"#, + r#"{"tool_calls":[{"index":0,"id":"late","function":{"name":"f","arguments":"{}"}}]}"#, + ] { + let stream = [ + event(1.0, r#"{"choices":[{"delta":{"content":"ok"}}]}"#), + event(2.0, r#"{"choices":[{"delta":{},"finish_reason":"stop"}]}"#), + event(3.0, &format!(r#"{{"choices":[{{"delta":{delta}}}]}}"#)), + event(4.0, "[DONE]"), + ]; + assert!(summarize(&stream).is_err_and(|e| e.contains("after finish"))); + } + let stream = [ + event( + 1.0, + r#"{"choices":[{"delta":{"content":"ok"},"finish_reason":"stop"}]}"#, + ), + event( + 2.0, + r#"{"choices":[],"usage":{"prompt_tokens":1,"prompt_tokens_details":{"cached_tokens":0}}}"#, + ), + event(3.0, "[DONE]"), + ]; + assert!( + summarize(&stream).is_ok(), + "usage after final delta is valid" + ); + } + + #[test] + fn fragmented_tool_arguments_are_checked() { + let stream = [ + event(1.0, r#"{"choices":[{"delta":{"role":"assistant"}}]}"#), + event( + 2.0, + r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call-a","type":"function","function":{"name":"weather","arguments":""}}]}}]}"#, + ), + event( + 3.0, + r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"city\":"}}]}}]}"#, + ), + event( + 4.0, + r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Seoul\"}"}}]}}]}"#, + ), + event( + 5.0, + r#"{"choices":[{"delta":{},"finish_reason":"tool_calls"}]}"#, + ), + event(6.0, "[DONE]"), + ]; + let output = summarize(&stream).unwrap(); + assert_eq!(output.ttft_ms, 2.0); + assert_eq!(output.first_content_ms, None); + assert_eq!( + serde_json::to_value(&output.tool_calls).unwrap(), + serde_json::json!([{"name":"weather","arguments":{"city":"Seoul"}}]) + ); + let mut broken = stream.to_vec(); + broken.remove(3); + assert!(summarize(&broken).is_err_and(|e| e.contains("arguments"))); + let mut wrong_finish = stream.to_vec(); + wrong_finish[4] = event(5.0, r#"{"choices":[{"delta":{},"finish_reason":"stop"}]}"#); + assert!(summarize(&wrong_finish).is_err_and(|e| e.contains("finish reason"))); + } +} diff --git a/crates/ds4-perf/src/serving/context.rs b/crates/ds4-perf/src/serving/context.rs new file mode 100644 index 0000000000000000000000000000000000000000..721aa1d3a91cfd16bbdc8431db656dcf729abc79 --- /dev/null +++ b/crates/ds4-perf/src/serving/context.rs @@ -0,0 +1,599 @@ +use super::*; +use crate::{csv, experiment::InputFile, runner}; +use std::{ + collections::BTreeMap, + io::BufReader, + process::{Command, Stdio}, + sync::atomic::{AtomicBool, Ordering}, + time::{Duration, Instant, SystemTime}, +}; + +const MEMORY_POLL_MS: u64 = 10; +// Sampling evidence tolerates scheduler jitter, but not an unobserved request. +const MEMORY_MAX_GAP_MS: f64 = 100.0; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct MemoryPoint { + elapsed_ms: f64, + available_bytes: u64, +} + +/// Keep samples, not just a claimed minimum, so profile selection can recheck it. +pub(super) fn measure( + dir: &Path, + action: impl FnOnce() -> Result, +) -> Result<(T, u64), String> { + let finished = AtomicBool::new(false); + let start = Instant::now(); + let (result, samples) = std::thread::scope(|scope| { + let observer = scope.spawn(|| { + let mut samples = Vec::new(); + loop { + samples.push(MemoryPoint { + elapsed_ms: start.elapsed().as_secs_f64() * 1000.0, + available_bytes: host_available()?, + }); + if finished.load(Ordering::Acquire) { + return Ok::<_, String>(samples); + } + std::thread::sleep(Duration::from_millis(MEMORY_POLL_MS)); + } + }); + let result = action(); + finished.store(true, Ordering::Release); + ( + result, + observer.join().map_err(|_| "host memory observer panicked"), + ) + }); + let samples = samples??; + let minimum = samples + .iter() + .map(|sample| sample.available_bytes) + .min() + .ok_or("no host memory samples")?; + fs::write( + dir.join("memory.json"), + serde_json::to_vec_pretty(&samples).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + Ok((result?, minimum)) +} + +pub(super) fn memory_min(dir: &Path, duration_ms: f64) -> Result { + let samples: Vec = + serde_json::from_slice(&fs::read(dir.join("memory.json")).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if !duration_ms.is_finite() + || duration_ms <= 0.0 + || samples.is_empty() + || samples + .iter() + .any(|p| !p.elapsed_ms.is_finite() || p.elapsed_ms < 0.0) + || samples.windows(2).any(|p| { + p[1].elapsed_ms <= p[0].elapsed_ms + || p[1].elapsed_ms - p[0].elapsed_ms > MEMORY_MAX_GAP_MS + }) + || samples + .first() + .is_some_and(|p| p.elapsed_ms > MEMORY_MAX_GAP_MS) + || samples + .last() + .is_some_and(|p| p.elapsed_ms + MEMORY_MAX_GAP_MS < duration_ms) + { + return Err("invalid host memory samples".into()); + } + samples + .iter() + .map(|sample| sample.available_bytes) + .min() + .ok_or("no host memory samples".into()) +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ClockRange { + min: u32, + max: u32, +} + +impl ClockRange { + pub(super) fn validate(&self) -> Result<(), String> { + if self.min == 0 || self.max < self.min { + return Err("invalid expected clock range".into()); + } + Ok(()) + } +} + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Server { + pid: u32, + boot_id: String, + start_ticks: u64, + executable: String, + executable_sha256: String, + argv_sha256: String, + #[serde(default)] + cwd: Option, + #[serde(default)] + environment: Option>, + #[serde(default)] + unreviewed_environment: Vec, + #[serde(default)] + source: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Identity { + server: Option, + inputs: BTreeMap, + expected_clock_range_mhz: Option, + limitations: Vec, +} + +impl Identity { + pub(super) fn capture( + args: &cli::Serving, + workload: &Workload, + checked: &mut artifact::Verification, + ) -> Result { + let server = args + .server_pid + .map(|pid| server(pid, &args.url)) + .transpose()?; + let mut limitations = Vec::new(); + if server.is_none() { + limitations.push("server PID/executable identity was not requested".into()); + } + if workload.inputs.is_empty() { + limitations.push("model/template/sidecar input hashes were not supplied".into()); + } + limitations.push("input manifest coverage is operator-declared; loaded model tensors are not introspected".into()); + let identity = Self { + server, + inputs: workload.inputs.clone(), + expected_clock_range_mhz: workload.expected_clock_range_mhz.clone(), + limitations, + }; + identity.verify_checked(args, checked)?; + if let Some(pid) = args.server_pid { + fs::write( + args.out.join("server.argv"), + fs::read(format!("/proc/{pid}/cmdline")).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + } + fs::write( + args.out.join("identity.json"), + serde_json::to_vec_pretty(&identity).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + Ok(identity) + } + + pub(super) fn verify(&self, args: &cli::Serving) -> Result<(), String> { + let mut checked = artifact::Verification::default(); + self.verify_checked(args, &mut checked)?; + checked.finish() + } + + fn verify_checked( + &self, + args: &cli::Serving, + checked: &mut artifact::Verification, + ) -> Result<(), String> { + if let Some(expected) = &self.server { + if &server(expected.pid, &args.url)? != expected { + return Err("server process/executable changed during serving collection".into()); + } + } + for (name, file) in &self.inputs { + if name.is_empty() { + return Err(format!("serving input changed: {name}")); + } + checked + .verify(&file.path, &file.sha256) + .map_err(|error| format!("serving input changed: {name}: {error}"))?; + } + Ok(()) + } + + pub(super) fn add_refs(&self, refs: &mut Vec) { + refs.extend(self.inputs.values().map(|input| artifact::Reference { + path: input.path.to_string_lossy().into(), + sha256: input.sha256.clone(), + })); + } +} + +fn server(pid: u32, origin: &str) -> Result { + let root = std::path::PathBuf::from(format!("/proc/{pid}")); + let stat = + fs::read_to_string(root.join("stat")).map_err(|e| format!("server PID {pid}: {e}"))?; + let fields = stat + .rsplit_once(')') + .ok_or("invalid process stat")? + .1 + .split_whitespace() + .collect::>(); + let start_ticks = fields + .get(19) + .ok_or("missing server start time")? + .parse::() + .map_err(|e| e.to_string())?; + let port = origin + .trim_end_matches('/') + .rsplit_once(':') + .ok_or("missing server port")? + .1 + .parse::() + .map_err(|e| e.to_string())?; + let ipv6 = origin.starts_with("http://[::1]:"); + let mut sockets = BTreeSet::new(); + for table in ["net/tcp", "net/tcp6"] { + let text = fs::read_to_string(root.join(table)).map_err(|e| e.to_string())?; + for line in text.lines().skip(1) { + let fields = line.split_whitespace().collect::>(); + if fields.len() < 10 || fields[3] != "0A" { + continue; + } + let address = fields[1].split(':').next().unwrap_or(""); + let matches_address = if ipv6 { + table == "net/tcp6" + && matches!( + address, + "00000000000000000000000000000000" | "00000000000000000000000001000000" + ) + } else { + table == "net/tcp" && matches!(address, "00000000" | "0100007F") + }; + if !matches_address { + continue; + } + if fields[1] + .rsplit_once(':') + .and_then(|(_, value)| u16::from_str_radix(value, 16).ok()) + == Some(port) + { + sockets.insert(format!("socket:[{}]", fields[9])); + } + } + } + let owns_listener = fs::read_dir(root.join("fd")) + .map_err(|e| e.to_string())? + .filter_map(Result::ok) + .filter_map(|entry| fs::read_link(entry.path()).ok()) + .any(|path| sockets.contains(path.to_string_lossy().as_ref())); + if !owns_listener { + return Err("server PID does not own the requested listening port".into()); + } + let exe = root.join("exe"); + let (environment, unreviewed_environment) = server_environment(&root)?; + Ok(Server { + pid, + start_ticks, + boot_id: fs::read_to_string("/proc/sys/kernel/random/boot_id") + .map_err(|e| e.to_string())? + .trim() + .into(), + executable: fs::read_link(&exe) + .map_err(|e| e.to_string())? + .to_string_lossy() + .into(), + executable_sha256: artifact::hash(&exe)?, + argv_sha256: artifact::hash(&root.join("cmdline"))?, + cwd: Some( + fs::read_link(root.join("cwd")) + .map_err(|e| e.to_string())? + .to_string_lossy() + .into(), + ), + environment: Some(environment), + unreviewed_environment, + source: source_snapshot(&root.join("cwd")).ok(), + }) +} + +/// Bind profiles to the actual runtime sources, including new native includes. +pub(super) fn source_snapshot(cwd: &Path) -> Result { + let output = Command::new("git") + .args(["rev-parse", "--show-toplevel"]) + .current_dir(cwd) + .output() + .map_err(|e| e.to_string())?; + if !output.status.success() { + return Err("profile requires a source checkout".into()); + } + let root = std::path::PathBuf::from( + String::from_utf8(output.stdout) + .map_err(|e| e.to_string())? + .trim(), + ); + let output = Command::new("git") + .args([ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + ]) + .current_dir(&root) + .output() + .map_err(|e| e.to_string())?; + if !output.status.success() { + return Err("cannot enumerate runtime sources".into()); + } + let mut files = BTreeMap::new(); + for bytes in output.stdout.split(|b| *b == 0).filter(|p| !p.is_empty()) { + let name = std::str::from_utf8(bytes).map_err(|e| e.to_string())?; + let path = Path::new(name); + if name.starts_with("crates/ds4-perf") || name.starts_with("tests/") { + continue; + } + if !matches!( + path.extension().and_then(|v| v.to_str()), + Some( + "rs" | "c" + | "h" + | "cu" + | "cuh" + | "inc" + | "cc" + | "cpp" + | "hpp" + | "m" + | "mm" + | "metal" + | "toml" + | "lock" + ) + ) && name != "Makefile" + && !name.starts_with("crates/") + { + continue; + } + files.insert(name, artifact::hash(&root.join(path))?); + } + if files.is_empty() { + return Err("no runtime sources in checkout".into()); + } + Ok( + serde_json::json!({"root": root, "sha256": artifact::hash_bytes(&serde_json::to_vec(&files).map_err(|e| e.to_string())?)}), + ) +} + +fn server_environment(root: &Path) -> Result<(BTreeMap, Vec), String> { + let mut environment = BTreeMap::new(); + let mut unreviewed = Vec::new(); + for assignment in fs::read(root.join("environ")) + .map_err(|e| e.to_string())? + .split(|b| *b == 0) + { + let Some(split) = assignment.iter().position(|b| *b == b'=') else { + continue; + }; + let Ok(name) = std::str::from_utf8(&assignment[..split]) else { + continue; + }; + if ![ + "DS4_", + "CUDA_", + "NVIDIA_", + "LD_", + "CUBLAS_", + "OMP_", + "OPENBLAS_", + ] + .iter() + .any(|prefix| name.starts_with(prefix)) + { + continue; + } + if !runner::reviewed_control(name) + && ![ + "DS4_CONT_PREFILL_CHUNK_LIVE", + "DS4_CONT_PREFILL_NOFENCE", + "DS4_DOTS3_BATCH", + "DS4_EXAONE_PREFILL_CHUNK", + "DS4_MEM_FLOOR_GB", + "DS4_MOTIF3_PREFILL_CHUNK", + "DS4_NATIVE_PREFILL_CHUNK", + "DS4_SERVER_COALESCE_MAX", + "DS4_SERVER_FORK", + "DS4_SERVER_FORK_PARTIAL", + "DS4_SERVER_PERSIST_MIN_TOKENS", + "DS4_STEP37_BATCH", + ] + .contains(&name) + { + unreviewed.push(name.into()); + continue; + } + match std::str::from_utf8(&assignment[split + 1..]) { + Ok(value) => { + environment.insert(name.into(), value.into()); + } + Err(_) => unreviewed.push(name.into()), + } + } + unreviewed.sort(); + Ok((environment, unreviewed)) +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct GpuSample { + label: String, + unix_ms: u64, + device_ordinal: usize, + uuid: Option, + name: Option, + driver: Option, + sm_clock_mhz: Option, + temperature_c: Option, + unavailable_reason: Option, +} + +impl GpuSample { + pub(super) fn timestamp_ms(&self) -> u64 { + self.unix_ms + } + + pub(super) fn verify_raw(&self, bytes: &[u8]) -> Result<(), String> { + let mut rows = Vec::new(); + csv::records(BufReader::new(bytes), |row| { + rows.push(row); + Ok(()) + })?; + if rows.len() != 1 || rows[0].len() != 5 { + return Err("invalid raw GPU telemetry".into()); + } + let row = &rows[0]; + if self.uuid.as_deref() != Some(row[0].trim()) + || self.name.as_deref() != Some(row[1].trim()) + || self.driver.as_deref() != Some(row[2].trim()) + || self.sm_clock_mhz != row[3].trim().parse().ok() + || self.temperature_c != row[4].trim().parse().ok() + { + return Err("raw GPU telemetry differs from summary".into()); + } + Ok(()) + } + + pub(super) fn failures(&self, range: Option<&ClockRange>) -> Vec { + let Some(range) = range else { + return Vec::new(); + }; + if self.unavailable_reason.is_some() || self.temperature_c.is_none() { + return vec![format!( + "{}: incomplete NVIDIA clock/temperature sample", + self.label + )]; + } + match self.sm_clock_mhz { + Some(clock) if (range.min..=range.max).contains(&clock) => Vec::new(), + Some(clock) => vec![format!( + "{}: observed clock {clock} MHz outside expected {}..{} MHz", + self.label, range.min, range.max + )], + None => vec![format!( + "{}: expected clock contract lacks observed NVIDIA clock", + self.label + )], + } + } +} + +pub(super) fn gpu(args: &cli::Serving, dir: &Path, label: &str) -> Result { + let mut sample = GpuSample { + label: label.into(), + unix_ms: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_err(|e| e.to_string())? + .as_millis() as u64, + device_ordinal: args.device, + uuid: None, + name: None, + driver: None, + sm_clock_mhz: None, + temperature_c: None, + unavailable_reason: None, + }; + let result = (|| { + let executable = + runner::resolve(std::ffi::OsStr::new("nvidia-smi")).ok_or("nvidia-smi unavailable")?; + let output = dir.join(format!("{label}.stdout")); + let mut command = Command::new(executable); + command + .arg(format!("--id={}", args.device)) + .args([ + "--query-gpu=uuid,name,driver_version,clocks.current.sm,temperature.gpu", + "--format=csv,noheader,nounits", + ]) + .stdout(Stdio::from( + fs::File::create(&output).map_err(|e| e.to_string())?, + )) + .stderr(Stdio::from( + fs::File::create(dir.join(format!("{label}.stderr"))).map_err(|e| e.to_string())?, + )); + let limits = process::Limits { + timeout: args.budget.limits().timeout.min(Duration::from_secs(10)), + bytes: args.budget.limits().bytes, + }; + if !process::run(&mut command, &args.out, limits)?.success() { + return Err("nvidia-smi query failed".into()); + } + let mut rows = Vec::new(); + csv::records( + BufReader::new(fs::File::open(&output).map_err(|e| e.to_string())?), + |row| { + rows.push(row); + Ok(()) + }, + )?; + if rows.len() != 1 || rows[0].len() != 5 { + return Err("unexpected NVIDIA query rows".into()); + } + let row = &rows[0]; + sample.uuid = Some(row[0].trim().into()); + sample.name = Some(row[1].trim().into()); + sample.driver = Some(row[2].trim().into()); + sample.sm_clock_mhz = Some(row[3].trim().parse::().map_err(|e| e.to_string())?); + sample.temperature_c = Some(row[4].trim().parse::().map_err(|e| e.to_string())?); + Ok::<_, String>(()) + })(); + if let Err(reason) = result { + if reason == "interrupted" { + return Err(reason); + } + sample.unavailable_reason = Some(reason); + } + process::check_bytes(&args.out, args.budget.limits())?; + fs::write( + dir.join(format!("{label}.json")), + serde_json::to_vec_pretty(&sample).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + Ok(sample) +} + +#[cfg(test)] +mod coverage_tests { + use super::*; + + #[test] + fn memory_window_needs_coverage() { + let dir = std::env::temp_dir().join(format!("ds4-memory-span-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + for (times, complete) in [ + (vec![0.0], false), + (vec![150.0, 200.0], false), + (vec![0.0, 200.0], false), + (vec![0.0, 50.0], false), + (vec![0.0, 50.0, 100.0, 150.0, 205.0], true), + ] { + let samples: Vec<_> = times + .iter() + .map(|&elapsed_ms| MemoryPoint { + elapsed_ms, + available_bytes: 1000, + }) + .collect(); + fs::write( + dir.join("memory.json"), + serde_json::to_vec(&samples).unwrap(), + ) + .unwrap(); + assert_eq!( + memory_min(&dir, 200.0).is_ok(), + complete, + "memory samples: {times:?}" + ); + } + let _ = fs::remove_dir_all(dir); + } +} diff --git a/crates/ds4-perf/src/serving/controls.rs b/crates/ds4-perf/src/serving/controls.rs new file mode 100644 index 0000000000000000000000000000000000000000..6d7dcba201d41c138cae18a2f2a9f2cbd545b0fb --- /dev/null +++ b/crates/ds4-perf/src/serving/controls.rs @@ -0,0 +1,80 @@ +use super::*; +use std::collections::BTreeMap; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Candidate { + arguments: BTreeMap, + qualified: bool, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Controls { + family: String, + capabilities: Value, + candidates: Vec, +} + +impl Payload for Controls { + const KIND: &'static str = "serving-controls"; + fn validate(&self) -> Result<(), String> { + if self.family.is_empty() + || self + .candidates + .iter() + .any(|c| c.qualified || c.arguments.len() != 1) + { + return Err("invalid unqualified serving controls".into()); + } + Ok(()) + } +} + +pub(super) fn run(args: &cli::ServingControls) -> Result<(), String> { + let value: Value = serde_json::from_slice(&fs::read(&args.plan).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + let plan = value.get("serving").unwrap_or(&value); + let family = plan["family"].as_str().ok_or("missing plan family")?; + let caps = plan + .get("controls") + .ok_or("plan lacks capability-derived controls")?; + let chunks = caps["scheduler_chunks"] + .as_array() + .ok_or("missing scheduler chunk controls")?; + let cap = plan["effective"]["native_chunk"].as_u64().unwrap_or(0); + let mut candidates = Vec::new(); + for (flag, current) in [ + ("--prefill-chunk", "sched_chunk"), + ("--prefill-chunk-live", "sched_chunk_live"), + ] { + let mut seen = BTreeSet::new(); + for chunk in chunks { + let width = chunk + .as_u64() + .filter(|v| *v > 0 && *v <= cap) + .ok_or("scheduler chunk outside native capability")?; + if !seen.insert(width) { + return Err("duplicate scheduler chunk control".into()); + } + if plan["effective"][current].as_u64() == Some(width) { + continue; + } + candidates.push(Candidate { + arguments: BTreeMap::from([(flag.into(), width)]), + qualified: false, + }); + } + } + let mut result = Artifact::new(Controls { + family: family.into(), + capabilities: caps.clone(), + candidates, + }); + fs::create_dir(&args.out).map_err(|e| e.to_string())?; + result + .inputs + .push(artifact::reference(&args.plan, &args.out)?); + result.warnings.push("Proposals only; every candidate requires its complete serving workload and fresh qualification.".into()); + artifact::save(&args.out.join("controls.json"), &result) +} diff --git a/crates/ds4-perf/src/serving/overlap.rs b/crates/ds4-perf/src/serving/overlap.rs new file mode 100644 index 0000000000000000000000000000000000000000..adb60077ed12d169426bc3f5b0ea4d277e4cc6da --- /dev/null +++ b/crates/ds4-perf/src/serving/overlap.rs @@ -0,0 +1,382 @@ +use super::*; +use std::{ + sync::mpsc, + time::{Duration, Instant}, +}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Workload { + name: String, + decode: Request, + prefill: Request, + min_decode_events_during_prefill: u32, + min_prefill_tokens: u64, + max_decode_gap_ms: f64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Request { + request: Value, + expect: Expected, + limits: Contract, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Expected { + content: String, + finish_reason: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Observation { + name: String, + repeat: u32, + decode_ttft_ms: f64, + decode_total_ms: f64, + prefill_dispatch_ms: f64, + prefill_ttft_ms: f64, + prefill_total_ms: f64, + decode_events_during_prefill: u32, + prefill_computed_tokens: u64, + decode_committed_tokens: u64, + max_decode_gap_ms: f64, + host_available_before_bytes: u64, + host_available_after_bytes: u64, + #[serde(default)] + host_min_available_bytes: Option, + failures: Vec, +} + +impl Observation { + pub(super) fn span_ms(&self) -> f64 { + self.decode_total_ms + .max(self.prefill_dispatch_ms + self.prefill_total_ms) + } + + pub(super) fn passed(&self) -> bool { + self.failures.is_empty() + } +} + +impl Workload { + pub(super) fn name(&self) -> &String { + &self.name + } + + pub(super) fn validate(&self) -> Result<(), String> { + if self.name.is_empty() + || self.min_decode_events_during_prefill == 0 + || self.min_prefill_tokens == 0 + || !self.max_decode_gap_ms.is_finite() + || self.max_decode_gap_ms <= 0.0 + { + return Err("overlap workload requires name and positive progress/gap limits".into()); + } + for request in [&self.decode, &self.prefill] { + if request.request["stream"] != true + || request.request["stream_options"]["include_usage"] != true + || request.request["messages"] + .as_array() + .is_none_or(Vec::is_empty) + || request + .request + .get("n") + .is_some_and(|n| n.as_u64() != Some(1)) + || request.expect.content.is_empty() + || !["stop", "length"].contains(&request.expect.finish_reason.as_str()) + || [request.limits.ttft_ms, request.limits.total_ms] + .iter() + .any(|v| !v.is_finite() || *v <= 0.0) + || request.limits.min_host_available_bytes == 0 + { + return Err("invalid overlap request/output/latency contract".into()); + } + } + Ok(()) + } +} + +fn usage(events: &[Event]) -> Result<(u64, u64), String> { + let usages = events + .iter() + .filter_map(|event| serde_json::from_str::(&event.data).ok()) + .filter_map(|value| value.get("usage").filter(|v| v.is_object()).cloned()) + .collect::>(); + if usages.len() != 1 { + return Err("overlap requires exactly one streaming usage record per request".into()); + } + let usage = &usages[0]; + let prompt = usage["prompt_tokens"] + .as_u64() + .ok_or("missing prompt token usage")?; + let cached = usage["prompt_tokens_details"]["cached_tokens"] + .as_u64() + .ok_or("missing cached token usage")?; + let completion = usage["completion_tokens"] + .as_u64() + .filter(|n| *n > 0) + .ok_or("missing completion token usage")?; + Ok(( + prompt + .checked_sub(cached) + .ok_or("cached tokens exceed prompt tokens")?, + completion, + )) +} + +fn prepare(dir: &Path, request: &Request) -> Result<(), String> { + fs::create_dir(dir).map_err(|e| e.to_string())?; + fs::write( + dir.join("request.json"), + serde_json::to_vec_pretty(&request.request).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string()) +} + +fn check( + output: &Completion, + request: &Request, + dispatch_ms: f64, + name: &str, + failures: &mut Vec, +) { + if output.content != request.expect.content + || output.finish_reason != request.expect.finish_reason + { + failures.push(format!( + "{name}: content/finish_reason differs from expected output" + )); + } + if output.ttft_ms - dispatch_ms > request.limits.ttft_ms { + failures.push(format!("{name}: TTFT exceeds contract")); + } + if output.total_ms - dispatch_ms > request.limits.total_ms { + failures.push(format!("{name}: total time exceeds contract")); + } +} + +pub(super) fn collect( + http: &Http, + workload: &Workload, + dir: &Path, + family: &str, + previous: Option<&Value>, + repeat: u32, +) -> Result<(Observation, Value), String> { + fs::create_dir(dir).map_err(|e| e.to_string())?; + let decode_dir = dir.join("decode"); + let prefill_dir = dir.join("prefill"); + prepare(&decode_dir, &workload.decode)?; + prepare(&prefill_dir, &workload.prefill)?; + let before = http.stats(dir, "before")?; + let current = plan(&before, family)?; + if previous.is_some_and(|previous| previous != ¤t) { + return Err("resolved plan changed before overlap workload".into()); + } + if current["effective"]["max_seqs"] + .as_u64() + .is_none_or(|n| n < 2) + { + return Err("overlap workload requires at least two effective banks".into()); + } + let available_before = host_available()?; + let start = Instant::now(); + let (sender, receiver) = mpsc::channel(); + let ((decode, prefill, dispatch_ms), available_min) = context::measure(dir, || { + Ok(std::thread::scope(|scope| { + let decoder = scope.spawn(|| http.stream_events(&decode_dir, start, Some(sender))); + // Start the long request only after decode has demonstrably begun. + let ready = loop { + match receiver.recv_timeout(Duration::from_millis(25)) { + Ok(event) if generated(&event) => break Ok(()), + Ok(_) | Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => { + break Err("decode ended before generating output".to_owned()) + } + } + }; + let dispatch_ms = start.elapsed().as_secs_f64() * 1000.0; + let prefill = ready.and_then(|()| http.stream_events(&prefill_dir, start, None)); + let decode = decoder + .join() + .map_err(|_| "decode collector panicked".to_owned()) + .and_then(|v| v); + (decode, prefill, dispatch_ms) + })) + })?; + let decode_events = decode?; + let prefill_events = prefill?; + let decode = summarize(&decode_events)?; + let prefill = summarize(&prefill_events)?; + let (prefill_computed_tokens, _) = usage(&prefill_events)?; + let (_, decode_committed_tokens) = usage(&decode_events)?; + let available_after = host_available()?; + let after = http.stats(dir, "after")?; + if current != plan(&after, family)? + || route_count(&after)?.checked_sub(route_count(&before)?) != Some(2) + { + return Err("overlap plan/route counters changed unexpectedly".into()); + } + let mut failures = Vec::new(); + check(&decode, &workload.decode, 0.0, "decode", &mut failures); + check( + &prefill, + &workload.prefill, + dispatch_ms, + "prefill", + &mut failures, + ); + if prefill_computed_tokens < workload.min_prefill_tokens { + failures.push(format!( + "peer computed {prefill_computed_tokens} prefill tokens; need {}", + workload.min_prefill_tokens + )); + } + let mut last = dispatch_ms; + let mut gap = 0f64; + let mut progress = 0; + for event in &decode_events { + if !generated(event) + || event.elapsed_ms <= dispatch_ms + || event.elapsed_ms >= prefill.ttft_ms + { + continue; + } + gap = gap.max(event.elapsed_ms - last); + last = event.elapsed_ms; + progress += 1; + } + gap = gap.max(prefill.ttft_ms - last); + if progress < workload.min_decode_events_during_prefill { + failures.push(format!( + "decode made {progress} SSE output events during peer prefill; need {}", + workload.min_decode_events_during_prefill + )); + } + if gap > workload.max_decode_gap_ms { + failures.push(format!( + "decode gap {gap:.3} ms exceeds {} ms during peer prefill", + workload.max_decode_gap_ms + )); + } + if decode.total_ms <= prefill.ttft_ms { + failures + .push("decode ended before peer prefill finished; use a longer decode request".into()); + } + if available_min.min(available_before).min(available_after) + < workload + .decode + .limits + .min_host_available_bytes + .max(workload.prefill.limits.min_host_available_bytes) + { + failures.push("host memory below contract at overlap boundary".into()); + } + let observation = Observation { + name: workload.name.clone(), + repeat, + decode_ttft_ms: decode.ttft_ms, + decode_total_ms: decode.total_ms, + prefill_dispatch_ms: dispatch_ms, + prefill_ttft_ms: prefill.ttft_ms - dispatch_ms, + prefill_total_ms: prefill.total_ms - dispatch_ms, + decode_events_during_prefill: progress, + prefill_computed_tokens, + decode_committed_tokens, + max_decode_gap_ms: gap, + host_available_before_bytes: available_before, + host_available_after_bytes: available_after, + host_min_available_bytes: Some(available_min), + failures, + }; + fs::write( + dir.join("result.json"), + serde_json::to_vec_pretty(&observation).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + Ok((observation, current)) +} + +pub(super) fn review( + run: &review::Run, + dir: &Path, + workload: &Workload, + observation: &Observation, + expected: &Value, + family: &str, + repeat: u32, +) -> Result { + run.stats(dir, expected, 2, family)?; + let raw: Value = run.json(&dir.join("result.json"))?; + let (decode_events, decode) = run.stream(&dir.join("decode"), &workload.decode.request)?; + let (prefill_events, prefill) = run.stream(&dir.join("prefill"), &workload.prefill.request)?; + let (computed, _) = usage(&prefill_events)?; + let (_, committed) = usage(&decode_events)?; + let memory = run.memory(dir, decode.total_ms.max(prefill.total_ms))?; + let dispatch = observation.prefill_dispatch_ms; + let mut failures = Vec::new(); + check(&decode, &workload.decode, 0.0, "decode", &mut failures); + check( + &prefill, + &workload.prefill, + dispatch, + "prefill", + &mut failures, + ); + let mut last = dispatch; + let mut gap = 0f64; + let mut progress = 0; + for event in &decode_events { + if generated(event) && event.elapsed_ms > dispatch && event.elapsed_ms < prefill.ttft_ms { + gap = gap.max(event.elapsed_ms - last); + last = event.elapsed_ms; + progress += 1; + } + } + gap = gap.max(prefill.ttft_ms - last); + if raw != serde_json::to_value(observation).map_err(|e| e.to_string())? + || observation.name != workload.name + || observation.repeat != repeat + || !observation.failures.is_empty() + || !failures.is_empty() + || !dispatch.is_finite() + || dispatch <= decode.ttft_ms + || dispatch >= prefill.ttft_ms + || decode.total_ms <= prefill.ttft_ms + || observation.decode_ttft_ms != decode.ttft_ms + || observation.decode_total_ms != decode.total_ms + || observation.prefill_ttft_ms != prefill.ttft_ms - dispatch + || observation.prefill_total_ms != prefill.total_ms - dispatch + || observation.decode_events_during_prefill != progress + || observation.max_decode_gap_ms != gap + || observation.prefill_computed_tokens != computed + || observation.decode_committed_tokens != committed + || progress < workload.min_decode_events_during_prefill + || computed < workload.min_prefill_tokens + || gap > workload.max_decode_gap_ms + || observation.host_min_available_bytes != Some(memory) + || memory + .min(observation.host_available_before_bytes) + .min(observation.host_available_after_bytes) + < workload + .decode + .limits + .min_host_available_bytes + .max(workload.prefill.limits.min_host_available_bytes) + { + return Err(format!("{}: raw overlap contract failed", workload.name)); + } + Ok([ + decode.ttft_ms / workload.decode.limits.ttft_ms, + decode.total_ms / workload.decode.limits.total_ms, + (prefill.ttft_ms - dispatch) / workload.prefill.limits.ttft_ms, + (prefill.total_ms - dispatch) / workload.prefill.limits.total_ms, + gap / workload.max_decode_gap_ms, + ] + .into_iter() + .fold(0.0, f64::max)) +} diff --git a/crates/ds4-perf/src/serving/profile.rs b/crates/ds4-perf/src/serving/profile.rs new file mode 100644 index 0000000000000000000000000000000000000000..63901c4f08459ce2ceb9cd5a25fb01065b7a140f --- /dev/null +++ b/crates/ds4-perf/src/serving/profile.rs @@ -0,0 +1,1290 @@ +use super::*; +use crate::experiment::InputFile; +use std::{collections::BTreeMap, ffi::OsString, path::PathBuf, process::Command}; + +const MIN_REPEATS: u32 = 3; +const SWITCHES: &[&str] = &[ + "--cuda", + "--cors", + "--no-update-check", + "--print-plan", + "--kv-cache-reject-different-quant", +]; + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Candidate { + name: String, + runs: Vec, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Selection { + workload: PathBuf, + candidates: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Launch { + executable: PathBuf, + executable_sha256: String, + arguments: Vec, + cwd: PathBuf, + environment: BTreeMap, + source: Value, + device: Value, + expected_clock_range_mhz: Value, + serving_plan: Value, +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Qualified { + name: String, + worst_latency_fraction: f64, + samples_per_workload: u32, + launch: Launch, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Profile { + selection: Selection, + qualified: bool, + scope: String, + selected: Option, + rejected: BTreeMap, +} +impl Payload for Profile { + const KIND: &'static str = "serving-profile"; + fn validate(&self) -> Result<(), String> { + if self.qualified != self.selected.is_some() + || self.scope != "declared-serving-workload-only" + || self.selected.as_ref().is_some_and(|s| { + s.samples_per_workload < MIN_REPEATS + || !s.worst_latency_fraction.is_finite() + || !(0.0..=1.0).contains(&s.worst_latency_fraction) + }) + { + return Err("invalid serving profile qualification".into()); + } + Ok(()) + } +} + +fn json(path: &Path) -> Result { + if fs::metadata(path).map_err(|e| e.to_string())?.len() > MAX_MANIFEST_BYTES { + return Err("profile input exceeds 16 MiB".into()); + } + serde_json::from_slice(&fs::read(path).map_err(|e| e.to_string())?).map_err(|e| e.to_string()) +} + +fn json_verified( + path: &Path, + checked: &mut artifact::Verification, +) -> Result { + if fs::metadata(path).map_err(|e| e.to_string())?.len() > MAX_MANIFEST_BYTES { + return Err("profile input exceeds 16 MiB".into()); + } + let bytes = fs::read(path).map_err(|e| e.to_string())?; + if bytes.len() as u64 > MAX_MANIFEST_BYTES { + return Err("profile input exceeds 16 MiB".into()); + } + checked.verify(path, &artifact::hash_bytes(&bytes))?; + serde_json::from_slice(&bytes).map_err(|e| e.to_string()) +} + +fn selection(path: &Path, checked: &mut artifact::Verification) -> Result { + let mut selection: Selection = json_verified(path, checked)?; + let mut names = BTreeSet::new(); + if selection.candidates.is_empty() || selection.candidates.len() > 64 { + return Err("selection requires 1..64 named candidates".into()); + } + for candidate in &selection.candidates { + if candidate.name.is_empty() + || !names.insert(&candidate.name) + || candidate.runs.is_empty() + || candidate.runs.len() > 100 + { + return Err("each unique candidate needs at least one run (maximum 100)".into()); + } + } + let base = path.parent().unwrap_or(Path::new(".")); + selection.workload = base + .join(&selection.workload) + .canonicalize() + .map_err(|e| e.to_string())?; + for candidate in &mut selection.candidates { + for run in &mut candidate.runs { + *run = base.join(&*run).canonicalize().map_err(|e| e.to_string())?; + } + if candidate.runs.iter().collect::>().len() != candidate.runs.len() { + return Err("duplicate run cannot supply additional repeats".into()); + } + } + Ok(selection) +} + +fn workload(path: &Path, checked: &mut artifact::Verification) -> Result { + let mut workload: Workload = json_verified(path, checked)?; + workload.validate()?; + if let Some(seed) = &mut workload.restart_from { + *seed = path + .parent() + .unwrap_or(Path::new(".")) + .join(&*seed) + .canonicalize() + .map_err(|e| e.to_string())?; + } + if workload.expected_clock_range_mhz.is_none() { + return Err("profile requires an expected clock range".into()); + } + for input in workload.inputs.values_mut() { + let consumed = path.parent().unwrap_or(Path::new(".")).join(&input.path); + checked + .verify(&consumed, &input.sha256) + .map_err(|error| format!("profile workload input changed: {error}"))?; + input.path = consumed.canonicalize().map_err(|e| e.to_string())?; + } + Ok(workload) +} + +fn text<'a>(value: &'a Value, key: &str) -> Result<&'a str, String> { + value[key] + .as_str() + .filter(|v| !v.is_empty()) + .ok_or_else(|| format!("profile lacks {key}")) +} + +// Only direct server launches are replayed. Unknown or duplicate options cannot +// introduce an untracked config file, wrapper, or later option override. +fn options(args: &[String]) -> Result, String> { + let mut result = BTreeMap::new(); + let mut args = args.iter(); + while let Some(flag) = args.next() { + let key = match flag.as_str() { + "-m" => "--model", + "-c" => "--ctx", + "-n" => "--tokens", + "-t" => "--threads", + "--cont-width" => "--max-seqs", + key => key, + }; + let value = match key { + key if SWITCHES.contains(&key) => "true".into(), + "--host" + | "--port" + | "--model-id" + | "--model" + | "--vision" + | "--mtp" + | "--mtp-mode" + | "--prefix-reuse" + | "--max-seqs" + | "--prefill-chunk" + | "--prefill-chunk-live" + | "--native-chunk" + | "--backend" + | "--tokens" + | "--mtp-draft" + | "--mtp-margin" + | "--ctx" + | "--threads" + | "--mem-floor-gb" + | "--kv-disk-dir" + | "--kv-disk-space-mb" + | "--kv-disk-space" + | "--kv-cache-min-tokens" + | "--kv-cache-cold-max-tokens" + | "--kv-cache-continued-interval-tokens" + | "--kv-cache-boundary-trim-tokens" + | "--kv-cache-boundary-align-tokens" + | "--expect-plan" => args.next().ok_or("missing server option value")?.clone(), + _ => return Err(format!("profile does not cover server option {flag}")), + }; + if result.insert(key.into(), value).is_some() { + return Err(format!("duplicate server option {key}")); + } + } + Ok(result) +} + +fn input_scope( + launch: &Launch, + workload: &Workload, + manifest: &Path, + checked: &mut artifact::Verification, +) -> Result<(), String> { + let options = options(&launch.arguments)?; + let count = options + .get("--max-seqs") + .and_then(|v| v.parse::().ok()); + if count.is_none() || count != launch.serving_plan["effective"]["max_seqs"].as_u64() { + return Err( + "profile needs explicit --max-seqs matching effective banks; auto cannot pin a launch" + .into(), + ); + } + let model = options + .get("--model") + .ok_or("profile needs a direct server --model argument")?; + let mut command: Vec = vec![ + "ds4-bench".into(), + "-m".into(), + launch.cwd.join(model).into(), + "--prompt-file".into(), + manifest.into(), + ]; + if let Some(mtp) = options.get("--mtp") { + command.extend(["--mtp".into(), launch.cwd.join(mtp).into()]); + } + let mut files = workload.inputs.clone(); + files.insert( + "prompt".into(), + InputFile { + path: manifest.into(), + sha256: artifact::hash(manifest)?, + }, + ); + let benchmark = crate::experiment::Workload { + protocol: "ds4-bench-v1".into(), + name: workload.name.clone(), + family: workload.family.clone(), + files, + shape: BTreeMap::new(), + cache_state: "serving".into(), + }; + let mut environment: BTreeMap = launch + .environment + .iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(); + for key in [ + "DS4_QWEN_PLE_DIR", + "DS4_CUDA_WEIGHT_IPC_MANIFEST", + "DS4_WEIGHT_SERVER", + ] { + if let Some(value) = environment.get_mut(std::ffi::OsStr::new(key)) { + *value = launch.cwd.join(&*value).into(); + } + } + for input in benchmark.files.values() { + checked.verify(&input.path, &input.sha256)?; + } + benchmark.verify_scope_checked(&command, &environment, checked)?; + let model = launch.cwd.join(model); + let parent = model.parent().ok_or("model has no parent directory")?; + // Either adjacent file changes the template loader's behavior, including + // when it first appears after the original capture. + for name in ["tokenizer_config.json", "chat_template.jinja"] { + let file = parent.join(name); + if file.exists() { + let canonical = file.canonicalize().map_err(|e| e.to_string())?; + let input = workload + .inputs + .values() + .find(|input| input.path == canonical) + .ok_or_else(|| format!("workload lacks template input: {name}"))?; + checked.verify(&file, &input.sha256)?; + } + } + if let Some(vision) = options.get("--vision") { + let consumed = launch.cwd.join(vision); + let path = consumed.canonicalize().map_err(|e| e.to_string())?; + let input = workload + .inputs + .values() + .find(|input| input.path == path) + .ok_or("workload lacks vision input")?; + checked.verify(&consumed, &input.sha256)?; + let mut vision_scope = benchmark.clone(); + vision_scope.files.insert( + "model".into(), + InputFile { + path: path.clone(), + sha256: input.sha256.clone(), + }, + ); + vision_scope.verify_scope_checked( + &[ + "ds4-bench".into(), + "-m".into(), + consumed.into(), + "--prompt-file".into(), + manifest.into(), + ], + &BTreeMap::new(), + checked, + )?; + } + if launch + .environment + .get("LD_PRELOAD") + .is_some_and(|v| !v.is_empty()) + { + return Err("preloaded libraries require a separate profile contract".into()); + } + // External owners can outlive or replace their manifest. Their process and + // device allocation identity are outside this direct-server profile. + for key in ["DS4_WEIGHT_SERVER", "DS4_CUDA_WEIGHT_IPC_MANIFEST"] { + if launch.environment.get(key).is_some_and(|v| !v.is_empty()) { + return Err("external owner identity is not covered by serving profiles".into()); + } + } + Ok(()) +} + +fn device(sample: &Value) -> Result { + for key in ["uuid", "name", "driver"] { + text(sample, key)?; + } + if sample["temperature_c"].as_u64().is_none() + || sample["device_ordinal"].as_u64().is_none() + || !sample["unavailable_reason"].is_null() + { + return Err("profile lacks complete clock/temperature/device evidence".into()); + } + Ok( + serde_json::json!({"uuid":sample["uuid"], "name":sample["name"], "driver":sample["driver"], "device_ordinal":sample["device_ordinal"]}), + ) +} + +fn launch( + run: &review::Run, + workload: &Workload, + manifest: &Path, + checked: &mut artifact::Verification, +) -> Result { + let data = run.data()?; + let identity: Value = run.json(Path::new("identity.json"))?; + if identity != serde_json::to_value(&data.identity).map_err(|e| e.to_string())? + || identity["inputs"] + != serde_json::to_value(&workload.inputs).map_err(|e| e.to_string())? + || identity["expected_clock_range_mhz"] + != serde_json::to_value(&workload.expected_clock_range_mhz) + .map_err(|e| e.to_string())? + { + return Err("raw identity differs from workload or summary".into()); + } + let server = &identity["server"]; + if server["unreviewed_environment"] + .as_array() + .is_none_or(|v| !v.is_empty()) + { + return Err("unreviewed server environment cannot qualify".into()); + } + let argv = run.bytes(Path::new("server.argv"))?; + if artifact::hash_bytes(&argv) != text(server, "argv_sha256")? { + return Err("server argv hash differs".into()); + } + let mut args = argv + .split(|v| *v == 0) + .filter(|v| !v.is_empty()) + .map(|v| String::from_utf8(v.into()).map_err(|e| e.to_string())) + .collect::, _>>()?; + if args.is_empty() { + return Err("empty server argv".into()); + } + args.remove(0); + let mut launch = Launch { + executable: text(server, "executable")?.into(), + executable_sha256: text(server, "executable_sha256")?.into(), + arguments: args, + cwd: text(server, "cwd")?.into(), + environment: serde_json::from_value(server["environment"].clone()) + .map_err(|e| e.to_string())?, + source: server["source"].clone(), + device: device( + &serde_json::to_value(data.gpu_samples.first().ok_or("missing device evidence")?) + .map_err(|e| e.to_string())?, + )?, + // Admission quotes include transient available memory. Only resolved + // options/capabilities define the launch; each run still checks its + // complete raw plan at every request boundary. + serving_plan: ["family", "requested", "effective", "qualified", "controls"] + .into_iter() + .map(|key| (key.to_owned(), data.serving_plan[key].clone())) + .collect::>() + .into(), + expected_clock_range_mhz: identity["expected_clock_range_mhz"].clone(), + }; + launch.serving_plan["quote"] = post_open_quote(&data.serving_plan); + normalize_guard(&mut launch, workload)?; + verify_static(&launch)?; + input_scope(&launch, workload, manifest, checked)?; + Ok(launch) +} + +fn normalize_guard(launch: &mut Launch, workload: &Workload) -> Result<(), String> { + let options = options(&launch.arguments)?; + let Some(path) = options.get("--expect-plan") else { + return Ok(()); + }; + let path = launch + .cwd + .join(path) + .canonicalize() + .map_err(|e| e.to_string())?; + let digest = artifact::hash(&path)?; + if !workload + .inputs + .values() + .any(|input| input.path == path && input.sha256 == digest) + || json::(&path)? != expected_plan(launch) + { + return Err("recorded --expect-plan is not pinned to the measured serving plan".into()); + } + // A prior profile's guard is evidence, not a reusable output path. The new + // application injects its own guard after validating the launch. + let mut index = 0; + while index < launch.arguments.len() { + if launch.arguments[index] == "--expect-plan" { + launch.arguments.drain(index..index + 2); + break; + } + index += if SWITCHES.contains(&launch.arguments[index].as_str()) { + 1 + } else { + 2 + }; + } + Ok(()) +} + +fn verify_static(launch: &Launch) -> Result<(), String> { + if artifact::hash(&launch.executable)? != launch.executable_sha256 { + return Err("profile server executable changed".into()); + } + let source_root = text(&launch.source, "root")?; + if context::source_snapshot(Path::new(source_root))? != launch.source { + return Err("profile runtime source changed".into()); + } + if !launch.cwd.is_dir() { + return Err("profile working directory disappeared".into()); + } + Ok(()) +} + +fn repeat_observations( + run: &review::Run, + root: &Path, + workload: &Workload, +) -> Result, String> { + let mut paths: Vec<_> = (0..workload.cases.len()) + .map(|index| format!("case-{index:03}")) + .collect(); + for index in 0..workload.overlaps.len() { + for stream in ["decode", "prefill"] { + paths.push(format!("overlap-{index:03}/{stream}")); + } + } + paths + .into_iter() + .map(|path| { + let events: Vec = run.json(&root.join(&path).join("events.json"))?; + let bytes = serde_json::to_vec(&events).map_err(|e| e.to_string())?; + Ok((path, artifact::hash_bytes(&bytes))) + }) + .collect() +} + +fn qualify( + candidate: &Candidate, + selection: &Selection, + workload: &Workload, + checked: &mut artifact::Verification, +) -> Result { + let mut selected = None; + let mut count = 0u32; + let mut worst = 0f64; + let mut hashes = BTreeSet::new(); + let mut observations = BTreeSet::new(); + for file in &candidate.runs { + if !hashes.insert(artifact::hash(file)?) { + return Err("copied evidence cannot supply additional repeats".into()); + } + let run = review::Run::load_verified(file, checked)?; + let data = run.data()?; + restart::review(&run, workload, checked)?; + if !data.passed + || !data.context_failures.is_empty() + || data.requested_repeats == 0 + || data.name != workload.name + || data.family != workload.family + || run.bytes(Path::new("workload.json"))? + != fs::read(&selection.workload).map_err(|e| e.to_string())? + || data.cases.len() != workload.cases.len() * data.requested_repeats as usize + || data.overlaps.len() != workload.overlaps.len() * data.requested_repeats as usize + || data.gpu_samples.len() != 2 * data.requested_repeats as usize + || data.gpu_windows.len() != data.requested_repeats as usize + { + return Err("incomplete workload, repeat, clock or correctness evidence".into()); + } + let launch = launch(&run, workload, &selection.workload, checked)?; + if selected + .as_ref() + .is_some_and(|previous| previous != &launch) + { + return Err("candidate runtime identity or effective plan differs across runs".into()); + } + for repeat in 0..data.requested_repeats { + let root = if data.requested_repeats == 1 { + PathBuf::new() + } else { + PathBuf::from(format!("repeat-{repeat:03}")) + }; + for observation in repeat_observations(&run, &root, workload)? { + if !observations.insert(observation) { + return Err( + "repeated runtime observations cannot supply independent samples".into(), + ); + } + } + for (offset, label) in ["gpu-before", "gpu-after"].iter().enumerate() { + let sample = &data.gpu_samples[repeat as usize * 2 + offset]; + let raw: Value = run.json(&root.join(format!("{label}.json")))?; + sample.verify_raw(&run.bytes(&root.join(format!("{label}.stdout")))?)?; + if raw != serde_json::to_value(sample).map_err(|e| e.to_string())? + || device(&raw)? != launch.device + || !sample + .failures(workload.expected_clock_range_mhz.as_ref()) + .is_empty() + { + return Err("raw clock/device evidence fails contract".into()); + } + } + let cases_start = repeat as usize * workload.cases.len(); + let overlaps_start = repeat as usize * workload.overlaps.len(); + let span_ms = data.cases[cases_start..cases_start + workload.cases.len()] + .iter() + .map(|case| case.total_ms) + .sum::() + + data.overlaps[overlaps_start..overlaps_start + workload.overlaps.len()] + .iter() + .map(overlap::Observation::span_ms) + .sum::(); + data.gpu_windows[repeat as usize].review( + &run, + &root, + workload.expected_clock_range_mhz.as_ref(), + &launch.device, + span_ms, + )?; + for (index, case) in workload.cases.iter().enumerate() { + let observed = &data.cases[repeat as usize * workload.cases.len() + index]; + if observed.name != case.name + || observed.repeat != repeat + || !observed.failures.is_empty() + || serde_json::to_value(&observed.scenario).map_err(|e| e.to_string())? + != serde_json::to_value(&case.scenario).map_err(|e| e.to_string())? + { + return Err("case identity or repeat mismatch".into()); + } + worst = worst.max(run.case( + &root.join(format!("case-{index:03}")), + case, + observed, + &data.serving_plan, + &workload.family, + )?); + } + for (index, overlap) in workload.overlaps.iter().enumerate() { + worst = worst.max(overlap::review( + &run, + &root.join(format!("overlap-{index:03}")), + overlap, + &data.overlaps[repeat as usize * workload.overlaps.len() + index], + &data.serving_plan, + &workload.family, + repeat, + )?); + } + } + count = count + .checked_add(data.requested_repeats) + .ok_or("repeat count overflow")?; + selected = Some(launch); + } + if count < MIN_REPEATS { + return Err(format!( + "profile needs at least {MIN_REPEATS} samples of every declared workload" + )); + } + Ok(Qualified { + name: candidate.name.clone(), + worst_latency_fraction: worst, + samples_per_workload: count, + launch: selected.ok_or("missing candidate runs")?, + }) +} + +#[cfg(test)] +fn choose(selection: Selection) -> Result { + let mut checked = artifact::Verification::default(); + let profile = choose_verified(selection, &mut checked)?; + checked.finish()?; + Ok(profile) +} + +fn choose_verified( + selection: Selection, + checked: &mut artifact::Verification, +) -> Result { + let workload = workload(&selection.workload, checked)?; + let mut selected: Option = None; + let mut rejected = BTreeMap::new(); + for candidate in &selection.candidates { + match qualify(candidate, &selection, &workload, checked) { + Ok(value) => { + if selected + .as_ref() + .is_some_and(|s| s.launch.device != value.launch.device) + { + rejected.insert( + candidate.name.clone(), + "candidate device differs from matched workload campaign".into(), + ); + continue; + } + if selected + .as_ref() + .is_none_or(|s| value.worst_latency_fraction < s.worst_latency_fraction) + { + selected = Some(value); + } + } + Err(error) => { + rejected.insert(candidate.name.clone(), error); + } + } + } + Ok(Profile { + selection, + qualified: selected.is_some(), + scope: "declared-serving-workload-only".into(), + selected, + rejected, + }) +} + +pub(super) fn run(args: &cli::ServingProfile) -> Result<(), String> { + let mut checked = artifact::Verification::default(); + let selection = selection(&args.plan, &mut checked)?; + let profile = choose_verified(selection, &mut checked)?; + let qualified = profile.qualified; + fs::create_dir(&args.out).map_err(|e| e.to_string())?; + let mut artifact = Artifact::new(profile); + artifact.warnings.push("Preflight quote comparison excluded: pre-open and post-open resident credits differ. Final startup and captured candidates compare post-open quotes with only available excluded.".into()); + let data = artifact.require()?; + let mut paths = vec![args.plan.clone(), data.selection.workload.clone()]; + paths.extend( + data.selection + .candidates + .iter() + .flat_map(|c| c.runs.clone()), + ); + for path in paths { + artifact.inputs.push(artifact::reference(&path, &args.out)?); + } + checked.finish()?; + artifact::save(&args.out.join("profile.json"), &artifact)?; + if !qualified { + return Err( + "no candidate meets every serving profile contract; see profile.json rejected reasons" + .into(), + ); + } + Ok(()) +} + +pub(super) fn apply(args: &cli::ApplyProfile) -> Result<(), String> { + let mut checked = artifact::Verification::default(); + let artifact: Artifact = artifact::load_verified(&args.profile, &mut checked)?; + let profile = artifact.require()?; + let selected = profile + .selected + .as_ref() + .ok_or("unqualified serving profile")?; + let replay = choose_verified(profile.selection.clone(), &mut checked)?; + if serde_json::to_value(&replay).map_err(|e| e.to_string())? + != serde_json::to_value(profile).map_err(|e| e.to_string())? + { + return Err("profile differs from revalidated evidence".into()); + } + fs::create_dir(&args.out).map_err(|e| e.to_string())?; + fs::write( + args.out.join("launch.json"), + serde_json::to_vec_pretty(&selected.launch).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + let expected = args.out.join("expected-plan.json"); + fs::write( + &expected, + serde_json::to_vec_pretty(&expected_plan(&selected.launch)).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + let expected = expected.canonicalize().map_err(|e| e.to_string())?; + checked.finish()?; + if !args.execute { + return Ok(()); + } + execute(&selected.launch, &args.out, &expected, &checked) +} + +fn command(launch: &Launch) -> Command { + let mut command = Command::new(&launch.executable); + command.args(&launch.arguments).current_dir(&launch.cwd); + for (key, _) in std::env::vars_os() { + if [ + "DS4_", + "CUDA_", + "NVIDIA_", + "LD_", + "CUBLAS_", + "OMP_", + "OPENBLAS_", + ] + .iter() + .any(|prefix| key.to_string_lossy().starts_with(prefix)) + { + command.env_remove(key); + } + } + command.envs(&launch.environment); + command +} + +fn expected_plan(launch: &Launch) -> Value { + serde_json::json!({ + "family":launch.serving_plan["family"], + "backend":launch.serving_plan["requested"]["backend"], + "effective":launch.serving_plan["effective"], + "qualified":launch.serving_plan["qualified"], + "controls":launch.serving_plan["controls"], + "post_open_quote":launch.serving_plan["quote"] + }) +} + +fn guarded(launch: &Launch, expected: &Path) -> Command { + let mut command = command(launch); + command.arg("--expect-plan").arg(expected); + command +} + +fn execute( + launch: &Launch, + out: &Path, + expected: &Path, + checked: &artifact::Verification, +) -> Result<(), String> { + // Re-run the server's model-free resolver before replacing this process. + let mut check = guarded(launch, expected); + check + .arg("--check-config") + .stdout(fs::File::create(out.join("check.json")).map_err(|e| e.to_string())?) + .stderr(fs::File::create(out.join("check.stderr")).map_err(|e| e.to_string())?); + let budget = cli::Budget { + timeout_seconds: 120, + max_output_mib: 64, + }; + if !process::run(&mut check, out, budget.limits())?.success() { + return Err("profile server configuration check failed".into()); + } + let plan: Value = json(&out.join("check.json"))?; + for key in ["family", "effective", "qualified", "controls"] { + if plan[key] != launch.serving_plan[key] { + return Err(format!("profile configuration changed: {key}")); + } + } + if plan["requested"]["backend"] != launch.serving_plan["requested"]["backend"] { + return Err("profile configuration changed: backend".into()); + } + let serving = cli::Serving { + budget, + url: "http://127.0.0.1:1".into(), + workload: PathBuf::new(), + out: out.into(), + repeats: 1, + server_pid: None, + device: launch.device["device_ordinal"] + .as_u64() + .ok_or("profile missing device ordinal")? as usize, + }; + let sample = context::gpu(&serving, out, "gpu-apply")?; + let range: context::ClockRange = + serde_json::from_value(launch.expected_clock_range_mhz.clone()) + .map_err(|e| e.to_string())?; + if !sample.failures(Some(&range)).is_empty() { + return Err("profile current GPU clock fails workload contract".into()); + } + if device(&serde_json::to_value(&sample).map_err(|e| e.to_string())?)? != launch.device { + return Err("profile device/driver changed".into()); + } + verify_static(launch)?; + checked.finish()?; + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + Err(guarded(launch, expected).exec().to_string()) + } + #[cfg(not(unix))] + { + Err("profile execution requires Unix".into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + struct Fixture(PathBuf); + impl Fixture { + fn new() -> Self { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = + std::env::temp_dir().join(format!("ds4-qualified-{}-{nonce}", std::process::id())); + fs::create_dir(&path).unwrap(); + assert!(Command::new("git") + .arg("init") + .arg("--quiet") + .arg(&path) + .status() + .unwrap() + .success()); + fs::write(path.join("runtime.rs"), "fixture source\n").unwrap(); + fs::write(path.join("model.gguf"), b"fixture model; no inference").unwrap(); + fs::copy("/bin/true", path.join("server")).unwrap(); + Self(path) + } + fn write(&self, path: &Path, value: &Value) { + fs::write(path, serde_json::to_vec_pretty(value).unwrap()).unwrap(); + } + fn workload(&self) -> PathBuf { + let file = self.0.join("workload.json"); + self.write(&file, &json!({"protocol":PROTOCOL,"name":"synthetic","family":"fixture", + "inputs":{"model":{"path":self.0.join("model.gguf"),"sha256":artifact::hash(&self.0.join("model.gguf")).unwrap()}}, + "expected_clock_range_mhz":{"min":300,"max":2200}, + "cases":[{"name":"cold","scenario":"kv_cold_prefill","request":{"model":"fixture","stream":true,"messages":[{"role":"user","content":"hello"}]}, + "expect":{"content":"hello","finish_reason":"stop","reuse_kind":"cold","effective_lane":"serial","speculation_active":false,"fallback_reason":null}, + "limits":{"ttft_ms":100.0,"total_ms":200.0,"min_host_available_bytes":900}}]})); + file + } + fn run( + &self, + manifest: &Path, + name: &str, + latency: f64, + available: u64, + repeats: u32, + ) -> PathBuf { + let root = self.0.join(name); + fs::create_dir(&root).unwrap(); + fs::copy(manifest, root.join("workload.json")).unwrap(); + let workload: Value = json_read(manifest); + let plan = json!({"family":"fixture","requested":{},"effective":{"max_seqs":1},"qualified":{},"issues":[],"controls":{}}); + let argv = format!( + "{}\0-m\0{}\0--max-seqs\01\0", + self.0.join("server").display(), + self.0.join("model.gguf").display() + ); + fs::write(root.join("server.argv"), &argv).unwrap(); + let identity = json!({"server":{"pid":1,"boot_id":"fixture","start_ticks":1, + "executable":self.0.join("server"),"executable_sha256":artifact::hash(&self.0.join("server")).unwrap(),"argv_sha256":artifact::hash_bytes(argv.as_bytes()), + "cwd":self.0,"environment":{},"unreviewed_environment":[],"source":context::source_snapshot(&self.0).unwrap()}, + "inputs":workload["inputs"],"expected_clock_range_mhz":workload["expected_clock_range_mhz"],"limitations":[]}); + self.write(&root.join("identity.json"), &identity); + let mut cases = Vec::new(); + let mut samples = Vec::new(); + let mut windows = Vec::new(); + for repeat in 0..repeats { + let parent = if repeats == 1 { + root.clone() + } else { + let p = root.join(format!("repeat-{repeat:03}")); + fs::create_dir(&p).unwrap(); + p + }; + let started = 1000 + repeat as u64 * 1000; + let finished = started + latency.ceil() as u64 + 3; + for (label, time) in [("gpu-before", started), ("gpu-after", finished + 1)] { + let sample = json!({"label":label,"unix_ms":time,"device_ordinal":0,"uuid":"fixture-uuid","name":"fixture GPU","driver":"fixture-driver","sm_clock_mhz":1000,"temperature_c":45,"unavailable_reason":null}); + self.write(&parent.join(format!("{label}.json")), &sample); + fs::write( + parent.join(format!("{label}.stdout")), + "fixture-uuid, fixture GPU, fixture-driver, 1000, 45\n", + ) + .unwrap(); + samples.push(sample); + } + let mut sample = samples.last().unwrap().clone(); + sample["label"] = json!("gpu-poll-00000"); + sample["unix_ms"] = json!(started + 1); + self.write(&parent.join("gpu-poll-00000.json"), &sample); + fs::write( + parent.join("gpu-poll-00000.stdout"), + "fixture-uuid, fixture GPU, fixture-driver, 1000, 45\n", + ) + .unwrap(); + let window = json!({"poll_interval_ms":500,"started_unix_ms":started,"finished_unix_ms":finished,"samples":[sample]}); + self.write(&parent.join("gpu-window.json"), &window); + windows.push(window); + let case = parent.join("case-000"); + fs::create_dir(&case).unwrap(); + self.write(&case.join("request.json"), &workload["cases"][0]["request"]); + let first = json!({"id":format!("{name}-{repeat}"),"choices":[{"delta":{"content":"hello"}}]}).to_string(); + let finish = r#"{"choices":[{"delta":{},"finish_reason":"stop"}]}"#; + self.write(&case.join("events.json"), &json!([{"elapsed_ms":latency,"data":first},{"elapsed_ms":latency+1.0,"data":finish},{"elapsed_ms":latency+2.0,"data":"[DONE]"}])); + fs::write( + case.join("response.sse"), + format!("data: {first}\n\ndata: {finish}\n\ndata: [DONE]\n\n"), + ) + .unwrap(); + fs::write(case.join("response.headers"), "HTTP/1.1 200 OK\r\n\r\n").unwrap(); + self.write(&case.join("memory.json"), &json!([{"elapsed_ms":0.0,"available_bytes":available},{"elapsed_ms":latency+3.0,"available_bytes":available}])); + let trace = json!({"reuse_kind":"cold","effective_lane":"serial","speculation_active":false,"fallback_reason":null}); + for (name, count) in [("before", 0), ("after", 1)] { + self.write(&case.join(format!("{name}.json")), &json!({"serving":plan,"queue_depth":0,"clients":0,"routes":{"serial":count},"last_request":trace})); + } + let observed = json!({"name":"cold","repeat":repeat,"scenario":"kv_cold_prefill","ttft_ms":latency,"first_content_ms":latency,"total_ms":latency+2.0,"content":"hello","finish_reason":"stop","host_available_before_bytes":available,"host_available_after_bytes":available,"host_min_available_bytes":available,"trace":trace,"failures":[]}); + self.write(&case.join("result.json"), &observed); + cases.push(observed); + } + let data: Evidence = serde_json::from_value(json!({"protocol":PROTOCOL,"name":"synthetic","family":"fixture","serving_plan":plan,"cases":cases,"overlaps":[],"requested_repeats":repeats,"identity":identity,"gpu_samples":samples,"gpu_windows":windows,"context_failures":[],"passed":true,"timing_method":"synthetic model-free test"})).unwrap(); + let mut evidence = Artifact::new(data); + references(&root, &root, &mut evidence.inputs).unwrap(); + evidence + .data + .as_ref() + .unwrap() + .identity + .add_refs(&mut evidence.inputs); + let path = root.join("serving.json"); + artifact::save(&path, &evidence).unwrap(); + path + } + } + impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + fn json_read(path: &Path) -> Value { + serde_json::from_slice(&fs::read(path).unwrap()).unwrap() + } + + #[test] + fn rewrapped_capture_is_not_repeat() { + let f = Fixture::new(); + let manifest = f.workload(); + let capture = f.run(&manifest, "single", 20.0, 1000, 1); + let mut runs = vec![capture.clone()]; + for index in 1..3 { + let mut copied = json_read(&capture); + copied["created_unix"] = json!(index); + copied["warnings"] = json!([format!("metadata copy {index}")]); + let path = capture.with_file_name(format!("copy-{index}.json")); + f.write(&path, &copied); + runs.push(path); + } + let profile = choose(Selection { + workload: manifest, + candidates: vec![Candidate { + name: "copied".into(), + runs, + }], + }) + .unwrap(); + assert!( + !profile.qualified, + "one runtime capture was counted three times" + ); + assert!(profile.rejected["copied"].contains("runtime observations")); + } + + #[test] + fn gpu_window_spans_actual_requests() { + let f = Fixture::new(); + let manifest = f.workload(); + let capture = f.run(&manifest, "short-window", 20.0, 1000, 3); + let mut evidence = json_read(&capture); + for (index, window) in evidence["data"]["gpu_windows"] + .as_array_mut() + .unwrap() + .iter_mut() + .enumerate() + { + window["finished_unix_ms"] = window["started_unix_ms"].clone(); + f.write( + &capture + .parent() + .unwrap() + .join(format!("repeat-{index:03}/gpu-window.json")), + window, + ); + } + for input in evidence["inputs"].as_array_mut().unwrap() { + if input["path"].as_str().unwrap().ends_with("gpu-window.json") { + let path = capture + .parent() + .unwrap() + .join(input["path"].as_str().unwrap()); + input["sha256"] = json!(artifact::hash(&path).unwrap()); + } + } + f.write(&capture, &evidence); + let profile = choose(Selection { + workload: manifest, + candidates: vec![Candidate { + name: "short-window".into(), + runs: vec![capture], + }], + }) + .unwrap(); + assert!(!profile.qualified); + assert!(profile.rejected["short-window"].contains("telemetry")); + } + + #[test] + fn selects_complete_contract_before_speed() { + let f = Fixture::new(); + let manifest = f.workload(); + let good = f.run(&manifest, "good", 20.0, 1000, 3); + let unsafe_fast = f.run(&manifest, "unsafe", 1.0, 100, 3); + let short = f.run(&manifest, "short", 2.0, 1000, 1); + let profile = choose(Selection { + workload: manifest, + candidates: vec![ + Candidate { + name: "unsafe".into(), + runs: vec![unsafe_fast], + }, + Candidate { + name: "short".into(), + runs: vec![short], + }, + Candidate { + name: "good".into(), + runs: vec![good], + }, + ], + }) + .unwrap(); + assert!(profile.qualified); + assert_eq!(profile.selected.unwrap().name, "good"); + assert!(profile.rejected["unsafe"].contains("memory")); + assert!(profile.rejected["short"].contains("at least 3")); + } + + #[test] + fn replay_rechecks_source_binary_and_inputs() { + let f = Fixture::new(); + let manifest = f.workload(); + let run = f.run(&manifest, "good", 20.0, 1000, 3); + let selection = Selection { + workload: manifest, + candidates: vec![Candidate { + name: "good".into(), + runs: vec![run], + }], + }; + let profile = choose(selection.clone()).unwrap(); + assert!(profile.qualified); + let launch = profile.selected.unwrap().launch; + fs::write(f.0.join("runtime.rs"), "changed source").unwrap(); + assert!(verify_static(&launch) + .unwrap_err() + .contains("source changed")); + fs::write(f.0.join("runtime.rs"), "fixture source\n").unwrap(); + fs::write(f.0.join("server"), b"changed binary").unwrap(); + assert!(verify_static(&launch) + .unwrap_err() + .contains("executable changed")); + fs::write(f.0.join("model.gguf"), b"changed model").unwrap(); + assert!(choose(selection).is_err_and(|e| e.contains("input changed"))); + } + + #[test] + fn missing_shard_and_late_override_are_rejected() { + let f = Fixture::new(); + let manifest = f.workload(); + let run = f.run(&manifest, "good", 20.0, 1000, 3); + let workload = workload(&manifest, &mut artifact::Verification::default()).unwrap(); + let mut launch = launch( + &review::Run::load(&run).unwrap(), + &workload, + &manifest, + &mut artifact::Verification::default(), + ) + .unwrap(); + launch.arguments.extend(["--max-seqs".into(), "64".into()]); + assert!(input_scope( + &launch, + &workload, + &manifest, + &mut artifact::Verification::default() + ) + .unwrap_err() + .contains("duplicate")); + launch.arguments.truncate(4); + launch.arguments[3] = "auto".into(); + assert!(input_scope( + &launch, + &workload, + &manifest, + &mut artifact::Verification::default() + ) + .unwrap_err() + .contains("explicit --max-seqs")); + launch.arguments[3] = "1".into(); + let first = f.0.join("model-00001-of-00002.gguf"); + let second = f.0.join("model-00002-of-00002.gguf"); + fs::write(&first, b"part1").unwrap(); + fs::write(&second, b"part2").unwrap(); + launch.arguments[1] = first.to_string_lossy().into(); + let mut workload = workload; + workload.inputs.insert( + "model".into(), + InputFile { + path: first.clone(), + sha256: artifact::hash(&first).unwrap(), + }, + ); + assert!(input_scope( + &launch, + &workload, + &manifest, + &mut artifact::Verification::default() + ) + .unwrap_err() + .contains("lacks consumed input")); + } + + #[test] + fn unpinned_template_sidecar_is_rejected() { + let f = Fixture::new(); + let manifest = f.workload(); + let run = f.run(&manifest, "good", 20.0, 1000, 3); + let workload = workload(&manifest, &mut artifact::Verification::default()).unwrap(); + let launch = launch( + &review::Run::load(&run).unwrap(), + &workload, + &manifest, + &mut artifact::Verification::default(), + ) + .unwrap(); + fs::write(f.0.join("chat_template.jinja"), "changed template").unwrap(); + assert!(input_scope( + &launch, + &workload, + &manifest, + &mut artifact::Verification::default() + ) + .is_err_and(|e| e.contains("template"))); + } + + #[cfg(unix)] + #[test] + fn launch_alias_is_rechecked_before_use() { + let fixture = Fixture::new(); + let manifest = fixture.workload(); + let run = fixture.run(&manifest, "good", 20.0, 1000, 3); + let mut checked = artifact::Verification::default(); + let workload = workload(&manifest, &mut checked).unwrap(); + let mut launch = launch( + &review::Run::load(&run).unwrap(), + &workload, + &manifest, + &mut checked, + ) + .unwrap(); + let alias = fixture.0.join("model-alias"); + std::os::unix::fs::symlink(fixture.0.join("model.gguf"), &alias).unwrap(); + launch.arguments[1] = alias.to_string_lossy().into(); + input_scope(&launch, &workload, &manifest, &mut checked).unwrap(); + let other = fixture.0.join("other-model"); + fs::copy(fixture.0.join("model.gguf"), &other).unwrap(); + fs::remove_file(&alias).unwrap(); + std::os::unix::fs::symlink(other, &alias).unwrap(); + assert!(checked + .finish() + .is_err_and(|error| error.contains("path changed"))); + } + + #[test] + fn checked_application_rederives_the_launch() { + let f = Fixture::new(); + let manifest = f.workload(); + let capture = f.run(&manifest, "good", 20.0, 1000, 3); + let selection = f.0.join("selection.json"); + f.write( + &selection, + &json!({"workload":manifest,"candidates":[{"name":"good","runs":[capture]}]}), + ); + let output = f.0.join("profile"); + run(&cli::ServingProfile { + plan: selection, + out: output.clone(), + }) + .unwrap(); + let profile = output.join("profile.json"); + let args = cli::ApplyProfile { + profile: profile.clone(), + out: f.0.join("checked"), + execute: false, + }; + apply(&args).unwrap(); + assert_eq!( + json_read(&args.out.join("launch.json"))["arguments"][3], + "1" + ); + let mut forged = json_read(&profile); + forged["data"]["selected"]["launch"]["arguments"][3] = json!("64"); + f.write(&profile, &forged); + assert!(apply(&cli::ApplyProfile { + profile, + out: f.0.join("forged"), + execute: false + }) + .is_err_and(|e| e.contains("differs from revalidated"))); + } + + #[test] + fn recorded_guard_must_be_pinned_and_replaced() { + let f = Fixture::new(); + let manifest = f.workload(); + let capture = f.run(&manifest, "good", 20.0, 1000, 3); + let mut workload = workload(&manifest, &mut artifact::Verification::default()).unwrap(); + let mut launch = launch( + &review::Run::load(&capture).unwrap(), + &workload, + &manifest, + &mut artifact::Verification::default(), + ) + .unwrap(); + let guard = f.0.join("expected.json"); + f.write(&guard, &expected_plan(&launch)); + launch + .arguments + .extend(["--expect-plan".into(), guard.to_string_lossy().into()]); + assert!(normalize_guard(&mut launch, &workload).is_err()); + workload.inputs.insert( + "guard".into(), + InputFile { + path: guard.clone(), + sha256: artifact::hash(&guard).unwrap(), + }, + ); + normalize_guard(&mut launch, &workload).unwrap(); + assert!(!launch.arguments.iter().any(|a| a == "--expect-plan")); + let command = guarded(&launch, &guard); + let arguments = command.get_args().collect::>(); + assert_eq!(arguments[arguments.len() - 2], "--expect-plan"); + assert_eq!(arguments[arguments.len() - 1], guard); + } +} diff --git a/crates/ds4-perf/src/serving/restart.rs b/crates/ds4-perf/src/serving/restart.rs new file mode 100644 index 0000000000000000000000000000000000000000..f7cd1cc5456b2a75e955b99087445cd61e1f3ef1 --- /dev/null +++ b/crates/ds4-perf/src/serving/restart.rs @@ -0,0 +1,202 @@ +//! Link an operator-controlled stop/start to the first disk-restore request. +use super::*; + +fn process_pair(previous: &Value, current: &Value) -> Result<(), String> { + let old = &previous["server"]; + let new = ¤t["server"]; + let pid = old["pid"] + .as_u64() + .ok_or("restart seed needs --server-pid")?; + let old_start = old["start_ticks"] + .as_u64() + .ok_or("seed start time missing")?; + let new_start = new["start_ticks"] + .as_u64() + .ok_or("restore needs --server-pid")?; + if old["boot_id"] == new["boot_id"] && (new_start <= old_start || old["pid"] == new["pid"]) { + return Err("restart restore requires a newer server process".into()); + } + for key in [ + "executable", + "executable_sha256", + "argv_sha256", + "cwd", + "environment", + "source", + "unreviewed_environment", + ] { + if old.get(key).is_none() || old[key] != new[key] { + return Err(format!("restart runtime changed: {key}")); + } + } + if previous["inputs"] != current["inputs"] { + return Err("restart input identities changed".into()); + } + if let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) { + let fields = stat + .rsplit_once(')') + .ok_or("invalid seed process stat")? + .1 + .split_whitespace() + .collect::>(); + if fields.get(19).and_then(|v| v.parse::().ok()) == Some(old_start) + && fields.first() != Some(&"Z") + { + return Err("seed server is still running; stop it before restart collection".into()); + } + } + Ok(()) +} + +fn seed_verified( + workload: &Workload, + identity: &context::Identity, + checked: &mut artifact::Verification, +) -> Result { + let path = workload + .restart_from + .as_ref() + .ok_or("missing restart seed")?; + let seed = review::Run::load_verified(path, checked)?; + let data = seed.data()?; + if !data.passed + || data.family != workload.family + || data.serving_plan["effective"]["disk"] != true + { + return Err("restart seed must pass on the same family with disk enabled".into()); + } + process_pair( + &serde_json::to_value(&data.identity).map_err(|e| e.to_string())?, + &serde_json::to_value(identity).map_err(|e| e.to_string())?, + )?; + let previous: Workload = seed.json(Path::new("workload.json"))?; + let restored = workload + .cases + .first() + .and_then(|case| case.request["messages"].as_array()) + .ok_or("restore request missing")?; + if !previous.cases.iter().any(|case| { + case.request["messages"] + .as_array() + .is_some_and(|messages| !messages.is_empty() && restored.starts_with(messages)) + }) { + return Err("restore conversation does not extend a seeded request".into()); + } + Ok(seed) +} + +pub(super) fn prepare( + workload: &Workload, + identity: &context::Identity, + out: &Path, + repeats: u32, + checked: &mut artifact::Verification, +) -> Result, String> { + let Some(path) = &workload.restart_from else { + return Ok(None); + }; + if repeats != 1 { + return Err( + "restart restore needs a fresh process per capture; --repeats must be 1".into(), + ); + } + seed_verified(workload, identity, checked)?; + Ok(Some(artifact::reference(path, out)?)) +} + +pub(super) fn first_request( + workload: &Workload, + current: &Value, + before: &Value, + index: usize, + repeat: u32, + checked: &mut artifact::Verification, +) -> Result<(), String> { + first_verified(workload, current, before, index, repeat, checked)?; + checked.finish() +} + +fn first_verified( + workload: &Workload, + current: &Value, + before: &Value, + index: usize, + repeat: u32, + checked: &mut artifact::Verification, +) -> Result<(), String> { + if index != 0 + || repeat != 0 + || route_count(before)? != 0 + || current["effective"]["disk"] != true + { + return Err( + "restart restore must be the new server's first request with disk enabled".into(), + ); + } + let seed: Artifact = artifact::load_verified( + workload + .restart_from + .as_ref() + .ok_or("restart seed missing")?, + checked, + )?; + let previous = &seed.require()?.serving_plan; + if post_open_quote(previous) != post_open_quote(current) { + return Err("restart post-open memory quote changed".into()); + } + for key in ["family", "requested", "effective", "qualified", "controls"] { + if previous[key] != current[key] { + return Err(format!("restart serving plan changed: {key}")); + } + } + Ok(()) +} + +pub(super) fn review( + run: &review::Run, + workload: &Workload, + checked: &mut artifact::Verification, +) -> Result<(), String> { + let data = run.data()?; + if workload.restart_from.is_none() { + if data.restart_from.is_some() { + return Err("unexpected restart linkage".into()); + } + return Ok(()); + } + let reference = data + .restart_from + .as_ref() + .ok_or("restart evidence lacks seed reference")?; + let bytes = run.bytes(Path::new(&reference.path))?; + let expected = workload + .restart_from + .as_ref() + .ok_or("restart seed missing")?; + if artifact::hash_bytes(&bytes) != reference.sha256 + || artifact::hash(expected)? != reference.sha256 + || data.requested_repeats != 1 + { + return Err("restart seed reference changed".into()); + } + seed_verified(workload, &data.identity, checked)?; + let before: Value = run.json(Path::new("case-000/before.json"))?; + first_verified(workload, &data.serving_plan, &before, 0, 0, checked) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + #[test] + fn requires_new_process_and_same_runtime() { + let old = json!({"server":{"pid":4294967000u64,"start_ticks":1,"boot_id":"fixture","executable":"server","executable_sha256":"binary","argv_sha256":"args","cwd":"/tmp","environment":{},"source":{},"unreviewed_environment":[]},"inputs":{}}); + assert!(process_pair(&old, &old).is_err_and(|e| e.contains("newer"))); + let mut new = old.clone(); + new["server"]["pid"] = json!(4294967001u64); + new["server"]["start_ticks"] = json!(2); + assert!(process_pair(&old, &new).is_ok()); + new["server"]["executable_sha256"] = json!("changed"); + assert!(process_pair(&old, &new).is_err_and(|e| e.contains("runtime changed"))); + } +} diff --git a/crates/ds4-perf/src/serving/review.rs b/crates/ds4-perf/src/serving/review.rs new file mode 100644 index 0000000000000000000000000000000000000000..8561fb57da9ea8312ebaccce1c7f05e5f006ef8a --- /dev/null +++ b/crates/ds4-perf/src/serving/review.rs @@ -0,0 +1,193 @@ +//! Recompute contracts from pinned raw records; summaries alone never qualify. +use super::*; +use serde::de::DeserializeOwned; +use std::path::PathBuf; + +pub(super) struct Run { + root: PathBuf, + evidence: Artifact, +} + +impl Run { + #[cfg(test)] + pub(super) fn load(path: &Path) -> Result { + Ok(Self { + root: path.parent().ok_or("run has no directory")?.into(), + evidence: artifact::load(path)?, + }) + } + + pub(super) fn load_verified( + path: &Path, + checked: &mut artifact::Verification, + ) -> Result { + Ok(Self { + root: path.parent().ok_or("run has no directory")?.into(), + evidence: artifact::load_verified(path, checked)?, + }) + } + + pub(super) fn data(&self) -> Result<&Evidence, String> { + self.evidence.require() + } + + pub(super) fn bytes(&self, path: &Path) -> Result, String> { + let file = self + .root + .join(path) + .canonicalize() + .map_err(|e| e.to_string())?; + if !self + .evidence + .inputs + .iter() + .any(|input| self.root.join(&input.path).canonicalize().ok().as_ref() == Some(&file)) + { + return Err(format!("raw evidence is not pinned: {}", path.display())); + } + fs::read(file).map_err(|e| e.to_string()) + } + + pub(super) fn json(&self, path: &Path) -> Result { + serde_json::from_slice(&self.bytes(path)?).map_err(|e| format!("{}: {e}", path.display())) + } + + pub(super) fn stream( + &self, + dir: &Path, + request: &Value, + ) -> Result<(Vec, Completion), String> { + // Remote media can change while the request JSON stays identical. + for message in request["messages"].as_array().into_iter().flatten() { + for part in message["content"].as_array().into_iter().flatten() { + let pinned = match part["type"].as_str() { + Some("image_url") => part["image_url"]["url"] + .as_str() + .is_some_and(|url| url.starts_with("data:image/")), + Some("input_audio") => part["input_audio"]["data"] + .as_str() + .is_some_and(|data| !data.is_empty()), + _ => true, + }; + if !pinned { + return Err("profile media must be embedded in the hashed request".into()); + } + } + } + let actual: Value = self.json(&dir.join("request.json"))?; + if &actual != request { + return Err("captured request differs from workload".into()); + } + let events: Vec = self.json(&dir.join("events.json"))?; + if events + .iter() + .any(|e| !e.elapsed_ms.is_finite() || e.elapsed_ms <= 0.0) + || events.windows(2).any(|e| e[1].elapsed_ms < e[0].elapsed_ms) + { + return Err("invalid SSE event timing".into()); + } + let raw = + String::from_utf8(self.bytes(&dir.join("response.sse"))?).map_err(|e| e.to_string())?; + let mut payloads = Vec::new(); + let mut lines = Vec::new(); + for line in raw.lines() { + if line.is_empty() { + if !lines.is_empty() { + payloads.push(lines.join("\n")); + lines.clear(); + } + } else if let Some(data) = line.strip_prefix("data:") { + lines.push(data.strip_prefix(' ').unwrap_or(data)); + } + } + if !lines.is_empty() + || payloads != events.iter().map(|e| e.data.clone()).collect::>() + { + return Err("SSE events differ from raw response".into()); + } + let headers = String::from_utf8(self.bytes(&dir.join("response.headers"))?) + .map_err(|e| e.to_string())?; + if headers + .lines() + .rfind(|l| l.starts_with("HTTP/")) + .and_then(|l| l.split_whitespace().nth(1)) + != Some("200") + { + return Err("captured response was not HTTP 200".into()); + } + let completion = summarize(&events)?; + Ok((events, completion)) + } + + pub(super) fn memory(&self, dir: &Path, duration_ms: f64) -> Result { + self.bytes(&dir.join("memory.json"))?; + context::memory_min(&self.root.join(dir), duration_ms) + } + + pub(super) fn stats( + &self, + dir: &Path, + expected: &Value, + routes: u64, + family: &str, + ) -> Result { + let before: Value = self.json(&dir.join("before.json"))?; + let after: Value = self.json(&dir.join("after.json"))?; + if &plan(&before, family)? != expected + || &plan(&after, family)? != expected + || route_count(&after)?.checked_sub(route_count(&before)?) != Some(routes) + { + return Err("raw serving plan or request attribution changed".into()); + } + Ok(after["last_request"].clone()) + } + + pub(super) fn case( + &self, + dir: &Path, + case: &Case, + observation: &Observation, + expected: &Value, + family: &str, + ) -> Result { + let (_, output) = self.stream(dir, &case.request)?; + let trace = self.stats(dir, expected, 1, family)?; + let memory = self.memory(dir, output.total_ms)?; + let raw: Value = self.json(&dir.join("result.json"))?; + if raw != serde_json::to_value(observation).map_err(|e| e.to_string())? + || output.ttft_ms != observation.ttft_ms + || output.total_ms != observation.total_ms + || output.first_content_ms != observation.first_content_ms + || output.content != observation.content + || output.finish_reason != observation.finish_reason + || output.content != case.expect.content + || output.finish_reason != case.expect.finish_reason + || output.tool_calls != observation.tool_calls + || output.tool_calls != case.expect.tool_calls + || output.cached_tokens != observation.cached_tokens + || case + .expect + .min_cached_tokens + .is_some_and(|minimum| output.cached_tokens.is_none_or(|cached| cached < minimum)) + || observation.host_min_available_bytes != Some(memory) + || memory + .min(observation.host_available_before_bytes) + .min(observation.host_available_after_bytes) + < case.limits.min_host_available_bytes + || output.ttft_ms > case.limits.ttft_ms + || output.total_ms > case.limits.total_ms + || observation.trace != trace + || trace["reuse_kind"] != case.expect.reuse_kind + || trace["effective_lane"] != case.expect.effective_lane + || trace["speculation_active"] != case.expect.speculation_active + || trace["fallback_reason"] + != serde_json::to_value(&case.expect.fallback_reason).map_err(|e| e.to_string())? + { + return Err(format!( + "{}: raw accuracy, latency, memory or trace contract failed", + case.name + )); + } + Ok((output.ttft_ms / case.limits.ttft_ms).max(output.total_ms / case.limits.total_ms)) + } +} diff --git a/crates/ds4-perf/src/serving/telemetry.rs b/crates/ds4-perf/src/serving/telemetry.rs new file mode 100644 index 0000000000000000000000000000000000000000..11bd8f42a81d3a6b16e9b03e4d157614ba6104a2 --- /dev/null +++ b/crates/ds4-perf/src/serving/telemetry.rs @@ -0,0 +1,201 @@ +//! Clock queries run beside the workload; every sample keeps its raw stdout. +use super::*; +use std::{ + sync::atomic::{AtomicBool, Ordering}, + time::{Duration, Instant, SystemTime}, +}; + +const POLL_MS: u64 = 500; +// Includes query and scheduling delay; larger gaps cannot qualify a run. +const MAX_GAP_MS: u64 = 3 * POLL_MS; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Window { + poll_interval_ms: u64, + started_unix_ms: u64, + finished_unix_ms: u64, + samples: Vec, +} +impl Window { + pub(super) fn failures(&self, range: Option<&context::ClockRange>) -> Vec { + if self.samples.is_empty() { + return vec!["no in-workload GPU telemetry".into()]; + } + let mut failures: Vec<_> = self + .samples + .iter() + .flat_map(|sample| sample.failures(range)) + .collect(); + if self.started_unix_ms == 0 + || self.finished_unix_ms < self.started_unix_ms + || self.samples.iter().any(|sample| { + let time = sample.timestamp_ms(); + time < self.started_unix_ms.saturating_sub(MAX_GAP_MS) + || time > self.finished_unix_ms.saturating_add(MAX_GAP_MS) + }) + || self.samples.windows(2).any(|pair| { + let first = pair[0].timestamp_ms(); + let next = pair[1].timestamp_ms(); + next <= first || next - first > MAX_GAP_MS + }) + || self.samples.first().is_some_and(|sample| { + sample.timestamp_ms() > self.started_unix_ms.saturating_add(MAX_GAP_MS) + }) + || self.samples.last().is_some_and(|sample| { + sample.timestamp_ms().saturating_add(MAX_GAP_MS) < self.finished_unix_ms + }) + { + failures.push("GPU telemetry chronology or workload coverage is incomplete".into()); + } + failures + } + pub(super) fn review( + &self, + run: &review::Run, + root: &Path, + range: Option<&context::ClockRange>, + expected_device: &Value, + minimum_ms: f64, + ) -> Result<(), String> { + let raw: Value = run.json(&root.join("gpu-window.json"))?; + if raw != serde_json::to_value(self).map_err(|e| e.to_string())? + || self.poll_interval_ms != POLL_MS + || !self.failures(range).is_empty() + || !minimum_ms.is_finite() + || minimum_ms < 0.0 + || self.finished_unix_ms.saturating_sub(self.started_unix_ms) as f64 + 1.0 < minimum_ms + { + return Err("in-workload GPU telemetry fails contract".into()); + } + let before: context::GpuSample = run.json(&root.join("gpu-before.json"))?; + let after: context::GpuSample = run.json(&root.join("gpu-after.json"))?; + if before.timestamp_ms() > self.started_unix_ms + || after.timestamp_ms() < self.finished_unix_ms + || self.samples.iter().any(|sample| { + sample.timestamp_ms() < before.timestamp_ms() + || sample.timestamp_ms() > after.timestamp_ms() + }) + { + return Err("GPU polling window lies outside its boundary samples".into()); + } + for (index, sample) in self.samples.iter().enumerate() { + let label = format!("gpu-poll-{index:05}"); + let value = serde_json::to_value(sample).map_err(|e| e.to_string())?; + for key in ["uuid", "name", "driver", "device_ordinal"] { + if value[key] != expected_device[key] { + return Err("GPU changed during workload".into()); + } + } + let raw: Value = run.json(&root.join(format!("{label}.json")))?; + if value != raw { + return Err("in-workload GPU sample summary changed".into()); + } + sample.verify_raw(&run.bytes(&root.join(format!("{label}.stdout")))?)?; + } + Ok(()) + } +} + +pub(super) fn observe( + args: &cli::Serving, + dir: &Path, + action: impl FnOnce() -> Result, +) -> Result<(T, Window), String> { + let finished = AtomicBool::new(false); + let started_unix_ms = unix_ms()?; + let (result, samples, finished_unix_ms) = std::thread::scope(|scope| { + let observer = scope.spawn(|| { + let mut samples = Vec::new(); + loop { + let start = Instant::now(); + let label = format!("gpu-poll-{:05}", samples.len()); + samples.push(context::gpu(args, dir, &label)?); + while !finished.load(Ordering::Acquire) + && start.elapsed() < Duration::from_millis(POLL_MS) + { + std::thread::sleep(Duration::from_millis(10)); + } + if finished.load(Ordering::Acquire) { + return Ok::<_, String>(samples); + } + } + }); + let result = action(); + let finished_unix_ms = unix_ms(); + finished.store(true, Ordering::Release); + ( + result, + observer.join().map_err(|_| "GPU observer panicked"), + finished_unix_ms, + ) + }); + let window = Window { + poll_interval_ms: POLL_MS, + started_unix_ms, + finished_unix_ms: finished_unix_ms?, + samples: samples??, + }; + fs::write( + dir.join("gpu-window.json"), + serde_json::to_vec_pretty(&window).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + Ok((result?, window)) +} + +fn unix_ms() -> Result { + Ok(SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_err(|e| e.to_string())? + .as_millis() as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + #[test] + fn intermediate_clock_spike_fails_contract() { + let range: context::ClockRange = + serde_json::from_value(json!({"min":300,"max":2200})).unwrap(); + let sample = |clock, time| json!({"label":"gpu-poll","unix_ms":time,"device_ordinal":0,"uuid":"fixture","name":"fixture","driver":"fixture","sm_clock_mhz":clock,"temperature_c":50,"unavailable_reason":null}); + let window: Window = serde_json::from_value( + json!({"poll_interval_ms":500,"started_unix_ms":1,"finished_unix_ms":1001,"samples":[sample(1000,1),sample(2500,501),sample(1000,1001)]}), + ) + .unwrap(); + assert_eq!(window.failures(Some(&range)).len(), 1); + } + + #[test] + fn gpu_window_needs_coverage() { + for (times, complete) in [ + (vec![1000, 900], false), + (vec![1000], false), + (vec![1000, 5000], false), + (vec![9000], false), + (vec![1000, 2000, 3000, 4000, 5000], true), + ] { + let samples: Vec<_> = times + .iter() + .map(|time| { + json!({ + "label":"gpu-poll","unix_ms":time,"device_ordinal":0,"uuid":"fixture", + "name":"fixture","driver":"fixture","sm_clock_mhz":1000, + "temperature_c":50,"unavailable_reason":null + }) + }) + .collect(); + let window: Window = serde_json::from_value(json!({ + "poll_interval_ms":500,"started_unix_ms":1000,"finished_unix_ms":5000, + "samples":samples + })) + .unwrap(); + assert_eq!( + window.failures(None).is_empty(), + complete, + "GPU samples: {times:?}" + ); + } + } +} diff --git a/crates/ds4-perf/src/serving/tools.rs b/crates/ds4-perf/src/serving/tools.rs new file mode 100644 index 0000000000000000000000000000000000000000..785d2167291df534c2abd9b2f3f94a555b0a2205 --- /dev/null +++ b/crates/ds4-perf/src/serving/tools.rs @@ -0,0 +1,116 @@ +//! Streamed function calls need an argument contract; text equality is insufficient. +use super::*; +use std::collections::BTreeMap; + +const MAX_CALLS: u64 = 32; +const MAX_ARGUMENT_BYTES: usize = 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Call { + name: String, + arguments: Value, +} +impl Call { + pub(super) fn valid(&self) -> bool { + !self.name.is_empty() && self.arguments.is_object() + } + pub(super) fn declared(&self, request: &Value) -> bool { + request["tools"].as_array().is_some_and(|tools| { + tools + .iter() + .any(|tool| tool["type"] == "function" && tool["function"]["name"] == self.name) + }) + } +} + +#[derive(Default)] +struct Partial { + id: Option, + name: String, + arguments: String, +} + +#[derive(Default)] +pub(super) struct Calls(BTreeMap); +impl Calls { + pub(super) fn add(&mut self, delta: &Value) -> Result { + let Some(value) = delta.get("tool_calls") else { + return Ok(false); + }; + let calls = value.as_array().ok_or("tool_calls delta is not an array")?; + let mut generated = false; + for call in calls { + let index = call["index"] + .as_u64() + .filter(|i| *i < MAX_CALLS) + .ok_or("invalid tool call index")?; + if call.get("type").is_some_and(|kind| kind != "function") { + return Err("unsupported tool call type".into()); + } + let partial = self.0.entry(index).or_default(); + if let Some(id) = call.get("id") { + let id = id + .as_str() + .filter(|id| !id.is_empty()) + .ok_or("invalid tool call id")?; + if partial.id.as_ref().is_some_and(|previous| previous != id) { + return Err("tool call id changed during stream".into()); + } + partial.id = Some(id.into()); + } + let function = call + .get("function") + .filter(|v| v.is_object()) + .ok_or("tool call lacks function delta")?; + for (key, output) in [ + ("name", &mut partial.name), + ("arguments", &mut partial.arguments), + ] { + if let Some(value) = function.get(key) { + let text = value + .as_str() + .ok_or("tool name/arguments delta must be text")?; + generated |= !text.is_empty(); + output.push_str(text); + if output.len() > MAX_ARGUMENT_BYTES { + return Err("tool call exceeds argument budget".into()); + } + } + } + } + Ok(generated) + } + + pub(super) fn finish(self) -> Result, String> { + let mut ids = BTreeSet::new(); + self.0 + .into_iter() + .enumerate() + .map(|(position, (index, partial))| { + if index != position as u64 + || !ids.insert(partial.id.ok_or("tool call id missing")?) + { + return Err("tool call indexes or ids are not unique/contiguous".into()); + } + let call = Call { + name: partial.name, + arguments: serde_json::from_str(&partial.arguments) + .map_err(|e| format!("incomplete tool arguments: {e}"))?, + }; + if !call.valid() { + return Err("tool call needs a name and JSON object arguments".into()); + } + Ok(call) + }) + .collect() + } +} + +pub(super) fn valid_finish(content: &str, reason: &str, calls: &[Call]) -> bool { + if calls.is_empty() { + !content.is_empty() && matches!(reason, "stop" | "length") + } else { + reason == "tool_calls" && calls.iter().all(Call::valid) + } +} diff --git a/crates/ds4-perf/src/serving_http.rs b/crates/ds4-perf/src/serving_http.rs new file mode 100644 index 0000000000000000000000000000000000000000..a7c6a3082c695d6021fc3246e68534bee9ed3a1c --- /dev/null +++ b/crates/ds4-perf/src/serving_http.rs @@ -0,0 +1,243 @@ +//! Curl owns HTTP framing. The observer timestamps complete SSE data events; +//! role-only events cannot masquerade as the first generated output. +use crate::{process, runner}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::{ + fs::{self, File}, + io::{BufRead, BufReader}, + path::{Path, PathBuf}, + process::{Command, Stdio}, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, + }, + time::{Duration, Instant}, +}; + +const POLL_MS: u64 = 1; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Event { + pub elapsed_ms: f64, + pub data: String, +} + +pub struct Http { + curl: PathBuf, + origin: String, + root: PathBuf, + limits: process::Limits, +} + +impl Http { + pub fn new(origin: &str, root: &Path, limits: process::Limits) -> Result { + let authority = origin + .strip_prefix("http://") + .ok_or("serving measurements require a local http:// origin")? + .trim_end_matches('/'); + let (host, port) = authority + .rsplit_once(':') + .ok_or("URL requires an explicit port")?; + if !["127.0.0.1", "localhost", "[::1]"].contains(&host) + || port.parse::().ok().filter(|n| *n != 0).is_none() + { + return Err("serving measurements require a loopback host and valid port".into()); + } + let host = if host == "localhost" { + "127.0.0.1" + } else { + host + }; + let curl = runner::resolve(std::ffi::OsStr::new("curl")).ok_or("curl is required")?; + Ok(Self { + curl, + origin: format!("http://{host}:{port}"), + root: root.into(), + limits, + }) + } + + fn command(&self, path: &str, dir: &Path, name: &str) -> Result { + let mut command = Command::new(&self.curl); + // Disable curlrc/proxy behavior so localhost collection is reproducible. + command + .args([ + "-q", + "--silent", + "--show-error", + "--no-buffer", + "--noproxy", + "*", + "--proto", + "=http", + "--fail-with-body", + "--max-time", + ]) + .arg(self.limits.timeout.as_secs_f64().to_string()) + .arg("--dump-header") + .arg(dir.join(format!("{name}.headers"))) + .arg(format!("{}{path}", self.origin)) + .stderr(Stdio::from( + File::create(dir.join(format!("{name}.stderr"))).map_err(|e| e.to_string())?, + )); + Ok(command) + } + + fn execute(&self, command: &mut Command, dir: &Path, name: &str) -> Result<(), String> { + let status = process::run(command, &self.root, self.limits)?; + if !status.success() { + return Err(format!( + "{name}: curl failed: {status}; see {}", + dir.join(format!("{name}.stderr")).display() + )); + } + let headers = + fs::read_to_string(dir.join(format!("{name}.headers"))).map_err(|e| e.to_string())?; + let status = headers.lines().rfind(|line| line.starts_with("HTTP/")); + if status.and_then(|line| line.split_whitespace().nth(1)) != Some("200") { + return Err(format!("{name}: expected HTTP 200")); + } + Ok(()) + } + + pub fn stats(&self, dir: &Path, name: &str) -> Result { + let file = dir.join(format!("{name}.json")); + let mut command = self.command("/v1/stats", dir, name)?; + command.stdout(Stdio::from(File::create(&file).map_err(|e| e.to_string())?)); + self.execute(&mut command, dir, name)?; + serde_json::from_slice(&fs::read(file).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string()) + } + + pub fn stream(&self, dir: &Path) -> Result, String> { + self.stream_events(dir, Instant::now(), None) + } + + pub fn stream_events( + &self, + dir: &Path, + start: Instant, + updates: Option>, + ) -> Result, String> { + let file = dir.join("response.sse"); + let mut command = self.command("/v1/chat/completions", dir, "response")?; + command + .args([ + "--header", + "Content-Type: application/json", + "--data-binary", + ]) + .arg(format!("@{}", dir.join("request.json").display())) + .stdout(Stdio::from(File::create(&file).map_err(|e| e.to_string())?)); + let input = File::open(file).map_err(|e| e.to_string())?; + let finished = AtomicBool::new(false); + let (result, observed) = std::thread::scope(|scope| { + let observer = scope + .spawn(|| observe(input, start, &finished, self.limits.bytes, updates.as_ref())); + let result = self.execute(&mut command, dir, "response"); + finished.store(true, Ordering::Release); + (result, observer.join().map_err(|_| "SSE observer panicked")) + }); + let events = observed??; + fs::write( + dir.join("events.json"), + serde_json::to_vec_pretty(&events).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + result?; + Ok(events) + } +} + +fn observe( + input: File, + start: Instant, + finished: &AtomicBool, + limit: u64, + updates: Option<&mpsc::Sender>, +) -> Result, String> { + let mut reader = BufReader::new(input); + let mut line = Vec::new(); + let mut data = Vec::new(); + let mut events = Vec::new(); + let mut bytes = 0u64; + loop { + let n = reader + .read_until(b'\n', &mut line) + .map_err(|e| e.to_string())?; + bytes += n as u64; + if bytes > limit { + return Err("SSE output budget exceeded".into()); + } + if n == 0 { + if finished.load(Ordering::Acquire) { + if !line.is_empty() || !data.is_empty() { + return Err("truncated SSE event".into()); + } + return Ok(events); + } + std::thread::sleep(Duration::from_millis(POLL_MS)); + continue; + } + if !line.ends_with(b"\n") { + continue; + } + let complete = std::str::from_utf8(&line) + .map_err(|e| e.to_string())? + .trim_end_matches(['\r', '\n']); + if complete.is_empty() && !data.is_empty() { + let event = Event { + elapsed_ms: start.elapsed().as_secs_f64() * 1000.0, + data: data.join("\n"), + }; + if let Some(updates) = updates { + let _ = updates.send(event.clone()); + } + events.push(event); + data.clear(); + } else if let Some(value) = complete.strip_prefix("data:") { + data.push(value.strip_prefix(' ').unwrap_or(value).to_owned()); + } + line.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn fragmented_utf8_event() { + let path = std::env::temp_dir().join(format!("ds4-sse-{}", std::process::id())); + let mut writer = File::create(&path).unwrap(); + let input = File::open(&path).unwrap(); + let finished = AtomicBool::new(false); + let data = "data: 안녕\r\n\r\n".as_bytes(); + writer.write_all(&data[..7]).unwrap(); + let events = std::thread::scope(|scope| { + let observer = scope.spawn(|| observe(input, Instant::now(), &finished, 1024, None)); + std::thread::sleep(Duration::from_millis(10)); + writer.write_all(&data[7..]).unwrap(); + finished.store(true, Ordering::Release); + observer.join().unwrap().unwrap() + }); + fs::remove_file(path).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].data, "안녕"); + } + + #[test] + fn rejects_nonlocal_origin() { + for origin in [ + "https://localhost:8002", + "http://example.com:8002", + "http://127.0.0.1:0", + "http://localhost:8002/path", + ] { + assert!(Http::new(origin, Path::new("."), process::Limits::default()).is_err()); + } + } +} diff --git a/crates/ds4-perf/src/timeline.rs b/crates/ds4-perf/src/timeline.rs new file mode 100644 index 0000000000000000000000000000000000000000..9b3afca6136b4b6df8d93942014504e2a1f04581 --- /dev/null +++ b/crates/ds4-perf/src/timeline.rs @@ -0,0 +1,193 @@ +use crate::{ + csv, + nsys::{phase_name, Phase}, +}; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::BufRead; +use std::path::Path; + +#[derive(Clone, Copy, Debug)] +struct Span { + start: f64, + end: f64, +} + +#[derive(Default)] +struct Window { + ranges: Vec, + busy: Vec, + memory: Vec, +} + +pub fn load(out: &Path, phases: &mut BTreeMap) -> Result<(), String> { + let ranges = csv::open(&out.join("nsys-ranges.csv")).map_err(|e| e.to_string())?; + let gpu = csv::open(&out.join("nsys-gpu-trace.csv")).map_err(|e| e.to_string())?; + analyze(ranges, gpu, phases) +} + +fn analyze( + ranges: impl BufRead, + gpu: impl BufRead, + phases: &mut BTreeMap, +) -> Result<(), String> { + let mut windows: BTreeMap = BTreeMap::new(); + let mut threads = BTreeSet::new(); + csv::table(ranges, "name", |header, row| { + let Some(name) = csv::value(row, header, &["name"]).and_then(phase_name) else { + return Ok(()); + }; + let start = csv::time(row, header, "Start").ok_or("range Start missing")?; + let duration = csv::time(row, header, "Duration").ok_or("range Duration missing")?; + let pid = csv::value(row, header, &["pid"]).ok_or("range PID missing")?; + let tid = csv::value(row, header, &["tid"]).ok_or("range TID missing")?; + threads.insert((pid.to_string(), tid.to_string())); + windows.entry(name.into()).or_default().ranges.push(Span { + start, + end: start + duration, + }); + Ok(()) + })?; + if windows.is_empty() || threads.len() != 1 { + return Err("need complete phase ranges from one benchmark thread".into()); + } + let mut all_ranges: Vec<_> = windows + .values() + .flat_map(|w| w.ranges.iter()) + .copied() + .collect(); + all_ranges.sort_by(|a, b| a.start.total_cmp(&b.start)); + if all_ranges.windows(2).any(|p| p[0].end > p[1].start) { + return Err("overlapping benchmark phases".into()); + } + let mut devices = BTreeSet::new(); + let mut events = 0; + csv::table(gpu, "name", |header, row| { + let start = csv::time(row, header, "Start").ok_or("GPU Start missing")?; + let duration = csv::time(row, header, "Duration").ok_or("GPU Duration missing")?; + let device = csv::value(row, header, &["device"]).ok_or("GPU Device missing")?; + let ctx = csv::value(row, header, &["ctx"]).ok_or("GPU Ctx missing")?; + let name = csv::value(row, header, &["name"]).ok_or("GPU Name missing")?; + // Nsight's formatter chooses B/KB/MB dynamically. Only presence, + // not byte magnitude, identifies memory operations in this timeline. + let bytes = header + .iter() + .position(|s| csv::key(s).starts_with("bytes (")) + .ok_or("GPU memory-operation column missing")?; + devices.insert((device.to_string(), ctx.to_string())); + events += 1; + let memory = row.get(bytes).is_some_and(|s| !s.trim().is_empty()) + || name.starts_with("[CUDA memcpy") + || name.starts_with("[CUDA memset"); + for window in windows.values_mut() { + for range in &window.ranges { + let span = Span { + start: start.max(range.start), + end: (start + duration).min(range.end), + }; + if span.end <= span.start { + continue; + } + window.busy.push(span); + if memory { + window.memory.push(span); + } + } + } + Ok(()) + })?; + // Missing traces must never become "0% GPU coverage". Multiple devices + // or contexts also lack a defensible single-GPU idle denominator. + if events == 0 || devices.len() != 1 { + return Err("empty or multiple-device/context GPU trace".into()); + } + for (name, window) in windows { + let ranges = union(window.ranges); + let busy = union(window.busy); + let memory = union(window.memory); + let mut largest: f64 = 0.0; + for range in &ranges { + let mut cursor = range.start; + for span in &busy { + if span.end <= range.start || span.start >= range.end { + continue; + } + largest = largest.max(span.start - cursor); + cursor = cursor.max(span.end); + } + largest = largest.max(range.end - cursor); + } + let phase = phases.entry(name.clone()).or_insert_with(|| Phase { + name, + ..Default::default() + }); + phase.wall_ns = Some(duration(&ranges)); + phase.busy_ns = Some(duration(&busy)); + phase.mem_ns = Some(duration(&memory)); + phase.gap_ns = Some(largest); + } + Ok(()) +} + +fn duration(spans: &[Span]) -> f64 { + spans.iter().map(|s| s.end - s.start).sum() +} + +fn union(mut spans: Vec) -> Vec { + spans.sort_by(|a, b| a.start.total_cmp(&b.start)); + let mut merged: Vec = Vec::new(); + for span in spans { + if let Some(last) = merged.last_mut() { + if span.start <= last.end { + last.end = last.end.max(span.end); + continue; + } + } + merged.push(span); + } + merged +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn nsight_scaled_memory_units() { + for unit in ["MB", "KB", "GiB"] { + let gpu = include_str!("../tests/fixtures/gpu-trace.csv") + .replace("Bytes (B)", &format!("Bytes ({unit})")); + let mut phases = BTreeMap::new(); + analyze( + include_bytes!("../tests/fixtures/ranges.csv").as_slice(), + gpu.as_bytes(), + &mut phases, + ) + .unwrap(); + assert_eq!(phases["ds4.prefill"].busy_ns, Some(600.0)); + assert_eq!(phases["ds4.prefill"].mem_ns, Some(100.0)); + } + } + + #[test] + fn clips_unions_and_keeps_gaps() { + let mut phases = BTreeMap::new(); + analyze( + include_bytes!("../tests/fixtures/ranges.csv").as_slice(), + include_bytes!("../tests/fixtures/gpu-trace.csv").as_slice(), + &mut phases, + ) + .unwrap(); + let p = &phases["ds4.prefill"]; + assert_eq!(p.wall_ns, Some(1000.0)); + assert_eq!(p.busy_ns, Some(600.0)); + assert_eq!(p.mem_ns, Some(100.0)); + assert_eq!(p.gap_ns, Some(300.0)); + assert_eq!(phases["ds4.decode"].busy_ns, Some(800.0)); + assert!(analyze( + include_bytes!("../tests/fixtures/ranges.csv").as_slice(), + b"".as_slice(), + &mut BTreeMap::new() + ) + .is_err()); + } +} diff --git a/crates/ds4-perf/src/trace.rs b/crates/ds4-perf/src/trace.rs new file mode 100644 index 0000000000000000000000000000000000000000..989f6b791d4063f6060ae61021e796135ebd6764 --- /dev/null +++ b/crates/ds4-perf/src/trace.rs @@ -0,0 +1,51 @@ +use serde::{Deserialize, Serialize}; + +/// Owned activity records: no CUPTI pointers survive a completed buffer callback. +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Event { + Start { + schema_version: u32, + api_version: u32, + pid: u32, + }, + End { + dropped: u64, + errors: u64, + }, + Error { + message: String, + }, + Kernel { + name: String, + start: u64, + end: u64, + device: u32, + context: u32, + stream: u32, + correlation: u32, + grid: [u32; 3], + block: [u32; 3], + registers: u32, + shared_bytes: u64, + graph_id: u32, + }, + Marker { + name: String, + domain: String, + timestamp: u64, + id: u32, + flags: u32, + pid: u32, + tid: u32, + }, + Memop { + operation: String, + start: u64, + end: u64, + bytes: u64, + device: u32, + context: u32, + stream: u32, + }, +} diff --git a/crates/ds4-perf/src/workload.rs b/crates/ds4-perf/src/workload.rs new file mode 100644 index 0000000000000000000000000000000000000000..41133396cfd0f3cb4a98c616ef9e6f15fae0898c --- /dev/null +++ b/crates/ds4-perf/src/workload.rs @@ -0,0 +1,421 @@ +use crate::experiment::Workload; +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsString, + path::{Path, PathBuf}, +}; + +fn shards(path: &Path) -> Result, String> { + let Some(name) = path + .file_name() + .and_then(|n| n.to_str()) + .and_then(|n| n.strip_suffix(".gguf")) + else { + return Ok(vec![path.into()]); + }; + let Some((prefix, count)) = name.rsplit_once("-of-") else { + return Ok(vec![path.into()]); + }; + let Some((base, index)) = prefix.rsplit_once('-') else { + return Err("invalid GGUF shard name".into()); + }; + let count = count + .parse::() + .map_err(|_| "invalid GGUF shard count")?; + let index = index + .parse::() + .map_err(|_| "invalid GGUF shard index")?; + if count == 0 || count > 10000 || index == 0 || index > count { + return Err("invalid GGUF shard set".into()); + } + Ok((1..=count) + .map(|i| path.with_file_name(format!("{base}-{i:05}-of-{count:05}.gguf"))) + .collect()) +} + +fn arguments(command: &[OsString]) -> Result, String> { + let mut paths = BTreeMap::new(); + let mut args = command.iter().skip(1); + while let Some(arg) = args.next() { + let arg = arg + .to_str() + .ok_or("workload protocol requires UTF-8 option names")?; + match arg { + "-m" | "--model" | "--prompt-file" | "--chat-prompt-file" | "--mtp" => { + let path = args.next().ok_or("benchmark path argument missing")?; + let key = match arg { + "-m" | "--model" => "model", + "--mtp" => "mtp", + _ => "prompt", + }; + paths.insert(key.into(), PathBuf::from(path)); + } + "--cuda" + | "--quality" + | "--warm-weights" + | "--ssd-streaming" + | "--ssd-streaming-cold" => {} + "-sys" + | "--system" + | "--backend" + | "-t" + | "--threads" + | "--ctx-start" + | "--ctx-max" + | "--ctx-alloc" + | "--step-incr" + | "--step-mul" + | "--gen-tokens" + | "--tokens" + | "-n" + | "--mtp-draft" + | "--mtp-margin" + | "--ssd-streaming-cache-experts" => { + args.next().ok_or("benchmark option value missing")?; + } + _ => { + return Err(format!( + "workload protocol ds4-bench-v1 does not cover option {arg}" + )) + } + } + } + if !paths.contains_key("model") || !paths.contains_key("prompt") { + return Err( + "workload protocol needs explicit -m and --prompt-file/--chat-prompt-file".into(), + ); + } + Ok(paths) +} + +pub fn model_argument(command: &[OsString]) -> Result { + arguments(command)? + .remove("model") + .ok_or("benchmark model argument missing".into()) +} + +impl Workload { + pub fn verify_scope( + &self, + command: &[OsString], + env: &BTreeMap, + ) -> Result<(), String> { + self.check_scope(command, env, None) + } + + pub fn verify_scope_checked( + &self, + command: &[OsString], + env: &BTreeMap, + checked: &mut ds4_perf::artifact::Verification, + ) -> Result<(), String> { + self.check_scope(command, env, Some(checked)) + } + + fn check_scope( + &self, + command: &[OsString], + env: &BTreeMap, + mut checked: Option<&mut ds4_perf::artifact::Verification>, + ) -> Result<(), String> { + if self.protocol != "ds4-bench-v1" { + return Err("unsupported workload protocol; use ds4-bench-v1".into()); + } + let declared: BTreeSet<_> = self.files.values().map(|v| v.path.clone()).collect(); + let mut require = |path: &Path| -> Result<(), String> { + let canonical = path + .canonicalize() + .map_err(|e| format!("{}: {e}", path.display()))?; + if !declared.contains(&canonical) { + return Err(format!("workload lacks consumed input: {}", path.display())); + } + if let Some(checked) = checked.as_deref_mut() { + let input = self + .files + .values() + .find(|input| input.path == canonical) + .ok_or("consumed input missing")?; + checked.verify(path, &input.sha256)?; + } + Ok(()) + }; + let arguments = arguments(command)?; + for (key, path) in &arguments { + require(path)?; + let canonical = path.canonicalize().map_err(|e| e.to_string())?; + if let Some(file) = self.files.get(key) { + if file.path != canonical { + return Err(format!("workload {key} differs from benchmark argument")); + } + } else { + return Err(format!("workload lacks {key} input")); + } + // Native split and sidecar lookup starts beside the consumed argv + // path. Canonicalizing the first shard must not change that root. + for shard in shards(path)? { + require(&shard)?; + } + if key != "model" { + continue; + } + // PLE is a sidecar read by the Qwen host even when weights are IPC. + let parent = path.parent().ok_or("model has no parent directory")?; + let selected = env + .get(std::ffi::OsStr::new("DS4_QWEN_PLE_DIR")) + .filter(|v| !v.is_empty()) + .map(PathBuf::from); + let ple = selected + .as_ref() + .map(|p| p.join("ple-manifest.json")) + .unwrap_or_else(|| parent.join("ple/ple-manifest.json")); + if selected.is_some() || ple.exists() { + require(&ple)?; + let manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(&ple).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + let fp8 = manifest.get("format_version").and_then(|v| v.as_u64()) == Some(2); + let root = if fp8 { + ple.parent().ok_or("PLE manifest has no parent")? + } else { + selected.as_deref().unwrap_or(parent) + }; + let parts = manifest + .get("logical_parts") + .and_then(|v| v.as_array()) + .ok_or("unrecognized PLE manifest")?; + for part in parts { + let file = part + .get("physical_file") + .and_then(|v| v.as_str()) + .ok_or("PLE physical file missing")?; + require(&root.join(file))?; + } + if fp8 { + let scale = manifest + .pointer("/quantization/scale/path") + .and_then(|v| v.as_str()) + .ok_or("PLE FP8 scale missing")?; + require(&root.join(scale))?; + } + } + } + for key in ["DS4_CUDA_WEIGHT_IPC_MANIFEST", "DS4_WEIGHT_SERVER"] { + if let Some(path) = env.get(std::ffi::OsStr::new(key)).filter(|v| !v.is_empty()) { + require(Path::new(path))?; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn glm_ssd_arguments() { + for cache in ["1024", "24GB", "0.5gb"] { + let args = [ + "ds4-bench-perf", + "--cuda", + "--ssd-streaming", + "--ssd-streaming-cold", + "--ssd-streaming-cache-experts", + cache, + "-m", + "model.gguf", + "--prompt-file", + "prompt.txt", + "--ctx-start", + "2048", + "--ctx-max", + "2048", + ] + .map(OsString::from); + let paths = arguments(&args).unwrap(); + assert_eq!(paths.len(), 2); + assert_eq!(paths["model"], PathBuf::from("model.gguf")); + assert_eq!(paths["prompt"], PathBuf::from("prompt.txt")); + } + } + + #[test] + fn glm_ssd_cache_missing() { + let args = [ + "ds4-bench-perf", + "-m", + "model.gguf", + "--prompt-file", + "prompt.txt", + "--ssd-streaming-cache-experts", + ] + .map(OsString::from); + assert_eq!( + arguments(&args).unwrap_err(), + "benchmark option value missing" + ); + } + + #[test] + fn glm_ssd_unknown_flag() { + let args = [ + "ds4-bench-perf", + "-m", + "model.gguf", + "--prompt-file", + "prompt.txt", + "--ssd-streaming-unknown", + ] + .map(OsString::from); + assert!(arguments(&args).is_err()); + } + + #[cfg(unix)] + #[test] + fn split_model_uses_consumed_alias_directory() { + use crate::experiment::InputFile; + let root = std::env::temp_dir().join(format!("ds4-split-alias-{}", std::process::id())); + let real = root.join("real"); + let alias = root.join("alias"); + std::fs::create_dir_all(&real).unwrap(); + std::fs::create_dir_all(&alias).unwrap(); + let first = "model-00001-of-00002.gguf"; + let second = "model-00002-of-00002.gguf"; + for path in [ + real.join(first), + real.join(second), + alias.join(second), + root.join("prompt"), + ] { + std::fs::write(path, b"fixture").unwrap(); + } + std::os::unix::fs::symlink(real.join(first), alias.join(first)).unwrap(); + let workload = Workload { + protocol: "ds4-bench-v1".into(), + name: "alias".into(), + family: "fixture".into(), + files: [ + ("model", real.join(first)), + ("shard", real.join(second)), + ("prompt", root.join("prompt")), + ] + .into_iter() + .map(|(key, path)| { + ( + key.into(), + InputFile { + path, + sha256: ds4_perf::artifact::hash_bytes(b"fixture"), + }, + ) + }) + .collect(), + shape: BTreeMap::new(), + cache_state: "cold".into(), + }; + let command = vec![ + "ds4-bench".into(), + "-m".into(), + alias.join(first).into(), + "--prompt-file".into(), + root.join("prompt").into(), + ]; + let result = workload.verify_scope(&command, &BTreeMap::new()); + std::fs::remove_dir_all(root).unwrap(); + assert!(result.is_err_and(|error| error.contains("lacks consumed input"))); + } + #[test] + fn sidecar_override_is_pinned() { + use crate::experiment::InputFile; + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::env::temp_dir().join(format!("ds4-fp8-{}-{nonce}", std::process::id())); + let sidecar = root.join("fp8"); + std::fs::create_dir_all(&sidecar).unwrap(); + for name in ["model", "prompt", "fp8/part.bin", "fp8/scale.bin"] { + std::fs::write(root.join(name), b"fixture").unwrap(); + } + std::fs::write( + sidecar.join("ple-manifest.json"), + br#"{ + "format_version":2,"logical_parts":[{"physical_file":"part.bin"}], + "quantization":{"scale":{"path":"scale.bin"}} + }"#, + ) + .unwrap(); + let mut workload = Workload { + protocol: "ds4-bench-v1".into(), + name: "fixture".into(), + family: "qwen4exp".into(), + files: BTreeMap::new(), + shape: BTreeMap::new(), + cache_state: "fresh".into(), + }; + for name in ["model", "prompt"] { + workload.files.insert( + name.into(), + InputFile { + path: root.join(name).canonicalize().unwrap(), + sha256: String::new(), + }, + ); + } + let command = vec![ + "bench".into(), + "-m".into(), + root.join("model").into(), + "--prompt-file".into(), + root.join("prompt").into(), + ]; + let env = BTreeMap::from([("DS4_QWEN_PLE_DIR".into(), sidecar.clone().into())]); + let missing_manifest = workload.verify_scope(&command, &env).is_err(); + for name in ["ple-manifest.json", "part.bin"] { + workload.files.insert( + name.into(), + InputFile { + path: sidecar.join(name).canonicalize().unwrap(), + sha256: String::new(), + }, + ); + } + let missing_scale = workload.verify_scope(&command, &env).is_err(); + workload.files.insert( + "scale".into(), + InputFile { + path: sidecar.join("scale.bin").canonicalize().unwrap(), + sha256: String::new(), + }, + ); + let complete = workload.verify_scope(&command, &env); + std::fs::remove_dir_all(root).unwrap(); + assert!(missing_manifest, "an override must pin its manifest"); + assert!(missing_scale, "FP8 scale bytes are a consumed input"); + complete.unwrap(); + } + + #[test] + fn expands_split_models() { + let result = shards(Path::new("/models/a-00001-of-00003.gguf")).unwrap(); + assert_eq!(result.len(), 3); + assert_eq!(result[2], Path::new("/models/a-00003-of-00003.gguf")); + assert!(shards(Path::new("/models/a-00004-of-00003.gguf")).is_err()); + } + #[test] + fn parses_values_as_values() { + let args = [ + "bench", + "--system", + "--mtp", + "-m", + "main.gguf", + "--prompt-file", + "prompt.txt", + ] + .map(Into::into); + let paths = arguments(&args).unwrap(); + assert!(!paths.contains_key("mtp")); + assert_eq!(paths["model"], Path::new("main.gguf")); + } +} diff --git a/crates/ds4-perf/tests/fixtures/bench.csv b/crates/ds4-perf/tests/fixtures/bench.csv new file mode 100644 index 0000000000000000000000000000000000000000..55c6e85c432aa1e428b858dca9452d969f1ec1e0 --- /dev/null +++ b/crates/ds4-perf/tests/fixtures/bench.csv @@ -0,0 +1,2 @@ +gen_tps,ctx_tokens,kvcache_bytes,prefill_tokens,gen_tokens,prefill_tps,first_token_sec,gen_tps_ss +24.3,8192,4294967296,8192,32,1502.8,0.0512,24.6 diff --git a/crates/ds4-perf/tests/fixtures/compare-live.json b/crates/ds4-perf/tests/fixtures/compare-live.json new file mode 100644 index 0000000000000000000000000000000000000000..c7d8acb2160157948a43778b3ff26e3c38222428 --- /dev/null +++ b/crates/ds4-perf/tests/fixtures/compare-live.json @@ -0,0 +1,80 @@ +{ + "verdict": "inconclusive", + "metrics": [ + { + "metric": "decode", + "ctx": 2048, + "unit": "seconds/token", + "baseline": [ + 0.040096230954290296, + 0.040048057669203045, + 0.040016006402561026 + ], + "candidate": [ + 0.04132231404958678, + 0.04101722723543889, + 0.040584415584415584 + ], + "baseline_median": 0.040048057669203045, + "candidate_median": 0.04101722723543889, + "median_percent": 2.4200164068908903, + "lower_percent": 1.2175324675324672, + "upper_percent": 3.2644628099173456 + }, + { + "metric": "first_token", + "ctx": 2048, + "unit": "seconds", + "baseline": [ + 0.0436, + 0.044, + 0.0442 + ], + "candidate": [ + 0.0455, + 0.0452, + 0.0445 + ], + "baseline_median": 0.044, + "candidate_median": 0.0452, + "median_percent": 2.7272727272727337, + "lower_percent": 0.6787330316742057, + "upper_percent": 4.357798165137616 + }, + { + "metric": "prefill", + "ctx": 2048, + "unit": "seconds/token", + "baseline": [ + 0.0008005507789359079, + 0.0007995906096078806, + 0.0008039360710036337 + ], + "candidate": [ + 0.0007272092617371576, + 0.0007278497135911377, + 0.0007299323352725202 + ], + "baseline_median": 0.0008005507789359079, + "candidate_median": 0.0007278497135911377, + "median_percent": -9.081380876476619, + "lower_percent": -9.543894351038439, + "upper_percent": -8.71174242147752 + } + ], + "correctness": { + "max_abs": 0.0, + "max_scaled": 0.0, + "bad": 0, + "checked": 1489920 + }, + "token_mismatches": 0, + "max_slowdown_percent": 3.0, + "logit_atol": 0.0001, + "logit_rtol": 0.0001, + "changes": { + "DS4_QWEN_PLE_WORKERS": "Some(\"16\") -> Some(\"32\")" + }, + "reasons": [], + "method": "sample-extrema-envelope-v1" +} diff --git a/crates/ds4-perf/tests/fixtures/gpu-trace.csv b/crates/ds4-perf/tests/fixtures/gpu-trace.csv new file mode 100644 index 0000000000000000000000000000000000000000..7158d607e7768008b1c6a767597f3335b8b7bc6b --- /dev/null +++ b/crates/ds4-perf/tests/fixtures/gpu-trace.csv @@ -0,0 +1,7 @@ +Start (ns),Duration (ns),Bytes (B),Device,Ctx,Name +0,900,100000,GB10 (0),1,[CUDA memcpy HtoD] +1100,300,,GB10 (0),1,ordinary_kernel +1300,300,,GB10 (0),1,"foo>" +1600,100,1024,GB10 (0),1,[CUDA memcpy DtoH] +3100,500,,GB10 (0),1,decode +3400,500,,GB10 (0),1,decode diff --git a/crates/ds4-perf/tests/fixtures/kernels.csv b/crates/ds4-perf/tests/fixtures/kernels.csv new file mode 100644 index 0000000000000000000000000000000000000000..09af335f33126e9788cd98128e79d64dd04790eb --- /dev/null +++ b/crates/ds4-perf/tests/fixtures/kernels.csv @@ -0,0 +1,5 @@ +NVTX Range,Style,PID,TID,NVTX Inst,Kern Inst,Total Time (ns),Avg (ns),Kernel Name +:ds4.prefill,PushPop,42,42,1,1,1000,1000,"foo>" +:ds4.prefill,PushPop,42,42,1,2,2000,1000,"foo>" +:ds4.prefill,PushPop,42,42,1,2,500,250,ordinary_kernel +:ds4.decode,PushPop,42,42,1,5,800,160,ordinary_kernel diff --git a/crates/ds4-perf/tests/fixtures/ncu-details.csv b/crates/ds4-perf/tests/fixtures/ncu-details.csv new file mode 100644 index 0000000000000000000000000000000000000000..c931e0cd3e7e39f9b1dc4620e3f848d0934e8aed --- /dev/null +++ b/crates/ds4-perf/tests/fixtures/ncu-details.csv @@ -0,0 +1,4 @@ +"ID","Process ID","Process Name","Host Name","thread Domain:Push/Pop_Range:PL_Type:PL_Value:CLR_Type:Color:Msg_Type:Msg","Id:Domain:Start/Stop_Range:PL_Type:PL_Value:CLR_Type:Color:Msg_Type:Msg","Kernel Name","Context","Stream","Block Size","Grid Size","Device","CC","Section Name","Metric Name","Metric Unit","Metric Value" +"0","232661","ds4-bench-perf","127.0.0.1","232661 "":ds4.prefill:none:none:none:none:none:none"" ","","ds4_moe_worklist_mmq_kernel","1","7","(32, 8, 1)","(48, 1, 1)","0","12.1","Command line profiler metrics","gpu__time_duration.sum","ns","3101888" +"0","232661","ds4-bench-perf","127.0.0.1","232661 "":ds4.prefill:none:none:none:none:none:none"" ","","ds4_moe_worklist_mmq_kernel","1","7","(32, 8, 1)","(48, 1, 1)","0","12.1","Command line profiler metrics","sm__throughput.avg.pct_of_peak_sustained_elapsed","%","23.55" +"0","232661","ds4-bench-perf","127.0.0.1","232661 "":ds4.prefill:none:none:none:none:none:none"" ","","ds4_moe_worklist_mmq_kernel","1","7","(32, 8, 1)","(48, 1, 1)","0","12.1","Command line profiler metrics","sm__warps_active.avg.pct_of_peak_sustained_active","%","16.75" diff --git a/crates/ds4-perf/tests/fixtures/ranges.csv b/crates/ds4-perf/tests/fixtures/ranges.csv new file mode 100644 index 0000000000000000000000000000000000000000..098c1280b18d3dfd662a97eed37792cc6182a512 --- /dev/null +++ b/crates/ds4-perf/tests/fixtures/ranges.csv @@ -0,0 +1,3 @@ +Start (ns),Duration (ns),Name,PID,TID +1000,1000,:ds4.prefill,42,42 +3000,1000,:ds4.decode,42,42 diff --git a/crates/ds4-perf/tests/inspect.rs b/crates/ds4-perf/tests/inspect.rs new file mode 100644 index 0000000000000000000000000000000000000000..1d1649ef706dd749dd56f0d308eb69724f297b8c --- /dev/null +++ b/crates/ds4-perf/tests/inspect.rs @@ -0,0 +1,56 @@ +use std::{fs, process::Command}; + +#[test] +fn inspect_without_cuda() { + let root = std::env::temp_dir().join(format!("ds4-inspect-{}", std::process::id())); + let out = root.join("machine"); + fs::create_dir(&root).unwrap(); + let result = Command::new(env!("CARGO_BIN_EXE_ds4-perf")) + .args(["inspect", "--out"]) + .arg(&out) + .arg("--gpu-helper") + .arg(root.join("not-installed")) + .env("PATH", &root) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let machine: serde_json::Value = + serde_json::from_slice(&fs::read(out.join("machine.json")).unwrap()).unwrap(); + assert_eq!(machine["kind"], "machine"); + assert_eq!(machine["schema_version"], 1); + assert_eq!(machine["data"]["gpu"], serde_json::Value::Null); + assert!(!machine["warnings"].as_array().unwrap().is_empty()); + // Evidence directories are immutable, including an incomplete inspection. + let again = Command::new(env!("CARGO_BIN_EXE_ds4-perf")) + .args(["inspect", "--out"]) + .arg(&out) + .output() + .unwrap(); + assert!(!again.status.success()); + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn four_command_help() { + for (command, flags) in [ + ("inspect", vec!["--calibrate"]), + ("scout", vec!["--fit", "--ncu", "--collector"]), + ("compare", vec!["--baseline", "--regression"]), + ("optimize", vec!["--auto"]), + ("serving", vec!["--url", "--workload", "--timeout-seconds"]), + ] { + let result = Command::new(env!("CARGO_BIN_EXE_ds4-perf")) + .args([command, "--help"]) + .output() + .unwrap(); + assert!(result.status.success(), "missing {command} help"); + let text = String::from_utf8_lossy(&result.stdout); + for flag in flags { + assert!(text.contains(flag), "missing {command} {flag}"); + } + } +} diff --git a/crates/ds4-perf/tests/profiles.rs b/crates/ds4-perf/tests/profiles.rs new file mode 100644 index 0000000000000000000000000000000000000000..e1fe8f06561f5b062074df1d5f9db7a07579511f --- /dev/null +++ b/crates/ds4-perf/tests/profiles.rs @@ -0,0 +1,90 @@ +use serde_json::{json, Value}; +use std::{fs, path::PathBuf, process::Command}; + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!("ds4-profile-{}-{nonce}", std::process::id())); + fs::create_dir(&path).unwrap(); + Self(path) + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +#[test] +fn controls_follow_runtime_capabilities() { + let f = Fixture::new(); + let plan = json!({"family":"family-without-perf-name-table", "requested":{}, "effective":{"native_chunk":1024,"sched_chunk":256,"sched_chunk_live":512}, "qualified":{}, "issues":[], + "controls":{"native_prefill_env":"DS4_EXAONE_PREFILL_CHUNK", "scheduler_chunks":[256,512,1024],"prefix_reuse":"partial","banks":"persistent","mtp":"none","disk":"qualified"}}); + fs::write(f.0.join("plan.json"), serde_json::to_vec(&plan).unwrap()).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_ds4-perf")) + .args(["serving-controls", "--plan"]) + .arg(f.0.join("plan.json")) + .arg("--out") + .arg(f.0.join("controls")) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let artifact: Value = + serde_json::from_slice(&fs::read(f.0.join("controls/controls.json")).unwrap()).unwrap(); + let candidates = artifact["data"]["candidates"].as_array().unwrap(); + assert_eq!(candidates.len(), 4); + for candidate in candidates { + assert_eq!(candidate["qualified"], false); + let options = candidate["arguments"].as_object().unwrap(); + assert_eq!(options.len(), 1); + let (key, value) = options.iter().next().unwrap(); + assert!(["--prefill-chunk", "--prefill-chunk-live"].contains(&key.as_str())); + assert!([256, 512, 1024].contains(&value.as_u64().unwrap())); + } +} + +#[test] +fn profile_refuses_unmeasured_and_override_arguments() { + let f = Fixture::new(); + fs::write( + f.0.join("selection.json"), + br#"{"workload":"missing.json","candidates":[{"name":"unmeasured","runs":[]}]}"#, + ) + .unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_ds4-perf")) + .args(["serving-profile", "--plan"]) + .arg(f.0.join("selection.json")) + .arg("--out") + .arg(f.0.join("selected")) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr).contains("at least one run"), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let output = Command::new(env!("CARGO_BIN_EXE_ds4-perf")) + .args([ + "apply-profile", + "--profile", + "missing.json", + "--out", + "missing", + "--", + "--max-seqs", + "64", + ]) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("unexpected argument")); +} diff --git a/crates/ds4-perf/tests/scout.rs b/crates/ds4-perf/tests/scout.rs new file mode 100644 index 0000000000000000000000000000000000000000..7f2efa349b68d2cce7bfef696155ef09496ce460 --- /dev/null +++ b/crates/ds4-perf/tests/scout.rs @@ -0,0 +1,788 @@ +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +struct Fixture(PathBuf); +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn script(dir: &Path, name: &str, body: &str) { + let path = dir.join(name); + fs::write(&path, format!("#!/bin/sh\nset -eu\n{body}\n")).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); +} + +impl Fixture { + fn new() -> Self { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = + std::env::temp_dir().join(format!("ds4-perf-test-{}-{nonce}", std::process::id())); + fs::create_dir(&dir).unwrap(); + script( + &dir, + "ds4-bench", + r#" +if [ "${1:-}" = --help ]; then printf 'NVTX: available\n'; exit 0; fi +if [ "${BENCH_NVTX:-available}" != unknown ]; then printf 'ds4-bench NVTX: %s\n' "${BENCH_NVTX:-available}" >&2; fi +printf '%s %s\n' "$$" "$DS4_QWEN_PLE_CACHE_MB" >> "$BENCH_CALLS" +printf '%s\000' "$@" >> "$BENCH_ARGS" +if [ "${BENCH_FAIL:-0}" = 1 ]; then echo 'intentional benchmark failure' >&2; exit 7; fi +cat "$FIXTURES/bench.csv" +"#, + ); + script( + &dir, + "nsys", + r#" +case "$1 ${2:-}" in + 'profile --help') echo "--trace cuda nvtx --cuda-graph-trace 'node' --sample --cpuctxsw"; exit 0;; + 'stats --help-reports') + printf 'The following built-in reports are available:\n' + printf 'nvtx_gpu_proj_sum -- projection\nnvtx_kern_sum[:base|:mangled] -- kernel\ncuda_gpu_kern_sum[:nvtx-name][:base|:mangled] -- kernel\ncuda_gpu_mem_time_sum -- mem\nnvtx_pushpop_trace -- ranges\ncuda_gpu_trace[:base|:mangled] -- trace\n' + exit 1;; + 'analyze --help-rules') echo 'The following built-in reports are available:'; echo 'cuda_api_sync[:options] -- sync'; exit 1;; +esac +case "$1" in + --version) echo 'fake nsys'; exit 0;; + profile) + shift + while [ "$#" -gt 0 ]; do + case "$1" in + --output) report=$2; shift 2;; + --*) shift;; + *) break;; + esac + done + "$@" + : > "$report.nsys-rep";; + stats|analyze) + if [ "${REPORT_FAIL:-0}" = 1 ]; then echo 'unsupported report fixture' >&2; exit 9; fi + shift + while [ "$#" -gt 0 ]; do + case "$1" in + --report|--rule) report=$2; shift 2;; + *) shift;; + esac + done + case "$report" in + cuda_gpu_kern_sum*) printf 'Name,Total Time (ns),Instances\nglobal_kernel,3500,4\n';; + esac + if [ "${BENCH_NVTX:-available}" = unknown ]; then exit 0; fi + case "$report" in + nvtx_kern_sum*) cat "$FIXTURES/kernels.csv";; + nvtx_pushpop_trace) cat "$FIXTURES/ranges.csv";; + cuda_gpu_trace*) cat "$FIXTURES/gpu-trace.csv";; + nvtx_gpu_proj_sum) printf 'Range,Total Proj Time (ns),Total Range Time (ns)\n:ds4.prefill,900,1000\n:ds4.decode,800,1000\n';; + *) echo 'SKIPPED: no data';; + esac;; +esac +"#, + ); + script( + &dir, + "nvidia-smi", + r#" +case "${1:-}" in + --query-compute-apps*) exit 0;; + --query-gpu*) echo 'Fake GPU, 12.1, fake-driver';; + *) echo 'CUDA Version: fake';; +esac +"#, + ); + script(&dir, "ncu", "exit 127"); + script(&dir, "nvcc", "echo 'fake CUDA'"); + Self(dir) + } + + fn command(&self, out: &str) -> Command { + self.options(out, &[]) + } + + fn options(&self, out: &str, options: &[&str]) -> Command { + let mut c = Command::new(env!("CARGO_BIN_EXE_ds4-perf")); + c.current_dir(&self.0) + .env("PATH", format!("{}:/usr/bin:/bin", self.0.display())) + .env( + "FIXTURES", + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"), + ) + .env("BENCH_CALLS", self.0.join("calls")) + .env("BENCH_ARGS", self.0.join("args")) + .env("DS4_QWEN_PLE_CACHE_MB", "1731") + .env("DS4_PRIVATE_SECRET", "must-not-be-recorded") + .args([ + "scout", + "--gpu-helper", + "/missing-test-helper", + "--out", + out, + ]) + .args(options) + .args(["--", "./ds4-bench", "a b", "'$(touch BAD)`id`", "--"]); + c + } + fn run(&self, out: &str) -> Output { + self.command(out).output().unwrap() + } +} + +#[test] +fn fresh_processes_and_artifacts() { + let f = Fixture::new(); + let result = f.run("run"); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let calls = fs::read_to_string(f.0.join("calls")).unwrap(); + let calls: Vec<_> = calls + .lines() + .map(|l| l.split_whitespace().collect::>()) + .collect(); + assert_eq!(calls.len(), 2); + assert_ne!(calls[0][0], calls[1][0]); + assert_eq!(calls[0][1], "1731"); + assert_eq!(calls[1][1], "1731"); + let expected = b"a b\0'$(touch BAD)`id`\0--\0"; + assert_eq!(fs::read(f.0.join("args")).unwrap(), expected.repeat(2)); + assert!(!f.0.join("BAD").exists()); + for file in [ + "manifest.txt", + "command.txt", + "command.argv", + "env.txt", + "bench.stdout", + "bench.stderr", + "baseline.csv", + "trace.nsys-rep", + "nsys-kernels.csv", + "nsys-nvtx.csv", + "nsys-sync.csv", + "report.txt", + "normalized-phases.csv", + "normalized-kernels.csv", + ] { + assert!(f.0.join("run").join(file).exists(), "missing {file}"); + } + let env = fs::read_to_string(f.0.join("run/env.txt")).unwrap(); + assert!(env.contains("1731")); + assert!(!env.contains("must-not-be-recorded")); + let report = fs::read_to_string(f.0.join("run/report.txt")).unwrap(); + assert!(report.contains("1502.80 tok/s")); + assert!(report.contains("GPU coverage 60.0 %")); + assert!(report.contains("foo>")); + assert!(report.contains("ncu unavailable")); + assert!(!f.run("run").status.success()); + assert_eq!( + fs::read_to_string(f.0.join("calls")) + .unwrap() + .lines() + .count(), + 2 + ); +} + +#[test] +fn failures_preserve_evidence() { + let f = Fixture::new(); + let result = f.command("failed").env("BENCH_FAIL", "1").output().unwrap(); + assert!(!result.status.success()); + assert!(fs::read_to_string(f.0.join("failed/report.txt")) + .unwrap() + .contains("INCOMPLETE SCOUT")); + assert_eq!( + fs::read_to_string(f.0.join("calls")) + .unwrap() + .lines() + .count(), + 1 + ); + assert!(fs::read_to_string(f.0.join("failed/status.txt")) + .unwrap() + .contains("FAILED")); + let result = f + .command("partial") + .env("REPORT_FAIL", "1") + .output() + .unwrap(); + assert!(result.status.success()); + assert!(f.0.join("partial/trace.nsys-rep").exists()); + assert!(fs::read_to_string(f.0.join("partial/report.txt")) + .unwrap() + .contains("UNKNOWN")); + assert!(fs::read_to_string(f.0.join("partial/nsys-kernels.stderr")) + .unwrap() + .contains("unsupported report fixture")); +} + +#[test] +fn target_capability_is_required() { + let f = Fixture::new(); + let result = f + .command("no-nvtx") + .env("BENCH_NVTX", "unavailable") + .output() + .unwrap(); + assert!(!result.status.success()); + assert!(String::from_utf8_lossy(&result.stderr).contains("make ds4-bench-perf")); + assert_eq!( + fs::read_to_string(f.0.join("calls")) + .unwrap() + .lines() + .count(), + 1 + ); + assert!(!f.0.join("no-nvtx/trace.nsys-rep").exists()); +} + +#[test] +fn unknown_target_keeps_global_cuda_evidence() { + let f = Fixture::new(); + let result = f + .command("legacy") + .env("BENCH_NVTX", "unknown") + .output() + .unwrap(); + assert!(result.status.success()); + let report = fs::read_to_string(f.0.join("legacy/report.txt")).unwrap(); + assert!(report.contains("CUDA-only analysis")); + assert!(report.contains("global_kernel")); + assert!(report.contains("ds4.prefill: UNKNOWN")); + assert!( + fs::read_to_string(f.0.join("legacy/normalized-kernels.csv")) + .unwrap() + .contains(",global_kernel,3500,4") + ); +} + +#[test] +fn doctor_checks_selected_benchmark() { + let f = Fixture::new(); + script(&f.0, "other-bench", "echo 'NVTX: unavailable'"); + let result = Command::new(env!("CARGO_BIN_EXE_ds4-perf")) + .current_dir(&f.0) + .env("PATH", format!("{}:/usr/bin:/bin", f.0.display())) + .args(["doctor", "--bench", "./other-bench"]) + .output() + .unwrap(); + assert!(result.status.success()); + let report = String::from_utf8_lossy(&result.stdout); + assert!(report.contains("./other-bench")); + assert!(report.contains("NVTX unavailable")); +} + +#[test] +fn requested_proof_is_required() { + let f = Fixture::new(); + let output = f + .options("no-proof", &["--proof", "--repeats", "3"]) + .output() + .unwrap(); + assert!( + !output.status.success(), + "missing requested proof must fail" + ); + let scout: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("no-proof/scout.json")).unwrap()).unwrap(); + assert_eq!(scout["complete"], false); +} + +impl Fixture { + fn workflow_files(&self) { + fs::write(self.0.join("model.gguf"), "model fixture").unwrap(); + fs::write(self.0.join("prompt.txt"), "prompt fixture").unwrap(); + let gpu = serde_json::json!({"ordinal":0,"name":"Fixture GPU","uuid":"fixture-uuid","compute_major":12,"compute_minor":1,"driver_version":13030,"total_memory_bytes":1073741824_u64,"free_memory_bytes":536870912_u64,"multiprocessors":48,"warp_size":32,"max_threads_per_block":1024,"max_threads_per_sm":1536,"max_blocks_per_sm":24,"registers_per_sm":65536,"shared_bytes_per_sm":102400,"shared_bytes_per_block":101376,"l2_bytes":25165824,"memory_bus_bits":256,"memory_clock_khz":8533000,"clock_khz":2418000,"unavailable":{}}); + fs::write(self.0.join("gpu.json"), serde_json::to_vec(&gpu).unwrap()).unwrap(); + script(&self.0, "gpu-helper", "cat gpu.json"); + script( + &self.0, + "ds4-bench", + r#" +if [ "${1:-}" = --help ]; then echo 'NVTX: available'; exit 0; fi +printf 'ds4-bench NVTX: available\n' >&2 +proof='' +while [ "$#" -gt 0 ]; do + case "$1" in --dump-frontier-logits-dir) proof=$2; shift;; esac + shift +done +tps=100 +ids='[1,1,1,1,1,1,1,1]' +case "${DS4_QWEN_PREFILL_CHUNK:-256}" in + 512) tps=150;; + 2048) ids='[1,1,1,1,1,1,1,2]';; + 4096) tps=80;; + 8192) echo 'candidate intentionally failed' >&2; exit 7;; +esac +if [ -n "$proof" ] && [ "${DS4_QWEN_PREFILL_CHUNK:-256}" != 16384 ]; then + mkdir -p "$proof" + printf '%s\n' '{"source":"ds4-bench","model":"model.gguf","backend":"cuda","quality":false,"quant_bits":2,"prompt_tokens":2048,"frontier_tokens":2048,"prefill_tokens":2048,"ctx":2057,"vocab":3,"argmax_id":1,"argmax_logit":2.0,"logits":[0.0,2.0,1.0]}' > "$proof/frontier_002048.logits.json" + printf '%s\n' "$ids" > "$proof/tokens-2048.json" +fi +printf 'ctx_tokens,prefill_tokens,prefill_tps,gen_tokens,gen_tps,first_token_sec,kvcache_bytes\n2048,2048,%s,8,20,0.05,1234\n' "$tps" +"#, + ); + let workload = serde_json::json!({"protocol":"ds4-bench-v1","name":"fixture","family":"qwen","files":{"model":{"path":"model.gguf","sha256":ds4_perf::artifact::hash(&self.0.join("model.gguf")).unwrap()},"prompt":{"path":"prompt.txt","sha256":ds4_perf::artifact::hash(&self.0.join("prompt.txt")).unwrap()}},"shape":{"ctx_tokens":2048,"gen_tokens":8},"cache_state":"fixture warmup process"}); + fs::write( + self.0.join("workload.json"), + serde_json::to_vec(&workload).unwrap(), + ) + .unwrap(); + } + + fn workflow(&self) -> Command { + let mut c = Command::new(env!("CARGO_BIN_EXE_ds4-perf")); + c.current_dir(&self.0) + .env("PATH", format!("{}:/usr/bin:/bin", self.0.display())) + .env( + "FIXTURES", + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"), + ) + .env("DS4_QWEN_PREFILL_CHUNK", "256"); + c + } + + fn source(&self) -> PathBuf { + self.workflow_files(); + let result = self + .workflow() + .args([ + "scout", + "--out", + "source", + "--gpu-helper", + "./gpu-helper", + "--proof", + "--repeats", + "3", + "--workload", + "workload.json", + "--cache-policy", + "warmup-then-fresh", + "--", + "./ds4-bench", + "--cuda", + "-m", + "model.gguf", + "--prompt-file", + "prompt.txt", + "--ctx-start", + "2048", + "--ctx-max", + "2048", + "--gen-tokens", + "8", + ]) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + self.0.join("source/scout.json") + } + + fn plan(&self, candidates: &[(&str, &str)]) -> PathBuf { + let plan = serde_json::json!({"candidates":candidates.iter().map(|(name,value)| serde_json::json!({"name":name,"environment":{"DS4_QWEN_PREFILL_CHUNK":value},"reason":"controlled test fixture"})).collect::>()}); + let path = self.0.join("plan.json"); + fs::write(&path, serde_json::to_vec(&plan).unwrap()).unwrap(); + path + } +} + +#[test] +fn auto_retains_only_proved_gain() { + let f = Fixture::new(); + let source = f.source(); + let plan = f.plan(&[ + ("fast", "512"), + ("bad-tokens", "2048"), + ("failed-process", "8192"), + ("unreached", "4096"), + ]); + let result = f + .workflow() + .args(["optimize", "--auto", "--scout"]) + .arg(source) + .arg("--plan") + .arg(plan) + .args(["--out", "auto", "--rounds", "3", "--repeats", "3"]) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let decision: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("auto/decision.json")).unwrap()).unwrap(); + assert_eq!(decision["complete"], true); + let data = &decision["data"]; + assert_eq!(data["trials"].as_array().unwrap().len(), 3); + assert_eq!(data["trials"][0]["accepted"], true); + assert_eq!(data["trials"][1]["accepted"], false); + assert_eq!(data["trials"][2]["accepted"], false); + assert_eq!( + data["selected"]["environment"]["DS4_QWEN_PREFILL_CHUNK"], + "512" + ); + assert_eq!(data["stop_reason"], "round budget reached"); + assert!(f.0.join("auto/candidate-03/warmup.stderr").exists()); + let compare: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("auto/compare-02-before/compare.json")).unwrap()) + .unwrap(); + assert_eq!(compare["data"]["verdict"], "incorrect"); +} + +#[test] +fn rejects_forged_timings() { + let f = Fixture::new(); + let source = f.source(); + let original: serde_json::Value = serde_json::from_slice(&fs::read(&source).unwrap()).unwrap(); + for metric in ["prefill_tps", "gen_tps", "first_token_sec", "unhashed"] { + let mut forged = original.clone(); + if metric == "unhashed" { + forged["inputs"] + .as_array_mut() + .unwrap() + .retain(|r| r["path"] != "bench.stdout"); + } else { + forged["data"]["samples"][0]["rows"][0][metric] = 123.0.into(); + } + fs::write(&source, serde_json::to_vec(&forged).unwrap()).unwrap(); + let out = format!("forged-{metric}"); + f.workflow() + .args(["compare", "--baseline"]) + .arg(&source) + .arg("--candidate") + .arg(&source) + .args(["--out", &out]) + .output() + .unwrap(); + let result: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join(out).join("compare.json")).unwrap()).unwrap(); + assert_eq!(result["data"]["verdict"], "incomparable", "{metric}"); + assert!(result["data"]["reasons"][0] + .as_str() + .unwrap() + .contains("raw benchmark")); + } +} + +#[test] +fn decode_does_not_tune_prefill() { + let f = Fixture::new(); + let source = f.source(); + let mut data: serde_json::Value = serde_json::from_slice(&fs::read(&source).unwrap()).unwrap(); + for (name, busy, gap, count) in [ + ("idle", 5_000_000, 5_000_000, 1), + ("fragmented", 9_000_000, 1_000, 1_000), + ] { + data["data"]["evidence"]["phases"] = serde_json::json!({ + "ds4.decode": {"name":"ds4.decode", "wall_ns":10_000_000, + "projected_ns":10_000_000, "busy_ns":busy, "mem_ns":0, + "gap_ns":gap, "kernels":[{"name":"decode", "total_ns":busy, + "count":count}]} + }); + fs::write(&source, serde_json::to_vec(&data).unwrap()).unwrap(); + let out = format!("decode-{name}"); + let result = f + .workflow() + .args(["optimize", "--scout"]) + .arg(&source) + .args(["--out", &out]) + .output() + .unwrap(); + assert!(result.status.success()); + let result: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join(out).join("decision.json")).unwrap()) + .unwrap(); + assert_eq!(result["data"]["candidates"], serde_json::json!([])); + } +} + +#[test] +fn geometry_signals_are_scoped() { + let f = Fixture::new(); + let source = f.source(); + let mut data: serde_json::Value = serde_json::from_slice(&fs::read(&source).unwrap()).unwrap(); + data["data"]["evidence"]["phases"] = serde_json::json!({ + "ds4.prefill": {"name":"ds4.prefill", "wall_ns":10_000_000, + "projected_ns":10_000_000, "busy_ns":9_000_000, "mem_ns":0, + "gap_ns":1_000, "kernels":[{"name":"prefill", "total_ns":9_000_000, + "count":1}]} + }); + fs::write(&source, serde_json::to_vec(&data).unwrap()).unwrap(); + for phase in ["ds4.decode", "ds4.prefill"] { + for kind in ["fit", "ncu"] { + let data = if kind == "fit" { + serde_json::json!({"calibration_status":"fixture", "workload_shape_status":"fixture", + "bounds":[{"operand_shape":null,"operand_shape_source":"unknown", + "launch":{"phase":phase,"kernel":"fixture","grid":[1,1,1],"block":[32,1,1], + "registers_per_thread":16,"shared_bytes":0,"instances":1,"total_ns":1000}, + "resident_blocks_upper":1,"occupancy_upper":0.1,"waves_lower":1, + "last_wave_fill":0.1,"block_limits":{}}],"unknown_launches":0, + "observed_copy_gb_s":null,"observed_fp32_gflop_s":null,"observed_launch_us":null, + "shape":{},"limits":[]}) + } else { + serde_json::json!({"targets":[{"selection":"fixture", "phase":phase, + "kernel":"fixture","captured":true,"launch":{"id":"0","kernel":"fixture", + "process":"1","device":"0","grid":"1,1,1","block":"32,1,1"}, + "metrics":[{"name":"sm__warps_active.avg.pct_of_peak_sustained_active", + "unit":"%","value":10.0,"unavailable":null}]}],"limits":[]}) + }; + let path = source.parent().unwrap().join(format!("{kind}.json")); + let artifact = serde_json::json!({"kind":kind,"schema_version":1,"producer_version":"0.1.1", + "created_unix":0,"complete":true,"inputs":[],"warnings":[],"data":data}); + fs::write(&path, serde_json::to_vec(&artifact).unwrap()).unwrap(); + let out = format!("geometry-{phase}-{kind}"); + let result = f + .workflow() + .args(["optimize", "--scout"]) + .arg(&source) + .args(["--out", &out]) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let result: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join(out).join("decision.json")).unwrap()) + .unwrap(); + let expected = if phase == "ds4.prefill" { "512" } else { "128" }; + assert_eq!( + result["data"]["candidates"][0]["environment"]["DS4_QWEN_PREFILL_CHUNK"], expected, + "{phase} {kind}" + ); + fs::remove_file(path).unwrap(); + } + } +} + +#[test] +fn rejects_ties_and_regressions() { + let f = Fixture::new(); + let source = f.source(); + let plan = f.plan(&[("tie", "1024"), ("slow", "4096")]); + let result = f + .workflow() + .args(["optimize", "--auto", "--scout"]) + .arg(&source) + .arg("--plan") + .arg(plan) + .args(["--out", "auto", "--rounds", "2", "--repeats", "3"]) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + let decision: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("auto/decision.json")).unwrap()).unwrap(); + assert_eq!(decision["data"]["selected"], serde_json::Value::Null); + let result = f + .workflow() + .args(["compare", "--regression", "--baseline"]) + .arg(&source) + .args([ + "--candidate", + "auto/candidate-02/scout.json", + "--out", + "regression", + ]) + .output() + .unwrap(); + assert!(!result.status.success()); + let comparison: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("regression/compare.json")).unwrap()).unwrap(); + assert_eq!(comparison["data"]["verdict"], "regressed"); + fs::write( + f.0.join("source/bench-proof/tokens-2048.json"), + "[2,2,2,2,2,2,2,2]", + ) + .unwrap(); + let result = f + .workflow() + .args(["compare", "--regression", "--baseline"]) + .arg(&source) + .args([ + "--candidate", + "auto/candidate-01/scout.json", + "--out", + "tampered", + ]) + .output() + .unwrap(); + assert!(!result.status.success()); + let comparison: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("tampered/compare.json")).unwrap()).unwrap(); + assert_eq!(comparison["data"]["verdict"], "incomparable"); +} + +#[test] +fn rejects_unused_flags() { + let f = Fixture::new(); + let result = f + .options("bad-flags", &["--cupti-sdk", "/unused"]) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!(String::from_utf8_lossy(&result.stderr).contains("require --collector cupti")); +} + +#[test] +fn records_nsys_replay_identity() { + let f = Fixture::new(); + assert!(f.run("identity").status.success()); + let scout: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("identity/scout.json")).unwrap()).unwrap(); + assert_eq!( + scout["data"]["replay"]["collector"]["path"], + f.0.join("nsys").to_str().unwrap() + ); +} + +#[test] +fn rejects_forged_process_scope() { + let f = Fixture::new(); + let source = f.source(); + let mut value: serde_json::Value = serde_json::from_slice(&fs::read(&source).unwrap()).unwrap(); + value["data"]["process_scope"]["before"] = serde_json::json!([999999]); + value["data"]["process_scope"]["verified"] = serde_json::json!(true); + fs::write(&source, serde_json::to_vec(&value).unwrap()).unwrap(); + let result = f + .workflow() + .args(["compare", "--regression", "--baseline"]) + .arg(&source) + .arg("--candidate") + .arg(&source) + .args(["--out", "forged"]) + .output() + .unwrap(); + assert!(!result.status.success()); + let comparison: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("forged/compare.json")).unwrap()).unwrap(); + assert_eq!(comparison["data"]["verdict"], "incomparable"); +} + +#[test] +fn bounds_ncu_queries() { + let f = Fixture::new(); + script( + &f.0, + "ncu", + r#"case "${1:-}" in --query-metrics) sleep 3;; *) echo 'fake ncu --launch-count --nvtx';; esac"#, + ); + let result = f + .options("query-budget", &["--ncu", "--timeout-seconds", "1"]) + .output() + .unwrap(); + assert!(!result.status.success()); + let scout: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("query-budget/scout.json")).unwrap()).unwrap(); + assert_eq!(scout["complete"], false); + let status = fs::read_to_string(f.0.join("query-budget/ncu/query.status.txt")).unwrap(); + assert!(status.contains("deadline"), "{status}"); +} + +#[test] +fn bounds_decision_publication() { + let f = Fixture::new(); + let source = f.source(); + let plan = f.plan(&[("large-reason", "512")]); + let mut value: serde_json::Value = serde_json::from_slice(&fs::read(&plan).unwrap()).unwrap(); + value["candidates"][0]["reason"] = serde_json::json!("x".repeat(1100000)); + fs::write(&plan, serde_json::to_vec(&value).unwrap()).unwrap(); + let result = f + .workflow() + .args(["optimize", "--scout"]) + .arg(source) + .arg("--plan") + .arg(plan) + .args(["--out", "large-decision", "--max-output-mib", "1"]) + .output() + .unwrap(); + assert!(!result.status.success()); + let decision: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("large-decision/decision.json")).unwrap()) + .unwrap(); + assert_eq!(decision["complete"], false); + assert!(decision["data"]["stop_reason"] + .as_str() + .unwrap() + .contains("budget")); +} + +#[test] +fn failed_probe_is_partial() { + let f = Fixture::new(); + script(&f.0, "ps", "exit 1"); + let result = f.run("probe-failed"); + assert!(!result.status.success()); + let value: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("probe-failed/scout.json")).unwrap()).unwrap(); + assert_eq!(value["complete"], false); +} + +#[test] +fn rejects_unbound_device() { + let f = Fixture::new(); + let result = f + .options("device-mismatch", &["--device", "1"]) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!(String::from_utf8_lossy(&result.stderr).contains("CUDA_VISIBLE_DEVICES")); +} + +#[test] +fn partial_fit_is_incomplete() { + let f = Fixture::new(); + f.workflow_files(); + let result = f + .workflow() + .args([ + "scout", + "--out", + "partial-fit", + "--fit", + "--gpu-helper", + "./gpu-helper", + "--workload", + "workload.json", + "--", + "./ds4-bench", + "--cuda", + "-m", + "model.gguf", + "--prompt-file", + "prompt.txt", + "--ctx-start", + "2048", + "--ctx-max", + "2048", + "--gen-tokens", + "8", + ]) + .output() + .unwrap(); + assert!(!result.status.success()); + for name in ["fit.json", "scout.json"] { + let value: serde_json::Value = + serde_json::from_slice(&fs::read(f.0.join("partial-fit").join(name)).unwrap()).unwrap(); + assert_eq!(value["complete"], false); + } +} diff --git a/crates/ds4-perf/tests/serving.rs b/crates/ds4-perf/tests/serving.rs new file mode 100644 index 0000000000000000000000000000000000000000..e6b535474912321f8fdcdda95f617df7cb71d2f9 --- /dev/null +++ b/crates/ds4-perf/tests/serving.rs @@ -0,0 +1,492 @@ +#![cfg(target_os = "linux")] + +use serde_json::{json, Value}; +use std::{ + fs, + io::{BufRead, BufReader, Read, Write}, + net::TcpListener, + path::PathBuf, + process::Command, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, + thread, + time::Duration, +}; + +struct Fixture { + root: PathBuf, + url: String, + stop: Arc, + worker: Option>, +} + +impl Fixture { + fn new() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let port = listener.local_addr().unwrap().port(); + let root = std::env::temp_dir().join(format!("ds4-serving-{}-{port}", std::process::id())); + fs::create_dir(&root).unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let finished = stop.clone(); + let worker = thread::spawn(move || { + let completed = Arc::new(AtomicUsize::new(0)); + let mut handlers = Vec::new(); + while !finished.load(Ordering::Relaxed) { + let Ok((mut stream, _)) = listener.accept() else { + thread::sleep(Duration::from_millis(2)); + continue; + }; + let completed = completed.clone(); + handlers.push(thread::spawn(move || { + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + let is_stats = line.starts_with("GET /v1/stats "); + let mut length = 0; + loop { + line.clear(); + reader.read_line(&mut line).unwrap(); + if line == "\r\n" { + break; + } + if let Some(value) = line.to_lowercase().strip_prefix("content-length:") { + length = value.trim().parse().unwrap(); + } + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).unwrap(); + if is_stats { + let completed = completed.load(Ordering::Relaxed); + let reuse = ["cold", "exact", "partial"][completed.saturating_sub(1).min(2)]; + let body = json!({ + "routes": {"chat_continuous": completed}, "queue_depth": 0, "clients": 1, + "serving": { + "family": "qwen4exp", "requested": {}, + "effective": {"prefix_reuse": "partial", "ctx": 4096, "max_seqs": 2}, + "qualified": {}, "issues": [] + }, + "last_request": {"effective_lane": "continuous", "reuse_kind": reuse, + "speculation_active": false, "fallback_reason": null} + }) + .to_string(); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .unwrap(); + return; + } + let request: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(request["stream"], true); + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n").unwrap(); + stream + .write_all(b"data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}\n\n") + .unwrap(); + if request["messages"][0]["content"] == "tool-call" { + stream.write_all(b"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call-a\",\"type\":\"function\",\"function\":{\"name\":\"weather\",\"arguments\":\"{\\\"city\\\":\"}}]}}]}\n\n").unwrap(); + stream.write_all(b"data: {\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"function\":{\"arguments\":\"\\\"Seoul\\\"}\"}}]}}]}\n\n").unwrap(); + } else if request["messages"][0]["content"] == "overlap-decode" { + for _ in 0..8 { + thread::sleep(Duration::from_millis(40)); + stream.write_all(b"data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n").unwrap(); + } + } else { + thread::sleep(Duration::from_millis(120)); + // The collector must handle an SSE event split across writes. + stream + .write_all(b"data: {\"choices\":[{\"delta\":{\"content\":\"ans") + .unwrap(); + thread::sleep(Duration::from_millis(10)); + stream.write_all(b"wer\"}}]}\n\n").unwrap(); + thread::sleep(Duration::from_millis(30)); + } + let finish = if request["messages"][0]["content"] == "tool-call" { "tool_calls" } else { "stop" }; + write!(stream, "data: {{\"choices\":[{{\"delta\":{{}},\"finish_reason\":\"{finish}\"}}]}}\n\n").unwrap(); + if request["stream_options"]["include_usage"] == true { + let decoding = request["messages"][0]["content"] == "overlap-decode"; + let usage = json!({"choices": [], "usage": {"prompt_tokens": if decoding {32} else {8192}, "prompt_tokens_details":{"cached_tokens":0}, "completion_tokens": if decoding {8} else {1}}}); + write!(stream, "data: {usage}\n\n").unwrap(); + } + stream.write_all(b"data: [DONE]\n\n").unwrap(); + completed.fetch_add(1, Ordering::Relaxed); + })); + } + for handler in handlers { + handler.join().unwrap(); + } + }); + Self { + root, + url: format!("http://127.0.0.1:{port}"), + stop, + worker: Some(worker), + } + } + + fn workload(&self) -> Value { + json!({ + "protocol": "ds4-serving-v1", "name": "three-conversation-paths", "family": "qwen4exp", + "cases": (["kv_cold_prefill", "warm_append", "partial_branch"].iter().zip(["cold", "exact", "partial"]) + .enumerate().map(|(i, (scenario, reuse))| json!({ + "name": format!("case-{i}"), "scenario": scenario, + "request": {"model": "fixture", "messages": [{"role":"user", "content": format!("turn {i}")}], "stream": true, "temperature": 0, "max_tokens": 8}, + "expect": {"content": "answer", "finish_reason": "stop", "reuse_kind": reuse, "effective_lane": "continuous", "speculation_active": false, "fallback_reason": null}, + "limits": {"ttft_ms": 2000, "total_ms": 3000, "min_host_available_bytes": 1} + })).collect::>()) + }) + } + + fn run(&self, workload: &Value) -> std::process::Output { + self.run_options(workload, &[]) + } + + fn run_options(&self, workload: &Value, options: &[&str]) -> std::process::Output { + fs::write( + self.root.join("workload.json"), + serde_json::to_vec(workload).unwrap(), + ) + .unwrap(); + Command::new(env!("CARGO_BIN_EXE_ds4-perf")) + .args(["serving", "--url", &self.url, "--workload"]) + .arg(self.root.join("workload.json")) + .arg("--out") + .arg(self.root.join("run")) + .args(["--timeout-seconds", "5"]) + .args(options) + .output() + .unwrap() + } +} + +#[test] +fn pins_identity_and_preserves_repeat_cache() { + let fixture = Fixture::new(); + let mut workload = fixture.workload(); + workload["cases"].as_array_mut().unwrap().truncate(1); + let model = fixture.root.join("model.fixture"); + fs::write(&model, b"model identity fixture").unwrap(); + workload["inputs"] = json!({"model": {"path": "model.fixture", "sha256": ds4_perf::artifact::hash(&model).unwrap()}}); + let pid = std::process::id().to_string(); + let output = fixture.run_options(&workload, &["--repeats", "2", "--server-pid", &pid]); + // Replaying a cold case must expose that the second request actually reused KV. + assert!(!output.status.success()); + let artifact: Value = + serde_json::from_slice(&fs::read(fixture.root.join("run/serving.json")).unwrap()).unwrap(); + assert_eq!(artifact["complete"], true, "{artifact}"); + assert_eq!(artifact["data"]["requested_repeats"], 2); + assert_eq!( + artifact["data"]["identity"]["server"]["pid"], + std::process::id() + ); + assert_eq!(artifact["data"]["cases"][0]["repeat"], 0); + assert_eq!(artifact["data"]["cases"][1]["repeat"], 1); + assert!(artifact["data"]["cases"][1]["failures"] + .to_string() + .contains("reuse_kind")); +} + +#[test] +fn decode_progress_during_peer_prefill() { + let fixture = Fixture::new(); + let mut workload = fixture.workload(); + let case = &workload["cases"][0]; + let mut decode = json!({"request": case["request"], "expect": {"content": "aaaaaaaa", "finish_reason": "stop"}, "limits": case["limits"]}); + decode["request"]["messages"][0]["content"] = json!("overlap-decode"); + decode["request"]["stream_options"] = json!({"include_usage":true}); + let mut prefill = json!({"request": case["request"], "expect": {"content": "answer", "finish_reason": "stop"}, "limits": case["limits"]}); + prefill["request"]["stream_options"] = json!({"include_usage":true}); + workload["cases"] = json!([]); + workload["overlaps"] = json!([{"name": "decode-peer-prefill", "decode": decode, "prefill": prefill, + "min_decode_events_during_prefill": 2, "min_prefill_tokens": 4096, "max_decode_gap_ms": 100}]); + let output = fixture.run(&workload); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let artifact: Value = + serde_json::from_slice(&fs::read(fixture.root.join("run/serving.json")).unwrap()).unwrap(); + let overlap = &artifact["data"]["overlaps"][0]; + assert!(overlap["decode_events_during_prefill"].as_u64().unwrap() >= 2); + assert!(overlap["max_decode_gap_ms"].as_f64().unwrap() <= 100.0); + assert_eq!(artifact["data"]["passed"], true); + assert_eq!(overlap["prefill_computed_tokens"], 8192); + assert_eq!(overlap["decode_committed_tokens"], 8); +} + +impl Drop for Fixture { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + self.worker.take().unwrap().join().unwrap(); + let _ = fs::remove_dir_all(&self.root); + } +} + +#[test] +fn captures_serving_latency_and_reuse() { + let fixture = Fixture::new(); + let output = fixture.run(&fixture.workload()); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let artifact: Value = + serde_json::from_slice(&fs::read(fixture.root.join("run/serving.json")).unwrap()).unwrap(); + assert_eq!(artifact["kind"], "serving"); + assert_eq!(artifact["complete"], true); + assert_eq!(artifact["data"]["passed"], true); + let cases = artifact["data"]["cases"].as_array().unwrap(); + assert_eq!(cases.len(), 3); + for case in cases { + assert!(case["ttft_ms"].as_f64().unwrap() >= 120.0); + assert!(case["total_ms"].as_f64().unwrap() >= 150.0); + assert_eq!(case["content"], "answer"); + assert!(case["host_min_available_bytes"] + .as_u64() + .is_some_and(|n| n > 0)); + assert!(case["failures"].as_array().unwrap().is_empty()); + } + for file in [ + "request.json", + "response.sse", + "events.json", + "before.json", + "after.json", + "memory.json", + ] { + assert!(fixture.root.join("run/case-000").join(file).is_file()); + } + assert!(!fixture.run(&fixture.workload()).status.success()); +} + +#[test] +fn refuses_incorrect_or_missed_reuse() { + for (key, value, expected) in [ + ("content", "wrong", "content"), + ("finish_reason", "length", "finish_reason"), + ("reuse_kind", "exact", "reuse_kind"), + ] { + let fixture = Fixture::new(); + let mut workload = fixture.workload(); + workload["cases"].as_array_mut().unwrap().truncate(1); + workload["cases"][0]["expect"][key] = json!(value); + if key == "reuse_kind" { + workload["cases"][0]["scenario"] = json!("warm_append"); + } + let output = fixture.run(&workload); + assert!(!output.status.success()); + let artifact: Value = + serde_json::from_slice(&fs::read(fixture.root.join("run/serving.json")).unwrap()) + .unwrap(); + assert_eq!(artifact["complete"], true); + assert_eq!(artifact["data"]["passed"], false); + assert!(artifact["data"]["cases"][0]["failures"] + .to_string() + .contains(expected)); + } +} + +#[test] +fn rejects_latency_and_memory_failure() { + let fixture = Fixture::new(); + let mut workload = fixture.workload(); + workload["cases"].as_array_mut().unwrap().truncate(1); + workload["cases"][0]["limits"]["ttft_ms"] = json!(1); + workload["cases"][0]["limits"]["min_host_available_bytes"] = json!(u64::MAX); + assert!(!fixture.run(&workload).status.success()); + let artifact: Value = + serde_json::from_slice(&fs::read(fixture.root.join("run/serving.json")).unwrap()).unwrap(); + let failures = artifact["data"]["cases"][0]["failures"].to_string(); + assert!(failures.contains("TTFT"), "{failures}"); + assert!(failures.contains("memory"), "{failures}"); +} + +#[test] +fn measures_and_checks_generated_tool_calls() { + for city in ["Seoul", "Busan"] { + let fixture = Fixture::new(); + let mut workload = fixture.workload(); + workload["cases"].as_array_mut().unwrap().truncate(1); + let case = &mut workload["cases"][0]; + case["scenario"] = json!("tool"); + case["request"]["messages"][0]["content"] = json!("tool-call"); + case["request"]["tools"] = json!([{"type":"function","function":{"name":"weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}]); + case["expect"]["content"] = json!(""); + case["expect"]["finish_reason"] = json!("tool_calls"); + case["expect"]["tool_calls"] = json!([{"name":"weather","arguments":{"city":city}}]); + let output = fixture.run(&workload); + assert_eq!( + output.status.success(), + city == "Seoul", + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let evidence: Value = + serde_json::from_slice(&fs::read(fixture.root.join("run/serving.json")).unwrap()) + .unwrap(); + assert_eq!( + evidence["data"]["cases"][0]["tool_calls"][0]["arguments"]["city"], + "Seoul" + ); + assert!(evidence["data"]["gpu_windows"][0]["samples"] + .as_array() + .is_some_and(|samples| !samples.is_empty())); + } +} + +#[test] +fn cache_usage_minimum_cannot_pass_on_a_cold_trace() { + let fixture = Fixture::new(); + let mut workload = fixture.workload(); + workload["cases"].as_array_mut().unwrap().truncate(1); + workload["cases"][0]["request"]["stream_options"] = json!({"include_usage":true}); + workload["cases"][0]["expect"]["min_cached_tokens"] = json!(1); + let output = fixture.run(&workload); + assert!(!output.status.success()); + let evidence: Value = + serde_json::from_slice(&fs::read(fixture.root.join("run/serving.json")).unwrap()).unwrap(); + assert_eq!(evidence["data"]["cases"][0]["cached_tokens"], 0); + assert!(evidence["data"]["cases"][0]["failures"] + .as_array() + .unwrap() + .iter() + .any(|failure| failure.as_str().unwrap().contains("cached token"))); +} + +// A separate process makes restart provenance testable without any model. +#[test] +fn restart_peer() { + let Some(root) = std::env::var_os("DS4_PERF_TEST_PEER") else { + return; + }; + let root = PathBuf::from(root); + let address = fs::read_to_string(root.join("address")).unwrap(); + let listener = TcpListener::bind(address.trim()).unwrap(); + let restored = root.join("seeded").exists(); + let mut completed = 0; + fs::write(root.join("ready"), b"ready").unwrap(); + for connection in listener.incoming() { + let mut stream = connection.unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + let stats = line.starts_with("GET /v1/stats "); + let mut length = 0; + loop { + line.clear(); + reader.read_line(&mut line).unwrap(); + if line == "\r\n" { + break; + } + if let Some(value) = line.to_lowercase().strip_prefix("content-length:") { + length = value.trim().parse().unwrap(); + } + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).unwrap(); + if stats { + let body = json!({"routes":{"chat":completed},"queue_depth":0,"clients":1, + "serving":{"family":"qwen4exp","requested":{},"effective":{"disk":true,"max_seqs":1},"qualified":{},"issues":[]}, + "last_request":{"reuse_kind":if restored {"exact"} else {"cold"},"effective_lane":"serial","speculation_active":false,"fallback_reason":null}}).to_string(); + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .unwrap(); + continue; + } + stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\ndata: {\"choices\":[{\"delta\":{\"content\":\"answer\"}}]}\n\ndata: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n").unwrap(); + let usage = json!({"choices":[],"usage":{"prompt_tokens":32,"prompt_tokens_details":{"cached_tokens":if restored {16} else {0}},"completion_tokens":1}}); + write!(stream, "data: {usage}\n\ndata: [DONE]\n\n").unwrap(); + fs::write(root.join("seeded"), b"checkpoint").unwrap(); + completed += 1; + } +} + +struct Peer(std::process::Child); +impl Drop for Peer { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} +fn start_peer(root: &std::path::Path) -> Peer { + let _ = fs::remove_file(root.join("ready")); + let child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "restart_peer", "--nocapture"]) + .env("DS4_PERF_TEST_PEER", root) + .current_dir(root) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + let peer = Peer(child); + for _ in 0..100 { + if root.join("ready").exists() { + return peer; + } + thread::sleep(Duration::from_millis(10)); + } + panic!("restart peer did not start"); +} + +#[test] +fn links_fresh_process_disk_restore() { + let fixture = Fixture::new(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + fs::write(fixture.root.join("address"), address.to_string()).unwrap(); + let mut workload = fixture.workload(); + workload["cases"].as_array_mut().unwrap().truncate(1); + workload["cases"][0]["expect"]["effective_lane"] = json!("serial"); + workload["cases"][0]["request"]["stream_options"] = json!({"include_usage":true}); + let collect = |name: &str, workload: &Value, pid: u32| { + let file = fixture.root.join(format!("{name}.json")); + fs::write(&file, serde_json::to_vec(workload).unwrap()).unwrap(); + Command::new(env!("CARGO_BIN_EXE_ds4-perf")) + .args(["serving", "--url"]) + .arg(format!("http://{address}")) + .arg("--workload") + .arg(file) + .arg("--out") + .arg(fixture.root.join(name)) + .arg("--server-pid") + .arg(pid.to_string()) + .args(["--timeout-seconds", "5"]) + .output() + .unwrap() + }; + let peer = start_peer(&fixture.root); + let seed = collect("seed", &workload, peer.0.id()); + assert!( + seed.status.success(), + "{}", + String::from_utf8_lossy(&seed.stderr) + ); + drop(peer); + // Same executable, argv and working directory; only the process changes. + let peer = start_peer(&fixture.root); + workload["restart_from"] = json!(fixture.root.join("seed/serving.json")); + workload["cases"][0]["scenario"] = json!("restart_restore"); + workload["cases"][0]["expect"]["reuse_kind"] = json!("exact"); + workload["cases"][0]["expect"]["min_cached_tokens"] = json!(16); + let restored = collect("restored", &workload, peer.0.id()); + assert!( + restored.status.success(), + "{}", + String::from_utf8_lossy(&restored.stderr) + ); + let second = collect("already-used", &workload, peer.0.id()); + assert!(!second.status.success()); + assert!(String::from_utf8_lossy(&second.stderr).contains("first request")); +} diff --git a/crates/ds4-server/Cargo.toml b/crates/ds4-server/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..8712c2ed7fb4540bd835c39522cd17d87f3fbcd4 --- /dev/null +++ b/crates/ds4-server/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "ds4-server" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "HTTP wire surfaces (Phase 7)" + +[lib] +path = "src/lib.rs" + +[[bin]] +name = "ds4-server-rs" +path = "src/bin/ds4-server-rs.rs" +required-features = ["native"] + +[features] +default = [] +native = [] + +[dependencies] +ds4-core = { path = "../ds4-core" } +ds4-dist = { path = "../ds4-dist" } +ds4-kv = { path = "../ds4-kv" } +ds4-sys = { path = "../ds4-sys" } +serde_json = { version = "1", features = ["arbitrary_precision"] } +sha2 = { version = "0.11", default-features = false } + +[dev-dependencies] +ds4-core = { path = "../ds4-core" } diff --git a/crates/ds4-server/examples/glm53_prompt.rs b/crates/ds4-server/examples/glm53_prompt.rs new file mode 100644 index 0000000000000000000000000000000000000000..c012ca4f2e2b6278c47cfe5a93f308b3c8290542 --- /dev/null +++ b/crates/ds4-server/examples/glm53_prompt.rs @@ -0,0 +1,19 @@ +//! Render a saved Chat Completions request without opening a model. +use ds4_core::chat_template::{RenderClock, Template}; +use ds4_server::{chat_input, parse_chat_request, ParseEnv}; + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().skip(1).collect(); + if args.len() != 3 { + return Err("usage: glm53_prompt TEMPLATE REQUEST OUTPUT".into()); + } + let source = std::fs::read_to_string(&args[0])?; + let body = std::fs::read_to_string(&args[1])?; + let template = Template::compile(&source, RenderClock::Fixed(0))?; + let parsed = parse_chat_request(&ParseEnv::default(), &body) + .map_err(|error| format!("request: {error:?}"))?; + let prompt = chat_input::render(&template, ds4_core::Variant::Glm53Flash as i32, &parsed) + .map_err(|error| format!("render: {error:?}"))?; + std::fs::write(&args[2], prompt)?; + Ok(()) +} diff --git a/crates/ds4-server/src/admit.rs b/crates/ds4-server/src/admit.rs new file mode 100644 index 0000000000000000000000000000000000000000..265cbc2ec16b3b061239c862d7f860a47d511dd6 --- /dev/null +++ b/crates/ds4-server/src/admit.rs @@ -0,0 +1,188 @@ +//! Admission bounds and job-id mint from `ds4_server.c` at v0.6.3-dfm. + +use crate::route::ReqKind; + +pub const SHED_CLIENTS: u8 = 0; +pub const SHED_QUEUE_DEPTH: u8 = 1; +pub const SHED_QUEUE_BYTES: u8 = 2; +pub const SHED_QUEUE_AGE: u8 = 3; +pub const SHED_SLOW_READER: u8 = 4; +pub const SHED_CONT_HOLD: u8 = 5; +pub const SHED_REASONS: usize = 6; + +pub const SHED_NAMES: [&str; SHED_REASONS] = [ + "clients", + "queue_depth", + "queue_bytes", + "queue_age", + "slow_reader", + "continuation_hold", +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EnqVerdict { + Ok = 0, + Stopping = 1, + ShedQueueDepth = 2, + ShedQueueBytes = 3, +} + +#[derive(Debug, Clone)] +pub struct AdmitState { + pub stopping: bool, + pub queued: i32, + pub inflight_body_bytes: u64, + pub max_queue: i32, + pub max_queue_bytes: u64, + pub max_clients: i32, + pub clients: i32, + pub seq: u64, +} + +impl Default for AdmitState { + fn default() -> Self { + Self { + stopping: false, + queued: 0, + inflight_body_bytes: 0, + max_queue: 0, + max_queue_bytes: 0, + max_clients: 0, + clients: 0, + seq: 0, + } + } +} + +/// Authoritative enqueue under the queue lock. Matches C `enqueue`. +pub fn enqueue(s: &mut AdmitState, body_bytes: u64) -> EnqVerdict { + if s.stopping { + return EnqVerdict::Stopping; + } + if s.max_queue > 0 && s.queued >= s.max_queue { + return EnqVerdict::ShedQueueDepth; + } + if s.max_queue_bytes > 0 && s.inflight_body_bytes + body_bytes > s.max_queue_bytes { + return EnqVerdict::ShedQueueBytes; + } + s.queued += 1; + s.inflight_body_bytes += body_bytes; + EnqVerdict::Ok +} + +pub fn queue_unlink_head(s: &mut AdmitState) { + if s.queued > 0 { + s.queued -= 1; + } +} + +pub fn enqueue_release(s: &mut AdmitState, body_bytes: u64) { + queue_unlink_head(s); + if s.inflight_body_bytes >= body_bytes { + s.inflight_body_bytes -= body_bytes; + } else { + s.inflight_body_bytes = 0; + } +} + +/// Pre-parse cheap bounds from `client_main`. `generation` covers `/v1/batch`. +pub fn preparse_shed( + s: &AdmitState, + inference: bool, + generation: bool, + incoming_body: u64, +) -> Option<(u8, i32, i32, &'static str)> { + if generation && s.max_clients > 0 && s.clients > s.max_clients { + return Some(( + SHED_CLIENTS, + 503, + 10, + "server connection capacity reached; retry later", + )); + } + if inference && s.max_queue > 0 && s.queued >= s.max_queue { + return Some(( + SHED_QUEUE_DEPTH, + 429, + 5, + "request queue is full; retry later", + )); + } + if inference + && s.max_queue_bytes > 0 + && s.inflight_body_bytes + incoming_body > s.max_queue_bytes + { + return Some(( + SHED_QUEUE_BYTES, + 429, + 5, + "server request-body budget exhausted; retry later", + )); + } + None +} + +pub fn mint_job_id(kind: ReqKind, seq: u64) -> String { + if kind == ReqKind::Chat { + format!("chatcmpl-{seq}") + } else { + format!("cmpl-{seq}") + } +} + +pub fn next_job_id(s: &mut AdmitState, kind: ReqKind) -> String { + s.seq += 1; + mint_job_id(kind, s.seq) +} + +/// C `client_main` enqueue Stopping body (`ds4_server.c`). +pub const SERVER_SHUTTING_DOWN: &str = "server shutting down"; + +pub fn enqueue_shed_error(v: EnqVerdict) -> Option<(u8, i32, Option, &'static str)> { + match v { + EnqVerdict::Ok => None, + // C client_main Stopping uses wire_http_error (no Retry-After). + EnqVerdict::Stopping => Some((SHED_CLIENTS, 503, None, SERVER_SHUTTING_DOWN)), + EnqVerdict::ShedQueueDepth => Some(( + SHED_QUEUE_DEPTH, + 429, + Some(5), + "request queue is full; retry later", + )), + EnqVerdict::ShedQueueBytes => Some(( + SHED_QUEUE_BYTES, + 429, + Some(5), + "server request-body budget exhausted; retry later", + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stopping_503_body_matches_c_client_main() { + // Given: C enqueue Stopping at ds4_server.c client_main + // When: the shed mapper runs + let shed = enqueue_shed_error(EnqVerdict::Stopping).expect("stopping sheds"); + + // Then: exact C production bytes, not the invented "is" + assert_eq!(shed.0, SHED_CLIENTS); + assert_eq!(shed.1, 503); + assert_eq!(shed.2, None); + assert_eq!(shed.3, "server shutting down"); + assert_ne!(shed.3, "server is shutting down"); + } + + #[test] + fn queue_full_429_keeps_c_retry_after() { + let depth = enqueue_shed_error(EnqVerdict::ShedQueueDepth).expect("depth sheds"); + assert_eq!(depth.1, 429); + assert_eq!(depth.2, Some(5)); + let bytes = enqueue_shed_error(EnqVerdict::ShedQueueBytes).expect("bytes sheds"); + assert_eq!(bytes.1, 429); + assert_eq!(bytes.2, Some(5)); + } +} diff --git a/crates/ds4-server/src/bin/ds4-server-rs.rs b/crates/ds4-server/src/bin/ds4-server-rs.rs new file mode 100644 index 0000000000000000000000000000000000000000..8980f59c510d3db620e62e0efd4d133968bbccec --- /dev/null +++ b/crates/ds4-server/src/bin/ds4-server-rs.rs @@ -0,0 +1,1347 @@ +//! Shadow HTTP host. GET surfaces are live; family decode uses the +//! native FFI when `-m` opens a model. Continuation registry is host-owned. +//! Incremental live DSML tool projection is host-owned. + +use ds4_core::{ + attach_host_quote, caps_from_ident, identify_gguf, probe_dspark_sidecar, probe_model_artifact, + probe_mtp_sidecar, probe_vision_sidecar, resolve_plan, Backend, DistributedConfig, + DistributedRole, Distribution, EngineFacts, GgufFile, Identified, MaxSeqs, Model, + ModelOpenOption, MtpMode, PrefixReuse, ProbeParts, ServingCaps, ServingRequest, Support, Vocab, + WeightSlice, +}; +use ds4_server::cache_identity::CacheIdentity; +use ds4_server::expected_plan::ExpectedPlan; +use ds4_server::kv_cli::DiskKvArgs; +use ds4_server::parse::EosPolicy; +use ds4_server::{ + accept_loop, accept_loop_with_engine, accept_loop_with_engine_cont, dist_weight_slice, + listen_if_allowed, model_id_from_gguf_path, run_assembled_worker, server_launch, ContLane, + DistArgs, ExactReplay, NativeDecode, ServerConfig, ServerLaunch, ToolMemoryConfig, + WORKER_REQUIRES_MODEL, +}; +use std::path::Path; + +fn distributed_config(opt: &ds4_dist::Options) -> Option { + let role = match opt.role { + ds4_dist::Role::None => return None, + ds4_dist::Role::Coordinator => DistributedRole::Coordinator, + ds4_dist::Role::Worker => DistributedRole::Worker, + }; + Some(DistributedConfig { + role, + layer_start: opt.layers.start, + layer_end: opt.layers.end, + has_output: opt.layers.has_output, + listen_host: opt.listen_host.clone(), + listen_port: opt.listen_port, + coordinator_host: opt.coordinator_host.clone(), + coordinator_port: opt.coordinator_port, + prefill_chunk: opt.prefill_chunk, + prefill_window: opt.prefill_window, + activation_bits: opt.activation_bits, + replay_check: opt.replay_check, + debug: opt.debug, + }) +} + +fn main() { + let mut cfg = ServerConfig::default(); + let mut model_path: Option = None; + let mut mtp_path: Option = None; + let mut backend = Backend::Cuda; + let mut n_threads = 0i32; + let mut serve_req = ServingRequest::from_env(); + let mut model_options = Vec::new(); + let mut vision_path: Option = None; + let mut kv = DiskKvArgs::default(); + let mut expected_plan: Option = None; + let mut dist = DistArgs::default(); + let mut ctx_set = false; + let mut v41_flags = V41Flags::default(); + let mut mem_budget_mb: Option = None; + // The reference's two exact-replay switches (`ds4_help.c:378-379`). + let mut tool_memory = ToolMemoryConfig::default(); + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + if dist + .parse_arg(&arg, &mut args) + .unwrap_or_else(|error| cli_error(&format!("ds4-server-rs: {error}"))) + { + continue; + } + if kv + .parse_arg(&arg, &mut args) + .unwrap_or_else(|error| cli_error(&error)) + { + continue; + } + match arg.as_str() { + "--host" => cfg.listen_host = args.next().unwrap_or_else(|| usage()), + "--port" => { + cfg.listen_port = args + .next() + .and_then(|p| p.parse().ok()) + .unwrap_or_else(|| usage()); + } + "--model-id" => cfg.model_id = args.next().unwrap_or_else(|| usage()), + "--model" | "-m" => { + let path = args.next().unwrap_or_else(|| usage()); + if let Some(id) = model_id_from_gguf_path(&path) { + if cfg.model_id == "ds4" { + cfg.model_id = id; + } + } + model_path = Some(path); + } + "--vision" | "--vision-gguf" => { + // "--vision" is the external encoder for GLM-5.3/Step/Ling/MiMo; + // "--vision-gguf" is the reference install.sh name for the V4.1 + // model's own tower sidecar. Both ride one native open slot; + // the native open picks the family from the loaded variant. + let path = args.next().unwrap_or_else(|| usage()); + vision_path = Some(path.clone()); + model_options.push(ModelOpenOption::Vision(path)); + } + "--ssd-streaming" => { + serve_req.ssd_streaming = true; + model_options.push(ModelOpenOption::SsdStreaming); + } + "--ssd-streaming-cold" => { + serve_req.ssd_streaming_cold = true; + model_options.push(ModelOpenOption::SsdStreamingCold); + } + "--ssd-streaming-cache-experts" => { + let value = args + .next() + .unwrap_or_else(|| cli_error("--ssd-streaming-cache-experts requires a value")); + let option = ModelOpenOption::ssd_cache(&value) + .unwrap_or_else(|error| cli_error(&error.message)); + match option { + ModelOpenOption::SsdCacheAuto => {} + ModelOpenOption::SsdCacheExperts(count) => { + serve_req.ssd_streaming_cache_experts = Some(count) + } + ModelOpenOption::SsdCacheBytes(bytes) => { + serve_req.ssd_streaming_cache_bytes = Some(bytes) + } + _ => unreachable!(), + } + model_options.push(option); + } + "--mtp" => { + let path = args.next().unwrap_or_else(|| usage()); + serve_req.mtp_path = Some(path.clone()); + mtp_path = Some(path); + } + "--mtp-mode" => { + serve_req.mtp_mode = MtpMode::parse(&args.next().unwrap_or_else(|| usage())) + .unwrap_or_else(|e| { + cli_error(&e); + }); + } + "--prefix-reuse" => { + serve_req.prefix_reuse = + PrefixReuse::parse(&args.next().unwrap_or_else(|| usage())) + .unwrap_or_else(|e| cli_error(&e)); + } + "--max-seqs" => { + serve_req.max_seqs = MaxSeqs::parse(&args.next().unwrap_or_else(|| usage())) + .unwrap_or_else(|e| cli_error(&e)); + } + "--prefill-chunk" => { + serve_req.sched_chunk = Some(positive_chunk(&arg, args.next())); + } + "--prefill-chunk-live" => { + serve_req.sched_chunk_live = Some(positive_chunk(&arg, args.next())); + } + "--native-chunk" => { + serve_req.native_chunk = Some(positive_chunk(&arg, args.next())); + } + "--print-plan" => serve_req.print_plan = true, + "--check-config" => serve_req.check_config = true, + "--expect-plan" => { + if expected_plan.is_some() { + cli_error("--expect-plan may be supplied only once"); + } + let path = args.next().unwrap_or_else(|| usage()); + expected_plan = Some( + ExpectedPlan::load(Path::new(&path)).unwrap_or_else(|error| cli_error(&error)), + ); + } + "--backend" => { + backend = match args.next().unwrap_or_else(|| usage()).as_str() { + "cuda" => Backend::Cuda, + "cpu" => Backend::Cpu, + "metal" => Backend::Metal, + other => { + eprintln!("ds4-server-rs: unknown backend {other}"); + std::process::exit(2); + } + }; + } + "--cuda" => backend = Backend::Cuda, + "--tokens" | "-n" => { + cfg.default_tokens = args + .next() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| usage()); + } + "--mtp-draft" => { + serve_req.mtp_draft = Some(positive_count(&arg, args.next())); + } + "--mtp-margin" => { + model_options.push(ModelOpenOption::MtpMargin(margin(&arg, args.next()))) + } + "--version" => { + println!("ds4-server v{}", env!("CARGO_PKG_VERSION")); + return; + } + "--no-update-check" => {} + "-c" | "--ctx" => { + cfg.ctx = args + .next() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| usage()); + serve_req.ctx = cfg.ctx; + ctx_set = true; + } + "-t" | "--threads" => { + n_threads = args + .next() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| usage()); + } + // Hidden rust-shadow alias for DS4_SERVER_COALESCE_MAX. + // Not a C flag; kept for rust-host-live scripts (e.g. --cont-width 1). + "--cont-width" => { + serve_req.max_seqs = + MaxSeqs::parse_coalesce(&args.next().unwrap_or_else(|| usage())) + .unwrap_or_else(|e| cli_error(&e)); + } + "--cors" => cfg.cors = true, + // The reference's exact-replay switches (`ds4_help.c:378-379`, + // parsed `ds4_server.c:15700-15703`): boot settings that every + // tool-memory entry point consults for itself. + "--disable-exact-dsml-tool-replay" => tool_memory.replay = ExactReplay::Off, + "--tool-memory-max-ids" => { + let raw = args.next().unwrap_or_else(|| usage()); + tool_memory.max_ids = raw.parse().unwrap_or_else(|_| { + cli_error(&format!( + "ds4-server-rs: --tool-memory-max-ids wants a count, got {raw}" + )) + }); + } + // The reference's debug trace (`ds4_help.c:323`; the file is opened + // `w` and announced at `ds4_server.c:16010-16019`). + "--trace" => { + let raw = args.next().unwrap_or_else(|| usage()); + ds4_server::trace::set_path(std::path::PathBuf::from(&raw)); + eprintln!("ds4-server-rs: tracing session to {raw}"); + } + "--ignore-eos-in-reasoning" => { + if cfg.eos_policy == EosPolicy::Default { + cfg.eos_policy = EosPolicy::Reasoning; + } + } + "--ignore-eos" => cfg.eos_policy = EosPolicy::Global, + "--mem-floor-gb" => { + let raw = args.next().unwrap_or_else(|| usage()); + cfg.apply_mem_floor_gb(&raw); + serve_req.mem_floor_gb = cfg.mem_floor_gb; + } + // Unit Z: the engine's process red line (core_util.c:198-200); + // install.sh boots with --mem-budget-mb 110000. Armed before + // the model open so the L1 gate sees it. + "--mem-budget-mb" => { + mem_budget_mb = Some( + args.next() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| usage()), + ); + } + // DeepSeek V4.1 (ds41) run surface — the engine's own spellings + // (cli_opts.c:258-395). They bind only when the opened model is + // V4.1; the boot below refuses --ctx for V4.1 (the context comes + // only from the metadata) and names the flags when the model is + // not V4.1. --zchain is not ported on this route. + "--engram-dir" => { + v41_flags.engram_dir = Some(args.next().unwrap_or_else(|| usage())); + } + "--v41-no-engram" => v41_flags.no_engram = true, + "--no-dspark" => v41_flags.no_dspark = true, + "--dspark" => v41_flags.dspark = true, + "--dspark-verify" => { + v41_flags.verify_k = args + .next() + .and_then(|v| v.parse().ok()) + .unwrap_or_else(|| usage()); + } + "--no-graph" => v41_flags.no_graph = true, + "--no-kv-reuse" => v41_flags.no_kv_reuse = true, + "--emit-trace" => v41_flags.emit_trace = true, + "--v41-prof" => v41_flags.prof = true, + // Unit Z: the sidecar mount the deployment boots with + // (install.sh: --zchain [--posttrain ]). + "--zchain" => { + v41_flags.zchain = Some(args.next().unwrap_or_else(|| usage())); + } + "--posttrain" => { + v41_flags.posttrain = Some(args.next().unwrap_or_else(|| usage())); + } + "-h" | "--help" => usage(), + other => { + eprintln!("ds4-server-rs: unknown argument {other}"); + std::process::exit(2); + } + } + } + kv.validate().unwrap_or_else(|error| cli_error(&error)); + if mtp_path.is_some() && model_path.is_none() { + cli_error("--mtp requires --model"); + } + if cfg.eos_policy != EosPolicy::Default && model_path.is_none() { + cli_error("EOS policy requires --model"); + } + dist.finish(&mut cfg.listen_host, &mut cfg.listen_port) + .unwrap_or_else(|error| cli_error(&format!("ds4-server-rs: {error}"))); + if cfg.model_name == "ds4" { + cfg.model_name = cfg.model_id.clone(); + } + serve_req.ctx = cfg.ctx; + serve_req.mem_floor_gb = cfg.mem_floor_gb; + serve_req.backend = backend; + serve_req.distribution = match distributed_config(&dist.opt) { + Some(_) => Distribution::Sliced, + None => Distribution::Single, + }; + if let Some(dir) = kv.dir() { + serve_req.kv_disk_dir = Some(dir.display().to_string()); + } + if kv.space_mb() > 0 { + serve_req.kv_disk_space_mb = Some(kv.space_mb()); + } + serve_req.kv_min_tokens = Some(kv.min_tokens()); + + let launch = server_launch(dist.opt.role, model_path.is_some()) + .unwrap_or_else(|error| cli_error(&error)); + launch.configure_serving(&mut serve_req); + // A sliced boot maps only its own layers, so the quote prices that + // interval instead of the whole sharded artifact. + let weight_slice = dist_weight_slice(dist.opt.role, &dist.opt.layers); + + let mut facts = EngineFacts::default(); + let mut kv_store = None; + if kv.dir().is_some() { + match kv.open() { + Some(store) => { + facts.disk_ready = Some(true); + kv_store = Some(store); + } + None => facts.disk_ready = Some(false), + } + } + let ident = model_path + .as_deref() + .and_then(|path| identify_gguf(std::path::Path::new(path)).ok()); + // A family whose context contract is its native maximum follows it + // unless --ctx was given (GLM VQ: 1,048,576 tokens). + if let Some(ctx) = ident + .as_ref() + .filter(|_| !ctx_set) + .and_then(|id| ds4_core::native_ctx_default(caps_from_ident(id))) + { + cfg.ctx = ctx as i32; + serve_req.ctx = cfg.ctx; + } + // V4.1 (ds41): the context comes only from the metadata + // (core_validate_v41.c:51-53) — refuse --ctx before the model load, not + // after it (the engine refuses it at parse time, cli_opts.c:285-293). + if ctx_set + && ident + .as_ref() + .is_some_and(|id| id.shape.variant == ds4_core::Variant::DeepSeek41Flash) + { + cli_error("ds4-server-rs: V4.1 context comes from the model metadata (deepseek4.context_length); --ctx is refused"); + } + let v41_wire = ident + .as_ref() + .filter(|id| id.shape.variant == ds4_core::Variant::DeepSeek41Flash) + .map(|id| { + let path = model_path.as_deref().unwrap(); + let gguf = GgufFile::open(Path::new(path)) + .unwrap_or_else(|error| cli_error(&format!("V4.1 GGUF: {error}"))); + let mut wire = ds4_core::V41Wire::load(&gguf, &id.shape) + .unwrap_or_else(|error| cli_error(&format!("V4.1 metadata: {error}"))); + let draft_block = wire + .mtp_armed() + .then_some(wire.mtp_block) + .flatten() + .filter(|block| *block > 0); + let native_default = if std::env::var_os("DS41_NO_DSPARK").is_some() { + MtpMode::Off + } else { + MtpMode::Auto + }; + v41_request(&v41_flags, &mut serve_req, draft_block, native_default) + .unwrap_or_else(|error| cli_error(&format!("ds4-server-rs: {error}"))); + if let Some(dir) = v41_flags.engram_dir.as_deref() { + wire.apply_engram_dir(dir); + } + cfg.ctx = wire.ctx as i32; + serve_req.ctx = cfg.ctx; + wire + }); + // Auto admission needs the requested workload before any cache is sized. + let mut preflight_options = model_options.clone(); + preflight_options.push(ModelOpenOption::ServingBudget(serve_req.clone())); + ds4_core::check_ssd_options( + &preflight_options, + ident.as_ref().map(|model| model.shape.family), + backend, + distributed_config(&dist.opt).as_ref(), + ) + .unwrap_or_else(|error| cli_error(&error.message)); + let caps = ident.as_ref().map(caps_from_ident); + if ident + .as_ref() + .is_some_and(|id| id.shape.family == ds4_core::ModelFamily::Glm53) + { + // Tensor/template validation is not qualification of the loaded model. + facts.artifact_qualified = Some(false); + } + if serve_req.ssd_streaming { + let identified = ident + .as_ref() + .unwrap_or_else(|| cli_error("SSD streaming model is not identified")); + let inventory = ds4_core::TensorInventory::open(Path::new(model_path.as_deref().unwrap())) + .unwrap_or_else(|error| cli_error(&format!("SSD inventory: {error}"))); + ds4_core::probe_ssd_quote(&mut facts, &serve_req, identified.shape, &inventory) + .unwrap_or_else(|error| cli_error(&error.message)); + } + let dist_probe = distributed_config(&dist.opt); + if let Some(path) = vision_path.as_deref() { + // The same rules the open applies: only a full GLM-5.3 or Step CUDA + // model takes an encoder, and then the artifact itself is opened. + if let Some(id) = ident.as_ref() { + facts.vision_path_ok = Some( + match probe_vision_sidecar(id.shape, backend, dist_probe.as_ref(), path) { + Ok(()) => true, + Err(error) => { + eprintln!("ds4-server-rs: --vision {path}: {error}"); + false + } + }, + ); + } + } + // The open still consumes this fallback, and only DeepSeek accepts a + // drafter at all, so the check has to look at it. + let dspark_path = std::env::var("DS4_DSPARK_MODEL") + .ok() + .filter(|path| !path.is_empty()); + if let (Some(id), Some(path)) = (ident.as_ref(), dspark_path.as_deref()) { + facts.dspark_ok = Some( + match probe_dspark_sidecar(id.shape, dist_probe.as_ref(), path) { + Ok(()) => true, + Err(error) => { + eprintln!("ds4-server-rs: DS4_DSPARK_MODEL {path}: {error}"); + false + } + }, + ); + } + if let Some(path) = mtp_path.as_deref() { + // The same attach the open performs: family acceptance, sidecar + // metadata, required tensors and layouts. A merely readable GGUF + // would let `--check-config` exit 0 on an artifact that cannot load. + if let Some(id) = ident.as_ref() { + facts.mtp_path_ok = Some( + match probe_mtp_sidecar(id.shape, dist_probe.as_ref(), path) { + Ok(()) => true, + Err(error) => { + eprintln!("ds4-server-rs: --mtp {path}: {error}"); + false + } + }, + ); + } + } + if serve_req.check_config { + // Nothing else opens the model on this path, so the check has to do + // the open's own pre-device validation itself. + if let Some(path) = model_path.as_deref() { + // MTP parts serve only with --mtp-mode on (auto keeps unqualified + // drafts off), so check what that open will map. + let parts = if serve_req.mtp_mode == MtpMode::On { + ProbeParts::WithMtp + } else { + ProbeParts::Trunk + }; + facts.artifact_ok = Some(match probe_model_artifact(path, parts) { + Ok(()) => true, + Err(error) => { + eprintln!("ds4-server-rs: -m {path}: {error}"); + false + } + }); + if cfg.eos_policy != EosPolicy::Default && facts.artifact_ok == Some(true) { + let family = ident + .as_ref() + .unwrap_or_else(|| cli_error("EOS policy requires an identified model")) + .shape + .family; + // A GVQ1 release directory keeps its tables beside the parts. + let vocab = Vocab::load_path(Path::new(path), family) + .unwrap_or_else(|error| cli_error(&format!("EOS policy vocab: {error}"))); + if vocab.eos_id < 0 { + cli_error("EOS policy requires a model EOS token"); + } + } + } + } + apply_host_quote( + &mut facts, + &serve_req, + caps, + ident.as_ref(), + model_path.as_deref(), + mtp_path.as_deref(), + vision_path.as_deref(), + dspark_path.as_deref(), + weight_slice, + vision_path.is_some(), + false, + ); + let plan = resolve_plan(&serve_req, caps, &facts); + plan.apply_env(); + cfg.adopt_plan(&plan); + eprint!("{}", plan.report()); + if serve_req.check_config { + if let Some(expected) = &expected_plan { + expected + .check_preflight(&plan) + .unwrap_or_else(|error| cli_error(&error)); + } + println!("{}", plan.to_json()); + std::process::exit(if plan.may_listen() { 0 } else { 2 }); + } + if !plan.may_listen() { + eprint!("{}", plan.report()); + cli_error("ds4-server-rs: serving plan rejected unsupported options"); + } + // The engine allocates a speculative runtime only above the family's + // draft minimum, so an unspecified draft takes the resolved one. + if let Some(draft) = ds4_core::open_draft_tokens( + plan.effective.mtp_mode, + serve_req.mtp_draft, + plan.effective.mtp_draft, + ) { + model_options.push(ModelOpenOption::MtpDraftTokens(draft)); + } + model_options.push(ModelOpenOption::ServingBudget(serve_req.clone())); + // A family whose caps refuse banks has no multi-sequence graph to open; + // asking native for one only reports the refusal. Bonsai (qwen35) is the + // resident case: its session is the trunk state itself. + let banks_available = plan + .caps + .is_some_and(|caps| caps.bank_support != Support::None); + let cont_width = + if serve_req.max_seqs == MaxSeqs::Off || plan.uses_serial_mtp() || !banks_available { + 0 + } else { + plan.effective.max_seqs as i32 + }; + + let native_dist = distributed_config(&dist.opt); + // Snapshot every artifact before native open, then recheck before enabling + // disk reads. Equal prompt tokens alone do not establish equal model state. + let cache_identity = model_path.as_deref().filter(|_| kv_store.is_some()).map(|path| { + let sidecars: Vec<_> = [mtp_path.as_deref(), vision_path.as_deref(), dspark_path.as_deref()] + .into_iter().flatten().map(Path::new).collect(); + let sources = engram_sources( + v41_wire.as_ref().map_or(&[], |wire| wire.engram_table_path.as_slice()), + &v41_flags, + ); + let settings = format!("backend={backend:?};threads={n_threads};options={model_options:?};dist={native_dist:?};v41={v41_flags:?}"); + CacheIdentity::capture_with_sources(Path::new(path), &sidecars, &sources, &settings) + .unwrap_or_else(|error| cli_error(&format!("disk KV identity: {error}"))) + }); + // --mem-budget-mb (unit Z) arms the engine's L1 load gate; it must be + // set before the model open reads it (core_util.c:238-256). + if let Some(mb) = mem_budget_mb { + ds4_core::set_mem_budget_mb(mb); + } + let mut v41_route: Option = None; + let model = match model_path.as_deref() { + Some(path) => { + let opened = match native_dist.as_ref() { + Some(config) => Model::open_distributed_options( + path, + backend, + n_threads, + true, + mtp_path.as_deref(), + None, + config, + &model_options, + ), + None => Model::open_with_support_options( + path, + backend, + n_threads, + true, + mtp_path.as_deref(), + None, + &model_options, + ), + }; + match opened { + Ok(m) => { + if cfg.eos_policy != EosPolicy::Default && m.vocab().eos_id < 0 { + cli_error("EOS policy requires a model EOS token"); + } + // V4.1 (ds41): no ds4_session exists for the family, so + // the serving path is the engine's push route + // (server_generate_v41.c) and the context comes only + // from the metadata (core_validate_v41.c:51-53) — + // --ctx is refused, never silently ignored. + if let Some(ctx) = m.v41_ctx() { + if ctx_set { + cli_error("ds4-server-rs: V4.1 context comes from the model metadata (deepseek4.context_length); --ctx is refused"); + } + cfg.ctx = ctx as i32; + serve_req.ctx = cfg.ctx; + // Unit H: the engine's request-omitted parse defaults + // (thinking off unless asked, effort HIGH, min_p 0.0) + // apply to the engine's own model set -- exactly this + // branch's model. + cfg.engine_defaults = true; + v41_route = Some(resolve_v41_route( + path, + &v41_flags, + &plan, + vision_path.as_deref(), + )); + eprintln!( + "ds4-server-rs: V4.1 route: ctx={} MTP={} verify={} prefix_reuse={}", + cfg.ctx, + plan.effective.mtp_mode.as_str(), + v41_flags.verify_k, + plan.effective.prefix_reuse.as_str(), + ); + } else if v41_flags.given() { + eprintln!( + "ds4-server-rs: warning: V4.1 flags given but the model is not V4.1; they are ignored" + ); + } + cfg.max_effort_default = matches!( + ds4_server::syntax_for_model_id(m.model_id()), + ds4_server::ModelSyntax::Glm53 | ds4_server::ModelSyntax::BeamVq + ); + cfg.have_engine = true; + Some(m) + } + Err(e) => { + eprintln!("ds4-server-rs: open {path}: {e}"); + std::process::exit(1); + } + } + } + None => None, + }; + let mut kv_store = if model.is_some() { kv_store } else { None }; + + let lane = if let Some(ref model) = model { + // What only the open engine knows. The refit re-resolves so a + // fitted-down width or a refused lane cannot stay silently claimed. + let mut opened = EngineFacts { + drafter_shared: Some(model.drafter_shared()), + mtp_loaded: mtp_path.is_some() || model.mtp().is_some(), + vision_loaded: model_options + .iter() + .any(|opt| matches!(opt, ModelOpenOption::Vision(_))), + ..facts.clone() + }; + model + .ssd_quote(&mut opened) + .unwrap_or_else(|error| cli_error(&error.message)); + if cont_width > 0 && backend == Backend::Cuda { + match model.batch_ctx_fit( + cfg.ctx, + cont_width, + plan.batch_max_total_tokens(cfg.ctx, cont_width), + ) { + Ok(batch) => { + eprintln!( + "ds4-server-rs: continuous lane ready (width={} seq_cap={})", + batch.max_seq(), + batch.seq_cap() + ); + let mut facts = EngineFacts { + banks_fitted: Some(batch.max_seq() as u32), + seq_cap: Some(batch.seq_cap() as u32), + cont_lane: Some(true), + partial_reuse: Some(batch.supports_partial_reuse()), + ..opened + }; + let vision = vision_path.is_some() || facts.vision_loaded; + apply_host_quote( + &mut facts, + &serve_req, + caps, + ident.as_ref(), + model_path.as_deref(), + mtp_path.as_deref(), + vision_path.as_deref(), + dspark_path.as_deref(), + weight_slice, + vision, + true, + ); + let fitted = resolve_plan(&serve_req, caps, &facts); + eprint!("{}", fitted.report()); + if !fitted.may_listen() { + cli_error("ds4-server-rs: fitted serving plan rejected"); + } + fitted.apply_env(); + let fitted_reuse = fitted.effective.prefix_reuse; + cfg.adopt_plan(&fitted); + Some( + ContLane::new( + batch, + model.vocab(), + model.model_id(), + model.routed_quant_bits(), + cfg.ctx, + model.token_eos(), + ) + .with_template(model.chat_template()) + .with_prefix_reuse(fitted_reuse) + .with_tool_memory(tool_memory), + ) + } + Err(e) => { + eprintln!("ds4-server-rs: continuous lane unavailable ({e}); serial only"); + // cont_lane=false also limits resident credit to model + // mappings: the failed batch left no live runtime. + let mut facts = EngineFacts { + banks_fitted: Some(1), + cont_lane: Some(false), + partial_reuse: Some(false), + ..opened + }; + let vision = vision_path.is_some() || facts.vision_loaded; + apply_host_quote( + &mut facts, + &serve_req, + caps, + ident.as_ref(), + model_path.as_deref(), + mtp_path.as_deref(), + vision_path.as_deref(), + dspark_path.as_deref(), + weight_slice, + vision, + true, + ); + let serial = resolve_plan(&serve_req, caps, &facts); + eprint!("{}", serial.report()); + if !serial.may_listen() { + cli_error("ds4-server-rs: serial fallback plan rejected"); + } + serial.apply_env(); + cfg.adopt_plan(&serial); + None + } + } + } else { + // Serial HTTP and distributed workers also need confirmed IPC + // ownership after open; their session graphs are still lazy. + let mut facts = EngineFacts { + banks_fitted: Some(1), + cont_lane: Some(false), + partial_reuse: Some(false), + ..opened + }; + let vision = vision_path.is_some() || facts.vision_loaded; + apply_host_quote( + &mut facts, + &serve_req, + caps, + ident.as_ref(), + model_path.as_deref(), + mtp_path.as_deref(), + vision_path.as_deref(), + dspark_path.as_deref(), + weight_slice, + vision, + true, + ); + let serial = resolve_plan(&serve_req, caps, &facts); + eprint!("{}", serial.report()); + if !serial.may_listen() { + cli_error("ds4-server-rs: opened serial plan rejected"); + } + serial.apply_env(); + cfg.adopt_plan(&serial); + None + } + } else { + None + }; + if let Some(expected) = &expected_plan { + expected + .check(cfg.serving_plan.as_ref().unwrap_or(&plan)) + .unwrap_or_else(|error| cli_error(&error)); + } + if launch == ServerLaunch::Worker { + let Some(ref model) = model else { + cli_error(WORKER_REQUIRES_MODEL); + }; + if serve_req.print_plan { + // A worker never fits a lane, so this is the whole plan it has. + print_plan(&cfg); + } + model.boot_prewarm(); + match run_assembled_worker(model, cfg.ctx, &dist.opt) { + Ok(rc) => std::process::exit(rc), + Err(e) => { + eprintln!("ds4-server-rs: {e}"); + std::process::exit(1); + } + } + } + // Print what serves, not what was asked: the native fit can still take + // banks, partial reuse and MTP away from the pre-open plan. + if serve_req.print_plan { + print_plan(&cfg); + } + if let Some(ref model) = model { + model.boot_prewarm(); + // KU-2: the V4.1 route pays its asset opens (sidecar mount, engram + // feed) inside request one unless they are paid here. A failure must + // not stop serving: the request path still opens them itself. + if let Some(route) = &v41_route { + if let Err(error) = model.v41_prewarm( + &route.model_path, + route.zchain.as_deref(), + route.posttrain.as_deref(), + route.engram_dir.as_deref(), + route.no_engram, + ) { + eprintln!("ds4-server-rs: V4.1 prewarm skipped: {error}"); + } + } + } + if let (Some(identity), Some(store)) = (cache_identity, kv_store.as_mut()) { + let effective = &cfg.serving_plan.as_ref().unwrap_or(&plan).effective; + let settings = format!( + "ctx={};banks={};native={:?};schedule={}/{};mtp={:?}/{}/{:?}", + effective.ctx, + effective.max_seqs, + effective.native_chunk, + effective.sched_chunk, + effective.sched_chunk_live, + effective.mtp_mode, + effective.mtp_weights, + effective.mtp_draft + ); + let digest = identity + .finish(&settings) + .unwrap_or_else(|error| cli_error(&format!("disk KV identity: {error}"))); + store.bind_identity(digest); + let hex: String = digest.iter().map(|byte| format!("{byte:02x}")).collect(); + eprintln!("disk KV identity: local-file-stat-v1 {hex} (file metadata, not full-content attestation)"); + } + + if !ds4_sys::install_stop_handlers() { + eprintln!("ds4-server-rs: failed to install stop handlers"); + std::process::exit(1); + } + cfg.stop_requested = Some(ds4_sys::stop_requested); + + let serving = cfg.serving_plan.as_ref().unwrap_or(&plan); + let listener = match listen_if_allowed(&cfg, serving) { + Ok(Some(listener)) => listener, + Ok(None) => cli_error("ds4-server-rs: serving plan rejected unsupported options"), + Err(e) => { + eprintln!( + "ds4-server-rs: listen {}:{}: {e}", + cfg.listen_host, cfg.listen_port + ); + std::process::exit(1); + } + }; + eprintln!( + "ds4-server-rs: listening on {}:{} model_id={} engine={} host_vocab={} (host continuation registry + incremental live DSML tool stream + corrective retry)", + cfg.listen_host, + cfg.listen_port, + cfg.model_id, + if cfg.have_engine { "open" } else { "none" }, + if model.is_some() { "yes" } else { "no" } + ); + + if let Some(ref model) = model { + // The fitted plan, not the pre-open one: the refit can downgrade + // reuse after the runtime says what it has. + let reuse = cfg + .serving_plan + .as_ref() + .map_or(plan.effective.prefix_reuse, |p| p.effective.prefix_reuse); + let mut engine = NativeDecode::new(model, cfg.ctx) + .with_vocab(model.vocab()) + .with_prefix_reuse(reuse) + .with_tool_memory(tool_memory); + if let Some(route) = v41_route.take() { + engine = engine.with_v41_route(route); + } + if let Some(store) = kv_store { + engine = engine.with_store(store); + } + match lane { + Some(mut lane) => accept_loop_with_engine_cont(listener, cfg, &mut engine, &mut lane), + None => accept_loop_with_engine(listener, cfg, &mut engine), + } + } else { + accept_loop(listener, cfg); + } +} + +/// A scheduler yield of zero is not a chunk, and the engine refuses a +/// nonpositive MTP draft: either would let `--check-config` approve a boot +/// failure. +fn positive_chunk(flag: &str, raw: Option) -> u32 { + u32::try_from(positive_count(flag, raw)).unwrap_or_else(|_| { + cli_error(&format!( + "ds4-server-rs: {flag} wants a positive token count" + )) + }) +} + +/// C `open_tuning` accepts 0 through 1000; anything else, NaN included, +/// aborts the open, so `--check-config` must not approve it. +fn margin(flag: &str, raw: Option) -> f32 { + match raw + .and_then(|v| v.parse::().ok()) + .filter(|m| (0.0..=1000.0).contains(m)) + { + Some(m) => m, + None => cli_error(&format!("ds4-server-rs: {flag} wants 0 to 1000")), + } +} + +fn positive_count(flag: &str, raw: Option) -> i32 { + match raw.and_then(|v| v.parse::().ok()).filter(|n| *n > 0) { + Some(n) => n, + None => cli_error(&format!("ds4-server-rs: {flag} wants a positive count")), + } +} + +fn print_plan(cfg: &ServerConfig) { + if let Some(plan) = cfg.serving_plan.as_ref() { + println!("{}", plan.to_json()); + } +} + +fn apply_host_quote( + facts: &mut EngineFacts, + req: &ServingRequest, + caps: Option, + ident: Option<&Identified>, + model_path: Option<&str>, + mtp_path: Option<&str>, + vision_path: Option<&str>, + dspark_path: Option<&str>, + slice: Option, + vision: bool, + resident: bool, +) { + let Some(caps) = caps else { + return; + }; + attach_host_quote( + facts, + req, + caps, + ident.map(|id| id.shape), + model_path.map(Path::new), + mtp_path.map(Path::new), + vision_path.map(Path::new), + dspark_path.map(Path::new), + ident.map(|id| id.split_count).unwrap_or(1), + slice, + vision, + resident, + ); +} + +fn cli_error(message: &str) -> ! { + eprintln!("{message}"); + std::process::exit(2); +} + +/// The DeepSeek V4.1 run switches the server accepts (the engine's +/// cli_opts.c:258-395 spellings). They bind only when the opened model is +/// V4.1; otherwise the boot warns and ignores them. +#[derive(Default, Debug)] +struct V41Flags { + engram_dir: Option, + no_engram: bool, + no_dspark: bool, + dspark: bool, + verify_k: i32, + no_graph: bool, + emit_trace: bool, + prof: bool, + no_kv_reuse: bool, + zchain: Option, + posttrain: Option, +} + +impl V41Flags { + fn given(&self) -> bool { + self.engram_dir.is_some() + || self.no_engram + || self.no_dspark + || self.dspark + || self.verify_k != 0 + || self.no_graph + || self.emit_trace + || self.prof + || self.no_kv_reuse + || self.zchain.is_some() + || self.posttrain.is_some() + } +} + +fn resolve_v41_route( + path: &str, + flags: &V41Flags, + plan: &ds4_core::ResolvedPlan, + vision_path: Option<&str>, +) -> ds4_server::V41ServeRoute { + ds4_server::V41ServeRoute { + model_path: std::path::PathBuf::from(path), + engram_dir: flags.engram_dir.clone(), + no_engram: flags.no_engram, + dspark: Some(match plan.effective.mtp_mode { + MtpMode::Off => 0, + MtpMode::Auto => 1, + MtpMode::On => 2, + }), + graph: flags.no_graph.then_some(false), + verify_k: flags.verify_k, + emit_trace: flags.emit_trace, + prof: flags.prof, + no_kv_reuse: plan.effective.prefix_reuse == ds4_core::ReuseKind::None, + zchain: flags.zchain.clone(), + posttrain: flags.posttrain.clone(), + // The V4.1 tower geometry + placeholder id, read from the sidecar the + // probe already validated; the request expands the placeholder with it. + vision: vision_path.map(|p| { + ds4_core::v41_media::V41VisionMeta::read(p) + .unwrap_or_else(|error| cli_error(&format!("--vision-gguf: {error}"))) + }), + } +} + +fn v41_request( + flags: &V41Flags, + req: &mut ServingRequest, + draft_block: Option, + native_default: MtpMode, +) -> Result<(), String> { + if flags.no_dspark && flags.dspark { + return Err("--no-dspark and --dspark are mutually exclusive".into()); + } + if req.mtp_draft.is_some() { + // The artifact fixes the draft block; verify_k controls verification, + // not allocation or the number of tokens the towers draft. + return Err("V4.1 --mtp-draft cannot change the artifact's draft block; use --dspark-verify to pin verification".into()); + } + if req.mtp_path.is_some() { + return Err("V4.1 uses embedded MTP towers; --mtp sidecars are unsupported".into()); + } + if (flags.no_dspark && req.mtp_mode == MtpMode::On) + || (flags.dspark && req.mtp_mode == MtpMode::Off) + { + return Err("V4.1 DSpark alias conflicts with --mtp-mode".into()); + } + if flags.no_kv_reuse && !matches!(req.prefix_reuse, PrefixReuse::Auto | PrefixReuse::Off) { + return Err("--no-kv-reuse conflicts with --prefix-reuse".into()); + } + if flags.verify_k < 0 { + return Err("--dspark-verify wants zero (automatic) or a positive count".into()); + } + if flags.no_dspark { + req.mtp_mode = MtpMode::Off; + } else if flags.dspark { + req.mtp_mode = MtpMode::On; + } + if req.mtp_mode == MtpMode::Auto && native_default == MtpMode::Off { + req.mtp_mode = MtpMode::Off; + } + if flags.no_kv_reuse { + req.prefix_reuse = PrefixReuse::Off; + } + if draft_block.is_none() { + if req.mtp_mode == MtpMode::On { + return Err("V4.1 artifact has no armed embedded MTP towers".into()); + } + req.mtp_mode = MtpMode::Off; + } + if flags.verify_k == 0 { + return Ok(()); + } + if req.mtp_mode == MtpMode::Off { + return Err("--dspark-verify requires enabled MTP".into()); + } + if flags.verify_k as u32 > draft_block.unwrap_or(0) { + return Err("--dspark-verify exceeds the artifact's draft block".into()); + } + Ok(()) +} + +fn engram_sources<'a>(paths: &'a [String], flags: &V41Flags) -> Vec<&'a Path> { + if flags.no_engram { + return Vec::new(); + } + paths.iter().map(Path::new).collect() +} + +#[cfg(test)] +mod v41_controls { + use super::*; + use ds4_core::{ModelFamily, ReuseKind, Variant}; + + fn plan(req: &ServingRequest) -> ds4_core::ResolvedPlan { + let caps = ds4_core::serving_caps(ModelFamily::DeepSeek4, Variant::DeepSeek41Flash); + resolve_plan(req, Some(caps), &EngineFacts::default()) + } + + #[test] + fn engram_selection_identity() { + let paths = vec![ + "/metadata/shard-a.safetensors".into(), + "/override/shard-b.safetensors".into(), + ]; + assert_eq!( + engram_sources(&paths, &V41Flags::default()), + vec![Path::new(&paths[0]), Path::new(&paths[1])] + ); + let flags = V41Flags { + no_engram: true, + ..V41Flags::default() + }; + assert!(engram_sources(&paths, &flags).is_empty()); + assert!(format!("{flags:?}").contains("no_engram: true")); + } + + #[test] + fn common_modes_reach_route() { + for (mode, native) in [(MtpMode::Off, 0), (MtpMode::On, 2)] { + let req = ServingRequest { + mtp_mode: mode, + ..ServingRequest::default() + }; + let plan = plan(&req); + assert!(!plan.has_errors()); + assert_eq!( + resolve_v41_route("model.gguf", &V41Flags::default(), &plan, None).dspark, + Some(native) + ); + } + } + + #[test] + fn reuse_off_reaches_route() { + let req = ServingRequest { + prefix_reuse: PrefixReuse::Off, + ..ServingRequest::default() + }; + let plan = plan(&req); + assert_eq!(plan.effective.prefix_reuse, ReuseKind::None); + assert!(resolve_v41_route("model.gguf", &V41Flags::default(), &plan, None).no_kv_reuse); + } + + #[test] + fn aliases_match_report() { + let mut req = ServingRequest::default(); + let flags = V41Flags { + no_dspark: true, + no_kv_reuse: true, + ..V41Flags::default() + }; + v41_request(&flags, &mut req, Some(3), MtpMode::Auto).unwrap(); + assert_eq!(req.mtp_mode, MtpMode::Off); + assert_eq!(req.prefix_reuse, PrefixReuse::Off); + let resolved = plan(&req); + let route = resolve_v41_route("model.gguf", &flags, &resolved, None); + assert_eq!(route.dspark, Some(0)); + assert!(route.no_kv_reuse); + assert_eq!(resolved.effective.mtp_mode, MtpMode::Off); + } + + #[test] + fn defaults_keep_native_reuse() { + let req = ServingRequest::default(); + let plan = plan(&req); + assert_eq!(plan.effective.prefix_reuse, ReuseKind::Exact); + assert_eq!(plan.effective.mtp_mode, MtpMode::Auto); + assert_eq!(plan.effective.mtp_draft, None); + assert_eq!( + resolve_v41_route("model.gguf", &V41Flags::default(), &plan, None).dspark, + Some(1) + ); + assert!(!resolve_v41_route("model.gguf", &V41Flags::default(), &plan, None).no_kv_reuse); + } + + #[test] + fn reject_discarded_draft() { + let mut req = ServingRequest { + mtp_draft: Some(2), + ..ServingRequest::default() + }; + assert!(v41_request(&V41Flags::default(), &mut req, Some(3), MtpMode::Auto).is_err()); + } + + #[test] + fn reject_conflicting_controls() { + let flags = V41Flags { + dspark: true, + ..V41Flags::default() + }; + let mut req = ServingRequest { + mtp_mode: MtpMode::Off, + ..ServingRequest::default() + }; + assert!(v41_request(&flags, &mut req, Some(3), MtpMode::Auto).is_err()); + let flags = V41Flags { + no_dspark: true, + ..V41Flags::default() + }; + req.mtp_mode = MtpMode::On; + assert!(v41_request(&flags, &mut req, Some(3), MtpMode::Auto).is_err()); + let flags = V41Flags { + no_kv_reuse: true, + ..V41Flags::default() + }; + req = ServingRequest { + prefix_reuse: PrefixReuse::Exact, + ..ServingRequest::default() + }; + assert!(v41_request(&flags, &mut req, Some(3), MtpMode::Auto).is_err()); + } + + #[test] + fn verify_is_bounded_and_active() { + for verify_k in [-1, 4] { + let flags = V41Flags { + verify_k, + ..V41Flags::default() + }; + assert!(v41_request( + &flags, + &mut ServingRequest::default(), + Some(3), + MtpMode::Auto + ) + .is_err()); + } + let flags = V41Flags { + no_dspark: true, + verify_k: 1, + ..V41Flags::default() + }; + assert!(v41_request( + &flags, + &mut ServingRequest::default(), + Some(3), + MtpMode::Auto + ) + .is_err()); + } + + #[test] + fn exact_reuse_only() { + let mut req = ServingRequest { + prefix_reuse: PrefixReuse::Exact, + ..ServingRequest::default() + }; + assert!(!plan(&req).has_errors()); + req.prefix_reuse = PrefixReuse::Partial; + assert!(plan(&req) + .issues + .iter() + .any(|issue| issue.code == "partial_unsupported")); + } + + #[test] + fn verify_alias_preserves_pin() { + let flags = V41Flags { + dspark: true, + verify_k: 3, + ..V41Flags::default() + }; + let mut req = ServingRequest::default(); + v41_request(&flags, &mut req, Some(3), MtpMode::Off).unwrap(); + let resolved = plan(&req); + let route = resolve_v41_route("model.gguf", &flags, &resolved, None); + assert_eq!(resolved.effective.mtp_mode, MtpMode::On); + assert_eq!(route.dspark, Some(2)); + assert_eq!(route.verify_k, 3); + } + + #[test] + fn reuse_off_disables_disk() { + let req = ServingRequest { + prefix_reuse: PrefixReuse::Off, + kv_disk_dir: Some("/tmp/v41-kv".into()), + ..ServingRequest::default() + }; + assert!(!plan(&req).effective.disk); + } + + #[test] + fn auto_honors_native_disable() { + let mut req = ServingRequest::default(); + v41_request(&V41Flags::default(), &mut req, Some(3), MtpMode::Off).unwrap(); + assert_eq!(req.mtp_mode, MtpMode::Off); + req.mtp_mode = MtpMode::On; + v41_request(&V41Flags::default(), &mut req, Some(3), MtpMode::Off).unwrap(); + assert_eq!(req.mtp_mode, MtpMode::On); + } + + #[test] + fn force_needs_towers() { + let mut req = ServingRequest { + mtp_mode: MtpMode::On, + ..ServingRequest::default() + }; + assert!(v41_request(&V41Flags::default(), &mut req, None, MtpMode::Auto).is_err()); + req.mtp_mode = MtpMode::Auto; + v41_request(&V41Flags::default(), &mut req, None, MtpMode::Auto).unwrap(); + assert_eq!(req.mtp_mode, MtpMode::Off); + } +} + +fn usage() -> ! { + eprintln!( + "usage: ds4-server-rs [--version] [--host HOST] [--port PORT] [--listen HOST PORT] [--model-id ID] [-m GGUF] [--vision GGUF] [--vision-gguf GGUF] [--mtp GGUF] [--mtp-mode off|auto|on] [--backend cuda|cpu|metal|--cuda] [--tokens N|-n N] [-c N] [--max-seqs N|auto] [--prefix-reuse off|exact|partial|auto] [--prefill-chunk N] [--prefill-chunk-live N] [--native-chunk N] [--print-plan] [--check-config] [-t N] [--mtp-draft N] [--mtp-margin N] [--mem-floor-gb N] [--cors]\n\ + [--ssd-streaming] [--ssd-streaming-cache-experts auto|N|GB] [--ssd-streaming-cold]\n\ + [--ignore-eos-in-reasoning] [--ignore-eos]\n\ +Disk KV: [--kv-disk-dir DIR] [--kv-disk-space-mb N] [--kv-disk-space 32G] [--kv-cache-min-tokens N]\n\ + [--kv-cache-cold-max-tokens N] [--kv-cache-continued-interval-tokens N]\n\ + [--kv-cache-boundary-trim-tokens N]\n\ + [--kv-cache-boundary-align-tokens N]\n\ + [--kv-cache-reject-different-quant]\n\ +Tool replay: [--disable-exact-dsml-tool-replay] [--tool-memory-max-ids N]\n\ + [--trace FILE]\n\ + V4.1 (ds41): --zchain DIR [--posttrain DIR] [--engram-dir DIR] [--v41-no-engram]\n\ + [--dspark] [--no-dspark] [--dspark-verify K] [--no-graph] [--no-kv-reuse] [--v41-prof]\n\ + [--emit-trace] [--mem-budget-mb N]\n\ + Distributed: [--role coordinator|worker] [--layers A:B] [--listen HOST PORT] [--coordinator HOST PORT]\n\ + [--dist-prefill-chunk N] [--dist-prefill-window N] [--dist-activation-bits N] [--dist-replay-check] [--debug]" + ); + std::process::exit(2); +} diff --git a/crates/ds4-server/src/cache_identity.rs b/crates/ds4-server/src/cache_identity.rs new file mode 100644 index 0000000000000000000000000000000000000000..5867ff05c5420596f548ddb3300a74b7c20ecfde --- /dev/null +++ b/crates/ds4-server/src/cache_identity.rs @@ -0,0 +1,540 @@ +//! Local file identity for disk KV, not a full weight-content attestation. + +use ds4_core::{identify_gguf, TensorInventory}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::io::Read; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +const SMALL_MAX: u64 = 16 * 1024 * 1024; + +#[derive(Clone)] +struct Input { + path: PathBuf, + contents: bool, + optional: bool, +} + +impl Input { + fn weights(path: PathBuf) -> Self { + Self { + path, + contents: false, + optional: false, + } + } + fn optional(path: PathBuf) -> Self { + Self { + path, + contents: true, + optional: true, + } + } + fn small(path: PathBuf) -> Self { + Self { + path, + contents: true, + optional: false, + } + } +} + +/// Source metadata captured before native open. Rechecked before enabling disk KV. +pub struct CacheIdentity { + inputs: Vec, + before: Value, + settings: String, +} + +impl CacheIdentity { + pub fn capture(model: &Path, sidecars: &[&Path], settings: &str) -> Result { + Self::capture_with_sources(model, sidecars, &[], settings) + } + + /// Bind non-GGUF inference inputs without reading their full contents. + pub fn capture_with_sources( + model: &Path, + sidecars: &[&Path], + sources: &[&Path], + settings: &str, + ) -> Result { + let roots = root_inputs(model, sidecars, sources); + let roots_before = snapshot(&roots)?; + // Engram shards are raw safetensors, not GGUF catalogs. Bind their + // file identity alongside the enumerated model inputs. + let mut inputs: Vec<_> = sources + .iter() + .map(|path| Input::weights((*path).to_owned())) + .collect(); + for path in std::iter::once(model).chain(sidecars.iter().copied()) { + let inventory = TensorInventory::open(path).map_err(|e| e.to_string())?; + inputs.extend( + inventory + .shards + .into_iter() + .map(|shard| Input::weights(shard.path)), + ); + let parent = path.parent().unwrap_or_else(|| Path::new(".")); + inputs.push(Input::optional(parent.join("chat_template.jinja"))); + inputs.push(Input::optional(parent.join("tokenizer_config.json"))); + } + let shape = identify_gguf(model).map_err(|e| e.to_string())?.shape; + let ple = if shape.family == ds4_core::ModelFamily::Qwen4Exp { + Some(add_ple(model, &mut inputs)?) + } else { + None + }; + inputs.sort_by(|a, b| a.path.cmp(&b.path)); + inputs.dedup_by(|a, b| a.path == b.path); + let settings = format!("{settings}\nshape={shape:?}\nenv={:?}", state_env()); + let before = snapshot(&inputs)?; + if snapshot(&roots)? != roots_before + || ple.is_some_and(|(input, expected)| snapshot(&[input]).ok() != Some(expected)) + { + return Err("artifact metadata changed while enumerating identity inputs".into()); + } + Ok(Self { + before, + inputs, + settings, + }) + } + + pub fn finish(self, effective: &str) -> Result<[u8; 32], String> { + if snapshot(&self.inputs)? != self.before { + return Err("model/tokenizer/template inputs changed while opening the model".into()); + } + let runtime = snapshot(&runtime_inputs()?)?; + let settings = format!( + "{}\neffective={effective}\nenv={:?}", + self.settings, + state_env() + ); + Ok(digest( + &json!({"sources": self.before, "runtime": runtime}), + &settings, + )) + } +} + +fn root_inputs(model: &Path, sidecars: &[&Path], sources: &[&Path]) -> Vec { + std::iter::once(model) + .chain(sidecars.iter().copied()) + .chain(sources.iter().copied()) + .map(|path| Input::weights(path.to_owned())) + .collect() +} + +fn digest(files: &Value, settings: &str) -> [u8; 32] { + Sha256::digest( + serde_json::to_vec(&json!({ + "method": "local-file-stat-v1", "files": files, "settings": settings, + })) + .expect("identity JSON is serializable"), + ) + .into() +} + +fn stamp(metadata: &fs::Metadata) -> Result { + #[cfg(unix)] + return Ok(json!({"device": metadata.dev(), "inode": metadata.ino(), + "bytes": metadata.len(), "mtime": [metadata.mtime(), metadata.mtime_nsec()], + "ctime": [metadata.ctime(), metadata.ctime_nsec()]})); + #[cfg(not(unix))] + { + let _ = metadata; + Err("local-file-stat-v1 requires Unix file identity".into()) + } +} + +fn snapshot(inputs: &[Input]) -> Result { + let mut out = Vec::new(); + for input in inputs { + let mut file = match fs::File::open(&input.path) { + Ok(file) => file, + Err(e) if input.optional && e.kind() == std::io::ErrorKind::NotFound => { + out.push(json!({"path": input.path, "missing": true})); + continue; + } + Err(e) => return Err(format!("{}: {e}", input.path.display())), + }; + let before = stamp(&file.metadata().map_err(|e| e.to_string())?)?; + let path = input.path.canonicalize().map_err(|e| e.to_string())?; + let hash = if input.contents { + let mut bytes = Vec::new(); + (&mut file) + .take(SMALL_MAX + 1) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if bytes.len() as u64 > SMALL_MAX { + return Err(format!( + "identity sidecar exceeds 16 MiB: {}", + path.display() + )); + } + Some( + Sha256::digest(&bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect::(), + ) + } else { + None + }; + let after = stamp(&file.metadata().map_err(|e| e.to_string())?)?; + if before != after + || before != stamp(&fs::metadata(&input.path).map_err(|e| e.to_string())?)? + { + return Err(format!( + "identity input changed while reading: {}", + path.display() + )); + } + out.push(json!({"path": path, "stat": before, "contents_sha256": hash})); + } + Ok(Value::Array(out)) +} + +fn state_env() -> BTreeMap> { + std::env::vars_os() + .filter_map(|(key, value)| { + Some((key.into_string().ok()?, value.as_encoded_bytes().to_vec())) + }) + .filter(|(key, _)| binds_env(key)) + .collect() +} + +fn binds_env(key: &str) -> bool { + let native = key.starts_with("DS4_") + || key.starts_with("DS41_") + || matches!(key, "CUDA_VISIBLE_DEVICES" | "NVIDIA_TF32_OVERRIDE"); + if !native { + return false; + } + + // Bind compute/KV/RoPE controls. Omit deployment, normalized scheduling + // and output destinations, which do not alter the saved state. + !key.starts_with("DS4_CUDA_WEIGHT_IPC") + && !key.starts_with("DS4_SERVER_") + && !key.starts_with("DS4_MEM") + && !matches!( + key, + "DS4_CONT_PREFILL_CHUNK" + | "DS4_CONT_PREFILL_CHUNK_LIVE" + | "DS4_SESSION_LAZY_GRAPH" + | "DS4_SESSION_GRAPH_FIT" + | "DS4_NO_BOOT_PREWARM" + | "DS4_BOOT_PREWARM" + | "DS4_GOV_TRACE" + | "DS41_GEN_LOGITS" + ) +} + +fn runtime_inputs() -> Result, String> { + let mut paths = BTreeSet::new(); + #[cfg(target_os = "linux")] + { + paths.insert(PathBuf::from("/proc/self/exe")); + // Native inference is linked into the executable. Include mapped + // shared libraries as well, including dynamically opened CUDA libraries. + let maps = fs::read_to_string("/proc/self/maps").map_err(|e| e.to_string())?; + for line in maps.lines() { + let name = line + .split_whitespace() + .skip(5) + .collect::>() + .join(" "); + if name.starts_with('/') && name.contains(".so") { + paths.insert(PathBuf::from(name)); + } + } + } + #[cfg(not(target_os = "linux"))] + { + paths.insert(std::env::current_exe().map_err(|e| e.to_string())?); + } + Ok(paths.into_iter().map(Input::weights).collect()) +} + +fn add_ple(model: &Path, inputs: &mut Vec) -> Result<(Input, Value), String> { + let parent = model.parent().ok_or("model has no parent")?; + let selected = std::env::var_os("DS4_QWEN_PLE_DIR") + .filter(|v| !v.is_empty()) + .map(PathBuf::from); + let path = selected + .as_ref() + .map(|p| p.join("ple-manifest.json")) + .unwrap_or_else(|| parent.join("ple/ple-manifest.json")); + ple_inputs(path, parent, selected.as_deref(), inputs) +} + +fn ple_inputs( + path: PathBuf, + parent: &Path, + selected: Option<&Path>, + inputs: &mut Vec, +) -> Result<(Input, Value), String> { + let manifest_input = Input::small(path.clone()); + let before = snapshot(std::slice::from_ref(&manifest_input))?; + let mut raw = Vec::new(); + fs::File::open(&path) + .map_err(|e| e.to_string())? + .take(SMALL_MAX + 1) + .read_to_end(&mut raw) + .map_err(|e| e.to_string())?; + if raw.len() as u64 > SMALL_MAX { + return Err("PLE manifest exceeds 16 MiB".into()); + } + let manifest: Value = serde_json::from_slice(&raw).map_err(|e| e.to_string())?; + let fp8 = manifest["format_version"].as_u64() == Some(2); + let root = if fp8 { + path.parent().ok_or("PLE manifest has no parent")? + } else { + selected.unwrap_or(parent) + }; + for part in manifest["logical_parts"] + .as_array() + .ok_or("PLE logical parts missing")? + { + let file = part["physical_file"] + .as_str() + .ok_or("PLE physical file missing")?; + inputs.push(Input::weights(root.join(file))); + } + if fp8 { + let scale = manifest + .pointer("/quantization/scale/path") + .and_then(Value::as_str) + .ok_or("PLE FP8 scale missing")?; + inputs.push(Input::small(root.join(scale))); + } + inputs.push(manifest_input.clone()); + Ok((manifest_input, before)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn v41_dump_keeps_cache() { + let identity = |path: &str| { + let env: BTreeMap<_, _> = [("DS41_GEN_LOGITS", path), ("DS41_ROW_SHIFT", "1")] + .into_iter() + .filter(|(key, _)| binds_env(key)) + .collect(); + digest(&Value::Null, &format!("env={env:?}")) + }; + assert_eq!(identity("a.rows"), identity("b.rows")); + assert!(binds_env("DS41_ROW_SHIFT")); + } + + #[test] + fn engram_sources_bind_identity() { + let dir = std::env::temp_dir().join(format!("ds4-cache-engram-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let model = dir.join("model.gguf"); + let shard = dir.join("engram.safetensors"); + fs::write(&model, b"model").unwrap(); + fs::File::create(&shard) + .unwrap() + .set_len(SMALL_MAX + 1) + .unwrap(); + + let inputs = root_inputs(&model, &[], &[&shard]); + assert_eq!( + inputs.len(), + 2, + "Engram must be an inference identity input" + ); + let before = snapshot(&inputs).unwrap(); + assert_eq!(before, snapshot(&inputs).unwrap()); + let replacement = dir.join("replacement"); + fs::File::create(&replacement) + .unwrap() + .set_len(SMALL_MAX + 1) + .unwrap(); + fs::rename(replacement, &shard).unwrap(); + assert_ne!(before, snapshot(&inputs).unwrap()); + assert_ne!(before, snapshot(&root_inputs(&model, &[], &[])).unwrap()); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn continuous_inference_controls_change_identity() { + for (key, before, after) in [ + ("DS4_CONT_MTP_DEPTH", Some("1"), Some("2")), + ("DS4_CONT_MTP_MODE", Some("0"), Some("2")), + ("DS4_CONT_MTP_BATCH_DRAFT", None, Some("1")), + ("DS4_CONT_DSPARK", Some("0"), Some("1")), + ("DS4_CONT_MTP_DRAFT_PROBE", None, Some("1")), + ("DS41_ROW_SHIFT", None, Some("1")), + ] { + let old = std::env::var_os(key); + let fingerprint = |value: Option<&str>| { + match value { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + digest(&Value::Null, &format!("env={:?}", state_env())) + }; + let before = fingerprint(before); + let after = fingerprint(after); + match old { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + assert_ne!(before, after, "native compute control was omitted: {key}"); + } + } + + #[test] + #[cfg(unix)] + fn non_utf8_env_is_bound() { + use std::os::unix::ffi::OsStringExt; + let key = "DS4_CACHE_TEST"; + let old = std::env::var_os(key); + std::env::set_var(key, std::ffi::OsString::from_vec(vec![0xff, 7])); + let result = std::panic::catch_unwind(state_env); + match old { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + assert!( + result.is_ok(), + "identity must preserve non-UTF8 environment bytes" + ); + assert_eq!(result.unwrap()[key], [0xff, 7]); + } + + #[test] + fn changed_shard_invalidates() { + let dir = std::env::temp_dir().join(format!("ds4-cache-identity-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let a = dir.join("one.gguf"); + let b = dir.join("two.gguf"); + std::fs::write(&a, b"weights-one").unwrap(); + std::fs::write(&b, b"weights-two").unwrap(); + let inputs = vec![Input::weights(a), Input::weights(b.clone())]; + let before = snapshot(&inputs).unwrap(); + assert_eq!(before, snapshot(&inputs).unwrap()); + let replacement = dir.join("replacement"); + std::fs::write(&replacement, b"weights-new").unwrap(); + std::fs::rename(replacement, &b).unwrap(); + assert_ne!(before, snapshot(&inputs).unwrap()); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn template_and_state_are_bound() { + let dir = std::env::temp_dir().join(format!("ds4-cache-template-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let template = dir.join("chat_template.jinja"); + let inputs = vec![Input::optional(template.clone())]; + let missing = snapshot(&inputs).unwrap(); + std::fs::write(&template, "{{ messages }}").unwrap(); + let old = snapshot(&inputs).unwrap(); + assert_ne!(missing, old); + std::fs::write(&template, "{{ tools }}").unwrap(); + let new = snapshot(&inputs).unwrap(); + assert_ne!(old, new); + assert_ne!(digest(&old, "rope=1"), digest(&old, "rope=2")); + assert_ne!(digest(&old, "runtime=1"), digest(&old, "runtime=2")); + assert_ne!(digest(&old, "runtime=1"), digest(&new, "runtime=1")); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn opening_input_change_fails() { + let dir = std::env::temp_dir().join(format!("ds4-cache-open-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let path = dir.join("weights"); + fs::write(&path, b"before").unwrap(); + let inputs = vec![Input::weights(path.clone())]; + let identity = CacheIdentity { + before: snapshot(&inputs).unwrap(), + inputs, + settings: "test".into(), + }; + fs::write(path, b"changed").unwrap(); + assert!(identity + .finish("ctx=2048") + .unwrap_err() + .contains("changed while opening")); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn bounded_sidecar_and_ple() { + let dir = std::env::temp_dir().join(format!("ds4-cache-ple-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let ple = dir.join("ple"); + fs::create_dir_all(&ple).unwrap(); + let part = ple.join("part.bin"); + let scale = ple.join("scale.bin"); + fs::write(&part, b"part").unwrap(); + fs::write(&scale, b"scale").unwrap(); + let path = ple.join("ple-manifest.json"); + fs::write( + &path, + serde_json::to_vec(&json!({"format_version":2, + "logical_parts":[{"physical_file":"part.bin"}], + "quantization":{"scale":{"path":"scale.bin"}}})) + .unwrap(), + ) + .unwrap(); + let mut inputs = Vec::new(); + let (manifest, baseline) = ple_inputs(path, &dir, None, &mut inputs).unwrap(); + assert_eq!(snapshot(&[manifest]).unwrap(), baseline); + assert_eq!(inputs.len(), 3); + let old = snapshot(&inputs).unwrap(); + fs::write(&part, b"PART").unwrap(); + let changed = snapshot(&inputs).unwrap(); + assert_ne!(old, changed); + fs::write(&scale, b"SCALE").unwrap(); + assert_ne!(changed, snapshot(&inputs).unwrap()); + fs::File::create(&scale) + .unwrap() + .set_len(SMALL_MAX + 1) + .unwrap(); + assert!(snapshot(&inputs).unwrap_err().contains("exceeds 16 MiB")); + fs::remove_file(part).unwrap(); + assert!(snapshot(&inputs).is_err()); + fs::remove_dir_all(dir).unwrap(); + } + + #[test] + #[ignore = "metadata-only: set DS4_IDENTITY_MODEL and optional DS4_IDENTITY_MTP"] + fn artifact_restart_identity() { + let model = std::env::var("DS4_IDENTITY_MODEL").unwrap(); + let mtp = std::env::var("DS4_IDENTITY_MTP").ok(); + let sidecars: Vec<_> = mtp.as_deref().map(Path::new).into_iter().collect(); + let first = CacheIdentity::capture(Path::new(&model), &sidecars, "cuda").unwrap(); + let expected: usize = std::env::var("DS4_IDENTITY_SHARDS") + .unwrap() + .parse() + .unwrap(); + assert_eq!( + first + .inputs + .iter() + .filter(|v| v.path.extension().is_some_and(|ext| ext == "gguf")) + .count(), + expected + ); + let first = first.finish("ctx=2048;mtp=on").unwrap(); + let second = CacheIdentity::capture(Path::new(&model), &sidecars, "cuda") + .unwrap() + .finish("ctx=2048;mtp=on") + .unwrap(); + assert_eq!(first, second); + } +} diff --git a/crates/ds4-server/src/chat_history_test.rs b/crates/ds4-server/src/chat_history_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..406521c74868373521155df3b83e5011590e45ba --- /dev/null +++ b/crates/ds4-server/src/chat_history_test.rs @@ -0,0 +1,327 @@ +use super::*; +use crate::parse::{ + ImageMime, ParseEnv, RequestAudio, RequestImage, RequestVideo, ToolCall, ToolChoice, +}; +use crate::tools::ParsedGenerated; +use crate::{parse_anthropic_request, parse_responses_request}; +use std::sync::Arc; + +const CALL_ID: &str = "call_saved"; +const FOREIGN_ID: &str = "call_foreign"; + +fn env() -> ParseEnv { + ParseEnv { + default_model: "inkling-small-mq85gb".into(), + default_effort: ThinkMode::High, + live_ids: vec![CALL_ID.into(), FOREIGN_ID.into()], + ..ParseEnv::default() + } +} + +fn original(api: Api) -> ParsedRequest { + match api { + Api::Responses => parse_responses_request( + &env(), + r#"{ + "instructions":"원래 시스템", + "input":[ + {"role":"user","content":"이전 질문"}, + {"role":"assistant","content":"이전 답"}, + {"role":"user","content":"서울을 찾아 줘."} + ], + "tools":[{"type":"function","name":"lookup","description":"장소 검색", + "parameters":{"type":"object","properties":{"city":{"type":"string"},"nested":{"type":"object"}},"required":["city"]} + }] + }"#, + ) + .unwrap(), + Api::Anthropic => parse_anthropic_request( + &env(), + r#"{ + "system":"원래 시스템", + "messages":[ + {"role":"user","content":"이전 질문"}, + {"role":"assistant","content":"이전 답"}, + {"role":"user","content":"서울을 찾아 줘."} + ], + "tools":[{"name":"lookup","description":"장소 검색", + "input_schema":{"type":"object","properties":{"city":{"type":"string"},"nested":{"type":"object"}},"required":["city"]} + }] + }"#, + ) + .unwrap(), + _ => panic!("unsupported fixture API"), + } +} + +fn generated() -> ParsedGenerated { + ParsedGenerated { + content: "확인하겠습니다.".as_bytes().to_vec(), + reasoning: "검증 후 호출한다.".as_bytes().to_vec(), + calls: vec![ToolCall { + id: CALL_ID.into(), + name: "lookup".into(), + arguments: r#"{"city":"서울","nested":{"z":2,"a":true}}"#.into(), + }], + raw_dsml: String::new(), + raw_tool_text: String::new(), + ok: true, + recovered: false, + } +} + +fn saved_messages() -> Vec { + vec![ + json!({"role":"system","content":"원래 시스템"}), + json!({"role":"user","content":"이전 질문"}), + json!({"role":"assistant","content":"이전 답"}), + json!({"role":"user","content":"서울을 찾아 줘."}), + json!({"role":"assistant","content":"확인하겠습니다.","reasoning_content":"검증 후 호출한다.", + "tool_calls":[{"id":"call_saved","type":"function","function":{"name":"lookup", + "arguments":{"city":"서울","nested":{"z":2,"a":true}} + }}] + }), + ] +} + +fn response_tail() -> ParsedRequest { + parse_responses_request( + &env(), + r#"{"input":[{"type":"function_call_output","call_id":"call_saved","output":"서울: 맑음"}]}"#, + ) + .unwrap() +} + +fn anthropic_tail() -> ParsedRequest { + parse_anthropic_request( + &env(), + r#"{"messages":[{"role":"user","content":[ + {"type":"tool_result","tool_use_id":"call_saved","content":"서울: 맑음"} + ]}]}"#, + ) + .unwrap() +} + +#[test] +fn response_tail_rebuild() { + let captured = original(Api::Responses); + let schemas = captured.tool_schemas.clone(); + let orders = format!("{:?}", captured.tool_orders); + let history = History::capture(captured, &generated()); + let mut next = response_tail(); + + assert!(history.matches(&next)); + assert!(history.restore(&mut next).unwrap()); + + let mut expected = saved_messages(); + expected.push(json!({"role":"tool","tool_call_id":"call_saved","content":"서울: 맑음"})); + assert_eq!(messages(&next).unwrap(), expected); + assert_eq!(next.tool_schemas, schemas); + assert_eq!(format!("{:?}", next.tool_orders), orders); + assert!(next.has_tools); +} + +#[test] +fn anthropic_tail_rebuild() { + let captured = original(Api::Anthropic); + let schemas = captured.tool_schemas.clone(); + let history = History::capture(captured, &generated()); + let mut next = anthropic_tail(); + + assert!(history.matches(&next)); + assert!(history.restore(&mut next).unwrap()); + + let mut expected = saved_messages(); + expected.push(json!({"role":"tool","tool_call_id":"call_saved","content":"서울: 맑음"})); + assert_eq!(messages(&next).unwrap(), expected); + assert_eq!(next.tool_schemas, schemas); +} + +#[test] +fn reject_foreign_live_ids() { + let history = History::capture(original(Api::Responses), &generated()); + let mut next = parse_responses_request( + &env(), + r#"{"input":[ + {"type":"function_call_output","call_id":"call_saved","output":"ours"}, + {"type":"function_call_output","call_id":"call_foreign","output":"unrelated"} + ]}"#, + ) + .unwrap(); + let before = format!("{next:?}"); + + assert!(!history.matches(&next)); + assert!(history.restore(&mut next).is_err()); + assert_eq!(format!("{next:?}"), before); +} + +#[test] +fn retain_caller_history() { + let history = History::capture(original(Api::Responses), &generated()); + let mut next = parse_responses_request( + &env(), + r#"{ + "instructions":"수정한 시스템", + "input":[ + {"role":"user","content":"수정한 질문"}, + {"role":"assistant","content":"수정한 답"}, + {"type":"function_call","call_id":"call_saved","name":"lookup", + "arguments":{"city":"서울","nested":{"z":2,"a":true}}}, + {"type":"function_call_output","call_id":"call_saved","output":"새 결과"} + ], + "tools":[{"type":"function","name":"new_tool","description":"새 도구", + "parameters":{"type":"object","properties":{}}}] + }"#, + ) + .unwrap(); + let schemas = next.tool_schemas.clone(); + let orders = format!("{:?}", next.tool_orders); + + assert!(history.matches(&next)); + history.restore(&mut next).unwrap(); + + assert_eq!( + messages(&next).unwrap(), + vec![ + json!({"role":"system","content":"수정한 시스템"}), + json!({"role":"user","content":"수정한 질문"}), + json!({"role":"assistant","content":"수정한 답","reasoning_content":"검증 후 호출한다.", + "tool_calls":[{"id":"call_saved","type":"function","function":{"name":"lookup", + "arguments":{"city":"서울","nested":{"z":2,"a":true}} + }}] + }), + json!({"role":"tool","tool_call_id":"call_saved","content":"새 결과"}), + ] + ); + assert_eq!(next.tool_schemas, schemas); + assert_eq!(format!("{:?}", next.tool_orders), orders); +} + +#[test] +fn override_system_and_tools() { + let history = History::capture(original(Api::Anthropic), &generated()); + let mut next = parse_anthropic_request( + &env(), + r#"{ + "system":"새 시스템", + "tool_choice":{"type":"none"}, + "messages":[{"role":"user","content":[ + {"type":"tool_result","tool_use_id":"call_saved","content":"서울: 맑음"} + ]}] + }"#, + ) + .unwrap(); + + assert!(history.restore(&mut next).unwrap()); + + let mut expected = saved_messages(); + expected[0] = json!({"role":"system","content":"새 시스템"}); + expected.push(json!({"role":"tool","tool_call_id":"call_saved","content":"서울: 맑음"})); + assert_eq!(messages(&next).unwrap(), expected); + assert_eq!(next.tool_choice, ToolChoice::None); + assert!(next.tool_schemas.is_empty()); + assert!(next.tool_orders.is_empty()); + assert!(!next.has_tools); +} + +fn image(data: &[u8]) -> RequestImage { + RequestImage { + mime: ImageMime::Png, + data: Arc::from(data), + } +} + +fn audio(data: &[u8]) -> RequestAudio { + RequestAudio { + data: Arc::from(data), + } +} + +fn video(data: &[u8]) -> RequestVideo { + RequestVideo { + data: Arc::from(data), + } +} + +#[test] +fn prepend_media_indices() { + let mut captured = original(Api::Anthropic); + captured.images = vec![image(b"old-image-a"), image(b"old-image-b")]; + captured.audios = vec![audio(b"old-audio")]; + captured.videos = vec![video(b"old-video")]; + let old_parts = vec![ + ChatPart::Image(1), + ChatPart::Audio(0), + ChatPart::Image(0), + ChatPart::Video(0), + ]; + captured + .messages + .iter_mut() + .find(|m| m.content == "서울을 찾아 줘.") + .unwrap() + .parts = old_parts.clone(); + let history = History::capture(captured, &generated()); + let mut next = anthropic_tail(); + next.images = vec![image(b"new-image-a"), image(b"new-image-b")]; + next.audios = vec![audio(b"new-audio")]; + next.videos = vec![video(b"new-video")]; + next.messages[0].parts.extend([ + ChatPart::Text("새 미디어".into()), + ChatPart::Image(0), + ChatPart::Audio(0), + ChatPart::Image(1), + ChatPart::Video(0), + ]); + + assert!(history.restore(&mut next).unwrap()); + + assert_eq!( + next.images, + vec![ + image(b"old-image-a"), + image(b"old-image-b"), + image(b"new-image-a"), + image(b"new-image-b") + ] + ); + assert_eq!(next.audios, vec![audio(b"old-audio"), audio(b"new-audio")]); + assert_eq!(next.videos, vec![video(b"old-video"), video(b"new-video")]); + let old = next + .messages + .iter() + .find(|m| m.content == "서울을 찾아 줘.") + .unwrap(); + assert_eq!(old.parts, old_parts); + let new = next + .messages + .iter() + .find(|m| m.tool_call_id == CALL_ID) + .unwrap(); + assert_eq!( + new.parts, + vec![ + ChatPart::ToolResult { + id: CALL_ID.into(), + content: "서울: 맑음".into() + }, + ChatPart::Text("새 미디어".into()), + ChatPart::Image(2), + ChatPart::Audio(1), + ChatPart::Image(3), + ChatPart::Video(1), + ] + ); +} + +#[test] +fn leave_ordinary_request() { + let history = History::capture(original(Api::Responses), &generated()); + let mut next = parse_responses_request(&env(), r#"{"input":"완전히 새로운 질문"}"#).unwrap(); + let before = format!("{next:?}"); + + assert!(next.live_call_ids.is_empty()); + assert!(!history.matches(&next)); + assert!(!history.restore(&mut next).unwrap()); + assert_eq!(format!("{next:?}"), before); +} diff --git a/crates/ds4-server/src/chat_input.rs b/crates/ds4-server/src/chat_input.rs new file mode 100644 index 0000000000000000000000000000000000000000..510a38e3e2274a5cab693d6301bdb0eaee23d4c2 --- /dev/null +++ b/crates/ds4-server/src/chat_input.rs @@ -0,0 +1,351 @@ +//! API normalization for the shared template adapter. This module carries +//! structured messages and tool values; model Jinja owns input grammar. + +use ds4_core::chat_template::{ChatOptions, Template}; +use ds4_core::{ChatThinkMode, Variant}; +use serde_json::{json, Value}; + +use crate::generate::GenerateError; +use crate::parse::{ChatMsg, ChatPart, ParsedRequest}; +use crate::route::{Api, ReqKind, ThinkMode}; + +pub(crate) fn render_model( + template: Option<&Template>, + model_id: i32, + parsed: &ParsedRequest, + system_token: &str, +) -> Result, GenerateError> { + if let Some(template) = template { + return render(template, model_id, parsed); + } + // The C-rendered DSML families ship no chat template — the engine's own + // renderer is their input grammar (server_parse_chat.c:205) — so the base + // V4 family (Flash) and V4.1 (DeepSeek41Flash) take render_prompt directly. + if model_id == Variant::Flash as i32 + || model_id == Variant::DeepSeek41Flash as i32 + || parsed.kind == ReqKind::Completion + { + return crate::generate::render_prompt_ex( + parsed, + model_id, + crate::render::dsml_head(model_id, system_token) + .with_effort(parsed.effort.head_budget()), + ); + } + Err(input_error( + "missing chat_template.jinja or tokenizer.chat_template", + )) +} + +pub fn render( + template: &Template, + model_id: i32, + parsed: &ParsedRequest, +) -> Result, GenerateError> { + if parsed.kind == ReqKind::Completion { + return Ok(parsed.prompt_text.clone().unwrap_or_default().into_bytes()); + } + let mut messages = messages(parsed)?; + if model_id != ds4_core::Variant::IQuestQ1 as i32 { + // IQuest assigns developer its own control token; older families use system. + for message in &mut messages { + if message["role"] == "developer" { + message["role"] = "system".into(); + } + } + } + prepare_image_text(model_id, &mut messages); + let tools = serde_json::Deserializer::from_str(&parsed.tool_schemas) + .into_iter::() + .map(|value| { + value.map(|function| { + let function = match function { + Value::Object(fields) => Value::Object( + fields + .into_iter() + // Responses puts the tool discriminator beside the + // function fields; Jinja receives a nested function. + .filter(|(key, _)| parsed.api != Api::Responses || key != "type") + .map(|(key, value)| { + ( + if key == "input_schema" { + "parameters".into() + } else { + key + }, + value, + ) + }) + .collect(), + ), + other => other, + }; + json!({"type":"function", "function":function}) + }) + }) + .collect::, _>>() + .map_err(input_error)?; + let mode = match parsed.think_mode { + ThinkMode::None => ChatThinkMode::None, + ThinkMode::Low => ChatThinkMode::Low, + ThinkMode::High => ChatThinkMode::High, + ThinkMode::Max => ChatThinkMode::Max, + }; + template + .render_chat(&messages, &tools, ChatOptions::new(model_id, mode)) + .map(String::into_bytes) + .map_err(input_error) +} + +fn prepare_image_text(model_id: i32, messages: &mut [Value]) { + use crate::render::{ + syntax_for_model_id, ModelSyntax, GLM_IMAGE, GLM_VISION_END, GLM_VISION_START, + }; + + if syntax_for_model_id(model_id) != ModelSyntax::Glm53 { + return; + } + // GLM's text template does not consume media arrays. The image processor + // supplies placeholders; native preparation later expands their token spans. + for message in messages { + let Some(parts) = message["content"].as_array() else { + continue; + }; + let mut text = String::new(); + for part in parts { + if part["type"] == "image" { + text.push_str(GLM_VISION_START); + text.push_str(GLM_IMAGE); + text.push_str(GLM_VISION_END); + } else if let Some(value) = part["text"].as_str() { + text.push_str(value); + } + } + message["content"] = text.into(); + } +} + +pub(crate) fn messages(parsed: &ParsedRequest) -> Result, GenerateError> { + let mut out = Vec::new(); + // Anthropic's system field is top-level; the legacy parser stores it last. + let ordered: Vec<_> = if parsed.api == Api::Anthropic { + parsed + .messages + .iter() + .filter(|m| m.role == "system") + .chain(parsed.messages.iter().filter(|m| m.role != "system")) + .collect() + } else { + parsed.messages.iter().collect() + }; + for msg in ordered { + if !msg + .parts + .iter() + .any(|p| matches!(p, ChatPart::ToolResult { .. })) + { + out.push(message(msg, &msg.parts, &msg.content)?); + continue; + } + let mut parts = Vec::new(); + for part in &msg.parts { + if let ChatPart::ToolResult { id, content } = part { + if !parts.is_empty() { + out.push(message(msg, &parts, "")?); + parts.clear(); + } + out.push(json!({"role":"tool", "tool_call_id":id, "content":content})); + } else { + parts.push(part.clone()); + } + } + if !parts.is_empty() { + out.push(message(msg, &parts, "")?); + } + } + Ok(out) +} + +fn message(msg: &ChatMsg, parts: &[ChatPart], fallback: &str) -> Result { + let role = &msg.role; + let mut out = json!({"role":role, "content":content(parts, fallback)}); + if !msg.name.is_empty() { + out["name"] = msg.name.clone().into(); + } + if !msg.reasoning.is_empty() { + out["reasoning_content"] = msg.reasoning.clone().into(); + } + if !msg.tool_call_id.is_empty() && role == "tool" { + out["tool_call_id"] = msg.tool_call_id.clone().into(); + } + if !msg.calls.is_empty() { + let calls = msg.calls.iter().map(|call| { + let arguments: Value = serde_json::from_str(if call.arguments.is_empty() { "{}" } else { &call.arguments }).map_err(input_error)?; + if !arguments.is_object() { + return Err(input_error("tool call arguments must be a JSON object")); + } + Ok(json!({"id":call.id, "type":"function", "function":{"name":call.name,"arguments":arguments}})) + }).collect::, GenerateError>>()?; + out["tool_calls"] = calls.into(); + } + Ok(out) +} + +fn content(parts: &[ChatPart], fallback: &str) -> Value { + if parts.is_empty() { + return fallback.into(); + } + if !parts.iter().any(|p| { + matches!( + p, + ChatPart::Image(_) | ChatPart::Audio(_) | ChatPart::Video(_) + ) + }) { + return parts + .iter() + .filter_map(|part| match part { + ChatPart::Text(text) => Some(text.as_str()), + _ => None, + }) + .collect::() + .into(); + } + parts + .iter() + .filter_map(|part| match part { + ChatPart::Text(text) => Some(json!({"type":"text", "text":text})), + ChatPart::Image(_) => Some(json!({"type":"image"})), + ChatPart::Audio(_) => Some(json!({"type":"audio"})), + ChatPart::Video(_) => Some(json!({"type":"video"})), + ChatPart::ToolResult { .. } => None, + }) + .collect::>() + .into() +} + +fn input_error(error: impl std::fmt::Display) -> GenerateError { + GenerateError::Engine(format!("chat input: {error}")) +} + +#[cfg(test)] +#[path = "chat_history_test.rs"] +mod history_tests; + +#[cfg(test)] +mod video_input { + use super::messages; + use crate::parse::{parse_chat_request, ParseEnv}; + + #[test] + fn video_part_is_a_video_block() { + let body = r#"{"messages":[{"role":"user","content":[{"type":"text","text":"look"},{"type":"video_url","video_url":{"url":"data:video/mp4;base64,AAAAAGZ0eXAAAAAAAAAAAA=="}}]}]}"#; + let parsed = parse_chat_request(&ParseEnv::default(), body).unwrap(); + let rendered = messages(&parsed).unwrap(); + assert_eq!(rendered[0]["content"][0]["type"], "text"); + assert_eq!(rendered[0]["content"][1]["type"], "video"); + } +} + +/// One retained tool frontier. The registry owns liveness/TTL; this owns the +/// structured context needed when a client sends only tool results. +#[derive(Clone, Debug)] +pub(crate) struct History { + request: ParsedRequest, + calls: Vec, +} + +impl History { + pub(crate) fn capture( + mut request: ParsedRequest, + generated: &crate::tools::ParsedGenerated, + ) -> Self { + let calls = generated.calls.iter().map(|call| call.id.clone()).collect(); + request.messages.push(ChatMsg { + role: "assistant".into(), + content: String::from_utf8_lossy(&generated.content).into_owned(), + reasoning: String::from_utf8_lossy(&generated.reasoning).into_owned(), + calls: generated.calls.clone(), + raw_dsml: generated.raw_dsml.clone(), + raw_tool_text: generated.raw_tool_text.clone(), + ..ChatMsg::default() + }); + Self { request, calls } + } + + pub(crate) fn matches(&self, parsed: &ParsedRequest) -> bool { + parsed.api == self.request.api + && !parsed.live_call_ids.is_empty() + && parsed + .live_call_ids + .iter() + .all(|id| self.calls.contains(id)) + } + + pub(crate) fn restore(&self, parsed: &mut ParsedRequest) -> Result { + if parsed.live_call_ids.is_empty() { + return Ok(false); + } + if !self.matches(parsed) { + return Err(input_error( + "tool continuation does not match retained chat history", + )); + } + let has_call = parsed + .messages + .iter() + .any(|msg| msg.calls.iter().any(|call| self.calls.contains(&call.id))); + if has_call { + // Full caller history is authoritative, including edits. Fill only + // reasoning omitted by a protocol that refers to our live frontier. + if let Some(saved) = self.request.messages.last() { + for msg in &mut parsed.messages { + if msg.reasoning.is_empty() + && msg.calls.iter().any(|call| self.calls.contains(&call.id)) + { + msg.reasoning.clone_from(&saved.reasoning); + } + } + } + } else { + let mut history = self.request.messages.clone(); + let mut tail = parsed.messages.clone(); + let current_system: Vec<_> = tail + .iter() + .filter(|m| m.role == "system" || m.role == "developer") + .cloned() + .collect(); + if !current_system.is_empty() { + history.retain(|m| m.role != "system" && m.role != "developer"); + history.splice(0..0, current_system); + } + tail.retain(|m| m.role != "system" && m.role != "developer"); + for msg in &mut tail { + for part in &mut msg.parts { + match part { + ChatPart::Image(index) => *index += self.request.images.len(), + ChatPart::Audio(index) => *index += self.request.audios.len(), + ChatPart::Video(index) => *index += self.request.videos.len(), + _ => {} + } + } + } + history.extend(tail); + parsed.messages = history; + parsed + .images + .splice(0..0, self.request.images.iter().cloned()); + parsed + .audios + .splice(0..0, self.request.audios.iter().cloned()); + parsed + .videos + .splice(0..0, self.request.videos.iter().cloned()); + } + if parsed.tool_schemas.is_empty() && parsed.tool_choice != crate::parse::ToolChoice::None { + parsed.tool_schemas.clone_from(&self.request.tool_schemas); + parsed.tool_orders.clone_from(&self.request.tool_orders); + parsed.has_tools = !parsed.tool_schemas.is_empty(); + } + Ok(true) + } +} diff --git a/crates/ds4-server/src/cont.rs b/crates/ds4-server/src/cont.rs new file mode 100644 index 0000000000000000000000000000000000000000..33b3c39529b136e187b921bcd40dbcd37a39ece3 --- /dev/null +++ b/crates/ds4-server/src/cont.rs @@ -0,0 +1,1224 @@ +//! Inc 5a/5b/5c continuation registry from `ds4_server.c` at v0.6.3-dfm. +//! Host-owned: publish / resolve / hold / pin / TTL / bank claim. +//! Native session still executes prefill; this decides reuse vs 409/503. + +use crate::route::Api; +use std::sync::OnceLock; +use std::time::Instant; + +pub const CONT_REGISTRY_MAX_DEFAULT: i32 = 64; +pub const CONT_GRACE_S: f64 = 60.0; +pub const CONT_TTL_S: f64 = 300.0; +pub const CONT_PIN_DEADLINE_S: f64 = 60.0; +pub const CONT_HOLD_SHED_S: f64 = 5.0; + +pub(crate) fn monotonic_now() -> f64 { + static EPOCH: OnceLock = OnceLock::new(); + EPOCH.get_or_init(Instant::now).elapsed().as_secs_f64() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ContState { + LiveFrontier = 0, + ReplayOnly = 1, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ContOwner { + SerialSession = 0, + BatchBank = 1, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ContPin(u64); + +#[derive(Clone, Debug)] +pub struct ContRecord { + record_id: u64, + pub state: ContState, + pub owner: ContOwner, + pub protocol: u8, + pub owner_id: i32, + pub owner_gen: u64, + pub frontier: i32, + pub call_ids: Vec, + pub publish_time: f64, + pub hard_refs: i32, + pub pin_expiry: f64, +} + +#[derive(Clone, Debug)] +pub struct ContRegistry { + records: Vec, + next_record_id: u64, + pub max_records: i32, + pub grace_s: f64, + pub ttl_s: f64, + pub pin_deadline_s: f64, + pub hold_shed_s: f64, + pub serial_live: Option, +} + +impl Default for ContRegistry { + fn default() -> Self { + Self { + records: Vec::new(), + next_record_id: 1, + max_records: CONT_REGISTRY_MAX_DEFAULT, + grace_s: CONT_GRACE_S, + ttl_s: CONT_TTL_S, + pin_deadline_s: CONT_PIN_DEADLINE_S, + hold_shed_s: CONT_HOLD_SHED_S, + serial_live: None, + } + } +} + +impl ContRegistry { + pub fn n_records(&self) -> i32 { + self.records.len() as i32 + } + + pub fn n_live(&self) -> i32 { + self.records + .iter() + .filter(|r| r.state == ContState::LiveFrontier) + .count() as i32 + } + + pub fn live_ids(&self, proto: Api) -> Vec { + let mut out = Vec::new(); + for rec in &self.records { + if rec.state != ContState::LiveFrontier || rec.protocol != proto as u8 { + continue; + } + for id in &rec.call_ids { + if !id.is_empty() && !out.iter().any(|x| x == id) { + out.push(id.clone()); + } + } + } + out + } + + fn key(proto: u8, id: &str) -> String { + let mut id = id; + if id.len() > 94 { + let mut end = 94; + while !id.is_char_boundary(end) { + end -= 1; + } + id = &id[..end]; + } + let mut s = String::with_capacity(2 + id.len()); + s.push(char::from(b'0' + proto)); + s.push('\u{001f}'); + s.push_str(id); + s + } + + fn find_idx(&self, proto: u8, id: &str) -> Option { + if id.is_empty() { + return None; + } + let k = Self::key(proto, id); + self.records + .iter() + .position(|r| r.call_ids.iter().any(|cid| Self::key(r.protocol, cid) == k)) + } + + fn set_eq(a: &[String], b: &[String]) -> bool { + a.len() == b.len() && a.iter().all(|id| b.iter().any(|x| x == id)) + } + + fn demote_idx(&mut self, idx: usize) { + if self.records[idx].state != ContState::LiveFrontier { + return; + } + self.records[idx].state = ContState::ReplayOnly; + if self.serial_live == Some(idx) { + self.serial_live = None; + } + } + + fn prune(&mut self) { + while self.n_records() > self.max_records { + let oldest = self + .records + .iter() + .enumerate() + .rev() + .find(|(_, r)| r.state == ContState::ReplayOnly && r.hard_refs <= 0); + match oldest { + Some((i, _)) => { + self.remove_idx(i); + } + None => break, + } + } + } + + fn remove_idx(&mut self, idx: usize) { + self.demote_idx(idx); + self.records.remove(idx); + if let Some(s) = self.serial_live { + if s == idx { + self.serial_live = None; + } else if s > idx { + self.serial_live = Some(s - 1); + } + } + } + + pub fn expire(&mut self, now: f64) { + if self.ttl_s <= 0.0 || self.n_live() == 0 { + return; + } + let mut i = 0; + while i < self.records.len() { + if self.records[i].state == ContState::LiveFrontier + && now - self.records[i].publish_time > self.ttl_s + { + self.demote_idx(i); + } + i += 1; + } + } + + pub fn publish( + &mut self, + proto: Api, + ids: &[String], + owner: ContOwner, + owner_id: i32, + gen: u64, + frontier: i32, + now: f64, + ) { + if ids.is_empty() || gen == 0 || frontier <= 0 { + return; + } + let ids: Vec = { + let mut out = Vec::new(); + for id in ids { + if !id.is_empty() && !out.iter().any(|x| x == id) { + out.push(id.clone()); + } + } + out + }; + if ids.is_empty() { + return; + } + if self.max_records <= 0 { + self.max_records = CONT_REGISTRY_MAX_DEFAULT; + } + match owner { + ContOwner::SerialSession => { + if let Some(i) = self.serial_live { + self.demote_idx(i); + } + } + ContOwner::BatchBank => { + if let Some(i) = self.records.iter().position(|r| { + r.state == ContState::LiveFrontier + && r.owner == ContOwner::BatchBank + && r.owner_id == owner_id + }) { + self.demote_idx(i); + } + } + } + let record_id = self.next_record_id; + self.next_record_id = self.next_record_id.wrapping_add(1).max(1); + let rec = ContRecord { + record_id, + state: ContState::LiveFrontier, + owner, + protocol: proto as u8, + owner_id: if owner == ContOwner::BatchBank { + owner_id + } else { + 0 + }, + owner_gen: gen, + frontier, + call_ids: ids, + publish_time: now, + hard_refs: 0, + pin_expiry: 0.0, + }; + self.records.insert(0, rec); + if owner == ContOwner::SerialSession { + self.serial_live = Some(0); + } else if let Some(s) = self.serial_live { + self.serial_live = Some(s + 1); + } + self.prune(); + } + + pub fn publish_serial( + &mut self, + proto: Api, + ids: &[String], + gen: u64, + frontier: i32, + now: f64, + ) { + self.publish(proto, ids, ContOwner::SerialSession, 0, gen, frontier, now); + } + + pub fn publish_bank( + &mut self, + proto: Api, + ids: &[String], + bank: i32, + gen: u64, + frontier: i32, + now: f64, + ) { + if bank < 0 { + return; + } + self.publish(proto, ids, ContOwner::BatchBank, bank, gen, frontier, now); + } + + pub fn demote_serial(&mut self) { + if let Some(i) = self.serial_live { + self.demote_idx(i); + } + } + + pub fn live_has_id(&mut self, proto: Api, id: &str, now: f64) -> bool { + self.expire(now); + self.find_idx(proto as u8, id) + .map(|i| self.records[i].state == ContState::LiveFrontier) + .unwrap_or(false) + } + + pub fn id_known(&self, id: &str) -> bool { + for proto in [ + Api::Openai as u8, + Api::Anthropic as u8, + Api::Responses as u8, + ] { + if self.find_idx(proto, id).is_some() { + return true; + } + } + false + } + + pub fn resolve_serial( + &mut self, + proto: Api, + ids: &[String], + session_gen: u64, + live_pos: i32, + now: f64, + ) -> bool { + if ids.is_empty() || session_gen == 0 { + return false; + } + self.expire(now); + let Some(i) = self.find_idx(proto as u8, &ids[0]) else { + return false; + }; + let rec = &self.records[i]; + rec.state == ContState::LiveFrontier + && rec.owner == ContOwner::SerialSession + && rec.protocol == proto as u8 + && Self::set_eq(&rec.call_ids, ids) + && rec.owner_gen == session_gen + && rec.frontier == live_pos + } + + pub fn bank_claim(&mut self, proto: Api, ids: &[String], now: f64) -> Option<(i32, u64, i32)> { + if ids.is_empty() { + return None; + } + self.expire(now); + let i = self.find_idx(proto as u8, &ids[0])?; + let rec = &self.records[i]; + if rec.state == ContState::LiveFrontier + && rec.owner == ContOwner::BatchBank + && rec.protocol == proto as u8 + && Self::set_eq(&rec.call_ids, ids) + { + Some((rec.owner_id, rec.owner_gen, rec.frontier)) + } else { + None + } + } + + /// C `cont_record_bank_protects`: an unverified native reference fails + /// closed for eviction, while a proven generation/frontier mismatch does not. + pub fn bank_protected(&self, bank: i32, live: Option<(u64, i32)>, now: f64) -> bool { + if bank < 0 { + return false; + } + self.records.iter().any(|record| { + if record.state != ContState::LiveFrontier + || record.owner != ContOwner::BatchBank + || record.owner_id != bank + { + return false; + } + if let Some((generation, frontier)) = live { + if generation == 0 + || frontier <= 0 + || record.owner_gen != generation + || record.frontier != frontier + { + return false; + } + } + let in_grace = self.grace_s > 0.0 && now - record.publish_time < self.grace_s; + let pinned = + record.hard_refs > 0 && self.pin_deadline_s > 0.0 && now < record.pin_expiry; + in_grace || pinned + }) + } + + pub fn bank_hold_retry(&self, bank: i32, live: Option<(u64, i32)>, now: f64) -> Option { + if !self.bank_protected(bank, live, now) { + return None; + } + let record = self.records.iter().find(|record| { + record.state == ContState::LiveFrontier + && record.owner == ContOwner::BatchBank + && record.owner_id == bank + && live.is_none_or(|(generation, frontier)| { + generation != 0 + && frontier > 0 + && record.owner_gen == generation + && record.frontier == frontier + }) + })?; + let grace_left = if self.grace_s > 0.0 { + self.grace_s - (now - record.publish_time) + } else { + 0.0 + }; + let pin_left = + if record.hard_refs > 0 && self.pin_deadline_s > 0.0 && now < record.pin_expiry { + record.pin_expiry - now + } else { + 0.0 + }; + let left = grace_left.max(pin_left); + let retry = (left + 0.999) as i32; + Some(retry.max(1)) + } + + pub fn pin_live(&mut self, proto: Api, id: &str, now: f64) -> Option { + self.expire(now); + let i = self.find_idx(proto as u8, id)?; + if self.records[i].state != ContState::LiveFrontier { + return None; + } + self.records[i].hard_refs += 1; + if self.pin_deadline_s > 0.0 { + let expiry = now + self.pin_deadline_s; + if expiry > self.records[i].pin_expiry { + self.records[i].pin_expiry = expiry; + } + } + Some(ContPin(self.records[i].record_id)) + } + + pub fn pin_owner(&self, pin: ContPin) -> Option { + self.records + .iter() + .find(|record| record.record_id == pin.0) + .map(|record| record.owner) + } + + pub fn unpin(&mut self, pin: ContPin) { + if let Some(rec) = self + .records + .iter_mut() + .find(|record| record.record_id == pin.0) + { + if rec.hard_refs > 0 { + rec.hard_refs -= 1; + } + } + } + + pub fn serial_hold(&mut self, proto: Api, req_ids: &[String], now: f64) -> Option { + self.expire(now); + let Some(i) = self.serial_live else { + return None; + }; + let rec = &self.records[i]; + if rec.protocol == proto as u8 + && Self::set_eq(&rec.call_ids, req_ids) + && !req_ids.is_empty() + { + return None; + } + let shed_w = if self.hold_shed_s < self.grace_s { + self.hold_shed_s + } else { + self.grace_s + }; + let shed_left = if shed_w > 0.0 { + shed_w - (now - rec.publish_time) + } else { + 0.0 + }; + let pinned = rec.hard_refs > 0 && self.pin_deadline_s > 0.0 && now < rec.pin_expiry; + if shed_left <= 0.0 && !pinned { + return None; + } + let mut left = shed_left; + if pinned && rec.pin_expiry - now > left { + left = rec.pin_expiry - now; + } + let mut retry = (left + 0.999) as i32; + if retry < 1 { + retry = 1; + } + Some(retry) + } + + pub fn serial_live_hard_refs(&self) -> i32 { + self.serial_live + .and_then(|i| self.records.get(i)) + .map(|r| r.hard_refs) + .unwrap_or(0) + } + + pub fn serial_live_state(&self) -> Option { + self.serial_live + .and_then(|i| self.records.get(i)) + .map(|r| r.state) + } + + pub fn set_serial_publish_time(&mut self, t: f64) { + if let Some(i) = self.serial_live { + self.records[i].publish_time = t; + } + } + + pub fn set_serial_pin_expiry(&mut self, t: f64) { + if let Some(i) = self.serial_live { + self.records[i].pin_expiry = t; + } + } + + pub fn rewind_live_publish(&mut self, delta: f64) { + for rec in &mut self.records { + if rec.state == ContState::LiveFrontier { + rec.publish_time -= delta; + } + } + } +} + +fn push_msg_tool_ids(ids: &mut Vec, m: &crate::parse::ChatMsg) { + if !m.tool_call_id.is_empty() && !ids.iter().any(|x| x == &m.tool_call_id) { + ids.push(m.tool_call_id.clone()); + } + for id in &m.tool_call_ids { + if !id.is_empty() && !ids.iter().any(|x| x == id) { + ids.push(id.clone()); + } + } +} + +fn anthropic_tool_result_tail(m: &crate::parse::ChatMsg) -> bool { + m.role == "user" && (!m.tool_call_id.is_empty() || !m.tool_call_ids.is_empty()) +} + +/// Call ids C `*_prepare_live_continuation` would bind (not the full history). +pub fn live_tool_result_ids(api: Api, messages: &[crate::parse::ChatMsg]) -> Vec { + match api { + Api::Anthropic => { + let mut tail_end = messages.len(); + while tail_end > 0 && crate::render::role_is_system(&messages[tail_end - 1].role) { + tail_end -= 1; + } + let mut tail_start = tail_end; + while tail_start > 0 && anthropic_tool_result_tail(&messages[tail_start - 1]) { + tail_start -= 1; + } + if tail_start == tail_end { + return Vec::new(); + } + let mut ids = Vec::new(); + for m in &messages[tail_start..tail_end] { + push_msg_tool_ids(&mut ids, m); + } + ids + } + Api::Responses => { + let mut tail_start = messages.len(); + while tail_start > 0 { + let role = messages[tail_start - 1].role.as_str(); + if role != "tool" && role != "function" { + break; + } + tail_start -= 1; + } + if tail_start == messages.len() { + return Vec::new(); + } + let mut ids = Vec::new(); + if tail_start > 0 { + let assistant = &messages[tail_start - 1]; + if assistant.role != "assistant" || assistant.calls.is_empty() { + return Vec::new(); + } + for c in &assistant.calls { + if !c.id.is_empty() && !ids.iter().any(|x| x == &c.id) { + ids.push(c.id.clone()); + } + } + return ids; + } + for m in &messages[tail_start..] { + push_msg_tool_ids(&mut ids, m); + } + ids + } + Api::Openai => Vec::new(), + } +} + +/// C `cont_bank_continuation_admit` equality: a live claim stays on its +/// bank only when generation and frontier still match. Any miss is the +/// protocol-native 409 full-replay surface. +pub fn place_bank_continuation( + claim: Option<(i32, u64, i32)>, + live: Option<(u64, i32)>, +) -> Result { + let Some((bank, generation, frontier)) = claim else { + return Err(BankContConflict); + }; + if bank < 0 || frontier <= 0 { + return Err(BankContConflict); + } + let Some((live_generation, live_frontier)) = live else { + return Err(BankContConflict); + }; + if live_generation != generation || live_frontier != frontier { + return Err(BankContConflict); + } + Ok(bank) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BankContConflict; + +fn csv(ids: &[&str]) -> Vec { + ids.iter().map(|s| (*s).to_string()).collect() +} + +fn hold_line(r: Option) -> String { + match r { + Some(n) => format!("HOLD 1 retry={n}"), + None => "HOLD 0".into(), + } +} + +/// Tape matching `tests/parity/cont_c_oracle` scripts. +pub fn dump_script(name: &str) -> String { + let mut out = String::new(); + match name { + "publish-resolve-demote" => { + let mut r = ContRegistry::default(); + let now = 1000.0; + r.publish_serial( + Api::Anthropic, + &csv(&["toolu_regA", "toolu_regB"]), + 7, + 100, + now, + ); + out.push_str(&format!( + "live_anth_a={}\n", + u32::from(r.live_has_id(Api::Anthropic, "toolu_regA", now)) + )); + out.push_str(&format!( + "live_anth_b={}\n", + u32::from(r.live_has_id(Api::Anthropic, "toolu_regB", now)) + )); + out.push_str(&format!( + "live_resp_a={}\n", + u32::from(r.live_has_id(Api::Responses, "toolu_regA", now)) + )); + let ids = csv(&["toolu_regA", "toolu_regB"]); + out.push_str(&format!( + "resolve_ok={}\n", + u32::from(r.resolve_serial(Api::Anthropic, &ids, 7, 100, now)) + )); + out.push_str(&format!( + "resolve_gen={}\n", + u32::from(r.resolve_serial(Api::Anthropic, &ids, 8, 100, now)) + )); + out.push_str(&format!( + "resolve_pos={}\n", + u32::from(r.resolve_serial(Api::Anthropic, &ids, 7, 101, now)) + )); + out.push_str(&format!( + "resolve_proto={}\n", + u32::from(r.resolve_serial(Api::Responses, &ids, 7, 100, now)) + )); + out.push_str(&format!( + "resolve_sub={}\n", + u32::from(r.resolve_serial(Api::Anthropic, &csv(&["toolu_regA"]), 7, 100, now)) + )); + out.push_str(&format!( + "resolve_sup={}\n", + u32::from(r.resolve_serial( + Api::Anthropic, + &csv(&["toolu_regA", "toolu_regB", "toolu_regC"]), + 7, + 100, + now + )) + )); + r.demote_serial(); + out.push_str(&format!( + "live_after_demote={}\n", + u32::from(r.live_has_id(Api::Anthropic, "toolu_regA", now)) + )); + out.push_str(&format!( + "resolve_after_demote={}\n", + u32::from(r.resolve_serial(Api::Anthropic, &ids, 7, 100, now)) + )); + out.push_str(&format!( + "known_after_demote={}\n", + u32::from(r.id_known("toolu_regA")) + )); + } + "supersede-cap" => { + let mut r = ContRegistry { + max_records: 4, + ..ContRegistry::default() + }; + let now = 1000.0; + for t in 1..=2 { + r.publish_serial( + Api::Anthropic, + &csv(&[&format!("toolu_turn{t}")]), + 3, + 50 * t, + now, + ); + } + out.push_str(&format!( + "live1={} live2={} n_live={} n_rec={}\n", + u32::from(r.live_has_id(Api::Anthropic, "toolu_turn1", now)), + u32::from(r.live_has_id(Api::Anthropic, "toolu_turn2", now)), + r.n_live(), + r.n_records() + )); + for t in 3..=8 { + r.publish_serial( + Api::Anthropic, + &csv(&[&format!("toolu_turn{t}")]), + 3, + 50 * t, + now, + ); + } + out.push_str(&format!( + "n_rec={} known1={} known2={} live8={} n_live={}\n", + r.n_records(), + u32::from(r.id_known("toolu_turn1")), + u32::from(r.id_known("toolu_turn2")), + u32::from(r.live_has_id(Api::Anthropic, "toolu_turn8", now)), + r.n_live() + )); + } + "grace-hold" => { + let mut r = ContRegistry::default(); + r.publish_serial(Api::Anthropic, &csv(&["toolu_hold"]), 4, 70, 1000.0); + out.push_str(&format!( + "{}\n", + hold_line(r.serial_hold(Api::Openai, &[], 1001.0)) + )); + out.push_str(&format!( + "{}\n", + hold_line(r.serial_hold(Api::Anthropic, &csv(&["toolu_hold"]), 1001.0)) + )); + out.push_str(&format!( + "{}\n", + hold_line(r.serial_hold(Api::Openai, &[], 1011.0)) + )); + out.push_str(&format!( + "still_live={}\n", + match r.serial_live_state() { + Some(ContState::LiveFrontier) => 1, + _ => 0, + } + )); + out.push_str(&format!( + "{}\n", + hold_line(r.serial_hold(Api::Openai, &[], 1131.0)) + )); + let pin = r.pin_live(Api::Anthropic, "toolu_hold", 1131.0); + out.push_str(&format!( + "{}\n", + hold_line(r.serial_hold(Api::Openai, &[], 1131.0)) + )); + r.set_serial_pin_expiry(1130.0); + out.push_str(&format!( + "{}\n", + hold_line(r.serial_hold(Api::Openai, &[], 1131.0)) + )); + if let Some(p) = pin { + r.unpin(p); + } + out.push_str(&format!("hard_refs={}\n", r.serial_live_hard_refs())); + } + "ttl" => { + let mut r = ContRegistry::default(); + r.publish_serial(Api::Anthropic, &csv(&["toolu_ttl"]), 4, 70, 1000.0); + out.push_str(&format!( + "live_before={}\n", + u32::from(r.live_has_id(Api::Anthropic, "toolu_ttl", 1000.0)) + )); + out.push_str(&format!( + "live_after={}\n", + u32::from(r.live_has_id(Api::Anthropic, "toolu_ttl", 1301.0)) + )); + out.push_str(&format!("n_live={}\n", r.n_live())); + out.push_str(&format!( + "resolve={}\n", + u32::from(r.resolve_serial(Api::Anthropic, &csv(&["toolu_ttl"]), 4, 70, 1301.0)) + )); + out.push_str(&format!("known={}\n", u32::from(r.id_known("toolu_ttl")))); + } + "bank-claim" => { + let mut r = ContRegistry::default(); + let now = 1000.0; + r.publish_bank(Api::Anthropic, &csv(&["toolu_bk1"]), 2, 7, 100, now); + out.push_str(&format!( + "live={} resp={} serial_live={}\n", + u32::from(r.live_has_id(Api::Anthropic, "toolu_bk1", now)), + u32::from(r.live_has_id(Api::Responses, "toolu_bk1", now)), + u32::from(r.serial_live.is_some()) + )); + let claim = r.bank_claim(Api::Anthropic, &csv(&["toolu_bk1"]), now); + out.push_str(&format!( + "claim={}\n", + match claim { + Some((b, g, f)) => format!("{b},{g},{f}"), + None => "-".into(), + } + )); + out.push_str(&format!( + "claim_resp={}\n", + u32::from( + r.bank_claim(Api::Responses, &csv(&["toolu_bk1"]), now) + .is_some() + ) + )); + out.push_str(&format!( + "resolve_serial={}\n", + u32::from(r.resolve_serial(Api::Anthropic, &csv(&["toolu_bk1"]), 7, 100, now)) + )); + r.publish_serial(Api::Anthropic, &csv(&["toolu_ser1"]), 9, 40, now); + out.push_str(&format!("n_live={}\n", r.n_live())); + r.demote_serial(); + out.push_str(&format!( + "n_live_after={} live_bk1={}\n", + r.n_live(), + u32::from(r.live_has_id(Api::Anthropic, "toolu_bk1", now)) + )); + r.publish_bank(Api::Anthropic, &csv(&["toolu_bk2"]), 2, 8, 120, now); + out.push_str(&format!( + "live_bk1={} live_bk2={}\n", + u32::from(r.live_has_id(Api::Anthropic, "toolu_bk1", now)), + u32::from(r.live_has_id(Api::Anthropic, "toolu_bk2", now)) + )); + r.publish_bank(Api::Responses, &csv(&["toolu_bk3"]), 3, 2, 80, now); + out.push_str(&format!("n_live={}\n", r.n_live())); + r.publish_bank(Api::Anthropic, &csv(&["toolu_bk_dead"]), 4, 0, 100, now); + r.publish_bank(Api::Anthropic, &csv(&["toolu_bk_dead"]), 4, 5, 0, now); + out.push_str(&format!( + "known_dead={}\n", + u32::from(r.id_known("toolu_bk_dead")) + )); + r.rewind_live_publish(301.0); + out.push_str(&format!( + "ttl_live={} n_live={} known_bk2={}\n", + u32::from(r.live_has_id(Api::Anthropic, "toolu_bk2", now)), + r.n_live(), + u32::from(r.id_known("toolu_bk2")) + )); + } + "bank-protection" => { + let mut r = ContRegistry::default(); + let now = 1001.0; + r.grace_s = 60.0; + r.ttl_s = 300.0; + r.pin_deadline_s = 20.0; + r.publish_bank( + Api::Anthropic, + &csv(&["toolu_protected"]), + 5, + 3, + 200, + 1000.0, + ); + out.push_str(&format!( + "current={}\n", + hold_line(r.bank_hold_retry(5, Some((3, 200)), now)) + )); + out.push_str(&format!( + "stale={}\n", + hold_line(r.bank_hold_retry(5, Some((4, 200)), now)) + )); + out.push_str(&format!( + "unknown={}\n", + hold_line(r.bank_hold_retry(5, None, now)) + )); + r.rewind_live_publish(100.0); + out.push_str(&format!( + "lapsed={}\n", + hold_line(r.bank_hold_retry(5, Some((3, 200)), now)) + )); + let pin = r.pin_live(Api::Anthropic, "toolu_protected", now); + out.push_str(&format!( + "pinned={}\n", + hold_line(r.bank_hold_retry(5, Some((3, 200)), now)) + )); + if let Some(pin) = pin { + r.unpin(pin); + } + out.push_str(&format!( + "unpinned={}\n", + hold_line(r.bank_hold_retry(5, Some((3, 200)), now)) + )); + } + _ => out.push_str("ERROR unknown-script\n"), + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::http_response_bytes; + use crate::generate::{GenerateError, GenerateOutcome, ScriptedDecode}; + use crate::serve::{handle_client_inner, ServerConfig, ServerInner}; + use crate::serve_cont::ContExec; + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::sync::{Arc, Mutex}; + use std::thread; + use std::time::Instant; + + #[test] + fn multibyte_call_id_is_bounded_on_a_utf8_boundary() { + let id = format!("{}한-tail", "a".repeat(93)); + let key = ContRegistry::key(Api::Responses as u8, &id); + assert_eq!(&key[2..], "a".repeat(93)); + + let mut registry = ContRegistry::default(); + registry.publish_bank(Api::Responses, &[id.clone()], 0, 1, 8, 1.0); + assert!(registry.live_has_id(Api::Responses, &id, 1.0)); + } + + #[test] + fn bank_protection_requires_a_current_reference_or_a_live_pin() { + let mut registry = ContRegistry::default(); + let ids = csv(&["toolu_bank"]); + registry.grace_s = 10.0; + registry.ttl_s = 100.0; + registry.pin_deadline_s = 20.0; + registry.publish_bank(Api::Anthropic, &ids, 2, 7, 100, 100.0); + + assert!(registry.bank_protected(2, Some((7, 100)), 105.0)); + assert!(!registry.bank_protected(2, Some((8, 100)), 105.0)); + assert!(!registry.bank_protected(2, Some((7, 101)), 105.0)); + assert!(registry.bank_protected(2, None, 105.0)); + + registry.rewind_live_publish(11.0); + assert!(!registry.bank_protected(2, Some((7, 100)), 105.0)); + let pin = registry + .pin_live(Api::Anthropic, "toolu_bank", 105.0) + .expect("the live bank record pins"); + assert!(registry.bank_protected(2, Some((7, 100)), 105.0)); + registry.unpin(pin); + assert!(!registry.bank_protected(2, Some((7, 100)), 105.0)); + } + + #[test] + fn pin_survives_duplicate_id_republish() { + let mut registry = ContRegistry::default(); + let ids = csv(&["toolu_same"]); + registry.publish_serial(Api::Anthropic, &ids, 1, 10, 1.0); + let pin = registry + .pin_live(Api::Anthropic, "toolu_same", 1.0) + .expect("live record should pin"); + + registry.publish_serial(Api::Anthropic, &ids, 2, 20, 2.0); + registry.unpin(pin); + + let old = registry + .records + .iter() + .find(|record| record.owner_gen == 1) + .expect("old record remains for replay"); + let new = registry + .records + .iter() + .find(|record| record.owner_gen == 2) + .expect("new record is live"); + assert_eq!(old.hard_refs, 0, "the originally pinned record is released"); + assert_eq!(new.hard_refs, 0, "republish is not accidentally unpinned"); + } + + #[test] + fn continuation_clock_is_monotonic() { + let first = monotonic_now(); + let second = monotonic_now(); + assert!(second >= first); + } + + #[test] + fn place_bank_continuation_stays_on_claimed_bank_when_live_matches() { + let mut registry = ContRegistry::default(); + let ids = csv(&["toolu_follow"]); + registry.publish_bank(Api::Anthropic, &ids, 2, 7, 100, 1000.0); + + let claim = registry.bank_claim(Api::Anthropic, &ids, 1000.0); + let placed = place_bank_continuation(claim, Some((7, 100))); + + assert_eq!(placed, Ok(2)); + } + + #[test] + fn place_bank_continuation_conflicts_when_generation_or_frontier_moved() { + let mut registry = ContRegistry::default(); + let ids = csv(&["toolu_follow"]); + registry.publish_bank(Api::Anthropic, &ids, 2, 7, 100, 1000.0); + let claim = registry.bank_claim(Api::Anthropic, &ids, 1000.0); + + assert_eq!( + place_bank_continuation(claim, Some((8, 100))), + Err(BankContConflict) + ); + assert_eq!( + place_bank_continuation(claim, Some((7, 101))), + Err(BankContConflict) + ); + } + + #[test] + fn pin_owner_reports_bank_owned_live_record() { + let mut registry = ContRegistry::default(); + registry.publish_bank(Api::Anthropic, &csv(&["toolu_pin"]), 3, 4, 50, 1.0); + let pin = registry + .pin_live(Api::Anthropic, "toolu_pin", 1.0) + .expect("live bank record pins"); + assert_eq!(registry.pin_owner(pin), Some(ContOwner::BatchBank)); + } + + struct BankLane { + live: Option<(u64, i32)>, + bank: i32, + generated: Arc>, + tool_ids: Vec, + generation: u64, + frontier: i32, + } + + impl ContExec for BankLane { + fn model_id(&self) -> i32 { + 0 + } + fn seq_cap(&self) -> i32 { + 8192 + } + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + vec![1] + } + fn encode_text(&self, _text: &str) -> Vec { + vec![1] + } + fn generate( + &mut self, + parsed: &crate::parse::ParsedRequest, + _job_id: &str, + _created: i64, + cors: bool, + _default_tokens: i32, + _t_arrive: Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut ds4_kv::Store>, + out: &mut dyn Write, + ) -> Result { + *self.generated.lock().expect("generated lock") = true; + if let Some(bank) = parsed.directed_bank { + assert_eq!(bank, self.bank); + } + out.write_all(&http_response_bytes( + 200, + Some("application/json"), + None, + cors, + "{}", + )) + .map_err(|_| GenerateError::Io)?; + Ok(GenerateOutcome { + tool_ids: self.tool_ids.clone(), + bank: Some(self.bank), + generation: self.generation, + frontier: self.frontier, + finish: if self.tool_ids.is_empty() { + "stop".into() + } else { + "tool_calls".into() + }, + ..GenerateOutcome::default() + }) + } + fn bank_live(&self, bank: i32) -> Option<(u64, i32)> { + if bank == self.bank { + self.live + } else { + None + } + } + } + + fn http_post(addr: std::net::SocketAddr, path: &str, body: &str) -> String { + let mut client = TcpStream::connect(addr).expect("connect"); + let req = format!( + "POST {path} HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + client.write_all(req.as_bytes()).expect("write"); + let mut out = Vec::new(); + client.read_to_end(&mut out).expect("read"); + String::from_utf8_lossy(&out).into_owned() + } + + #[test] + fn streaming_anthropic_tool_turn_stays_on_its_bank_for_output_only_follow_up() { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 16, + ..ServerConfig::default() + }; + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + let generated = Arc::new(Mutex::new(false)); + let seen = Arc::clone(&generated); + let h = thread::spawn(move || { + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let mut engine = ScriptedDecode::from_pieces(&[b"ok"]); + let mut cont = BankLane { + live: None, + bank: 2, + generated: Arc::clone(&seen), + tool_ids: vec!["toolu_bankturn".into()], + generation: 7, + frontier: 100, + }; + { + let (mut first, _) = listener.accept().expect("accept first"); + handle_client_inner(&cfg, &inner, &mut first, Some(&mut engine), Some(&mut cont)); + } + { + let mut g = inner.lock().expect("inner"); + assert_eq!( + g.creg + .bank_claim(Api::Anthropic, &["toolu_bankturn".into()], monotonic_now()), + Some((2, 7, 100)) + ); + } + cont.live = Some((7, 100)); + cont.tool_ids.clear(); + *seen.lock().expect("generated lock") = false; + { + let (mut second, _) = listener.accept().expect("accept follow-up"); + handle_client_inner( + &cfg, + &inner, + &mut second, + Some(&mut engine), + Some(&mut cont), + ); + } + assert!(*seen.lock().expect("generated lock")); + let mut g = inner.lock().expect("inner"); + assert_eq!( + g.creg + .bank_claim(Api::Anthropic, &["toolu_bankturn".into()], monotonic_now()), + Some((2, 7, 100)) + ); + }); + let tools = r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":16,"stream":true,"thinking":{"type":"disabled"},"tools":[{"name":"bash","input_schema":{"type":"object"}}]}"#; + let first = http_post(addr, "/v1/messages", tools); + assert!(first.starts_with("HTTP/1.1 200 "), "{first}"); + let follow = http_post( + addr, + "/v1/messages", + r#"{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bankturn","content":"ok"}]}],"max_tokens":8}"#, + ); + h.join().expect("server thread"); + assert!(follow.starts_with("HTTP/1.1 200 "), "{follow}"); + } + + #[test] + fn streaming_anthropic_bank_follow_up_conflicts_when_frontier_moved() { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 16, + ..ServerConfig::default() + }; + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + let generated = Arc::new(Mutex::new(false)); + let seen = Arc::clone(&generated); + let h = thread::spawn(move || { + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + { + let mut g = inner.lock().expect("inner"); + g.creg.publish_bank( + Api::Anthropic, + &["toolu_moved".into()], + 2, + 7, + 100, + monotonic_now(), + ); + } + let mut engine = ScriptedDecode::from_pieces(&[b"ok"]); + let mut cont = BankLane { + live: Some((8, 100)), + bank: 2, + generated: seen, + tool_ids: Vec::new(), + generation: 8, + frontier: 100, + }; + let (mut s, _) = listener.accept().expect("accept"); + handle_client_inner(&cfg, &inner, &mut s, Some(&mut engine), Some(&mut cont)); + assert!(!*generated.lock().expect("generated lock")); + }); + let follow = http_post( + addr, + "/v1/messages", + r#"{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_moved","content":"ok"}]}],"max_tokens":8}"#, + ); + h.join().expect("server thread"); + assert!(follow.starts_with("HTTP/1.1 409 "), "{follow}"); + assert!( + follow.contains("Anthropic continuation state is not available"), + "{follow}" + ); + assert!( + follow.contains("replaying the full messages history"), + "{follow}" + ); + } +} diff --git a/crates/ds4-server/src/dist_cli.rs b/crates/ds4-server/src/dist_cli.rs new file mode 100644 index 0000000000000000000000000000000000000000..327d55767a2fe0ebfaffb23d774780a31737c2ca --- /dev/null +++ b/crates/ds4-server/src/dist_cli.rs @@ -0,0 +1,113 @@ +//! Distributed CLI flags for `ds4-server-rs`, matching C `ds4_dist_parse_cli_arg`. + +use ds4_dist::{parse_cli_arg, prepare_engine_options, CliResult, Options, Role}; + +#[derive(Debug, Default)] +pub struct DistArgs { + pub opt: Options, +} + +impl DistArgs { + pub fn parse_arg( + &mut self, + option: &str, + args: &mut impl Iterator, + ) -> Result { + match parse_cli_arg(option, args, &mut self.opt)? { + CliResult::Matched => Ok(true), + CliResult::NotMatched => Ok(false), + CliResult::Error => unreachable!(), + } + } + + /// C uses `--listen` for the coordinator data port and `--host`/`--port` + /// for HTTP. The Rust shadow used `--listen` for HTTP; if no `--role` is + /// set, keep that meaning. + pub fn finish(&mut self, http_host: &mut String, http_port: &mut u16) -> Result<(), String> { + if self.opt.role == Role::None { + if let Some(host) = self.opt.listen_host.take() { + if self.opt.listen_port != 0 { + *http_host = host; + *http_port = u16::try_from(self.opt.listen_port) + .map_err(|_| "ds4-server-rs: --listen port is out of range".to_string())?; + self.opt.listen_port = 0; + } + } + } + prepare_engine_options(&self.opt)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::DistArgs; + use ds4_dist::Role; + + fn parse(args: &[&str]) -> Result<(DistArgs, String, u16), String> { + let mut dist = DistArgs::default(); + let mut host = "127.0.0.1".to_string(); + let mut port = 8000u16; + let mut rest = args.iter().map(|s| (*s).to_string()); + while let Some(arg) = rest.next() { + if !dist.parse_arg(&arg, &mut rest)? { + return Err(format!("unmatched {arg}")); + } + } + dist.finish(&mut host, &mut port)?; + Ok((dist, host, port)) + } + + #[test] + fn listen_without_role_stays_http() { + let (dist, host, port) = parse(&["--listen", "0.0.0.0", "9001"]).unwrap(); + assert_eq!(dist.opt.role, Role::None); + assert!(dist.opt.listen_host.is_none()); + assert_eq!(dist.opt.listen_port, 0); + assert_eq!(host, "0.0.0.0"); + assert_eq!(port, 9001); + } + + #[test] + fn coordinator_keeps_listen_for_workers() { + let (dist, host, port) = parse(&[ + "--role", + "coordinator", + "--layers", + "0:19", + "--listen", + "169.254.1.1", + "1234", + ]) + .unwrap(); + assert_eq!(dist.opt.role, Role::Coordinator); + assert_eq!(dist.opt.listen_host.as_deref(), Some("169.254.1.1")); + assert_eq!(dist.opt.listen_port, 1234); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 8000); + } + + #[test] + fn worker_requires_coordinator() { + let err = parse(&["--role", "worker", "--layers", "20:output"]).unwrap_err(); + assert!(err.contains("--coordinator")); + } + + #[test] + fn worker_parses_coordinator_address() { + let (dist, _, _) = parse(&[ + "--role", + "worker", + "--layers", + "20:output", + "--coordinator", + "127.0.0.1", + "1234", + ]) + .unwrap(); + assert_eq!(dist.opt.role, Role::Worker); + assert_eq!(dist.opt.coordinator_host.as_deref(), Some("127.0.0.1")); + assert_eq!(dist.opt.coordinator_port, 1234); + assert!(dist.opt.layers.has_output); + } +} diff --git a/crates/ds4-server/src/dsml.rs b/crates/ds4-server/src/dsml.rs new file mode 100644 index 0000000000000000000000000000000000000000..a0e46ada281e5bed3fe17359aaff4c940631c8b9 --- /dev/null +++ b/crates/ds4-server/src/dsml.rs @@ -0,0 +1,1018 @@ +//! DSML decode tracker + `sem_accum_sampling_override` from `ds4_server.c` +//! at v0.6.3-dfm. Structural tool syntax is greedy; payload keeps the +//! request sampler. Required tool / think-end prefixes force exact tokens. + +use crate::parse::ToolChoice; +use crate::route::ThinkMode; +use crate::stream::ChatFormat; +use crate::tool_stream::DsmlSyntax; +use crate::tools::{ + dsml_attr, DSML_INVOKE_END, DSML_INVOKE_END_SHORT, DSML_INVOKE_END_SPACED, DSML_INVOKE_START, + DSML_INVOKE_START_SHORT, DSML_INVOKE_START_SPACED, DSML_PARAM_END, DSML_PARAM_END_SHORT, + DSML_PARAM_END_SPACED, DSML_PARAM_START, DSML_PARAM_START_SHORT, DSML_PARAM_START_SPACED, + DSML_TOOL_CALLS_END, DSML_TOOL_CALLS_END_SHORT, DSML_TOOL_CALLS_END_SPACED, + DSML_TOOL_CALLS_START, DSML_TOOL_CALLS_START_SHORT, DSML_TOOL_CALLS_START_SPACED, +}; + +pub const SAMPLE_OVERRIDE_NONE: i32 = 0; +pub const SAMPLE_OVERRIDE_GREEDY: i32 = 1; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DsmlDecodeState { + Outside = 0, + Structural = 1, + StringBody = 2, + JsonStructural = 3, + JsonString = 4, +} + +impl DsmlDecodeState { + pub fn is_tool(self) -> bool { + self != DsmlDecodeState::Outside + } + pub fn uses_payload_sampling(self) -> bool { + matches!( + self, + DsmlDecodeState::StringBody | DsmlDecodeState::JsonString + ) + } + pub fn name(self) -> &'static str { + match self { + Self::Outside => "outside", + Self::Structural => "structural", + Self::StringBody => "string-body", + Self::JsonStructural => "json-structural", + Self::JsonString => "json-string", + } + } +} + +/// The engine's dsml_decode_state_is_syntax (server_generate_v41.c): under +/// sampling the tool-call syntax positions (the tag / parameter head / JSON +/// punctuation) take the raw argmax while the string value body and the JSON +/// string keep the request's draw. True = greedy at this position. +pub fn dsml_decode_state_is_syntax(state: DsmlDecodeState) -> bool { + !matches!( + state, + DsmlDecodeState::Outside | DsmlDecodeState::StringBody | DsmlDecodeState::JsonString + ) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TrackMode { + Search, + Structural, + StringBody, + JsonParam, + Done, +} + +/// The tool dialect the tracker recognizes: C `dsml_decode_tracker::model_syntax` +/// (ds4_server.c:7364). GLM's `//` block is +/// deliberately NOT a member of the DSML syntax table; the reference selects it +/// by model syntax (ds4_server.c:7595-7611) so the dialects never recognize each +/// other's bytes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ToolSyntax { + #[default] + Dsml, + Glm, +} + +/// GLM's tool block (ds4_server.c:7376 `glm_tool_syntax`): the function name +/// sits between `` and the first ``, and argument values are +/// text, so every value body is a string body. +const GLM_SYNTAX: DsmlSyntax = DsmlSyntax { + tool_calls_start: b"", + tool_calls_end: b"", + invoke_start: b"", + invoke_end: b"", + param_start: b"", + param_end: b"", + entity_values: false, +}; + +const SYNTAXES: [DsmlSyntax; 4] = [ + DsmlSyntax { + tool_calls_start: DSML_TOOL_CALLS_START.as_bytes(), + tool_calls_end: DSML_TOOL_CALLS_END.as_bytes(), + invoke_start: DSML_INVOKE_START.as_bytes(), + invoke_end: DSML_INVOKE_END.as_bytes(), + param_start: DSML_PARAM_START.as_bytes(), + param_end: DSML_PARAM_END.as_bytes(), + entity_values: true, + }, + DsmlSyntax { + tool_calls_start: DSML_TOOL_CALLS_START_SHORT.as_bytes(), + tool_calls_end: DSML_TOOL_CALLS_END_SHORT.as_bytes(), + invoke_start: DSML_INVOKE_START_SHORT.as_bytes(), + invoke_end: DSML_INVOKE_END_SHORT.as_bytes(), + param_start: DSML_PARAM_START_SHORT.as_bytes(), + param_end: DSML_PARAM_END_SHORT.as_bytes(), + entity_values: true, + }, + DsmlSyntax { + tool_calls_start: b"", + tool_calls_end: b"", + invoke_start: b"", + param_start: b"", + entity_values: true, + }, + // The official V4.1 encoding.py spelling (4294136): the block name and the + // tag names open with a space (engine server_types.h DS4_*_SPACED). + DsmlSyntax { + tool_calls_start: DSML_TOOL_CALLS_START_SPACED.as_bytes(), + tool_calls_end: DSML_TOOL_CALLS_END_SPACED.as_bytes(), + invoke_start: DSML_INVOKE_START_SPACED.as_bytes(), + invoke_end: DSML_INVOKE_END_SPACED.as_bytes(), + param_start: DSML_PARAM_START_SPACED.as_bytes(), + param_end: DSML_PARAM_END_SPACED.as_bytes(), + entity_values: false, + }, +]; + +#[derive(Debug, Clone)] +pub struct DsmlDecodeTracker { + mode: TrackMode, + pub decode: DsmlDecodeState, + syn: Option, + pos: usize, + json_in_string: bool, + json_escaped: bool, + syntax: ToolSyntax, +} + +impl Default for DsmlDecodeTracker { + fn default() -> Self { + Self { + mode: TrackMode::Search, + decode: DsmlDecodeState::Outside, + syn: None, + pos: 0, + json_in_string: false, + json_escaped: false, + syntax: ToolSyntax::Dsml, + } + } +} + +impl DsmlDecodeTracker { + /// C: `tracker.model_syntax = ...`, assigned before the first update + /// (ds4_server.c:20591). A dialect must be chosen before the block opens: + /// the start search is dialect-specific. + pub fn set_tool_syntax(&mut self, syntax: ToolSyntax) { + self.syntax = syntax; + } + + /// The `` bare spellings (f302be7) belong to the + /// DSML dialects; GLM's block has ``/`` only, so its + /// dialect must not accept the aliases. + fn alias(&self) -> bool { + self.syntax == ToolSyntax::Dsml + } + + /// C `tracker->syn` (ds4_server.c:7758): a tool block has been opened since + /// the last reset. + pub fn block_started(&self) -> bool { + self.syn.is_some() + } + + /// The dialect the tracker was told to recognize. + pub fn tool_syntax(&self) -> ToolSyntax { + self.syntax + } + + /// C `tracker->mode == DSML_TRACK_DONE` (ds4_server.c:7759): the block that + /// was opened has been closed by its own end marker. + pub fn block_closed(&self) -> bool { + self.mode == TrackMode::Done + } + + pub fn update(&mut self, raw: &[u8]) { + loop { + if self.mode == TrackMode::Done { + self.decode = DsmlDecodeState::Outside; + return; + } + if self.mode == TrackMode::Search { + let found = match self.syntax { + // The GLM dialect searches only its own start marker + // (ds4_server.c:7595-7611): `` is not one of the + // DSML spellings, and the DSML list must not see GLM bytes. + ToolSyntax::Glm => { + let from = self.pos.min(raw.len()); + find_lit_bounded(&raw[from..], GLM_SYNTAX.tool_calls_start) + .map(|rel| (from + rel + GLM_SYNTAX.tool_calls_start.len(), GLM_SYNTAX)) + } + ToolSyntax::Dsml => dsml_find_tool_start_from(raw, self.pos), + }; + match found { + None => { + let hold = dsml_max_tool_start_len(); + self.pos = if raw.len() > hold { + raw.len() - hold + } else { + 0 + }; + self.decode = DsmlDecodeState::Outside; + return; + } + Some((pos, syn)) => { + self.syn = Some(syn); + self.pos = pos; + self.mode = TrackMode::Structural; + self.decode = DsmlDecodeState::Structural; + } + } + } + if self.mode == TrackMode::StringBody { + let syn = self.syn.unwrap(); + while self.pos < raw.len() { + let end_len = param_close_len(&syn, raw, self.pos, self.alias()); + if end_len > 0 { + self.pos += end_len; + self.mode = TrackMode::Structural; + self.decode = DsmlDecodeState::Structural; + break; + } + if param_close_partial_any(&syn, raw, self.pos, self.alias()) { + // A tail that is a proper prefix of the close spelling + // holds the position: the stream may cut the marker + // between two tokens, and the next update re-reads it + // (ds4_server.c:7635-7641). A lone '<' is payload. + self.decode = if raw.len() - self.pos >= 2 { + DsmlDecodeState::Structural + } else { + DsmlDecodeState::StringBody + }; + return; + } + self.pos += 1; + } + if self.mode == TrackMode::StringBody { + self.decode = DsmlDecodeState::StringBody; + return; + } + } + if self.mode == TrackMode::JsonParam { + let syn = self.syn.unwrap(); + while self.pos < raw.len() { + if !self.json_in_string { + let end_len = param_close_len(&syn, raw, self.pos, self.alias()); + if end_len > 0 { + self.pos += end_len; + self.mode = TrackMode::Structural; + self.decode = DsmlDecodeState::Structural; + break; + } + if param_close_partial(&syn, raw, self.pos, 2, self.alias()) { + self.decode = DsmlDecodeState::Structural; + return; + } + } + let c = raw[self.pos]; + self.pos += 1; + if self.json_in_string { + if self.json_escaped { + self.json_escaped = false; + } else if c == b'\\' { + self.json_escaped = true; + } else if c == b'"' { + self.json_in_string = false; + } + } else if c == b'"' { + self.json_in_string = true; + } + } + if self.mode == TrackMode::JsonParam { + self.decode = if self.json_in_string { + DsmlDecodeState::JsonString + } else { + DsmlDecodeState::JsonStructural + }; + return; + } + } + while self.mode == TrackMode::Structural { + let syn = self.syn.unwrap(); + while self.pos < raw.len() && raw[self.pos].is_ascii_whitespace() { + self.pos += 1; + } + if self.pos >= raw.len() { + self.decode = DsmlDecodeState::Structural; + return; + } + if raw_full_lit(raw, self.pos, syn.tool_calls_end) { + self.mode = TrackMode::Done; + self.pos += syn.tool_calls_end.len(); + self.decode = DsmlDecodeState::Outside; + return; + } + if raw_full_lit(raw, self.pos, syn.invoke_end) { + self.pos += syn.invoke_end.len(); + continue; + } + if raw_full_lit(raw, self.pos, syn.invoke_start) { + match raw[self.pos..].iter().position(|&c| c == b'>') { + None => { + self.decode = DsmlDecodeState::Structural; + return; + } + Some(rel) => { + self.pos += rel + 1; + continue; + } + } + } + if param_open_full(&syn, raw, self.pos, self.alias()) { + let tag_start = self.pos; + let Some(rel) = raw[self.pos..].iter().position(|&c| c == b'>') else { + self.decode = DsmlDecodeState::Structural; + return; + }; + let tag_after = self.pos + rel + 1; + // GLM argument values carry no type attribute: every value + // body is text (ds4_server.c:7731-7739 over glm_tool_syntax). + let string_value = self.syntax == ToolSyntax::Glm + || dsml_attr_is_string_true(raw, tag_start, tag_after); + self.pos = tag_after; + if string_value { + self.mode = TrackMode::StringBody; + self.decode = DsmlDecodeState::StringBody; + } else { + self.mode = TrackMode::JsonParam; + self.json_in_string = false; + self.json_escaped = false; + self.decode = DsmlDecodeState::JsonStructural; + } + break; + } + if raw_partial_lit(raw, self.pos, syn.tool_calls_end) + || raw_partial_lit(raw, self.pos, syn.invoke_start) + || raw_partial_lit(raw, self.pos, syn.invoke_end) + || param_open_partial(&syn, raw, self.pos, self.alias()) + || param_close_partial(&syn, raw, self.pos, 1, self.alias()) + { + self.decode = DsmlDecodeState::Structural; + return; + } + if self.syntax == ToolSyntax::Glm { + // GLM puts the function and argument names between tags; + // unknown structural bytes advance (ds4_server.c:7746-7752). + self.pos += 1; + continue; + } + self.decode = DsmlDecodeState::Structural; + return; + } + } + } +} + +fn dsml_attr_is_string_true(raw: &[u8], tag_start: usize, tag_end: usize) -> bool { + if tag_end <= tag_start || tag_end > raw.len() { + return false; + } + dsml_attr(&raw[tag_start..tag_end], "string").is_some_and(|v| v == b"true") +} + +fn dsml_max_tool_start_len() -> usize { + SYNTAXES + .iter() + .map(|s| s.tool_calls_start.len()) + .max() + .unwrap_or(0) +} + +fn dsml_find_tool_start(raw: &[u8]) -> Option<(usize, DsmlSyntax)> { + let mut best: Option<(usize, DsmlSyntax)> = None; + for syn in SYNTAXES { + if let Some(i) = find_lit_bounded(raw, syn.tool_calls_start) { + if best.map(|(p, _)| i < p).unwrap_or(true) { + best = Some((i + syn.tool_calls_start.len(), syn)); + } + } + } + best +} + +fn dsml_find_tool_start_from(raw: &[u8], start: usize) -> Option<(usize, DsmlSyntax)> { + if start > raw.len() { + return None; + } + dsml_find_tool_start(&raw[start..]).map(|(rel, syn)| (start + rel, syn)) +} + +fn raw_full_lit(raw: &[u8], pos: usize, lit: &[u8]) -> bool { + pos <= raw.len() && raw.len() - pos >= lit.len() && raw[pos..].starts_with(lit) +} + +fn raw_partial_lit(raw: &[u8], pos: usize, lit: &[u8]) -> bool { + if pos > raw.len() || raw.len() - pos >= lit.len() { + return false; + } + raw[pos..] == lit[..raw.len() - pos] +} + +fn raw_partial_lit_min(raw: &[u8], pos: usize, lit: &[u8], min_len: usize) -> bool { + if pos > raw.len() || raw.len() - pos >= lit.len() { + return false; + } + let avail = raw.len() - pos; + avail >= min_len && raw[pos..] == lit[..avail] +} + +fn find_lit_bounded(s: &[u8], lit: &[u8]) -> Option { + if lit.is_empty() { + return Some(0); + } + s.windows(lit.len()).position(|w| w == lit) +} + +/// The engine's bare-parameter aliases (f302be7, server_dsml_parse.c:5-47): a +/// decorated or spaced block also accepts the plain spellings, because the +/// artifact mixes them inside one block (decorated invoke, bare parameters). +/// The plain syntax already is the bare form, so the alias never stacks on it. +/// The open alias carries the trailing space: "" is not an open. +const BARE_PARAM_START: &[u8] = b""; + +fn bare_param_alias(syn: &DsmlSyntax) -> bool { + syn.param_start != &b" bool { + raw_full_lit(raw, pos, syn.param_start) || (alias && raw_full_lit(raw, pos, BARE_PARAM_START)) +} + +fn param_open_partial(syn: &DsmlSyntax, raw: &[u8], pos: usize, alias: bool) -> bool { + raw_partial_lit(raw, pos, syn.param_start) + || (alias && raw_partial_lit(raw, pos, BARE_PARAM_START)) +} + +/// Length of the close at pos in either spelling (f302be7, +/// server_dsml_parse.c:31-38); 0 when neither matches. +fn param_close_len(syn: &DsmlSyntax, raw: &[u8], pos: usize, alias: bool) -> usize { + if raw_full_lit(raw, pos, syn.param_end) { + return syn.param_end.len(); + } + if alias && raw_full_lit(raw, pos, BARE_PARAM_END) { + return BARE_PARAM_END.len(); + } + 0 +} + +fn param_close_partial( + syn: &DsmlSyntax, + raw: &[u8], + pos: usize, + min_len: usize, + alias: bool, +) -> bool { + raw_partial_lit_min(raw, pos, syn.param_end, min_len) + || (alias && raw_partial_lit_min(raw, pos, BARE_PARAM_END, min_len)) +} + +/// Any proper prefix of the close spelling (C `raw_partial_lit`, ds4_server.c: +/// 7636): the tracker holds its position on one, so a marker cut between tokens +/// is re-read by the next update. `param_close_partial` is the >= min_len form +/// used where only the decoded state matters. +fn param_close_partial_any(syn: &DsmlSyntax, raw: &[u8], pos: usize, alias: bool) -> bool { + raw_partial_lit(raw, pos, syn.param_end) || (alias && raw_partial_lit(raw, pos, BARE_PARAM_END)) +} + +/// Earliest close in either spelling from `from`, as (offset, len) +/// (f302be7, server_dsml_parse.c:42-47). +fn param_close_find(syn: &DsmlSyntax, raw: &[u8], from: usize) -> Option<(usize, usize)> { + let rest = &raw[from..]; + let own = find_lit_bounded(rest, syn.param_end).map(|r| (r, syn.param_end.len())); + if !bare_param_alias(syn) { + return own; + } + let bare = find_lit_bounded(&rest[..own.map_or(rest.len(), |(r, _)| r)], BARE_PARAM_END) + .map(|r| (r, BARE_PARAM_END.len())); + bare.or(own) +} + +/// The buffer tail is a proper prefix of either close spelling (>= min_len +/// bytes): the stream holds it back until the next token proves it (f302be7, +/// server_dsml_stream.c:104-110). +fn raw_suffix_partial_close(syn: &DsmlSyntax, raw: &[u8], min_len: usize, alias: bool) -> bool { + let mut n = min_len; + while n <= raw.len() && n < 32 { + if param_close_partial(syn, raw, raw.len() - n, min_len, alias) { + return true; + } + n += 1; + } + false +} + +fn dsml_decode_scan_json_param( + raw: &[u8], + mut pos: usize, + syn: DsmlSyntax, + alias: bool, +) -> DsmlDecodeState { + let mut in_string = false; + let mut escaped = false; + while pos < raw.len() { + if !in_string && param_close_len(&syn, raw, pos, alias) > 0 { + return DsmlDecodeState::Structural; + } + let c = raw[pos]; + pos += 1; + if in_string { + if escaped { + escaped = false; + } else if c == b'\\' { + escaped = true; + } else if c == b'"' { + in_string = false; + } + } else if c == b'"' { + in_string = true; + } + } + if !in_string && raw_suffix_partial_close(&syn, raw, 2, bare_param_alias(&syn)) { + return DsmlDecodeState::Structural; + } + if in_string { + DsmlDecodeState::JsonString + } else { + DsmlDecodeState::JsonStructural + } +} + +/// Slow reference recognizer used by C tests (`dsml_decode_state_for_text`). +pub fn dsml_decode_state_for_text(raw: &[u8]) -> DsmlDecodeState { + if raw.is_empty() { + return DsmlDecodeState::Outside; + } + let Some((mut pos, syn)) = dsml_find_tool_start(raw) else { + return DsmlDecodeState::Outside; + }; + loop { + while pos < raw.len() && raw[pos].is_ascii_whitespace() { + pos += 1; + } + if pos >= raw.len() { + return DsmlDecodeState::Structural; + } + if raw_full_lit(raw, pos, syn.tool_calls_end) { + return DsmlDecodeState::Outside; + } + if raw_full_lit(raw, pos, syn.invoke_end) { + pos += syn.invoke_end.len(); + continue; + } + if raw_full_lit(raw, pos, syn.invoke_start) { + match raw[pos..].iter().position(|&c| c == b'>') { + None => return DsmlDecodeState::Structural, + Some(rel) => { + pos += rel + 1; + continue; + } + } + } + if param_open_full(&syn, raw, pos, bare_param_alias(&syn)) { + let tag_start = pos; + let Some(rel) = raw[pos..].iter().position(|&c| c == b'>') else { + return DsmlDecodeState::Structural; + }; + let tag_end = pos + rel + 1; + let string_value = dsml_attr_is_string_true(raw, tag_start, tag_end); + pos = tag_end; + if string_value { + match param_close_find(&syn, raw, pos) { + None => { + if raw_suffix_partial_close(&syn, raw, 2, bare_param_alias(&syn)) { + return DsmlDecodeState::Structural; + } + return DsmlDecodeState::StringBody; + } + Some((end, end_len)) => { + pos += end + end_len; + continue; + } + } + } + let json_state = dsml_decode_scan_json_param(raw, pos, syn, bare_param_alias(&syn)); + if json_state == DsmlDecodeState::Structural { + match param_close_find(&syn, raw, pos) { + None => return DsmlDecodeState::Structural, + Some((end, end_len)) => { + pos += end + end_len; + continue; + } + } + } + return json_state; + } + for s in SYNTAXES { + if raw_partial_lit(raw, pos, s.tool_calls_end) + || raw_partial_lit(raw, pos, s.invoke_start) + || raw_partial_lit(raw, pos, s.invoke_end) + || raw_partial_lit(raw, pos, s.param_start) + || raw_partial_lit(raw, pos, s.param_end) + { + return DsmlDecodeState::Structural; + } + } + return DsmlDecodeState::Structural; + } +} + +pub fn agent_turn_reasoning_cap(think_mode: ThinkMode, max_tokens: i32) -> i32 { + let reserve = 64; + let budget_cap = if max_tokens > reserve { + max_tokens - reserve + } else { + 0 + }; + let effort_cap = match think_mode { + ThinkMode::High => 256, + ThinkMode::Max => 512, + _ => 64, + }; + budget_cap.min(effort_cap) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SampleOverride { + None, + Greedy, + Token(i32), +} + +impl SampleOverride { + pub fn as_c_int(self) -> i32 { + match self { + Self::None => SAMPLE_OVERRIDE_NONE, + Self::Greedy => SAMPLE_OVERRIDE_GREEDY, + Self::Token(t) => t + 2, + } + } +} + +pub struct SamplePolicy<'a> { + pub tool_choice: ToolChoice, + pub has_tool_results: bool, + pub think_mode: ThinkMode, + pub max_tokens: i32, + pub required_tool_prefix: &'a [i32], + pub required_think_end_prefix: &'a [i32], +} + +pub fn sampling_override( + track_tools: bool, + saw_tool_start: bool, + thinking_inside: bool, + completion: i32, + chat_format: ChatFormat, + dsml: DsmlDecodeState, + tool_pos: &mut i32, + think_pos: &mut i32, + p: &SamplePolicy<'_>, +) -> SampleOverride { + let prefix_in_flight = *tool_pos > 0 && (*tool_pos as usize) < p.required_tool_prefix.len(); + let required_pending = p.tool_choice == ToolChoice::Required + && track_tools + && (!saw_tool_start || prefix_in_flight); + // MiMo auto chooses the marker; only then complete its XML head. + let auto_body_pending = p.tool_choice == ToolChoice::Auto + && track_tools + && saw_tool_start + && !thinking_inside + && (*tool_pos as usize) < p.required_tool_prefix.len(); + let reserve_post_thinking = required_pending || p.has_tool_results; + if reserve_post_thinking && thinking_inside { + if completion >= agent_turn_reasoning_cap(p.think_mode, p.max_tokens) + && (*think_pos as usize) < p.required_think_end_prefix.len() + { + let token = p.required_think_end_prefix[*think_pos as usize]; + *think_pos += 1; + return SampleOverride::Token(token); + } + return SampleOverride::None; + } + if (required_pending || auto_body_pending) + && (*tool_pos as usize) < p.required_tool_prefix.len() + { + let token = p.required_tool_prefix[*tool_pos as usize]; + *tool_pos += 1; + return SampleOverride::Token(token); + } + let state = if track_tools && chat_format == ChatFormat::DeepSeek { + dsml + } else { + DsmlDecodeState::Outside + }; + if state.is_tool() && !state.uses_payload_sampling() { + return SampleOverride::Greedy; + } + SampleOverride::None +} + +pub fn dump_script(name: &str) -> String { + match name { + "state-prefix" => dump_state(state_prefix()), + "state-path-param" => dump_state(state_path_param()), + "state-path-closing" => dump_state(state_path_closing()), + "state-json-struct" => dump_state(state_json_struct()), + "state-json-string" => dump_state(state_json_string()), + "state-done" => dump_state(state_done()), + "override-required" => dump_override_required(), + "override-think-cap" => dump_override_think_cap(), + "override-tool-result" => dump_override_tool_result(), + "cap-low-128" => format!("{}\n", agent_turn_reasoning_cap(ThinkMode::Low, 128)), + "cap-high-128" => format!("{}\n", agent_turn_reasoning_cap(ThinkMode::High, 128)), + "cap-max-600" => format!("{}\n", agent_turn_reasoning_cap(ThinkMode::Max, 600)), + "glm-boundaries-token" => dump_glm_boundaries(false), + "glm-boundaries-byte" => dump_glm_boundaries(true), + "glm-control-text" => dump_glm_control_text(), + _ => "ERROR unknown-script\n".into(), + } +} + +/// The reference's GLM fragments, ds4_server.c:20572 +/// (`test_glm_decode_tracker_boundaries` in /data/ds4). +fn glm_parts() -> [&'static str; 7] { + [ + "prose bashcommand", + "printf ''", + "<", + "/arg_", + "value>", + "descriptiontest", + "", + ] +} + +/// Feed the accumulated text fragment by fragment against the GLM dialect and +/// print the state after each one, so a dialect drift shows as a tape line. +fn dump_glm_boundaries(bytewise: bool) -> String { + let mut tracker = DsmlDecodeTracker::default(); + tracker.set_tool_syntax(ToolSyntax::Glm); + let mut raw = Vec::new(); + let mut states = Vec::new(); + for part in glm_parts() { + if bytewise { + for byte in part.bytes() { + raw.push(byte); + tracker.update(&raw); + } + } else { + raw.extend_from_slice(part.as_bytes()); + tracker.update(&raw); + } + states.push(tracker.decode.name()); + } + format!("states={}\n", states.join(",")) +} + +/// ds4_server.c:20608 (`test_tool_control_text_inside_arguments`): control +/// markers written into an argument value are payload, so the block stays open. +fn dump_glm_control_text() -> String { + let raw = format!( + "reasonwritecontent\ + literal {DSML_TOOL_CALLS_END} " + ); + let mut tracker = DsmlDecodeTracker::default(); + tracker.set_tool_syntax(ToolSyntax::Glm); + for n in 1..=raw.len() { + tracker.update(&raw.as_bytes()[..n]); + } + format!( + "tracker={} start={} end={}\n", + tracker.decode.name(), + u8::from(tracker.block_started()), + u8::from(tracker.block_closed()) + ) +} + +fn state_prefix() -> String { + format!( + "{}\n{} name=\"edit\">\n", + DSML_TOOL_CALLS_START, DSML_INVOKE_START + ) +} +fn state_path_param() -> String { + format!( + "{}\n{} name=\"edit\">\n{} name=\"path\" string=\"true\">/tmp/a.py", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ) +} +fn state_path_closing() -> String { + format!( + "{}\n{} name=\"edit\">\n{} name=\"path\" string=\"true\">/tmp/a.py String { + format!( + "{}\n{} name=\"edit\">\n{} name=\"edits\" string=\"false\">[{{", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ) +} +fn state_json_string() -> String { + format!( + "{}\n{} name=\"edit\">\n{} name=\"edits\" string=\"false\">[{{\"newText\":\"for i in", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ) +} +fn state_done() -> String { + format!( + "{}\n{} name=\"edit\">\n{} name=\"edits\" string=\"false\">[]{}\n{}\n{}", + DSML_TOOL_CALLS_START, + DSML_INVOKE_START, + DSML_PARAM_START, + DSML_PARAM_END, + DSML_INVOKE_END, + DSML_TOOL_CALLS_END + ) +} + +fn dump_state(raw: String) -> String { + let ref_state = dsml_decode_state_for_text(raw.as_bytes()); + let mut tr = DsmlDecodeTracker::default(); + tr.update(raw.as_bytes()); + format!("ref={} tracker={}\n", ref_state.name(), tr.decode.name()) +} + +fn dump_override_required() -> String { + let p = SamplePolicy { + tool_choice: ToolChoice::Required, + has_tool_results: false, + think_mode: ThinkMode::None, + max_tokens: 128, + required_tool_prefix: &[101, 202], + required_think_end_prefix: &[], + }; + let mut tool_pos = 0; + let mut think_pos = 0; + let a = sampling_override( + true, + false, + false, + 0, + ChatFormat::DeepSeek, + DsmlDecodeState::Outside, + &mut tool_pos, + &mut think_pos, + &p, + ); + let b = sampling_override( + true, + false, + false, + 0, + ChatFormat::DeepSeek, + DsmlDecodeState::Outside, + &mut tool_pos, + &mut think_pos, + &p, + ); + format!("{} {}\n", a.as_c_int(), b.as_c_int()) +} + +fn dump_override_think_cap() -> String { + let p = SamplePolicy { + tool_choice: ToolChoice::Required, + has_tool_results: false, + think_mode: ThinkMode::Low, + max_tokens: 128, + required_tool_prefix: &[], + required_think_end_prefix: &[303], + }; + let mut tool_pos = 0; + let mut think_pos = 0; + let none = sampling_override( + true, + false, + true, + 0, + ChatFormat::DeepSeek, + DsmlDecodeState::Outside, + &mut tool_pos, + &mut think_pos, + &p, + ); + let forced = sampling_override( + true, + false, + true, + 64, + ChatFormat::DeepSeek, + DsmlDecodeState::Outside, + &mut tool_pos, + &mut think_pos, + &p, + ); + format!("{} {}\n", none.as_c_int(), forced.as_c_int()) +} + +fn dump_override_tool_result() -> String { + let p = SamplePolicy { + tool_choice: ToolChoice::Auto, + has_tool_results: true, + think_mode: ThinkMode::Low, + max_tokens: 128, + required_tool_prefix: &[], + required_think_end_prefix: &[303], + }; + let mut tool_pos = 0; + let mut think_pos = 0; + let a = sampling_override( + false, + false, + true, + 63, + ChatFormat::DeepSeek, + DsmlDecodeState::Outside, + &mut tool_pos, + &mut think_pos, + &p, + ); + let b = sampling_override( + false, + false, + true, + 64, + ChatFormat::DeepSeek, + DsmlDecodeState::Outside, + &mut tool_pos, + &mut think_pos, + &p, + ); + format!("{} {}\n", a.as_c_int(), b.as_c_int()) +} + +#[cfg(test)] +mod override_tests { + use super::{sampling_override, DsmlDecodeState, SampleOverride, SamplePolicy}; + use crate::parse::ToolChoice; + use crate::route::ThinkMode; + use crate::stream::ChatFormat; + + #[test] + fn required_prefix_finishes_after_tool_start() { + let policy = SamplePolicy { + tool_choice: ToolChoice::Required, + has_tool_results: false, + think_mode: ThinkMode::None, + max_tokens: 128, + required_tool_prefix: &[101, 202], + required_think_end_prefix: &[], + }; + let mut tool_pos = 0; + let mut think_pos = 0; + let mut sample = |saw_tool_start| { + sampling_override( + true, + saw_tool_start, + false, + tool_pos, + ChatFormat::Qwen4Exp, + DsmlDecodeState::Outside, + &mut tool_pos, + &mut think_pos, + &policy, + ) + }; + assert_eq!(sample(false), SampleOverride::Token(101)); + assert_eq!(sample(true), SampleOverride::Token(202)); + } + + #[test] + fn auto_tool_start_forces_function_prefix() { + let policy = SamplePolicy { + tool_choice: ToolChoice::Auto, + has_tool_results: false, + think_mode: ThinkMode::None, + max_tokens: 128, + required_tool_prefix: &[202], + required_think_end_prefix: &[], + }; + let mut tool_pos = 0; + let mut think_pos = 0; + let mut sample = |saw_tool_start| { + sampling_override( + true, + saw_tool_start, + false, + 0, + ChatFormat::Qwen4Exp, + DsmlDecodeState::Outside, + &mut tool_pos, + &mut think_pos, + &policy, + ) + }; + assert_eq!(sample(false), SampleOverride::None); + assert_eq!(sample(true), SampleOverride::Token(202)); + assert_eq!(sample(true), SampleOverride::None); + } +} diff --git a/crates/ds4-server/src/error.rs b/crates/ds4-server/src/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..6225630922dc97473af4618b7ab6af2a1b983a22 --- /dev/null +++ b/crates/ds4-server/src/error.rs @@ -0,0 +1,184 @@ +//! Endpoint-native error envelopes. Do not unify the families. + +use crate::json::json_escape; +use crate::route::WireSurface; + +pub fn http_reason(code: i32) -> &'static str { + match code { + 200 => "OK", + 204 => "No Content", + 400 => "Bad Request", + 404 => "Not Found", + 409 => "Conflict", + 429 => "Too Many Requests", + 500 => "Internal Server Error", + 503 => "Service Unavailable", + _ => "Error", + } +} + +pub fn openai_error_type(code: i32) -> &'static str { + if code == 429 { + "rate_limit_error" + } else if code >= 500 { + "server_error" + } else { + "invalid_request_error" + } +} + +pub fn anthropic_error_type(code: i32) -> &'static str { + if code == 429 { + "rate_limit_error" + } else if code == 404 { + "not_found_error" + } else if code == 503 { + "overloaded_error" + } else if code >= 500 { + "api_error" + } else { + "invalid_request_error" + } +} + +pub fn openai_error_body(code: i32, msg: &str) -> String { + format!( + "{{\"error\":{{\"message\":{},\"type\":\"{}\"}}}}\n", + json_escape(msg), + openai_error_type(code) + ) +} + +pub fn anthropic_error_body(code: i32, msg: &str) -> String { + format!( + "{{\"type\":\"error\",\"error\":{{\"type\":\"{}\",\"message\":{}}}}}\n", + anthropic_error_type(code), + json_escape(msg) + ) +} + +pub fn wire_error_body(surface: WireSurface, code: i32, msg: &str) -> String { + if surface == WireSurface::Anthropic { + anthropic_error_body(code, msg) + } else { + openai_error_body(code, msg) + } +} + +pub fn cors_headers() -> &'static str { + "Access-Control-Allow-Origin: *\r\nAccess-Control-Allow-Methods: GET, POST, OPTIONS\r\nAccess-Control-Allow-Headers: *\r\n" +} + +pub fn retry_after_header(sec: i32) -> String { + format!("Retry-After: {sec}\r\n") +} + +pub fn http_head( + code: i32, + content_type: Option<&str>, + extra_headers: Option<&str>, + cors: bool, + body_len: usize, +) -> String { + let mut h = format!( + "HTTP/1.1 {code} {}\r\nContent-Length: {body_len}\r\n", + http_reason(code) + ); + if let Some(t) = content_type { + if !t.is_empty() { + h.push_str("Content-Type: "); + h.push_str(t); + h.push_str("\r\n"); + } + } + if let Some(e) = extra_headers { + if !e.is_empty() { + h.push_str(e); + } + } + if cors { + h.push_str(cors_headers()); + } + h.push_str("Connection: close\r\n\r\n"); + h +} + +pub fn http_response_bytes( + code: i32, + content_type: Option<&str>, + extra_headers: Option<&str>, + cors: bool, + body: &str, +) -> Vec { + let mut out = http_head(code, content_type, extra_headers, cors, body.len()).into_bytes(); + out.extend_from_slice(body.as_bytes()); + out +} + +pub fn wire_http_error_bytes( + surface: WireSurface, + code: i32, + msg: &str, + cors: bool, + retry_after: Option, +) -> Vec { + let extra = retry_after.map(retry_after_header); + let body = wire_error_body(surface, code, msg); + http_response_bytes( + code, + Some("application/json"), + extra.as_deref(), + cors, + &body, + ) +} + +pub(crate) fn wire_stream_error_bytes( + surface: WireSurface, + msg: &str, + responses_sequence: i32, +) -> Vec { + let message = if msg.is_empty() { + "internal server error" + } else { + msg + }; + match surface { + WireSurface::Anthropic => format!( + "event: error\ndata: {{\"type\":\"error\",\"error\":{{\"type\":\"api_error\",\"message\":{}}}}}\n\n", + json_escape(message) + ) + .into_bytes(), + WireSurface::Responses => format!( + "data: {{\"type\":\"error\",\"sequence_number\":{responses_sequence},\"code\":\"server_error\",\"message\":{},\"param\":null}}\n\n", + json_escape(message) + ) + .into_bytes(), + WireSurface::OpenaiChat | WireSurface::OpenaiCompletion => format!( + "event: error\ndata: {{\"error\":{{\"message\":{},\"type\":\"server_error\"}}}}\n\n", + json_escape(message) + ) + .into_bytes(), + } +} + +#[cfg(test)] +mod stream_error_tests { + use super::*; + + #[test] + fn stream_errors_keep_each_surface_wire_shape() { + assert_eq!( + wire_stream_error_bytes(WireSurface::OpenaiChat, "boom", 0), + b"event: error\ndata: {\"error\":{\"message\":\"boom\",\"type\":\"server_error\"}}\n\n" + ); + assert_eq!( + wire_stream_error_bytes(WireSurface::Anthropic, "boom", 0), + b"event: error\ndata: {\"type\":\"error\",\"error\":{\"type\":\"api_error\",\"message\":\"boom\"}}\n\n" + ); + assert_eq!( + wire_stream_error_bytes(WireSurface::Responses, "boom", 7), + b"data: {\"type\":\"error\",\"sequence_number\":7,\"code\":\"server_error\",\"message\":\"boom\",\"param\":null}\n\n" + ); + } +} diff --git a/crates/ds4-server/src/expected_plan.rs b/crates/ds4-server/src/expected_plan.rs new file mode 100644 index 0000000000000000000000000000000000000000..6e92405083dd066ad3e3e8061f9e3dc19e9e9fb0 --- /dev/null +++ b/crates/ds4-server/src/expected_plan.rs @@ -0,0 +1,170 @@ +//! Profiles bind the resolved runtime, including facts learned after native open. +use ds4_core::ResolvedPlan; +use serde_json::Value; +use std::path::Path; + +const MAX_PLAN_BYTES: u64 = 64 * 1024; +const KEYS: [&str; 5] = ["family", "backend", "effective", "qualified", "controls"]; + +pub struct ExpectedPlan { + value: Value, +} + +impl ExpectedPlan { + pub fn load(path: &Path) -> Result { + let file = std::fs::File::open(path).map_err(|e| format!("expected serving plan: {e}"))?; + if file.metadata().map_err(|e| e.to_string())?.len() > MAX_PLAN_BYTES { + return Err("expected serving plan exceeds 64 KiB".into()); + } + let value: Value = + serde_json::from_reader(file).map_err(|e| format!("expected serving plan: {e}"))?; + Self::parse(value) + } + + fn parse(value: Value) -> Result { + if value.as_object().is_none_or(|v| v.len() != KEYS.len() + 1) + || value + .get("post_open_quote") + .is_none_or(|v| !v.is_null() && !v.is_object()) + || ["family", "backend"] + .iter() + .any(|key| value[key].as_str().is_none_or(str::is_empty)) + || ["effective", "qualified", "controls"] + .iter() + .any(|key| !value[key].is_object()) + { + return Err( + "expected serving plan requires family/backend/effective/qualified/controls/post_open_quote".into(), + ); + } + Ok(Self { value }) + } + + /// Preflight has not earned resident-memory credits; check policy here. + pub fn check_preflight(&self, actual: &ResolvedPlan) -> Result<(), String> { + if !actual.may_listen() { + return Err("expected serving plan rejected: runtime has configuration errors".into()); + } + let actual = actual.to_json(); + for key in KEYS { + let value = if key == "backend" { + &actual["requested"]["backend"] + } else { + &actual[key] + }; + if self.value[key] != *value { + return Err(format!( + "expected serving plan mismatch: {key}; refusing to listen" + )); + } + } + Ok(()) + } + + pub fn check(&self, actual: &ResolvedPlan) -> Result<(), String> { + self.check_preflight(actual)?; + if self.value["post_open_quote"] != quote(&actual.to_json()) { + return Err( + "expected serving plan mismatch: post-open memory quote; refusing to listen".into(), + ); + } + Ok(()) + } +} + +fn quote(plan: &Value) -> Value { + let mut quote = plan["quote"].clone(); + if let Some(fields) = quote.as_object_mut() { + fields.remove("available"); + } + quote +} + +#[cfg(test)] +mod tests { + use super::*; + use ds4_core::{ + resolve_plan, serving_caps, EngineFacts, MaxSeqs, ModelFamily, ServingRequest, Variant, + }; + use serde_json::json; + + fn actual() -> ResolvedPlan { + let request = ServingRequest { + max_seqs: MaxSeqs::Fixed(1), + ..ServingRequest::default() + }; + resolve_plan( + &request, + Some(serving_caps( + ModelFamily::Qwen4Exp, + Variant::Qwen38FlashNext, + )), + &EngineFacts::default(), + ) + } + fn expected(actual: &ResolvedPlan) -> Value { + let value = actual.to_json(); + json!({"family":value["family"],"backend":value["requested"]["backend"],"effective":value["effective"],"qualified":value["qualified"],"controls":value["controls"],"post_open_quote":quote(&value)}) + } + #[test] + fn rejects_post_open_drift() { + let actual = actual(); + for (section, key, changed) in [ + ("effective", "max_seqs", json!(2)), + ( + "effective", + "mtp_weights", + json!(!actual.effective.mtp_weights), + ), + ("qualified", "banks_n", json!(999)), + ] { + let mut value = expected(&actual); + value[section][key] = changed; + assert!( + ExpectedPlan::parse(value).unwrap().check(&actual).is_err(), + "{section}.{key}" + ); + } + let mut value = expected(&actual); + value["backend"] = json!("cpu"); + assert!(ExpectedPlan::parse(value).unwrap().check(&actual).is_err()); + } + #[test] + fn accepts_exact_contract_and_rejects_partial_schema() { + let actual = actual(); + assert!(ExpectedPlan::parse(expected(&actual)) + .unwrap() + .check(&actual) + .is_ok()); + assert!(ExpectedPlan::parse(json!({"effective":{}})).is_err()); + } + + #[test] + fn quote_comparison_respects_accounting_phase() { + let mut actual = actual(); + actual.quote = Some(ds4_core::ServingQuote { + expert_cache: 0, + expert_staging: 0, + expert_metadata: 0, + shared_weights: 1, + per_bank: 2, + mtp_state: 3, + scratch: 4, + checkpoint_pool: 5, + ple: 6, + media_reserve: 7, + floor: 8, + available: 100, + banks: 1, + total: 36, + }); + let guard = ExpectedPlan::parse(expected(&actual)).unwrap(); + actual.quote.as_mut().unwrap().available = 200; + assert!(guard.check(&actual).is_ok()); + actual.quote.as_mut().unwrap().shared_weights = 99; + assert!(guard.check_preflight(&actual).is_ok()); + assert!(guard + .check(&actual) + .is_err_and(|e| e.contains("post-open memory quote"))); + } +} diff --git a/crates/ds4-server/src/format.rs b/crates/ds4-server/src/format.rs new file mode 100644 index 0000000000000000000000000000000000000000..8831196e76947b3740827ccfa87328ad539a0b78 --- /dev/null +++ b/crates/ds4-server/src/format.rs @@ -0,0 +1,180 @@ +//! v0.6.3 schema-format refusal (ds4-on-spark#10). + +use crate::json::{json_skip_value, json_string, Json}; +use crate::route::{parse_reasoning_effort_name, RequestedEffort, ThinkMode}; + +pub fn output_format_type_supported(field: &str, typ: &str) -> Result<(), String> { + if typ == "text" { + return Ok(()); + } + if typ == "json_object" || typ == "json_schema" { + return Err(format!( + "{field} type '{typ}' is not supported: structured output is \ + unsupported; omit {field} or use type \"text\"" + )); + } + Err(format!("{field} type '{typ}' is not supported")) +} + +pub fn parse_output_format_value(p: &mut Json<'_>, field: &str) -> Result<(), String> { + p.ws(); + if p.lit("null") { + return Ok(()); + } + if p.peek() == Some(b'"') { + let typ = json_string(p).ok_or_else(|| String::new())?; + return output_format_type_supported(field, &typ); + } + if p.bump() != Some(b'{') { + return Err(String::new()); + } + p.ws(); + let mut typ: Option = None; + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p).ok_or_else(|| String::new())?; + p.ws(); + if p.bump() != Some(b':') { + return Err(String::new()); + } + if key == "type" { + p.ws(); + typ = Some(json_string(p).ok_or_else(|| String::new())?); + } else if !json_skip_value(p) { + return Err(String::new()); + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return Err(String::new()); + } + if let Some(t) = typ { + output_format_type_supported(field, &t)?; + } + Ok(()) +} + +pub fn parse_responses_text_value(p: &mut Json<'_>) -> Result<(), String> { + p.ws(); + if p.lit("null") { + return Ok(()); + } + if p.peek() != Some(b'{') { + if json_skip_value(p) { + return Ok(()); + } + return Err(String::new()); + } + p.i += 1; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p).ok_or_else(|| String::new())?; + p.ws(); + if p.bump() != Some(b':') { + return Err(String::new()); + } + if key == "format" { + parse_output_format_value(p, "text.format")?; + } else if !json_skip_value(p) { + return Err(String::new()); + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return Err(String::new()); + } + Ok(()) +} + +/// The `reasoning_effort` value: a name from the shared vocabulary, or the +/// oracle's numeric form -- an integer in [1,100] renders as itself +/// (encoding.py:456-473). A number turns thinking on and takes the engine's +/// collapsed HIGH mode: only the head line carries the number. +pub fn parse_reasoning_effort_value(p: &mut Json<'_>) -> Result, String> { + p.ws(); + if p.lit("null") { + return Ok(None); + } + if let Some(budget) = json_effort_budget(p) { + return Ok(Some(RequestedEffort { + mode: ThinkMode::High, + budget: Some(budget), + })); + } + let name = json_string(p).ok_or_else(|| String::new())?; + parse_reasoning_effort_name(&name) + .map(RequestedEffort::named) + .map(Some) + .ok_or_else(|| String::new()) +} + +/// A bare integer token in [1,100]: the oracle asserts `type(effort) is int` +/// (encoding.py:464), so a fraction or an exponent is not a budget and falls +/// through to the refusal. An integer outside the range is refused there too. +fn json_effort_budget(p: &mut Json<'_>) -> Option { + p.ws(); + let start = p.i; + let mut v: u32 = 0; + while let Some(c) = p.peek() { + if !c.is_ascii_digit() { + break; + } + v = (v * 10 + (c - b'0') as u32).min(u32::from(u8::MAX)); + p.i += 1; + } + if p.i == start || matches!(p.peek(), Some(b'.' | b'e' | b'E')) { + p.i = start; + return None; + } + (1..=100).contains(&v).then_some(v as u8) +} + +pub fn parse_output_config_effort(p: &mut Json<'_>) -> Result, String> { + p.ws(); + if p.lit("null") { + return Ok(None); + } + if p.peek() != Some(b'{') { + if json_skip_value(p) { + return Ok(None); + } + return Err(String::new()); + } + p.i += 1; + p.ws(); + let mut effort = None; + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p).ok_or_else(|| String::new())?; + p.ws(); + if p.bump() != Some(b':') { + return Err(String::new()); + } + if key == "effort" { + effort = parse_reasoning_effort_value(p)?; + } else if key == "format" { + parse_output_format_value(p, "output_config.format")?; + } else if !json_skip_value(p) { + return Err(String::new()); + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return Err(String::new()); + } + Ok(effort) +} + +pub fn parse_output_config_format(p: &mut Json<'_>) -> Result<(), String> { + parse_output_config_effort(p).map(|_| ()) +} diff --git a/crates/ds4-server/src/generate.rs b/crates/ds4-server/src/generate.rs new file mode 100644 index 0000000000000000000000000000000000000000..d4eaa3eb43df1c56ed272c75322414aa686fa25b --- /dev/null +++ b/crates/ds4-server/src/generate.rs @@ -0,0 +1,7361 @@ +//! Serial decode driver: render (including tool-schema / invoke reconstruct) +//! → host Vocab tokenize (FFI fallback) → host SessionLedger pos/generation +//! → native prefill/eval → SemAccum + generated-message parse → stream +//! projectors. +//! Incremental live DSML tool projection, required-prefix / structural +//! greedy sampling, and corrective retry (`decode_again` / model-visible +//! tool error) are host-owned. Continuation publish/hold/resolve is +//! host-owned (`cont`). + +use std::cell::RefCell; +use std::io::Write; +#[cfg(any(feature = "native", test))] +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, Instant}; +#[cfg(any(feature = "native", test))] +use std::time::{SystemTime, UNIX_EPOCH}; + +use ds4_core::{ReuseMiss, ReuseTaken}; +use ds4_kv::PrefixAnswer; +use ds4_kv::Store as KvStore; +#[cfg(any(feature = "native", test))] +use ds4_kv::{ + chat_anchor_pos as kv_chat_anchor_pos, continued_store_target_from_host, + store_len as kv_store_len, Header as KvHeader, HostKvView, Reason as KvReason, + EXT_THINKING_VISIBLE, EXT_TOOL_MAP, +}; + +use crate::dsml::{SampleOverride, SamplePolicy}; +use crate::parse::{fixed_thinking_sampling, DEFAULT_MIN_P, DEFAULT_TEMPERATURE, DEFAULT_TOP_P}; +use crate::parse::{ChatMsg, ChatPart, EosPolicy, ParsedRequest, ToolCall, ToolChoice}; +use crate::render::{ + render_chat_choice_ex, syntax_for_model_id, tool_start_marker, ModelSyntax, RenderError, + DSML_EOS, QWEN_IM_END, SOLAR_IM_END, +}; +use crate::retry::{ + build_recovery_suffix, parse_failure_should_retry, terminal_finish, truncation_outcome, + TruncationOutcome, +}; +use crate::route::{decode_budget, think_mode_enabled, Api, ReqKind}; +use crate::stream::{ + anthropic_final_response, anthropic_sse_finish_live, anthropic_sse_start_live, + anthropic_sse_stream_update, final_response, openai_sse_finish_live, openai_sse_stream_update, + openai_stream_start, responses_final_response, responses_sse_created, + responses_sse_finish_live, responses_sse_stream_update, responses_stream_init, sse_chunk, + sse_done, sse_headers, stream_error, stream_heartbeat_if_due, think_end, think_start, + AnthropicStream, ChatFormat, OpenaiStream, ReqTimings, ResponsesStream, StreamReq, ThinkBlock, + Writer, +}; +#[cfg(feature = "native")] +use crate::tool_memory::ToolMemory; +use crate::tools::{assign_tool_ids, parse_generated_for_response, SemAccum}; + +#[derive(Debug)] +pub enum GenerateError { + /// The request cannot be served as asked (HTTP 400): the engine answers it + /// before any work, never by silently dropping part of the input. + BadRequest(&'static str), + Unsupported(&'static str), + ContinuationHold { + retry_after: i32, + }, + Engine(String), + Streamed(String), + Io, +} + +impl std::fmt::Display for GenerateError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + GenerateError::BadRequest(s) => f.write_str(s), + GenerateError::Unsupported(s) => f.write_str(s), + GenerateError::ContinuationHold { .. } => { + f.write_str("batch capacity is reserved for live tool continuations") + } + GenerateError::Engine(s) => write!(f, "{s}"), + GenerateError::Streamed(s) => write!(f, "{s}"), + GenerateError::Io => f.write_str("client stream write failed"), + } + } +} + +impl From for GenerateError { + fn from(e: RenderError) -> Self { + GenerateError::Unsupported(e.0) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NativeGraphFit { + /// C quote verdict at the probed ctx (`ds4_engine_session_graph_fits`). + /// Family fit checks (EXAONE/Solar/Motif/dots3) report only this bit; + /// their byte fields stay zero. + pub fits: bool, + pub need_bytes: u64, + pub avail_bytes: u64, + pub headroom_bytes: u64, + pub deficit_bytes: u64, + pub fail_open: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VisionProbe { + pub token_count: u32, +} + +#[derive(Clone, Debug)] +pub struct VisionPromptInput { + pub data: Arc<[u8]>, + pub token_offset: u32, + /// Packed MiMo frames from the prepare pass. Empty for still images + /// and for a video the sync path still has to decode itself. + pub frames: Vec, +} + +#[derive(Clone, Debug)] +pub struct AudioPromptInput { + pub data: Arc<[u8]>, + pub token_offset: u32, +} + +/// C `serial_session_ensure_fit` view of the serial session lane. +/// `None` from [`DecodeIo::serial_session_probe`] means the engine has no +/// native serial session (stub/test engines): the host must pass native and +/// never invent a rightsize or a refuse. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SerialSessionProbe { + /// Current session ctx; the boot `-c` when no session exists yet (the + /// Rust host creates the serial session lazily, which is C's boot-shape + /// lazy session by another name). + pub cur_ctx: i32, + /// C `ds4_session_graph_pending`: true when the graph alloc is still + /// deferred (including "no session yet"). A pending session can be + /// re-created at another ctx for free. + pub graph_pending: bool, +} + +pub trait DecodeIo { + fn model_id(&self) -> i32; + /// True when this engine serves the V4.1 (ds41) family: no ds4_session + /// exists for it, so the serial path takes the push-based route instead + /// of the pull loop (server_generate_v41.c; the dispatch mirrors the + /// engine's one-line check, server_generate.c:19). + fn is_v41(&self) -> bool { + false + } + /// The V4.1 push entry: the engine's one-shot generate with callbacks + /// (`ds4_engine_v41_generate_argmax`). `sampling` is the per-request + /// decode face (the engine's ds4_engine_set_decode_sampling, + /// server_generate_v41.c:409); None resets to bare argmax. `emit` + /// returns false to stop, `progress` returns false to abort the prefill; + /// `greedy` answers the tool-syntax greedy hook once per position (the + /// engine's v41_syntax_greedy, c6e2706): true = that position takes the + /// raw argmax. Only an engine with a loaded v41 route answers; the + /// default refuses by name. + fn v41_generate( + &self, + _prompt: &[i32], + _n_predict: i32, + _sampling: Option, + _images: Option<&ds4_core::v41_media::V41Images>, + _greedy: &mut dyn FnMut(*const i32, u32) -> bool, + _emit: &mut dyn FnMut(i32) -> bool, + _progress: &mut dyn FnMut(&str, i32, i32) -> bool, + ) -> Result<(), GenerateError> { + Err(GenerateError::Unsupported( + "V4.1 generation is not loaded on this engine", + )) + } + /// The V4.1 vision tower's geometry and placeholder id, read from the + /// `--vision-gguf` sidecar (`core_validate_v41.c:48-66`); None on an engine + /// with no sidecar, so a request never expands a placeholder it cannot run. + fn v41_vision(&self) -> Option { + None + } + /// The last V4.1 run's speculation account (rounds, offered, accepted; + /// the engine's `ds4_engine_v41_last_spec_stats`, core_v41_api.c:41-46). + /// Zero for engines that do not speculate. + fn v41_last_spec_stats(&self) -> (i32, i32, i32) { + (0, 0, 0) + } + /// The last V4.1 run's restored token count (U-B; the engine's + /// `ds4_engine_v41_last_reused`, core_v41_kvreuse.c:28), reported as usage + /// `cached_tokens` (server_generate_v41.c:99-102). Zero on engines + /// without the route. + fn v41_kvreuse_last_reused(&self) -> i32 { + 0 + } + fn template(&self) -> Option<&ds4_core::chat_template::Template> { + None + } + /// C `ds4_chat_system_token()` (core_engine_api.c:94-96): the DSML prompt + /// head token the loaded vocab owns, "" when it has none (the V4 fallback). + fn chat_system_token(&self) -> &'static str { + "" + } + fn render_request(&self, parsed: &ParsedRequest) -> Result, GenerateError> { + if let Some(template) = self.template() { + return crate::chat_input::render(template, self.model_id(), parsed); + } + render_prompt_ex( + parsed, + self.model_id(), + crate::render::dsml_head(self.model_id(), self.chat_system_token()) + .with_effort(parsed.effort.head_budget()), + ) + } + fn restore_chat(&self, _parsed: &mut ParsedRequest) -> Result<(), GenerateError> { + Ok(()) + } + fn remember_chat( + &mut self, + _parsed: &ParsedRequest, + _generated: &crate::tools::ParsedGenerated, + ) { + } + fn kv_store_mut(&mut self) -> Option<&mut KvStore> { + None + } + fn tokenize_text(&self, text: &str) -> Result, GenerateError>; + fn tokenize_rendered_chat(&self, text: &[u8]) -> Result, GenerateError>; + fn tokenizes_control_literals(&self) -> bool { + true + } + fn token_text(&self, token: i32) -> Result, GenerateError>; + fn token_is_stop(&self, token: i32) -> bool; + fn eos_id(&self) -> i32 { + -1 + } + fn eot_id(&self) -> i32 { + -1 + } + fn vision_probe(&self, _data: &[u8]) -> Result { + Err(GenerateError::Unsupported("vision encoder is not loaded")) + } + fn vision_tokens(&self, data: &[u8]) -> Result, GenerateError> { + let marker = match syntax_for_model_id(self.model_id()) { + ModelSyntax::Glm53 => 154854, + ModelSyntax::Inkling => 200054, + _ => { + return Err(GenerateError::Unsupported( + "image token format is unavailable", + )) + } + }; + Ok(vec![marker; self.vision_probe(data)?.token_count as usize]) + } + fn sync_vision_prompt( + &mut self, + _tokens: &[i32], + _images: &[VisionPromptInput], + ) -> Result<(), GenerateError> { + Err(GenerateError::Unsupported("vision encoder is not loaded")) + } + fn audio_probe(&self, _data: &[u8]) -> Result { + Err(GenerateError::Unsupported("audio encoder is not loaded")) + } + fn sync_media_prompt( + &mut self, + tokens: &[i32], + images: &[VisionPromptInput], + audios: &[AudioPromptInput], + ) -> Result<(), GenerateError> { + if audios.is_empty() { + return self.sync_vision_prompt(tokens, images); + } + Err(GenerateError::Unsupported("audio encoder is not loaded")) + } + fn sync_mimo_prompt( + &mut self, + tokens: &[i32], + images: &[VisionPromptInput], + audios: &[AudioPromptInput], + videos: &[VisionPromptInput], + ) -> Result<(), GenerateError> { + if !videos.is_empty() { + return Err(GenerateError::Unsupported("video input requires MiMo")); + } + self.sync_media_prompt(tokens, images, audios) + } + fn sync(&mut self, tokens: &[i32]) -> Result<(), GenerateError>; + /// Start this request's reuse trace. A corrective retry re-syncs inside + /// the same request, so the sync itself must not erase what the first + /// one reported. + fn begin_trace(&mut self) {} + fn sync_prompt( + &mut self, + _prompt: &[u8], + tokens: &[i32], + _disk_eligible: bool, + _thinking_visible_eligible: bool, + ) -> Result { + self.sync(tokens)?; + Ok(0) + } + fn prompt_sync_elapsed(&self) -> Option { + None + } + fn restore_tool_replay(&mut self, _messages: &mut [ChatMsg]) {} + fn sync_tool_replay_prompt( + &mut self, + prompt: &[u8], + tokens: &[i32], + ) -> Result { + self.sync_prompt(prompt, tokens, true, false) + } + fn remember_tool_replay(&mut self, _calls: &[ToolCall], _raw_dsml: &str) {} + fn maybe_store_continued(&mut self) -> Result<(), GenerateError> { + Ok(()) + } + fn shutdown(&mut self) -> Result<(), GenerateError> { + Ok(()) + } + fn eval(&mut self, token: i32) -> Result<(), GenerateError>; + /// Commit a nonempty greedy prefix bounded by the positive budget. + /// Implementations may include EOS as the final accepted token. + fn eval_greedy(&mut self, first: i32, _budget: i32) -> Result, GenerateError> { + self.eval(first)?; + Ok(vec![first]) + } + /// Drop the committed tail after an output boundary. Engines without + /// a recorded prefix state retain the conservative invalidation path. + fn trim_greedy(&mut self, _pos: i32) -> Result<(), GenerateError> { + self.invalidate(); + Ok(()) + } + fn last_eval_speculated(&self) -> bool { + false + } + /// The mechanism the last prompt sync used, for the request trace. + fn last_reuse(&self) -> ReuseTaken { + ReuseTaken::Cold + } + /// Why the last prompt sync refused a candidate, for the request trace. + fn last_miss(&self) -> ReuseMiss { + ReuseMiss::None + } + fn sample( + &mut self, + temperature: f32, + top_k: i32, + top_p: f32, + min_p: f32, + rng: &mut u64, + ) -> i32; + fn sample_excluding( + &mut self, + temperature: f32, + top_k: i32, + top_p: f32, + min_p: f32, + rng: &mut u64, + excluded_id: i32, + ) -> i32 { + if excluded_id >= 0 { + return -1; + } + self.sample(temperature, top_k, top_p, min_p, rng) + } + fn native_graph_fit(&self, _ctx: i32) -> Option { + None + } + /// C `serial_session_ensure_fit` inputs. `None` = no native serial + /// session lane; the host ensure-fit passes native. + fn serial_session_probe(&self) -> Option { + None + } + /// C `ds4_session_free` + `ds4_session_create(target)`: replace the + /// serial session with a right-sized one. The old session's live + /// records die with it (the caller demotes the registry). + fn serial_session_rightsize(&mut self, _target_ctx: i32) -> Result<(), GenerateError> { + Ok(()) + } + /// C refusal shape: free the session so the server keeps its boot + /// invariant and later requests re-probe fresh (the Rust host restores + /// lazily on next use instead of eagerly re-creating at boot ctx). + fn serial_session_reset(&mut self) {} + fn pos(&self) -> i32; + fn ctx(&self) -> i32; + fn generation(&self) -> u64; + fn session_tokens(&self) -> Vec { + Vec::new() + } + fn remember_thinking_visible_checkpoint(&mut self, _text: Vec) {} + fn invalidate(&mut self); +} + +#[cfg(any(feature = "native", test))] +#[derive(Clone, Copy)] +struct DiskSyncPolicy { + save_current: bool, + load: bool, +} + +#[cfg(any(feature = "native", test))] +struct ThinkingVisibleCheckpoint { + text: Vec, + frontier: i32, +} + +#[cfg(any(feature = "native", test))] +fn settle_thinking_visible_checkpoint( + checkpoint: &mut Option, + sync_succeeded: bool, +) { + if sync_succeeded { + *checkpoint = None; + } +} + +#[cfg(any(feature = "native", test))] +trait SerialKvIo { + fn ctx(&self) -> i32; + fn chat_token_ids(&self) -> (i32, i32) { + (-1, -1) + } + fn live_len(&self) -> i32; + fn live_tokens(&self) -> Vec; + /// The native plan may replay part or all of a matching token prefix. + fn sync_start(&self, _tokens: &[i32]) -> i32 { + self.live_len() + } + fn render_tokens(&self, tokens: &[i32]) -> Result, GenerateError>; + fn checkpoint_trailer(&self, _text: &[u8]) -> Option> { + Some(Vec::new()) + } + fn tokenize_suffix(&mut self, suffix: &[u8]) -> Result, GenerateError>; + /// Record which mechanism produced the reuse. Cached/computed counts + /// cannot tell an appended frontier turn from a checkpoint replay. + fn note_reuse(&mut self, _taken: ReuseTaken) {} + /// Record why a candidate was refused, so the trace can say it. The + /// first reason wins: a later, broader one would mask it. + fn note_miss(&mut self, _miss: ReuseMiss) {} + fn sync(&mut self, tokens: &[i32]) -> Result<(), GenerateError>; + fn sync_with_prefill_checkpoints( + &mut self, + tokens: &[i32], + _store: &mut KvStore, + _identity: (u8, u8, u32), + _cached_floor: i32, + ) -> Result<(), GenerateError> { + self.sync(tokens) + } + fn save_payload(&mut self, path: &Path) -> Result<(), GenerateError>; + fn load_payload_range( + &mut self, + path: &Path, + offset: u64, + length: u64, + ) -> Result<(), GenerateError>; + fn invalidate(&mut self); +} + +#[cfg(any(feature = "native", test))] +fn write_checkpoint( + store: &mut KvStore, + header: KvHeader, + text: &[u8], + trailer: &[u8], + save_payload: impl FnOnce(&Path) -> Result<(), GenerateError>, +) -> Result<(), GenerateError> { + if store + .reuse_compatible(header.clone(), text, trailer) + .map_err(|error| GenerateError::Engine(error.to_string()))? + .is_some() + { + return Ok(()); + } + let payload = store + .payload_temp() + .map_err(|error| GenerateError::Engine(error.to_string()))?; + save_payload(payload.path())?; + store + .write_payload_file(header, text, payload.path(), trailer) + .map_err(|error| GenerateError::Engine(error.to_string()))?; + Ok(()) +} + +#[cfg(any(feature = "native", test))] +fn kv_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0) +} + +#[cfg(any(feature = "native", test))] +pub(crate) fn kv_identity(model_id: i32, quant_bits: i32, ctx: i32) -> Option<(u8, u8, u32)> { + let model_id = u8::try_from(model_id).ok()?; + let quant_bits = match quant_bits { + 2 | 4 => quant_bits as u8, + _ => return None, + }; + let ctx = u32::try_from(ctx).ok().filter(|ctx| *ctx > 0)?; + Some((model_id, quant_bits, ctx)) +} + +#[cfg(any(feature = "native", test))] +fn intermediate_prefill_eligible( + deepseek: bool, + cuda: bool, + disk_eligible: bool, + tool_replay: bool, +) -> bool { + disk_eligible && !tool_replay && deepseek && cuda +} + +#[cfg(any(feature = "native", test))] +pub(crate) fn kv_header(model_id: u8, quant_bits: u8, ctx: u32, tokens: u32) -> KvHeader { + let now = kv_now(); + KvHeader { + quant_bits, + reason: KvReason::Evict, + ext_flags: 0, + model_id, + tokens, + hits: 0, + ctx_size: ctx, + created_at: now, + last_used: now, + payload_bytes: 0, + text_bytes: 0, + } +} + +#[cfg(any(feature = "native", test))] +fn try_store_live( + io: &mut impl SerialKvIo, + store: &mut KvStore, + model_id: u8, + quant_bits: u8, + ctx: u32, + reason: KvReason, + checkpoint: Option<&ThinkingVisibleCheckpoint>, +) -> Result { + let live = io.live_tokens(); + if live.len() < store.opt.min_tokens.max(0) as usize { + return Ok(false); + } + let tokens = u32::try_from(live.len()) + .map_err(|_| GenerateError::Engine("KVC token count exceeds u32".into()))?; + let (text, ext_flags) = match checkpoint { + Some(checkpoint) if usize::try_from(checkpoint.frontier).ok() == Some(live.len()) => { + (checkpoint.text.clone(), EXT_THINKING_VISIBLE) + } + _ => (io.render_tokens(&live)?, 0), + }; + let trailer = if ext_flags == 0 { + let Some(trailer) = io.checkpoint_trailer(&text) else { + return Ok(false); + }; + trailer + } else { + Vec::new() + }; + let mut header = kv_header(model_id, quant_bits, ctx, tokens); + header.reason = reason; + header.ext_flags = ext_flags; + write_checkpoint(store, header, &text, &trailer, |path| io.save_payload(path))?; + Ok(true) +} + +#[cfg(any(feature = "native", test))] +fn continued_target(store: &KvStore, live_tokens: i32) -> i32 { + continued_store_target_from_host( + &store.opt, + HostKvView { + live_tokens, + stored_tokens: store.continued_last_store_tokens, + }, + ) +} + +#[cfg(any(feature = "native", test))] +fn try_store_continued( + io: &mut impl SerialKvIo, + store: &mut KvStore, + identity: (u8, u8, u32), +) -> Result { + let target = continued_target(store, io.live_len()); + if target == 0 { + return Ok(false); + } + let (model_id, quant_bits, ctx) = identity; + if !try_store_live( + io, + store, + model_id, + quant_bits, + ctx, + KvReason::Continued, + None, + )? { + return Ok(false); + } + store.continued_last_store_tokens = store.continued_last_store_tokens.max(target); + Ok(true) +} + +fn store_continued_best_effort(engine: &mut dyn DecodeIo) { + if let Err(error) = engine.maybe_store_continued() { + eprintln!("ds4-server-rs: continued KV checkpoint failed: {error}"); + } +} + +fn continued_decode_allowed(acc: &SemAccum) -> bool { + !(acc.track_tools && (acc.saw_tool_start || acc.dsml_state().is_tool())) +} + +#[cfg(any(feature = "native", test))] +fn sync_maybe_checkpoint( + io: &mut impl SerialKvIo, + tokens: &[i32], + store: Option<&mut KvStore>, + identity: Option<(u8, u8, u32)>, + cached_floor: i32, + enabled: bool, +) -> Result<(), GenerateError> { + match (enabled, store, identity) { + (true, Some(store), Some(identity)) => { + io.sync_with_prefill_checkpoints(tokens, store, identity, cached_floor) + } + _ => io.sync(tokens), + } +} + +#[cfg(any(feature = "native", test))] +fn suppress_continued(store: &mut KvStore, target: i32) -> Option { + if continued_target(store, target) != target { + return None; + } + let old = store.continued_last_store_tokens; + store.continued_last_store_tokens = target; + Some(old) +} + +#[cfg(any(feature = "native", test))] +fn restore_suppressed_continued(store: &mut KvStore, old: Option, target: i32) { + if let Some(old) = old { + if store.continued_last_store_tokens == target { + store.continued_last_store_tokens = old; + } + } +} + +#[cfg(any(feature = "native", test))] +fn cold_sync(io: &mut impl SerialKvIo, tokens: &[i32]) -> Result { + io.note_reuse(ReuseTaken::Cold); + io.sync(tokens)?; + Ok(0) +} + +#[cfg(any(feature = "native", test))] +fn record_prefix_reuse(io: &mut impl SerialKvIo, tokens: &[i32], prefix: usize) -> i32 { + let prefix = i32::try_from(prefix.min(tokens.len())).unwrap_or(0); + let cached = io.sync_start(tokens).clamp(0, prefix); + if cached < prefix { + io.note_miss(ReuseMiss::StateReplay); + } + io.note_reuse(if cached == 0 { + ReuseTaken::Cold + } else if cached < prefix { + ReuseTaken::Partial + } else { + ReuseTaken::Exact + }); + cached +} + +#[cfg(any(feature = "native", test))] +fn cold_sync_and_store( + io: &mut impl SerialKvIo, + store: &mut KvStore, + identity: (u8, u8, u32), + tokens: &[i32], + prefill_checkpoints: bool, +) -> Result { + io.note_reuse(ReuseTaken::Cold); + let (model_id, quant_bits, ctx) = identity; + let Ok(full_len) = i32::try_from(tokens.len()) else { + sync_maybe_checkpoint( + io, + tokens, + Some(store), + Some(identity), + 0, + prefill_checkpoints, + )?; + return Ok(0); + }; + if full_len < store.opt.min_tokens { + io.note_miss(ReuseMiss::BelowThreshold); + } + if full_len < store.opt.min_tokens + || store.opt.cold_max_tokens <= 0 + || full_len > store.opt.cold_max_tokens + { + sync_maybe_checkpoint( + io, + tokens, + Some(store), + Some(identity), + 0, + prefill_checkpoints, + )?; + return Ok(0); + } + let (user_id, assistant_id) = io.chat_token_ids(); + let anchor = kv_chat_anchor_pos(&store.opt, tokens, user_id, assistant_id); + let target = if anchor >= store.opt.min_tokens { + anchor + } else { + kv_store_len(&store.opt, full_len) + }; + if target < store.opt.min_tokens || target > full_len { + sync_maybe_checkpoint( + io, + tokens, + Some(store), + Some(identity), + 0, + prefill_checkpoints, + )?; + return Ok(0); + } + let target_i32 = target; + let target = target_i32 as usize; + let suppressed = suppress_continued(store, target_i32); + let first_sync = if target < tokens.len() { + sync_maybe_checkpoint( + io, + &tokens[..target], + Some(store), + Some(identity), + 0, + prefill_checkpoints, + ) + } else { + sync_maybe_checkpoint( + io, + tokens, + Some(store), + Some(identity), + 0, + prefill_checkpoints, + ) + }; + if let Err(error) = first_sync { + restore_suppressed_continued(store, suppressed, target_i32); + return Err(error); + } + let cold_stored = matches!( + try_store_live(io, store, model_id, quant_bits, ctx, KvReason::Cold, None,), + Ok(true) + ); + if cold_stored { + store.continued_last_store_tokens = store.continued_last_store_tokens.max(target_i32); + } else { + restore_suppressed_continued(store, suppressed, target_i32); + } + if target < tokens.len() { + sync_maybe_checkpoint( + io, + tokens, + Some(store), + Some(identity), + 0, + prefill_checkpoints, + )?; + } + Ok(0) +} + +#[cfg(any(feature = "native", test))] +fn discard_loaded(store: &mut KvStore, io: &mut impl SerialKvIo, path: &Path) { + store.continued_last_store_tokens = 0; + let _ = store.discard(path); + io.invalidate(); +} + +#[cfg(any(feature = "native", test))] +fn disk_sync_template( + io: &mut impl SerialKvIo, + mut store: Option<&mut KvStore>, + model_id: i32, + quant_bits: i32, + prompt: &[u8], + tokens: &[i32], + policy: DiskSyncPolicy, +) -> Result { + let history = policy + .load + .then(|| history_frontier(model_id, prompt, tokens, |text| io.tokenize_suffix(text))) + .flatten() + .filter(|(_, prefix)| { + store + .as_ref() + .is_some_and(|store| prefix.len() >= store.opt.min_tokens.max(1) as usize) + }); + if let Some((text, prefix)) = history { + // Keep an already deeper, matching live frontier. Otherwise sync to + // the history boundary before the generation-only think pair: both + // the saved token ledger and the native KV now describe that prefix. + let live = io.live_tokens(); + if live.len() <= prefix.len() || !tokens.starts_with(&live) { + let cached = disk_sync_prompt_impl( + io, + store.as_deref_mut(), + model_id, + quant_bits, + &text, + &prefix, + None, + false, + policy, + false, + PromptReuse::Tokens, + )?; + if let (Some(store), Some((model_id, quant_bits, ctx)), Some(trailer)) = ( + store, + kv_identity(model_id, quant_bits, io.ctx()), + io.checkpoint_trailer(&text), + ) { + let header = kv_header(model_id, quant_bits, ctx, prefix.len() as u32); + if let Err(error) = + write_checkpoint(store, header, &text, &trailer, |path| io.save_payload(path)) + { + eprintln!("ds4-server-rs: history KV checkpoint failed: {error}"); + } + } + io.sync(tokens)?; + return Ok(cached); + } + } + // Rendered-text identity is not token identity: an official template + // that drops a block still held in KV would continue from a sequence + // the client never sent. Jinja families reuse on tokens only. + disk_sync_prompt_impl( + io, + store, + model_id, + quant_bits, + prompt, + tokens, + None, + false, + policy, + false, + PromptReuse::Tokens, + ) +} + +#[cfg(any(feature = "native", test))] +pub(crate) fn history_frontier( + model_id: i32, + prompt: &[u8], + tokens: &[i32], + encode: impl FnOnce(&[u8]) -> Result, GenerateError>, +) -> Option<(Vec, Vec)> { + let (suffix, boundary): (&[u8], &[u8]) = match syntax_for_model_id(model_id) { + ModelSyntax::Step37 => (b"assistant\n\n\n", b"<|im_start|>"), + ModelSyntax::Motif3 => (b"", b"<|startofturn|><|assistant|>"), + _ => return None, + }; + let text = prompt.strip_suffix(suffix)?; + if !text.ends_with(boundary) { + return None; + } + let prefix = encode(text).ok()?; + // Both official templates omit the empty think pair on history replay. + // Stop at a control token: content whitespace may be trimmed (Motif) or + // merge with the role newline (Step). Validate the full token prefix too. + if prefix.is_empty() || prefix.len() >= tokens.len() || !tokens.starts_with(&prefix) { + return None; + } + Some((text.to_vec(), prefix)) +} + +#[cfg(any(feature = "native", test))] +#[derive(Clone, Copy, PartialEq, Eq)] +enum PromptReuse { + Tokens, + LegacyText, + Off, +} + +#[cfg(any(feature = "native", test))] +fn disk_sync_prompt( + io: &mut impl SerialKvIo, + store: Option<&mut KvStore>, + model_id: i32, + quant_bits: i32, + prompt: &[u8], + canonical_tokens: &[i32], + checkpoint: Option<&ThinkingVisibleCheckpoint>, + thinking_visible_eligible: bool, + policy: DiskSyncPolicy, +) -> Result { + disk_sync_prompt_impl( + io, + store, + model_id, + quant_bits, + prompt, + canonical_tokens, + checkpoint, + thinking_visible_eligible, + policy, + false, + PromptReuse::LegacyText, + ) +} + +#[cfg(any(feature = "native", test))] +fn disk_sync_tool_replay( + io: &mut impl SerialKvIo, + store: Option<&mut KvStore>, + model_id: i32, + quant_bits: i32, + prompt: &[u8], + canonical_tokens: &[i32], + policy: DiskSyncPolicy, +) -> Result { + disk_sync_prompt_impl( + io, + store, + model_id, + quant_bits, + prompt, + canonical_tokens, + None, + false, + policy, + true, + PromptReuse::LegacyText, + ) +} + +#[cfg(any(feature = "native", test))] +fn disk_sync_prompt_impl( + io: &mut impl SerialKvIo, + mut store: Option<&mut KvStore>, + model_id: i32, + quant_bits: i32, + prompt: &[u8], + canonical_tokens: &[i32], + checkpoint: Option<&ThinkingVisibleCheckpoint>, + thinking_visible_eligible: bool, + policy: DiskSyncPolicy, + allow_tool_map: bool, + reuse: PromptReuse, +) -> Result { + let identity = kv_identity(model_id, quant_bits, io.ctx()); + let prefill_checkpoints = policy.load && !allow_tool_map; + if reuse == PromptReuse::Off { + if policy.save_current { + if let (Some(store), Some((model_id, quant_bits, ctx))) = + (store.as_deref_mut(), identity) + { + let _ = try_store_live( + io, + store, + model_id, + quant_bits, + ctx, + KvReason::Evict, + checkpoint, + ); + } + } + return cold_sync(io, canonical_tokens); + } + let live = io.live_tokens(); + if !live.is_empty() && canonical_tokens.starts_with(&live) { + let cached = record_prefix_reuse(io, canonical_tokens, live.len()); + sync_maybe_checkpoint( + io, + canonical_tokens, + store.as_deref_mut(), + identity, + cached, + prefill_checkpoints, + )?; + return Ok(cached); + } + if reuse == PromptReuse::LegacyText && thinking_visible_eligible { + if let Some(checkpoint) = checkpoint { + if !live.is_empty() + && usize::try_from(checkpoint.frontier).ok() == Some(live.len()) + && checkpoint.text.len() < prompt.len() + && prompt.starts_with(&checkpoint.text) + { + let prefix = live.len(); + let mut effective = live; + effective.extend(io.tokenize_suffix(&prompt[checkpoint.text.len()..])?); + let cached = record_prefix_reuse(io, &effective, prefix); + sync_maybe_checkpoint( + io, + &effective, + store.as_deref_mut(), + identity, + cached, + prefill_checkpoints, + )?; + return Ok(cached); + } + } + } + if reuse == PromptReuse::LegacyText && !live.is_empty() { + let rendered = io.render_tokens(&live)?; + if prompt.starts_with(&rendered) { + let prefix = live.len(); + let mut effective = live; + effective.extend(io.tokenize_suffix(&prompt[rendered.len()..])?); + let cached = record_prefix_reuse(io, &effective, prefix); + sync_maybe_checkpoint( + io, + &effective, + store.as_deref_mut(), + identity, + cached, + prefill_checkpoints, + )?; + return Ok(cached); + } + } + + // A live session that holds this conversation, whose render moved: the + // text still leads here, the token sequence no longer does. A session + // about a different conversation is not that, and says nothing. + if !live.is_empty() && prompt.starts_with(&io.render_tokens(&live)?) { + io.note_miss(ReuseMiss::RenderedPrefix); + } + + let Some(store) = store else { + return cold_sync(io, canonical_tokens); + }; + store.continued_last_store_tokens = 0; + let Some((model_id, quant_bits, ctx)) = identity else { + return cold_sync(io, canonical_tokens); + }; + if policy.save_current { + let _ = try_store_live( + io, + store, + model_id, + quant_bits, + ctx, + KvReason::Evict, + checkpoint, + ); + } + if !policy.load { + return cold_sync(io, canonical_tokens); + } + let candidate = match store.text_prefix_candidate(prompt, model_id, quant_bits, ctx) { + Ok(candidate) => candidate, + Err(_) => { + // A record whose envelope will not read is refused by its + // payload as surely as one whose tokens disagree. + io.note_miss(ReuseMiss::PayloadMismatch); + return cold_sync_and_store( + io, + store, + (model_id, quant_bits, ctx), + canonical_tokens, + prefill_checkpoints, + ); + } + }; + let Some((path, envelope)) = candidate else { + // Order matters: a record that exists but cannot be used, then a + // conversation too short to have been stored, then plain absence. + let short = + i32::try_from(canonical_tokens.len()).unwrap_or(i32::MAX) < store.opt.min_tokens; + io.note_miss( + match store.prefix_answer(prompt, model_id, quant_bits, ctx) { + PrefixAnswer::Mismatch => ReuseMiss::PayloadMismatch, + // A record written before the minimum was raised is skipped + // by every search since, whatever this prompt's length. + PrefixAnswer::Shallow => ReuseMiss::BelowThreshold, + _ if short => ReuseMiss::BelowThreshold, + _ => ReuseMiss::NoCheckpoint, + }, + ); + return cold_sync_and_store( + io, + store, + (model_id, quant_bits, ctx), + canonical_tokens, + prefill_checkpoints, + ); + }; + let extension_ok = match envelope.header.ext_flags { + 0 => envelope.trailer_bytes == 0, + EXT_THINKING_VISIBLE => envelope.trailer_bytes == 0, + EXT_TOOL_MAP => allow_tool_map && envelope.trailer_bytes > 0, + _ => false, + }; + if !extension_ok { + io.note_miss(ReuseMiss::PayloadMismatch); + return cold_sync_and_store( + io, + store, + (model_id, quant_bits, ctx), + canonical_tokens, + prefill_checkpoints, + ); + } + if store + .open_payload(&path, &envelope) + .map_err(|error| GenerateError::Engine(error.to_string())) + .and_then(|payload| { + io.load_payload_range( + payload.path(), + envelope.payload_offset, + envelope.header.payload_bytes, + ) + }) + .is_err() + { + // The candidate was chosen and then could not be read. That is a + // refusal by its payload, not an absence. + io.note_miss(ReuseMiss::PayloadMismatch); + io.invalidate(); + return cold_sync_and_store( + io, + store, + (model_id, quant_bits, ctx), + canonical_tokens, + prefill_checkpoints, + ); + } + let loaded = io.live_tokens(); + if loaded.len() != envelope.header.tokens as usize { + io.note_miss(ReuseMiss::PayloadMismatch); + io.invalidate(); + let _ = store.discard(&path); + return cold_sync_and_store( + io, + store, + (model_id, quant_bits, ctx), + canonical_tokens, + prefill_checkpoints, + ); + } + if reuse == PromptReuse::Tokens && !canonical_tokens.starts_with(&loaded) { + // The template re-rendered this conversation differently, so the + // stored tokens are no longer a prefix of the prompt. + io.note_miss(ReuseMiss::RenderedPrefix); + io.invalidate(); + return cold_sync_and_store( + io, + store, + (model_id, quant_bits, ctx), + canonical_tokens, + prefill_checkpoints, + ); + } + let prefix = loaded.len(); + store.continued_last_store_tokens = prefix as i32; + // Token identity permits restore, but native state may require replay + // below that frontier (e.g. an unaligned Dots3 MTP append). + if reuse == PromptReuse::Tokens { + let cached = record_prefix_reuse(io, canonical_tokens, prefix); + if cached > 0 { + let _ = store.touch_hit(&path); + } + sync_maybe_checkpoint( + io, + canonical_tokens, + Some(store), + identity, + cached, + prefill_checkpoints, + )?; + return Ok(cached); + } + let mut effective = loaded; + let suffix = &prompt[envelope.text.len()..]; + let suffix_tokens = match io.tokenize_suffix(suffix) { + Ok(tokens) => tokens, + Err(error) => { + discard_loaded(store, io, &path); + return Err(error); + } + }; + effective.extend(suffix_tokens); + let cached = record_prefix_reuse(io, &effective, prefix); + if cached > 0 { + let _ = store.touch_hit(&path); + } + if let Err(error) = sync_maybe_checkpoint( + io, + &effective, + Some(store), + Some((model_id, quant_bits, ctx)), + cached, + prefill_checkpoints, + ) { + discard_loaded(store, io, &path); + return Err(error); + } + Ok(cached) +} + +#[derive(Debug, Clone, Default)] +pub struct GenerateOutcome { + pub tool_ids: Vec, + pub bank: Option, + pub generation: u64, + pub frontier: i32, + pub finish: String, + pub timings: ReqTimings, + pub lane: Option<&'static str>, + pub speculation_active: bool, + pub reuse: ReuseTaken, + pub reuse_miss: ReuseMiss, + pub fallback_reason: Option, +} + +pub fn generation_blocked(parsed: &ParsedRequest, model_id: i32) -> Option<&'static str> { + let syntax = syntax_for_model_id(model_id); + if !parsed.videos.is_empty() && syntax != ModelSyntax::Mimo2 { + return Some("video input requires MiMo"); + } + if !parsed.audios.is_empty() && !matches!(syntax, ModelSyntax::Inkling | ModelSyntax::Mimo2) { + return Some("audio input requires Inkling or MiMo"); + } + if parsed.images.is_empty() { + return None; + } + let v41 = + syntax == ModelSyntax::DeepSeek && model_id == ds4_core::Variant::DeepSeek41Flash as i32; + let image_gate = match syntax { + ModelSyntax::Glm53 + | ModelSyntax::Inkling + | ModelSyntax::Step37 + | ModelSyntax::Ling3Vl + | ModelSyntax::Mimo2 => None, + ModelSyntax::Qwen4Exp => Some("image input requires continuous runtime"), + // V4.1 carries its own vision tower as a sidecar; whether that + // sidecar is open is checked in prepare_serial_prompt, which knows + // the engine (the 400 lives there, 80296d3). + ModelSyntax::DeepSeek if v41 => None, + _ => { + Some("image input is supported only by Qwen4Exp, GLM-5.3, Inkling, Step, Ling or MiMo") + } + }; + if image_gate.is_some() { + return image_gate; + } + // A head that admits images may still not place one inside a tool result: + // that placement exists in the DSML V4.1 render only (render.rs + // append_dsml_user_content for the Anthropic tool_result, and the tool + // branch for a Chat Completions tool/function message). Every other head + // renders tool results from the ToolResult part, which carries the plain + // text (parse.rs json_tool_result_content), so the image could never reach + // its prompt. Fail closed instead of letting the media expansion reject + // it later, or letting the placeholder token leak into a head that cannot + // place it. Two shapes carry a tool image: the Anthropic tool_result (an + // image on a user message that has a ToolResult part) and a Chat + // Completions tool/function message. + if !v41 + && parsed.messages.iter().any(|m| { + m.parts.iter().any(|p| matches!(p, ChatPart::Image(_))) + && (m.role != "user" + || m.parts + .iter() + .any(|p| matches!(p, ChatPart::ToolResult { .. }))) + }) + { + return Some("images inside a tool result are supported only by DeepSeek V4.1 Flash"); + } + None +} + +pub fn chat_format_for_syntax(syntax: ModelSyntax) -> ChatFormat { + match syntax { + ModelSyntax::SolarOpen2 => ChatFormat::SolarOpen2, + ModelSyntax::Exaone => ChatFormat::Exaone, + // Only the generated thinking/tool envelope is shared with Qwen. + ModelSyntax::Qwen4Exp + | ModelSyntax::Step37 + | ModelSyntax::Mimo2 + | ModelSyntax::Qwen35 + | ModelSyntax::NaiveN05 => ChatFormat::Qwen4Exp, + ModelSyntax::K2Horizon => ChatFormat::K2Horizon, + ModelSyntax::Inkling => ChatFormat::Inkling, + ModelSyntax::IQuestQ1 => ChatFormat::IQuestQ1, + ModelSyntax::BeamVq => ChatFormat::Beam, + // Ling shares GLM's thinking and tool-call XML. + ModelSyntax::DeepSeek + | ModelSyntax::Motif3 + | ModelSyntax::Dots3 + | ModelSyntax::Glm53 + | ModelSyntax::Ling3Vl => ChatFormat::DeepSeek, + } +} + +pub fn stream_req_from_parsed(parsed: &ParsedRequest, model_id: i32) -> StreamReq { + let syntax = syntax_for_model_id(model_id); + StreamReq { + kind: parsed.kind, + api: parsed.api, + model: parsed.model.clone(), + think_mode: parsed.think_mode, + has_tools: parsed.has_tools, + stream: parsed.stream, + stream_include_usage: parsed.stream_include_usage, + reasoning_summary_emit: parsed.reasoning_summary_emit, + chat_format: chat_format_for_syntax(syntax), + syntax, + cache_read_tokens: 0, + cache_write_tokens: 0, + timings: ReqTimings::default(), + tool_orders: parsed.tool_orders.clone(), + } +} + +pub fn render_prompt(parsed: &ParsedRequest, model_id: i32) -> Result, GenerateError> { + render_prompt_ex(parsed, model_id, crate::render::DsmlHead::V4) +} + +pub fn render_prompt_ex( + parsed: &ParsedRequest, + model_id: i32, + head: crate::render::DsmlHead, +) -> Result, GenerateError> { + match parsed.kind { + ReqKind::Completion => Ok(parsed.prompt_text.clone().unwrap_or_default().into_bytes()), + ReqKind::Chat => Ok(render_chat_choice_ex( + syntax_for_model_id(model_id), + &parsed.messages, + &parsed.tool_schemas, + &parsed.tool_orders, + parsed.think_mode, + parsed.tool_choice, + head, + )?), + } +} + +pub(crate) fn ordinary_disk_cache_eligible(parsed: &ParsedRequest) -> bool { + parsed.api == Api::Openai + && !think_mode_enabled(parsed.think_mode) + && !parsed.has_tools + && !parsed.has_tool_results + && parsed.live_call_ids.is_empty() +} + +/// Whether this request may carry exact sampled DSML. +/// +/// On DeepSeek V4.1 the lane follows the reference, which attaches in all +/// three chat-shaped parsers — OpenAI chat (`ds4_server.c:4272`), Anthropic +/// (`:4498`), Responses (`:5529`) — and refuses nothing but its disable +/// switch (`tool_memory_attach_to_messages`, `:10682`), so thinking mode, +/// live call ids and the surface do not narrow it there. The legacy +/// `/v1/completions` surface carries no messages and stays out. +/// +/// Every other family keeps the narrow lane (OpenAI chat, non-thinking, no +/// live ids, the three syntaxes whose renderers honour the bytes): widening +/// it moves their thinking-visible and intermediate prefill checkpoints off +/// the request, and no reuse or output gate has measured that for them. +fn tool_replay_scope(parsed: &ParsedRequest, model_id: i32) -> bool { + if parsed.kind != ReqKind::Chat { + return false; + } + if model_id == ds4_core::Variant::DeepSeek41Flash as i32 { + return true; + } + parsed.api == Api::Openai + && !think_mode_enabled(parsed.think_mode) + && parsed.live_call_ids.is_empty() + && matches!( + syntax_for_model_id(model_id), + ModelSyntax::DeepSeek | ModelSyntax::SolarOpen2 | ModelSyntax::Qwen4Exp + ) +} + +fn tool_replay_disk_cache_eligible(parsed: &ParsedRequest, model_id: i32) -> bool { + tool_replay_scope(parsed, model_id) + && parsed.messages.iter().any(|message| { + !message.calls.is_empty() + || !message.tool_call_id.is_empty() + || !message.tool_call_ids.is_empty() + }) +} + +fn tool_replay_producer_eligible(parsed: &ParsedRequest, model_id: i32) -> bool { + tool_replay_scope(parsed, model_id) && parsed.has_tools +} + +fn thinking_visible_cache_eligible(parsed: &ParsedRequest) -> bool { + parsed.kind == ReqKind::Chat && parsed.api != Api::Responses +} + +pub(crate) fn thinking_visible_key( + prompt: &[u8], + content: &[u8], + syntax: ModelSyntax, + format: ChatFormat, + terminal: bool, +) -> Option> { + if matches!( + syntax, + ModelSyntax::Step37 | ModelSyntax::Ling3Vl | ModelSyntax::IQuestQ1 | ModelSyntax::BeamVq + ) { + // Removing reasoning changes these families' history grammar. Re-render + // the structured history with Jinja instead of inventing a prefix. + return None; + } + let mut visible = if format == ChatFormat::K2Horizon { + if !prompt.ends_with(b"\n") { + return None; + } + let content = content.trim_ascii(); + let mut visible = Vec::with_capacity(prompt.len() + 16 + content.len()); + visible.extend_from_slice(prompt); + visible.extend_from_slice(b""); + visible.extend_from_slice(content); + visible + } else if format == ChatFormat::Qwen4Exp || syntax == ModelSyntax::Exaone { + if !prompt.ends_with(b"\n") { + return None; + } + let content = content.trim_ascii(); + let mut visible = Vec::with_capacity(prompt.len() + 12 + content.len()); + visible.extend_from_slice(prompt); + visible.extend_from_slice(b"\n\n\n"); + visible.extend_from_slice(content); + visible + } else { + let start = think_start(format).as_bytes(); + if !prompt.ends_with(start) { + return None; + } + // GLM history replays `` + stripped content. + let glm = syntax == ModelSyntax::Glm53; + let prefix = if format == ChatFormat::SolarOpen2 || glm { + prompt + } else { + &prompt[..prompt.len() - start.len()] + }; + let content = if glm { content.trim_ascii() } else { content }; + let mut visible = + Vec::with_capacity(prefix.len() + think_end(format).len() + content.len()); + visible.extend_from_slice(prefix); + visible.extend_from_slice(think_end(format).as_bytes()); + visible.extend_from_slice(content); + visible + }; + if terminal { + match syntax { + ModelSyntax::Qwen4Exp | ModelSyntax::Qwen35 => { + visible.extend_from_slice(QWEN_IM_END.as_bytes()) + } + ModelSyntax::K2Horizon => { + visible.extend_from_slice(crate::render::K2_IM_END.as_bytes()) + } + ModelSyntax::Exaone => visible.extend_from_slice(b"<|endofturn|>"), + ModelSyntax::Motif3 => visible.extend_from_slice(b"<|endofturn|>"), + ModelSyntax::SolarOpen2 => visible.extend_from_slice(SOLAR_IM_END.as_bytes()), + ModelSyntax::Glm53 => {} + _ => visible.extend_from_slice(DSML_EOS.as_bytes()), + } + if matches!( + syntax, + ModelSyntax::Qwen4Exp + | ModelSyntax::Qwen35 + | ModelSyntax::Exaone + | ModelSyntax::SolarOpen2 + ) { + visible.push(b'\n'); + } + } + Some(visible) +} + +fn motif3_no_think_visible_checkpoint( + parsed: &ParsedRequest, + syntax: ModelSyntax, + prompt: &[u8], + content: &[u8], + finish: &str, +) -> Option> { + if parsed.kind != ReqKind::Chat + || syntax != ModelSyntax::Motif3 + || !ordinary_disk_cache_eligible(parsed) + || finish == "error" + || finish == "length" + { + return None; + } + let prefix = prompt.strip_suffix(b"")?; + let content = content.trim_ascii(); + let mut visible = Vec::with_capacity(prefix.len() + content.len()); + visible.extend_from_slice(prefix); + visible.extend_from_slice(content); + Some(visible) +} + +pub(crate) fn prepare_required_prefixes( + parsed: &mut ParsedRequest, + syntax: ModelSyntax, + tokenize: impl Fn(&[u8]) -> Result, GenerateError>, +) -> Result<(), GenerateError> { + let mimo_auto = syntax == ModelSyntax::Mimo2 + && parsed.has_tools + && parsed.tool_choice == ToolChoice::Auto + && !think_mode_enabled(parsed.think_mode); + if parsed.tool_choice != ToolChoice::Required && !parsed.has_tool_results && !mimo_auto { + return Ok(()); + } + let format = chat_format_for_syntax(syntax); + if !mimo_auto && parsed.required_think_end_prefix.is_empty() { + let toks = tokenize(think_end(format).as_bytes())?; + if toks.is_empty() { + return Err(GenerateError::Engine( + "failed to tokenize thinking control prefix".into(), + )); + } + parsed.required_think_end_prefix = toks; + } + if (parsed.tool_choice == ToolChoice::Required || mimo_auto) + && parsed.required_tool_prefix.is_empty() + { + // MiMo can continue a bare tool marker as prose without thinking. + let prefix = if mimo_auto { + b" Result<(), GenerateError> { + if !w.out.is_empty() { + out.write_all(&w.out).map_err(|_| GenerateError::Io)?; + w.out.clear(); + } + out.flush().map_err(|_| GenerateError::Io) +} + +fn append_recovery_suffix(engine: &mut dyn DecodeIo, suffix: &[u8]) -> Result { + if suffix.is_empty() { + return Ok(0); + } + let before = engine.pos(); + let mut target = engine.session_tokens(); + let extra = engine.tokenize_rendered_chat(suffix)?; + target.extend(extra); + engine.sync(&target)?; + let delta = engine.pos() - before; + Ok(if delta > 0 { delta } else { 0 }) +} + +fn retry_chat( + engine: &mut dyn DecodeIo, + parsed: &mut ParsedRequest, + prompt: &mut Vec, + acc: &SemAccum, + detail: &str, +) -> Result<(), GenerateError> { + if engine.template().is_none() { + let syntax = syntax_for_model_id(engine.model_id()); + let format = chat_format_for_syntax(syntax); + return append_recovery_suffix( + engine, + &build_recovery_suffix(syntax, format, parsed.think_mode, prompt, acc, detail), + ) + .map(|_| ()); + } + + // Discard the invalid generation. Re-render the valid conversation with a + // correction, so recovery never invents a model's role or tool delimiters. + let mut retry = parsed.clone(); + let correction = format!("\n\nTool error: {detail}. The previous attempt was not executed. Emit a valid tool call matching the provided schema, or answer normally if no tool is needed."); + if let Some(last) = retry.messages.last_mut().filter(|m| m.role == "user") { + last.content.push_str(&correction); + if !last.parts.is_empty() { + last.parts.push(crate::parse::ChatPart::Text(correction)); + } + } else { + retry.messages.push(crate::parse::ChatMsg { + role: "user".into(), + content: correction, + ..Default::default() + }); + } + let rendered = engine.render_request(&retry)?; + let tokens = engine.tokenize_rendered_chat(&rendered)?; + let media = prepare_media(engine, &retry, tokens)?; + engine.invalidate(); + if media.vision.is_empty() && media.audios.is_empty() && media.videos.is_empty() { + engine.sync(&media.tokens)?; + } else { + engine.sync_mimo_prompt(&media.tokens, &media.vision, &media.audios, &media.videos)?; + } + *parsed = retry; + *prompt = rendered; + Ok(()) +} + +fn decode_pass( + engine: &mut dyn DecodeIo, + parsed: &ParsedRequest, + req: &StreamReq, + job_id: &str, + acc: &mut SemAccum, + finish: &mut &'static str, + max_tokens: i32, + rng: &mut u64, + w: &mut Writer, + out: &mut impl Write, + mut oa: Option<&mut OpenaiStream>, + mut anth: Option<&mut AnthropicStream>, + mut resp: Option<&mut ResponsesStream>, + first_tok: &mut Option, + decode_steps: &mut i32, + speculation: &mut bool, + stop_requested: Option bool>, + token_ids: &mut Vec, +) -> Result<(), GenerateError> { + let mut last_heartbeat = Instant::now(); + 'decode: while acc.completion < max_tokens && engine.pos() < engine.ctx() { + if stop_requested.is_some_and(|stop| stop()) { + *finish = "error"; + break; + } + out.flush().map_err(|_| GenerateError::Io)?; + if continued_decode_allowed(acc) { + store_continued_best_effort(engine); + } + let mut temperature = parsed.temperature; + let mut top_k = parsed.top_k; + let mut top_p = parsed.top_p; + let mut min_p = parsed.min_p; + // IQuest and GLM-5.3 use the requested sampler in both channels. + // Greedy thinking must remain eligible for their embedded MTP paths. + if think_mode_enabled(parsed.think_mode) + && fixed_thinking_sampling(syntax_for_model_id(engine.model_id())) + { + temperature = DEFAULT_TEMPERATURE; + top_k = 0; + top_p = DEFAULT_TOP_P; + min_p = DEFAULT_MIN_P; + } + let policy = SamplePolicy { + tool_choice: parsed.tool_choice, + has_tool_results: parsed.has_tool_results, + think_mode: parsed.think_mode, + max_tokens: parsed.max_tokens, + required_tool_prefix: &parsed.required_tool_prefix, + required_think_end_prefix: &parsed.required_think_end_prefix, + }; + let ov = acc.sampling_override(&policy); + if matches!(ov, SampleOverride::Greedy) { + temperature = 0.0; + } + let eos = engine.eos_id(); + if parsed.eos_policy != EosPolicy::Default && eos < 0 { + return Err(GenerateError::Engine( + "EOS policy has no model EOS token".into(), + )); + } + let excluded = parsed.eos_policy.excluded(eos, acc.thinking_inside()); + let excluded_eot = excluded.and_then(|_| { + let eot = engine.eot_id(); + (eot >= 0 && engine.token_is_stop(eot)).then_some(eot) + }); + let token = if let SampleOverride::Token(t) = ov { + if excluded == Some(t) || excluded_eot == Some(t) { + return Err(GenerateError::Engine( + "forced token conflicts with EOS policy".into(), + )); + } + t + } else if let Some(excluded_id) = excluded { + engine.sample_excluding(temperature, top_k, top_p, min_p, rng, excluded_id) + } else { + engine.sample(temperature, top_k, top_p, min_p, rng) + }; + if token < 0 && excluded.is_some() { + return Err(GenerateError::Engine( + "EOS policy left no sample token".into(), + )); + } + if token < 0 || engine.token_is_stop(token) { + if let Some(text) = + crate::tools::beam::stop_text(req.chat_format, token, engine.eot_id()) + { + let _ = acc.feed_stop(text, &parsed.stops); + } + *finish = "stop"; + break; + } + let budget = (max_tokens - acc.completion).min(engine.ctx() - engine.pos()); + // Forced control prefixes can change policy between tokens. Only + // an unconstrained greedy segment may be committed ahead of output. + let prefix_start = engine.pos(); + let accepted = if temperature <= 0.0 + && parsed.eos_policy == EosPolicy::Default + && matches!(ov, SampleOverride::None) + && parsed.required_tool_prefix.is_empty() + && parsed.required_think_end_prefix.is_empty() + { + let accepted = engine.eval_greedy(token, budget)?; + if engine.last_eval_speculated() { + *speculation = true; + } + accepted + } else { + engine.eval(token)?; + vec![token] + }; + if accepted.is_empty() || accepted.len() > budget as usize || accepted[0] != token { + engine.invalidate(); + return Err(GenerateError::Engine("invalid greedy prefix result".into())); + } + let count = accepted.len(); + for (index, token) in accepted.into_iter().enumerate() { + if stop_requested.is_some_and(|stop| stop()) { + *finish = "error"; + engine.invalidate(); + break 'decode; + } + if token < 0 { + engine.invalidate(); + return Err(GenerateError::Engine("invalid greedy prefix token".into())); + } + if engine.token_is_stop(token) { + if let Some(text) = + crate::tools::beam::stop_text(req.chat_format, token, engine.eot_id()) + { + let _ = acc.feed_stop(text, &parsed.stops); + } + *finish = "stop"; + if index + 1 < count { + engine.invalidate(); + } + break 'decode; + } + if parsed.return_token_ids { + token_ids.push(token); + } + // A speculative prefix advances multiple tokens in one decode step. + if index == 0 { + *decode_steps += 1; + } + if first_tok.is_none() { + *first_tok = Some(Instant::now()); + } + let piece = engine.token_text(token).inspect_err(|_| { + engine.invalidate(); + })?; + let feed = acc.feed(&piece, &parsed.stops); + + if req.stream { + let view = &acc.text[..feed.emit_limit.min(acc.text.len())]; + match req.api { + Api::Openai if req.kind == ReqKind::Completion => { + if let Some(delta) = last_delta(&acc.text, feed.emit_limit, piece.len()) { + sse_chunk(w, req, job_id, Some(delta), None); + } + } + Api::Openai => { + if let Some(st) = oa.as_mut() { + openai_sse_stream_update(w, req, job_id, st, view, false); + } + } + Api::Anthropic => { + if let Some(st) = anth.as_mut() { + if !anthropic_sse_stream_update(w, req, job_id, st, view, false) { + engine.invalidate(); + return Err(GenerateError::Io); + } + } + } + Api::Responses => { + if let Some(st) = resp.as_mut() { + if !responses_sse_stream_update(w, req, st, view, false) { + engine.invalidate(); + return Err(GenerateError::Io); + } + } + } + } + stream_heartbeat_if_due( + w, + req, + resp.as_deref_mut(), + &mut last_heartbeat, + Instant::now(), + ": decode\n\n", + ); + flush(w, out).inspect_err(|_| { + engine.invalidate(); + })?; + } + + if feed.hit_stop { + *finish = "stop"; + engine.invalidate(); + break 'decode; + } + if acc.track_tools && acc.saw_tool_end && req.chat_format == ChatFormat::DeepSeek { + *finish = "tool_calls"; + if index + 1 < count { + engine.trim_greedy(prefix_start + index as i32 + 1)?; + } + break 'decode; + } + } + } + Ok(()) +} + +pub fn generate_and_write( + engine: &mut dyn DecodeIo, + parsed: &ParsedRequest, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + out: &mut impl Write, +) -> Result { + generate_and_write_at( + engine, + parsed, + job_id, + created, + cors, + default_tokens, + Instant::now(), + out, + ) +} + +/// Queue-aware serial entry point. `t_arrive` is captured by the HTTP owner; +/// callers outside the queued server should use [`generate_and_write`]. +pub fn generate_and_write_at( + engine: &mut dyn DecodeIo, + parsed: &ParsedRequest, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + t_arrive: Instant, + out: &mut impl Write, +) -> Result { + let (outcome, terminal) = generate_terminal_at( + engine, + parsed, + job_id, + created, + cors, + default_tokens, + t_arrive, + out, + )?; + out.write_all(&terminal).map_err(|_| GenerateError::Io)?; + Ok(outcome) +} + +/// Serial phase 1: everything before the prompt sync — blocked check, tool +/// replay restore, required prefixes, render, tokenize. Split out so the +/// server's `ensure_serial_session_fit` (C `serial_session_ensure_fit`) can +/// size the session from the exact sync-bound token count without a second +/// render+tokenize pass. +pub(crate) struct PreparedSerialPrompt { + pub(crate) parsed: ParsedRequest, + pub(crate) tool_replay: bool, + pub(crate) prompt: Vec, + pub(crate) tokens: Vec, + pub(crate) vision: Vec, + audios: Vec, + videos: Vec, +} + +struct MediaPrompt { + tokens: Vec, + vision: Vec, + audios: Vec, + videos: Vec, +} + +fn prepare_media( + engine: &dyn DecodeIo, + parsed: &ParsedRequest, + tokens: Vec, +) -> Result { + if parsed.images.is_empty() && parsed.audios.is_empty() && parsed.videos.is_empty() { + return Ok(MediaPrompt { + tokens, + vision: Vec::new(), + audios: Vec::new(), + videos: Vec::new(), + }); + } + if syntax_for_model_id(engine.model_id()) == ModelSyntax::Mimo2 { + return prepare_mimo(engine, parsed, tokens); + } + if !parsed.videos.is_empty() { + return Err(GenerateError::Unsupported("video input requires MiMo")); + } + // V4.1 owns its tower as a sidecar and expands the <|deepseek_image|> + // placeholder in the push route (generate_terminal_v41), not here: this + // seam probes session-based encoders. Pass the tokens through untouched. + if engine.model_id() == ds4_core::Variant::DeepSeek41Flash as i32 { + if !parsed.audios.is_empty() { + return Err(GenerateError::Unsupported( + "audio input is not supported by DeepSeek V4.1", + )); + } + return Ok(MediaPrompt { + tokens, + vision: Vec::new(), + audios: Vec::new(), + videos: Vec::new(), + }); + } + const GLM_IMAGE_TOKEN: i32 = 154854; + const INKLING_IMAGE_TOKEN: i32 = 200054; + const STEP_IMAGE_TOKEN: i32 = 128001; + const LING3VL_IMAGE_TOKEN: i32 = 157157; + const MIMO_IMAGE_TOKEN: i32 = 151655; + const INKLING_AUDIO_TOKEN: i32 = 200053; + const MIMO_AUDIO_TOKEN: i32 = 151669; + let image_token = match syntax_for_model_id(engine.model_id()) { + ModelSyntax::Glm53 => GLM_IMAGE_TOKEN, + ModelSyntax::Inkling => INKLING_IMAGE_TOKEN, + ModelSyntax::Step37 => STEP_IMAGE_TOKEN, + ModelSyntax::Ling3Vl => LING3VL_IMAGE_TOKEN, + ModelSyntax::Mimo2 => MIMO_IMAGE_TOKEN, + _ => { + return Err(GenerateError::Unsupported( + "serial images require GLM-5.3, Inkling, Step or Ling", + )) + } + }; + if parsed.images.len() + parsed.audios.len() + parsed.videos.len() > 4 { + return Err(GenerateError::Unsupported( + "serial media supports 1 to 4 inputs", + )); + } + let mut image_spans = Vec::with_capacity(parsed.images.len()); + let mut expanded_len = tokens.len(); + for image in &parsed.images { + let span = engine.vision_tokens(&image.data)?; + if span.is_empty() { + return Err(GenerateError::Engine( + "image probe returned zero tokens".into(), + )); + } + expanded_len = expanded_len + .checked_add(span.len() - 1) + .ok_or_else(|| GenerateError::Engine("expanded image prompt is too large".into()))?; + image_spans.push(span); + } + let mut audio_counts = Vec::with_capacity(parsed.audios.len()); + for audio in &parsed.audios { + let count = engine.audio_probe(&audio.data)?; + if count == 0 { + return Err(GenerateError::Engine( + "audio probe returned zero tokens".into(), + )); + } + expanded_len = expanded_len + .checked_add(count as usize - 1) + .ok_or_else(|| GenerateError::Engine("expanded media prompt is too large".into()))?; + audio_counts.push(count); + } + let mut expanded = Vec::with_capacity(expanded_len); + let mut images = Vec::with_capacity(parsed.images.len()); + let mut image_index = 0usize; + let mut audios = Vec::with_capacity(parsed.audios.len()); + let mut audio_index = 0usize; + for token in tokens { + if (image_token == INKLING_IMAGE_TOKEN && token == INKLING_AUDIO_TOKEN) + || (image_token == MIMO_IMAGE_TOKEN && token == MIMO_AUDIO_TOKEN) + { + let Some((audio, count)) = parsed + .audios + .get(audio_index) + .zip(audio_counts.get(audio_index)) + else { + return Err(GenerateError::Engine( + "ambiguous audio placeholder in prompt".into(), + )); + }; + let token_offset = u32::try_from(expanded.len()) + .map_err(|_| GenerateError::Engine("expanded media prompt is too large".into()))?; + expanded.extend(std::iter::repeat_n(token, *count as usize)); + audios.push(AudioPromptInput { + data: audio.data.clone(), + token_offset, + }); + audio_index += 1; + continue; + } + if token != image_token { + expanded.push(token); + continue; + } + let Some((image, span)) = parsed + .images + .get(image_index) + .zip(image_spans.get(image_index)) + else { + return Err(GenerateError::Engine( + "ambiguous image placeholder in prompt".into(), + )); + }; + let token_offset = u32::try_from(expanded.len()) + .map_err(|_| GenerateError::Engine("expanded image prompt is too large".into()))?; + expanded.extend_from_slice(span); + images.push(VisionPromptInput { + data: image.data.clone(), + token_offset, + frames: Vec::new(), + }); + image_index += 1; + } + if image_index != parsed.images.len() + || audio_index != parsed.audios.len() + || expanded.len() != expanded_len + { + return Err(GenerateError::Engine( + "media placeholder count does not match payloads".into(), + )); + } + Ok(MediaPrompt { + tokens: expanded, + vision: images, + audios, + videos: Vec::new(), + }) +} + +/// 2 fps frames and a temporal patch of 2, so each pair is one second. +const MIMO_SECONDS_PER_PAIR: f32 = 1.0; + +fn prepare_mimo( + engine: &dyn DecodeIo, + parsed: &ParsedRequest, + tokens: Vec, +) -> Result { + if parsed.images.len() + parsed.audios.len() + parsed.videos.len() > 4 { + return Err(GenerateError::Unsupported( + "serial media supports 1 to 4 inputs", + )); + } + + let mut pieces = Vec::new(); + let mut vision = Vec::new(); + let mut audios = Vec::new(); + let mut videos = Vec::new(); + // Pair a video only with the audio part that follows it in the same + // message. The next message's audio is a separate clip; the rendered + // tokens have a boundary between the two stubs. + for message in &parsed.messages { + let parts = &message.parts; + let mut index = 0usize; + while index < parts.len() { + match &parts[index] { + ChatPart::Text(_) | ChatPart::ToolResult { .. } => index += 1, + ChatPart::Image(slot) => { + let image = parsed.images.get(*slot).ok_or_else(|| { + GenerateError::Engine("image reference is missing".into()) + })?; + let span = engine.vision_tokens(&image.data)?; + if span.is_empty() { + return Err(GenerateError::Engine( + "image probe returned zero tokens".into(), + )); + } + pieces.push(ds4_core::MediaPiece::Image { + count: span.len() as u32, + }); + vision.push(VisionPromptInput { + data: image.data.clone(), + token_offset: 0, + frames: Vec::new(), + }); + index += 1; + } + ChatPart::Audio(slot) => { + let audio = parsed.audios.get(*slot).ok_or_else(|| { + GenerateError::Engine("audio reference is missing".into()) + })?; + let count = engine.audio_probe(&audio.data)?; + if count == 0 { + return Err(GenerateError::Engine( + "audio probe returned zero tokens".into(), + )); + } + pieces.push(ds4_core::MediaPiece::Audio { count }); + audios.push(AudioPromptInput { + data: audio.data.clone(), + token_offset: 0, + }); + index += 1; + } + ChatPart::Video(slot) => { + let video = parsed.videos.get(*slot).ok_or_else(|| { + GenerateError::Engine("video reference is missing".into()) + })?; + let (_duration, packed) = ds4_core::load_video(&video.data) + .map_err(|error| GenerateError::Engine(error.to_string()))?; + if packed.is_empty() { + return Err(GenerateError::Engine("video produced no frames".into())); + } + + // The audio that follows a video is that video's track. Text + // between them leaves a separate audio part. + let audio_len = match parts.get(index + 1) { + Some(ChatPart::Audio(audio_slot)) => { + let audio = parsed.audios.get(*audio_slot).ok_or_else(|| { + GenerateError::Engine("audio reference is missing".into()) + })?; + let count = engine.audio_probe(&audio.data)?; + if count == 0 { + return Err(GenerateError::Engine( + "audio probe returned zero tokens".into(), + )); + } + audios.push(AudioPromptInput { + data: audio.data.clone(), + token_offset: 0, + }); + count + } + _ => 0, + }; + let mut pairs = Vec::with_capacity(packed.len()); + for (pair_index, visual) in packed.iter().enumerate() { + let start_s = pair_index as f32 * MIMO_SECONDS_PER_PAIR; + let timestamp_ids = + engine.tokenize_text(&ds4_core::format_timestamp(start_s))?; + if timestamp_ids.is_empty() { + return Err(GenerateError::Engine("video timestamp is empty".into())); + } + let audio_tokens = if audio_len == 0 { + 0 + } else { + // The last pair keeps the remaining codec rows. Duration + // times 6.25 Hz is not the feature length. + let end_s = if pair_index + 1 == packed.len() { + start_s + audio_len as f32 + } else { + (pair_index as f32 + 1.0) * MIMO_SECONDS_PER_PAIR + }; + ds4_core::audio_interval(start_s, end_s, audio_len) + .map_err(|error| GenerateError::Engine(error.to_string()))? + }; + pairs.push(ds4_core::VideoPair { + timestamp_s: start_s, + timestamp_ids, + height: visual.height(), + width: visual.width(), + audio_tokens, + }); + } + if audio_len == 0 { + pieces.push(ds4_core::MediaPiece::Video { pairs }); + index += 1; + } else { + pieces.push(ds4_core::MediaPiece::Joint { pairs }); + index += 2; + } + videos.push(VisionPromptInput { + data: video.data.clone(), + token_offset: 0, + frames: packed, + }); + } + } + } + } + + let (expanded, spans) = ds4_core::expand_pieces(&tokens, &pieces) + .map_err(|error| GenerateError::Engine(error.to_string()))?; + ds4_core::check_span_budget(spans.len()) + .map_err(|error| GenerateError::Engine(error.to_string()))?; + Ok(MediaPrompt { + tokens: expanded, + vision, + audios, + videos, + }) +} + +pub(crate) fn prepare_serial_prompt( + engine: &mut dyn DecodeIo, + parsed: &ParsedRequest, +) -> Result { + if let Some(msg) = generation_blocked(parsed, engine.model_id()) { + return Err(GenerateError::Unsupported(msg)); + } + // V4.1 images are served by the model's own tower, which is a sidecar + // (--vision-gguf). A request that carries an image while no tower is open + // is refused with a 400 before any work, never silently dropped (80296d3). + if engine.model_id() == ds4_core::Variant::DeepSeek41Flash as i32 + && !parsed.images.is_empty() + && engine.v41_vision().is_none() + { + return Err(GenerateError::BadRequest( + "image input requires the DeepSeek V4.1 vision sidecar (--vision-gguf)", + )); + } + + let mut parsed = parsed.clone(); + engine.restore_chat(&mut parsed)?; + let syntax = syntax_for_model_id(engine.model_id()); + let tool_replay = parsed.images.is_empty() + && parsed.audios.is_empty() + && parsed.videos.is_empty() + && tool_replay_disk_cache_eligible(&parsed, engine.model_id()); + if tool_replay { + engine.restore_tool_replay(&mut parsed.messages); + } + if engine.tokenizes_control_literals() { + prepare_required_prefixes(&mut parsed, syntax, |literal| { + engine.tokenize_rendered_chat(literal) + })?; + } + + let prompt = engine.render_request(&parsed)?; + let tokens = match parsed.kind { + ReqKind::Completion => { + let text = std::str::from_utf8(&prompt).unwrap_or(""); + engine.tokenize_text(text)? + } + ReqKind::Chat => engine.tokenize_rendered_chat(&prompt)?, + }; + let MediaPrompt { + tokens, + vision, + audios, + videos, + } = prepare_media(engine, &parsed, tokens)?; + Ok(PreparedSerialPrompt { + parsed, + tool_replay, + prompt, + tokens, + vision, + audios, + videos, + }) +} + +/// Runs serial generation while withholding only the final wire terminal. +/// Streaming headers/deltas still flow through `out` as they are produced. +pub(crate) fn generate_terminal_at( + engine: &mut dyn DecodeIo, + parsed: &ParsedRequest, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + t_arrive: Instant, + out: &mut impl Write, +) -> Result<(GenerateOutcome, Vec), GenerateError> { + let prep = prepare_serial_prompt(engine, parsed)?; + generate_terminal_prepared( + engine, + prep, + job_id, + created, + cors, + default_tokens, + t_arrive, + None, + out, + ) +} + +/// Serial phase 2: prompt sync onward, on a [`PreparedSerialPrompt`]. +pub(crate) fn generate_terminal_prepared( + engine: &mut dyn DecodeIo, + prep: PreparedSerialPrompt, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + t_arrive: Instant, + stop_requested: Option bool>, + out: &mut impl Write, +) -> Result<(GenerateOutcome, Vec), GenerateError> { + let PreparedSerialPrompt { + mut parsed, + tool_replay, + mut prompt, + tokens, + vision, + audios, + videos, + } = prep; + // The engine's one-line dispatch (server_generate.c:19): a V4.1 engine + // has no ds4_session, so the pull loop below cannot run for it — the + // push route takes over with the same prepared prompt. + if engine.is_v41() { + return generate_terminal_v41( + engine, + &parsed, + &prompt, + &tokens, + job_id, + created, + cors, + default_tokens, + t_arrive, + stop_requested, + out, + ); + } + let syntax = syntax_for_model_id(engine.model_id()); + let mut req = stream_req_from_parsed(&parsed, engine.model_id()); + let mut w = Writer::new(created); + if req.stream { + w.out.extend_from_slice(&sse_headers(cors)); + flush(&mut w, out)?; + } + engine.begin_trace(); + let t_prefill = Instant::now(); + let sync_result = if !vision.is_empty() || !audios.is_empty() || !videos.is_empty() { + engine + .sync_mimo_prompt(&tokens, &vision, &audios, &videos) + .map(|()| 0) + } else if tool_replay { + engine.sync_tool_replay_prompt(&prompt, &tokens) + } else { + engine.sync_prompt( + &prompt, + &tokens, + ordinary_disk_cache_eligible(&parsed), + thinking_visible_cache_eligible(&parsed), + ) + }; + let cached = match sync_result { + Ok(cached) => cached, + Err(error) if req.stream => { + let message = error.to_string(); + stream_error(&mut w, &req, None, &message); + flush(&mut w, out)?; + return Err(GenerateError::Streamed(message)); + } + Err(error) => return Err(error), + }; + store_continued_best_effort(engine); + let decode_t0 = Instant::now(); + let prefill_elapsed = engine + .prompt_sync_elapsed() + .unwrap_or_else(|| decode_t0.duration_since(t_prefill)); + let mut first_tok = None; + let mut decode_steps = 0i32; + let mut speculation = false; + let mut token_ids = Vec::new(); + + let prompt_n = engine.pos(); + let mut rng = parsed.seed; + req.cache_read_tokens = cached.clamp(0, prompt_n); + req.cache_write_tokens = prompt_n - req.cache_read_tokens; + let mut acc; + let mut finish; + let mut recovery_attempted = false; + + let mut oa = if req.stream && req.api == Api::Openai && req.kind == ReqKind::Chat { + let mut stream = openai_stream_start(&req); + stream.tool.use_random_ids(); + Some(stream) + } else { + None + }; + let mut anth = if req.stream && req.api == Api::Anthropic { + let mut stream = anthropic_sse_start_live(&mut w, &req, job_id, prompt_n); + stream.tool.use_random_ids(); + Some(stream) + } else { + None + }; + let mut resp = if req.stream && req.api == Api::Responses { + let (rid, rsid, mid) = responses_ids(job_id); + let mut st = responses_stream_init(&req, &rid, &rsid, &mid); + responses_sse_created(&mut w, &req, &mut st, created); + Some(st) + } else { + None + }; + + if req.stream { + if req.api == Api::Openai && req.kind == ReqKind::Chat { + sse_chunk(&mut w, &req, job_id, None, None); + } + flush(&mut w, out)?; + } + + let mut parsed_gen; + loop { + let mut max_tokens = + decode_budget(parsed.max_tokens_set, parsed.max_tokens, default_tokens); + let room = engine.ctx() - engine.pos(); + if room >= 0 && max_tokens > room { + max_tokens = room; + } + acc = SemAccum::init( + parsed.kind == ReqKind::Chat, + parsed.has_tools, + think_mode_enabled(parsed.think_mode), + req.chat_format, + &prompt, + ); + // The request's tools, for the start scan that recognises the + // artifact's quoted root (find_tool_start). + acc.set_tool_orders(&parsed.tool_orders); + // The tool dialect is a property of the served family, chosen before the + // first update (ds4_server.c:20591); GLM's block is not a DSML spelling. + acc.set_model_syntax(syntax); + finish = "length"; + token_ids.clear(); + + let decoded = decode_pass( + engine, + &parsed, + &req, + job_id, + &mut acc, + &mut finish, + max_tokens, + &mut rng, + &mut w, + out, + oa.as_mut(), + anth.as_mut(), + resp.as_mut(), + &mut first_tok, + &mut decode_steps, + &mut speculation, + stop_requested, + &mut token_ids, + ); + if let Err(error) = decoded { + if !req.stream || matches!(&error, GenerateError::Io) { + return Err(error); + } + let message = error.to_string(); + stream_error(&mut w, &req, resp.as_mut(), &message); + flush(&mut w, out)?; + return Err(GenerateError::Streamed(message)); + } + + finish = terminal_finish(finish); + match truncation_outcome( + syntax, + req.chat_format, + parsed.kind == ReqKind::Chat, + parsed.has_tools, + acc.saw_tool_start, + acc.saw_tool_end, + finish, + parsed.stream, + recovery_attempted, + &acc.text, + &parsed.tool_orders, + ) { + TruncationOutcome::Repair(text) => { + acc.text = text; + acc.saw_tool_end = true; + } + TruncationOutcome::RetryUnterminated => { + if retry_chat( + engine, + &mut parsed, + &mut prompt, + &acc, + "unterminated tool call", + ) + .is_ok() + { + recovery_attempted = true; + continue; + } + finish = "error"; + } + TruncationOutcome::ErrorUnterminated => { + finish = "error"; + } + TruncationOutcome::None => {} + } + + parsed_gen = if parsed.kind == ReqKind::Chat { + let (pg, recovered_finish) = parse_generated_for_response( + syntax, + &acc.text, + parsed.has_tools, + acc.saw_tool_start, + think_mode_enabled(parsed.think_mode), + req.chat_format, + &parsed.tool_orders, + finish, + ); + finish = recovered_finish; + pg + } else { + crate::tools::ParsedGenerated { + content: acc.text.clone(), + ok: true, + ..Default::default() + } + }; + if !parsed_gen.ok + && parse_failure_should_retry( + syntax, + parsed.stream, + recovery_attempted, + finish, + parsed_gen.recovered, + parsed.has_tools, + acc.saw_tool_start, + ) + { + if retry_chat(engine, &mut parsed, &mut prompt, &acc, "invalid tool call").is_ok() { + recovery_attempted = true; + continue; + } + finish = "error"; + } + break; + } + + let completion = acc.completion; + req.timings.prefill_tokens = prompt_n - req.cache_read_tokens; + req.timings.prefill_cached = req.cache_read_tokens; + req.timings.decode_tokens = completion; + req.timings.decode_steps = decode_steps; + if completion > 0 { + if let Some(t_first) = first_tok { + req.timings = ReqTimings { + valid: true, + ttft_ms: t_first.duration_since(t_arrive).as_secs_f64() * 1e3, + prefill_ms: prefill_elapsed.as_secs_f64() * 1e3, + decode_ms: Instant::now().duration_since(t_first).as_secs_f64() * 1e3, + prefill_tokens: prompt_n - req.cache_read_tokens, + prefill_cached: req.cache_read_tokens, + decode_tokens: completion, + decode_steps, + }; + } + } + if !parsed_gen.calls.is_empty() { + if let Some(st) = oa.as_ref() { + st.tool.apply_ids(&mut parsed_gen.calls); + } + if let Some(st) = anth.as_ref() { + st.tool.apply_ids(&mut parsed_gen.calls); + } + assign_tool_ids( + &mut parsed_gen.calls, + if parsed.api == Api::Anthropic { + "toolu_" + } else { + "call_" + }, + ); + if tool_replay_producer_eligible(&parsed, engine.model_id()) { + engine.remember_tool_replay(&parsed_gen.calls, &parsed_gen.raw_dsml); + } + finish = "tool_calls"; + } + let visible = if parsed_gen.calls.is_empty() + && parsed.kind == ReqKind::Chat + && think_mode_enabled(parsed.think_mode) + && !matches!(finish, "error" | "length") + && !acc.thinking_inside() + { + thinking_visible_key(&prompt, &parsed_gen.content, syntax, req.chat_format, true) + } else { + None + } + .or_else(|| { + motif3_no_think_visible_checkpoint(&parsed, syntax, &prompt, &parsed_gen.content, finish) + }); + if let Some(visible) = visible { + engine.remember_thinking_visible_checkpoint(visible); + } + let matched_stop = acc.matched_stop.clone(); + let terminal = if req.stream { + match req.api { + Api::Openai if req.kind == ReqKind::Completion => { + sse_chunk(&mut w, &req, job_id, None, Some(finish)); + sse_done(&mut w, &req, job_id, prompt_n, completion); + } + Api::Openai => { + if let Some(st) = oa.as_mut() { + openai_sse_finish_live( + &mut w, + &req, + job_id, + st, + &acc.text, + finish, + prompt_n, + completion, + &parsed_gen.calls, + ); + } + } + Api::Anthropic => { + if let Some(st) = anth.as_mut() { + if !anthropic_sse_finish_live( + &mut w, + &req, + job_id, + st, + &acc.text, + finish, + matched_stop.as_deref(), + completion, + &parsed_gen.calls, + ) { + return Err(GenerateError::Io); + } + } + } + Api::Responses => { + if let Some(st) = resp.as_mut() { + if !responses_sse_finish_live( + &mut w, + &req, + st, + &acc.text, + finish, + prompt_n, + completion, + acc.reasoning_tokens, + created, + &parsed_gen.calls, + ) { + return Err(GenerateError::Io); + } + } + } + } + std::mem::take(&mut w.out) + } else { + let bytes = match req.api { + Api::Anthropic => anthropic_final_response( + &req, + job_id, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + finish, + matched_stop.as_deref(), + prompt_n, + completion, + cors, + &parsed_gen.calls, + ), + Api::Responses => { + let (rid, rsid, mid) = responses_ids(job_id); + let think = if acc.thinking_inside() { + ThinkBlock::Open + } else { + ThinkBlock::Closed + }; + responses_final_response( + &req, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + finish, + think, + prompt_n, + completion, + acc.reasoning_tokens, + created, + cors, + &rid, + &rsid, + &mid, + &parsed_gen.calls, + ) + } + Api::Openai => final_response( + &req, + job_id, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + finish, + prompt_n, + completion, + created, + cors, + &parsed_gen.calls, + if parsed.return_token_ids { + &token_ids + } else { + &[] + }, + ), + }; + bytes + }; + if finish != "error" && finish != "length" { + engine.remember_chat(&parsed, &parsed_gen); + } + // The reference's --trace section (`ds4_server.c:12011-12017`), written + // after the wire bytes: the turn above holds the engine, and an engine call + // here cannot disturb what the client already received. + if crate::trace::enabled() { + let prompt_ids = engine.tokenize_rendered_chat(&prompt).unwrap_or_default(); + let mut text_of = |token: i32| engine.token_text(token).unwrap_or_default(); + crate::trace::write_request( + &crate::trace::TraceRequest { + api: req.api, + kind: parsed.kind, + stream: req.stream, + model_id: engine.model_id(), + model: &req.model, + tools: req.tool_orders.len(), + think: req.think_mode, + prompt: &prompt, + tokens: &prompt_ids, + cached: req.cache_read_tokens, + prompt_tokens: prompt_n, + max_tokens: parsed.max_tokens, + temperature: parsed.temperature, + top_k: parsed.top_k, + top_p: parsed.top_p, + min_p: parsed.min_p, + seed: parsed.seed, + finish, + completion, + saw_tool_start: acc.saw_tool_start, + saw_tool_end: acc.saw_tool_end, + elapsed_sec: t_arrive.elapsed().as_secs_f64(), + content: &parsed_gen.content, + reasoning: &parsed_gen.reasoning, + calls: &parsed_gen.calls, + }, + &mut text_of, + ); + } + let outcome = GenerateOutcome { + tool_ids: parsed_gen + .calls + .iter() + .map(|c| c.id.clone()) + .filter(|id| !id.is_empty()) + .collect(), + bank: None, + generation: engine.generation(), + frontier: engine.pos(), + finish: finish.to_string(), + timings: req.timings, + speculation_active: speculation, + reuse: engine.last_reuse(), + reuse_miss: engine.last_miss(), + lane: None, + fallback_reason: None, + }; + Ok((outcome, terminal)) +} + +/// The V4.1 callbacks' shared state (the engine's `v41_gen`, +/// server_generate_v41.c:15-...): the emit and progress closures both need +/// the writer and the client probe, so one struct carries the per-request +/// streaming state and the two closures reach it through a RefCell — the +/// native calls them sequentially, never reentrantly. +struct V41EmitState<'a, W: Write> { + engine: &'a dyn DecodeIo, + parsed: &'a ParsedRequest, + req: StreamReq, + job_id: &'a str, + stop_requested: Option bool>, + max_tokens: i32, + acc: SemAccum, + w: &'a mut Writer, + out: &'a mut W, + oa: Option, + anth: Option, + resp: Option, + first_tok: Option, + decode_steps: i32, + token_ids: Vec, + finish: &'static str, + last_heartbeat: Instant, + err: Option, +} + +impl V41EmitState<'_, W> { + /// Flush the accumulated bytes; the job sink's flush is also the + /// disconnect probe (`serve.rs` JobSink::flush), so a dead peer aborts + /// generation the same way it does on the pull path. + fn flush_out(&mut self) -> bool { + if flush(self.w, self.out).is_ok() { + return true; + } + self.err = Some(GenerateError::Io); + false + } + + /// `v41_emit` (server_generate_v41.c:84-206): one pushed token, the same + /// per-token work as the pull loop's body (`decode_pass`), in the + /// engine's order — EOS and the budget check run before the token + /// counts, so neither produces text (:114-118). + fn emit_token(&mut self, token: i32) -> bool { + if self.err.is_some() { + return false; + } + if self.first_tok.is_none() { + self.first_tok = Some(Instant::now()); + } + if self.stop_requested.is_some_and(|stop| stop()) { + self.finish = "error"; + self.err = Some(GenerateError::Engine("shutdown requested".into())); + return false; + } + if self.parsed.return_token_ids { + self.token_ids.push(token); + } + if token < 0 || self.engine.token_is_stop(token) { + self.finish = "stop"; + return false; + } + if self.acc.completion >= self.max_tokens { + return false; + } + self.decode_steps += 1; + let piece = match self.engine.token_text(token) { + Ok(piece) => piece, + Err(error) => { + self.err = Some(error); + return false; + } + }; + let feed = self.acc.feed(&piece, &self.parsed.stops); + if self.req.stream { + let view = &self.acc.text[..feed.emit_limit.min(self.acc.text.len())]; + match self.req.api { + Api::Openai if self.req.kind == ReqKind::Completion => { + if let Some(delta) = last_delta(&self.acc.text, feed.emit_limit, piece.len()) { + sse_chunk(self.w, &self.req, self.job_id, Some(delta), None); + } + } + Api::Openai => { + if let Some(st) = self.oa.as_mut() { + openai_sse_stream_update(self.w, &self.req, self.job_id, st, view, false); + } + } + Api::Anthropic => { + if let Some(st) = self.anth.as_mut() { + if !anthropic_sse_stream_update( + self.w, + &self.req, + self.job_id, + st, + view, + false, + ) { + self.err = Some(GenerateError::Io); + return false; + } + } + } + Api::Responses => { + if let Some(st) = self.resp.as_mut() { + if !responses_sse_stream_update(self.w, &self.req, st, view, false) { + self.err = Some(GenerateError::Io); + return false; + } + } + } + } + stream_heartbeat_if_due( + self.w, + &self.req, + self.resp.as_mut(), + &mut self.last_heartbeat, + Instant::now(), + ": decode\n\n", + ); + if !self.flush_out() { + return false; + } + } + if feed.hit_stop { + self.finish = "stop"; + return false; + } + if self.acc.track_tools + && self.acc.saw_tool_end + && self.req.chat_format == ChatFormat::DeepSeek + { + self.finish = "tool_calls"; + return false; + } + true + } + + /// The engine's v41_syntax_greedy (server_generate_v41.c, c6e2706): under + /// sampling the tool-call syntax positions (the DSML tags / the parameter + /// head / the JSON punctuation) take the raw argmax while the string value + /// body and the JSON string keep the request's draw. The engine rebuilds a + /// copy of the tracker and the thinking state over its emitted text + the + /// ahead tokens; the port's text lives here, so the accumulator answers + /// through SemAccum::syntax_greedy. The request gate is the engine's own: + /// only a chat request with tools asks at all. + fn syntax_greedy(&mut self, ahead: *const i32, n: u32) -> bool { + if self.parsed.kind != ReqKind::Chat || !self.parsed.has_tools { + return false; + } + let mut pieces: Vec> = Vec::with_capacity(n as usize); + for i in 0..n as usize { + let token = unsafe { *ahead.add(i) }; + if self.engine.token_is_stop(token) { + break; /* the engine's own stop: the ahead run ends there */ + } + match self.engine.token_text(token) { + Ok(piece) => pieces.push(piece), + Err(_) => break, + } + } + self.acc.syntax_greedy(&pieces) + } + + /// `v41_progress_cb` (server_generate_v41.c:22-32): the prefill + /// keepalive (`: prefill`, the engine's 5 s interval) and the client + /// probe; returning false aborts the prefill (the engine's nonzero + /// return, `ds41_forward.h`). + fn prefill_progress(&mut self, _event: &str, _current: i32, _total: i32) -> bool { + if self.req.stream { + stream_heartbeat_if_due( + self.w, + &self.req, + self.resp.as_mut(), + &mut self.last_heartbeat, + Instant::now(), + ": prefill\n\n", + ); + } + self.flush_out() + } +} + +/// V4.1 (ds41) serial generation — the engine's push-based route +/// (`server_generate_v41.c`). Render and tokenize already happened +/// (`prepare_serial_prompt`); the engine's one-shot generate pushes tokens +/// through the emit callback, the per-token work mirrors `v41_emit` +/// (:84-206) on the shared machinery (`SemAccum::feed` carries text, +/// thinking, DSML and stops), and the finish mirrors `v41_finish` (:211-316) +/// minus the pieces the engine itself drops on this route: no +/// invalid-DSML retry (needs a session), no tool replay / live bindings / +/// thinking checkpoints (all session-addressed), and no tool-memory write +/// (the port's tool memory is wired to the session store this route never +/// creates; the engine's v41 path keeps its own). +/// Expand the V4.1 prompt for its images: each `<|deepseek_image|>` placeholder +/// becomes its span and each image is preprocessed (the engine's +/// `ds4_v41_images_prepare`, core_v41_vision.c:186-218). None when the engine +/// has no vision sidecar or the request carries no image, so a text request +/// never touches the tower and stays byte-identical. +fn expand_v41_images( + engine: &dyn DecodeIo, + parsed: &ParsedRequest, + tokens: &[i32], +) -> Result, GenerateError> { + let Some(meta) = engine.v41_vision() else { + // No sidecar: an image request is refused by name, never silently + // dropped (the engine's 400, 80296d3: "image_url 进不了 sink 时改为 + // 400, 不再静默丢图"). prepare_serial_prompt answers this first; this + // guard is the backstop for a caller that skipped the preflight. + if !parsed.images.is_empty() { + return Err(GenerateError::BadRequest( + "image input requires the DeepSeek V4.1 vision sidecar (--vision-gguf)", + )); + } + return Ok(None); + }; + if parsed.images.is_empty() { + return Ok(None); + } + // The images in prompt order: the renderer wrote one placeholder per image + // part in message/part order, so collect the bytes the same way. + let mut order: Vec<&[u8]> = Vec::with_capacity(parsed.images.len()); + for m in &parsed.messages { + for part in &m.parts { + if let ChatPart::Image(idx) = part { + order.push(parsed.images[*idx].data.as_ref()); + } + } + } + let imgs = + ds4_core::v41_media::V41Images::expand(tokens, &order, &meta, engine.ctx().max(0) as usize) + .map_err(|_| { + GenerateError::BadRequest( + "invalid V4.1 images or expanded image request exceeds context/memory budget", + ) + })?; + Ok(Some(imgs)) +} + +#[allow(clippy::too_many_arguments)] +fn generate_terminal_v41( + engine: &mut dyn DecodeIo, + parsed: &ParsedRequest, + prompt: &[u8], + tokens: &[i32], + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + t_arrive: Instant, + stop_requested: Option bool>, + out: &mut impl Write, +) -> Result<(GenerateOutcome, Vec), GenerateError> { + // Vision (V4.1): expand each <|deepseek_image|> placeholder into its span + // and preprocess the images before anything sizes the prompt (the engine's + // ds4_v41_images_prepare, core_v41_vision.c:186-218). A text request, or + // an engine with no --vision-gguf sidecar, keeps the tokens untouched. + let images = expand_v41_images(engine, parsed, tokens)?; + let prompt_tokens: &[i32] = images.as_ref().map_or(tokens, |i| &i.tokens); + let prompt_n = prompt_tokens.len() as i32; + if prompt_n < 1 { + // v41_gen_begin (:325): an empty prompt never reaches the engine. + return Err(GenerateError::Engine("empty prompt".into())); + } + let syntax = syntax_for_model_id(engine.model_id()); + let mut req = stream_req_from_parsed(parsed, engine.model_id()); + // v41_gen_begin (:342-345): the budget clamps to the metadata context + // (the only context source; --ctx is refused at boot), and the request + // starts as an all-write claim -- the KV reuse's own split replaces it + // after the run (U-B, below). + let mut max_tokens = decode_budget(parsed.max_tokens_set, parsed.max_tokens, default_tokens); + let room = engine.ctx() - prompt_n; + if max_tokens > room { + max_tokens = room; + } + if max_tokens < 0 { + max_tokens = 0; + } + // v41_gen_begin (:342-345): the route starts with no reuse claimed; the + // native's restored count replaces both after the run (U-B, below). + req.cache_read_tokens = 0; + req.cache_write_tokens = prompt_n; + let mut w = Writer::new(created); + if req.stream { + w.out.extend_from_slice(&sse_headers(cors)); + } + let oa = if req.stream && req.api == Api::Openai && req.kind == ReqKind::Chat { + let mut stream = openai_stream_start(&req); + stream.tool.use_random_ids(); + Some(stream) + } else { + None + }; + let anth = if req.stream && req.api == Api::Anthropic { + let mut stream = anthropic_sse_start_live(&mut w, &req, job_id, prompt_n); + stream.tool.use_random_ids(); + Some(stream) + } else { + None + }; + let resp = if req.stream && req.api == Api::Responses { + let (rid, rsid, mid) = responses_ids(job_id); + let mut st = responses_stream_init(&req, &rid, &rsid, &mid); + responses_sse_created(&mut w, &req, &mut st, created); + Some(st) + } else { + None + }; + if req.stream { + if req.api == Api::Openai && req.kind == ReqKind::Chat { + sse_chunk(&mut w, &req, job_id, None, None); + } + flush(&mut w, out)?; + } + + let t0 = Instant::now(); + let chat_format = req.chat_format; + let mut acc = SemAccum::init( + parsed.kind == ReqKind::Chat, + parsed.has_tools, + think_mode_enabled(parsed.think_mode), + chat_format, + prompt, + ); + // The request's tools, for the start scan that recognises the artifact's + // quoted root (find_tool_start). + acc.set_tool_orders(&parsed.tool_orders); + // The tool dialect is a property of the served family, chosen before the + // first update (ds4_server.c:20591); GLM's block is not a DSML spelling. + acc.set_model_syntax(syntax); + // The trace's own copy of the cache account: the state below takes the + // request, and the engine borrow it holds outlives this line. + let trace_cached = req.cache_read_tokens; + let state = RefCell::new(V41EmitState { + engine, + parsed, + req, + job_id, + stop_requested, + max_tokens, + acc, + w: &mut w, + out, + oa, + anth, + resp, + first_tok: None, + decode_steps: 0, + token_ids: Vec::new(), + finish: "length", + last_heartbeat: Instant::now(), + err: None, + }); + // The per-request sampling face (server_generate_v41.c:355-366): request + // values win, omitted fields keep the parse's defaults (temp 1.0 / top_p + // 1.0 / min_p 0.0 for this model set); DRY follows dry_set with the + // engine's base / allowed fallbacks (1.75 / 2; the service dry defaults + // have no port flags yet, and their values are the same defaults). + let sampling = ds4_core::V41Sampling { + temperature: parsed.temperature, + top_p: parsed.top_p, + min_p: parsed.min_p, + top_k: parsed.top_k, + seed: parsed.seed, + freq_penalty: parsed.frequency_penalty, + presence_penalty: parsed.presence_penalty, + dry_multiplier: if parsed.dry_set { + parsed.dry_multiplier + } else { + 0.0 + }, + dry_base: if parsed.dry_set && parsed.dry_base > 1.0 { + parsed.dry_base + } else { + 1.75 + }, + dry_allowed_length: if parsed.dry_set && parsed.dry_allowed_length > 0 { + parsed.dry_allowed_length + } else { + 2 + }, + }; + let result = { + let mut emit = |token: i32| state.borrow_mut().emit_token(token); + let mut progress = |event: &str, current: i32, total: i32| { + state.borrow_mut().prefill_progress(event, current, total) + }; + let mut greedy = |ahead: *const i32, n: u32| state.borrow_mut().syntax_greedy(ahead, n); + engine.v41_generate( + prompt_tokens, + max_tokens, + Some(sampling), + images.as_ref(), + &mut greedy, + &mut emit, + &mut progress, + ) + }; + let mut st = state.into_inner(); + + if let Some(error) = st.err.take() { + if !st.req.stream || matches!(error, GenerateError::Io) { + return Err(error); + } + let message = error.to_string(); + stream_error(st.w, &st.req, st.resp.as_mut(), &message); + flush(st.w, st.out)?; + return Err(GenerateError::Streamed(message)); + } + if let Err(error) = result { + // The native's own refusal (context over-run, a failed feed block). + if !st.req.stream || matches!(error, GenerateError::Io) { + return Err(error); + } + let message = error.to_string(); + stream_error(st.w, &st.req, st.resp.as_mut(), &message); + flush(st.w, st.out)?; + return Err(GenerateError::Streamed(message)); + } + + // The KV prefix reuse's own split (U-B; core_v41_kvreuse.c:28): the native + // knows at prefill end how much of this prompt came from the checkpoint, + // and the engine reports exactly that as usage cached/cache_write tokens + // (server_generate_v41.c:99-102). Engines without the route answer 0. + let kv_reused = engine.v41_kvreuse_last_reused().clamp(0, prompt_n); + st.req.cache_read_tokens = kv_reused; + st.req.cache_write_tokens = prompt_n - kv_reused; + + let completion = st.acc.completion; + let mut finish = terminal_finish(st.finish); + // v41_finish (:230-235): an unterminated tool call is an error on this + // route -- the V4 repair needs a session. A budget cut is the exception, + // and the deviation is named against the C reference: its + // tool_parse_failure_recovery_finish (ds4_server.c:6776-6784) returns the + // incoming "length" unchanged because "callers can distinguish truncation + // from a completed turn", and maps every other non-error finish to "stop" + // (test_incomplete_tool_call_keeps_stop_reason, :19613). Overwriting the + // pre-set "length" here sent a cut turn to the wire as "stop" (CTR 8.7, + // NC-07), so the signal never reached the writer. The error arm below is + // what the client's closed enum needs; "length" never takes it. + if parsed.kind == ReqKind::Chat + && parsed.has_tools + && st.acc.saw_tool_start + && !st.acc.saw_tool_end + && finish != "error" + && finish != "length" + { + finish = "error"; + } + let mut parsed_gen = if parsed.kind == ReqKind::Chat { + let (pg, recovered_finish) = parse_generated_for_response( + syntax, + &st.acc.text, + parsed.has_tools, + st.acc.saw_tool_start, + think_mode_enabled(parsed.think_mode), + chat_format, + &parsed.tool_orders, + finish, + ); + finish = recovered_finish; + pg + } else { + crate::tools::ParsedGenerated { + content: st.acc.text.clone(), + ok: true, + ..Default::default() + } + }; + // The V4.1 artifact mixes DSML tag spellings inside one block -- the + // wrapper and invoke opens carry the U+FF5C decoration while the + // parameter tags stay plain (measured 2026-10-10, docs 6.23; the tool + // shim exists for exactly this). The strict style match above rejects + // the mix and returns the block as text; recover the calls with the + // shim's tolerant scan. + if parsed_gen.calls.is_empty() && parsed.kind == ReqKind::Chat && parsed.has_tools { + if let Some(tolerant) = crate::tools::parse_dsml_tolerant( + &st.acc.text, + &parsed.tool_orders, + think_mode_enabled(parsed.think_mode), + ) { + parsed_gen = tolerant; + } + } + if !parsed_gen.calls.is_empty() { + if let Some(stream) = st.oa.as_ref() { + stream.tool.apply_ids(&mut parsed_gen.calls); + } + if let Some(stream) = st.anth.as_ref() { + stream.tool.apply_ids(&mut parsed_gen.calls); + } + assign_tool_ids( + &mut parsed_gen.calls, + if parsed.api == Api::Anthropic { + "toolu_" + } else { + "call_" + }, + ); + finish = "tool_calls"; + } + // The OpenAI finish_reason set has no "error": open-grok rejected the + // whole chunk with "serialization error: unknown variant `error`" when + // the route's unterminated-tool-call state (engine + // server_generate_v41.c:230-234) reached the wire (2026-10-10 23:17 + // session evidence). The client surfaces carry "stop" instead; the + // engine keeps its internal distinction in the logs. + if finish == "error" { + finish = "stop"; + } + st.req.timings.prefill_tokens = prompt_n; + st.req.timings.prefill_cached = kv_reused; + st.req.timings.decode_tokens = completion; + // The engine's speculation account is the round count + // (`ds4_engine_v41_last_spec_stats`, core_v41_api.c:41-46); the sink's + // per-token counter is not a step counter -- a speculative run advances + // several tokens per round, and counting tokens made tok_per_step read + // 1.00 on a run that moved 2.8 tokens per round (2026-10-12). The + // timing block below rebuilds the whole struct, so both writes take + // this one value. + let (rounds, _, _) = engine.v41_last_spec_stats(); + let decode_steps = if rounds > 0 { rounds } else { st.decode_steps }; + st.req.timings.decode_steps = decode_steps; + if completion > 0 { + if let Some(t_first) = st.first_tok { + st.req.timings = ReqTimings { + valid: true, + ttft_ms: t_first.duration_since(t_arrive).as_secs_f64() * 1e3, + prefill_ms: t_first.duration_since(t0).as_secs_f64() * 1e3, + decode_ms: Instant::now().duration_since(t_first).as_secs_f64() * 1e3, + prefill_tokens: prompt_n, + prefill_cached: kv_reused, + decode_tokens: completion, + decode_steps, + }; + } + } + let matched_stop = st.acc.matched_stop.clone(); + let terminal = if st.req.stream { + match st.req.api { + Api::Openai if st.req.kind == ReqKind::Completion => { + sse_chunk(st.w, &st.req, job_id, None, Some(finish)); + sse_done(st.w, &st.req, job_id, prompt_n, completion); + std::mem::take(&mut st.w.out) + } + Api::Openai => { + if let Some(stream) = st.oa.as_mut() { + openai_sse_finish_live( + st.w, + &st.req, + job_id, + stream, + &st.acc.text, + finish, + prompt_n, + completion, + &parsed_gen.calls, + ); + } + std::mem::take(&mut st.w.out) + } + Api::Anthropic => { + if let Some(stream) = st.anth.as_mut() { + if !anthropic_sse_finish_live( + st.w, + &st.req, + job_id, + stream, + &st.acc.text, + finish, + matched_stop.as_deref(), + completion, + &parsed_gen.calls, + ) { + return Err(GenerateError::Io); + } + } + std::mem::take(&mut st.w.out) + } + Api::Responses => { + if let Some(stream) = st.resp.as_mut() { + if !responses_sse_finish_live( + st.w, + &st.req, + stream, + &st.acc.text, + finish, + prompt_n, + completion, + st.acc.reasoning_tokens, + created, + &parsed_gen.calls, + ) { + return Err(GenerateError::Io); + } + } + std::mem::take(&mut st.w.out) + } + } + } else { + match st.req.api { + Api::Anthropic => anthropic_final_response( + &st.req, + job_id, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + finish, + matched_stop.as_deref(), + prompt_n, + completion, + cors, + &parsed_gen.calls, + ), + Api::Responses => { + let (rid, rsid, mid) = responses_ids(job_id); + let think = if st.acc.thinking_inside() { + ThinkBlock::Open + } else { + ThinkBlock::Closed + }; + responses_final_response( + &st.req, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + finish, + think, + prompt_n, + completion, + st.acc.reasoning_tokens, + created, + cors, + &rid, + &rsid, + &mid, + &parsed_gen.calls, + ) + } + Api::Openai => final_response( + &st.req, + job_id, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + finish, + prompt_n, + completion, + created, + cors, + &parsed_gen.calls, + if parsed.return_token_ids { + &st.token_ids + } else { + &[] + }, + ), + } + }; + let outcome = GenerateOutcome { + tool_ids: parsed_gen + .calls + .iter() + .map(|c| c.id.clone()) + .filter(|id| !id.is_empty()) + .collect(), + bank: None, + generation: engine.generation(), + frontier: prompt_n + completion, + finish: finish.to_string(), + timings: st.req.timings, + speculation_active: engine.v41_last_spec_stats().0 > 0, + reuse: ReuseTaken::Cold, + reuse_miss: ReuseMiss::None, + lane: None, + fallback_reason: None, + }; + // The reference writes the sampled block on the shared finish path + // (`tool_memory_remember`, ds4_server.c:14524, after `assign_tool_call_ids` + // at :10743, so the keys are the ids that reached the wire). This route + // builds its own outcome and returns before that path, so without the write + // here the next turn's attach reads an empty map. It sits after the wire + // bytes because the emit state holds the engine for the whole run + // (`V41EmitState`, :3164). + // Read the tracker's verdicts before the engine is touched again: the emit + // state holds the engine for the whole run, so its last use must come first. + let (saw_tool_start, saw_tool_end) = (st.acc.saw_tool_start, st.acc.saw_tool_end); + engine.remember_tool_replay(&parsed_gen.calls, &parsed_gen.raw_dsml); + // The same rule for the trace: it is written after the wire bytes, and this + // route's own prompt ids are the ones the engine preprint used. + if crate::trace::enabled() { + let mut text_of = |token: i32| engine.token_text(token).unwrap_or_default(); + crate::trace::write_request( + &crate::trace::TraceRequest { + api: parsed.api, + kind: parsed.kind, + stream: parsed.stream, + model_id: engine.model_id(), + model: &parsed.model, + tools: parsed.tool_orders.len(), + think: parsed.think_mode, + prompt, + tokens: prompt_tokens, + cached: trace_cached, + prompt_tokens: prompt_n, + max_tokens: parsed.max_tokens, + temperature: parsed.temperature, + top_k: parsed.top_k, + top_p: parsed.top_p, + min_p: parsed.min_p, + seed: parsed.seed, + finish, + completion, + saw_tool_start, + saw_tool_end, + elapsed_sec: t_arrive.elapsed().as_secs_f64(), + content: &parsed_gen.content, + reasoning: &parsed_gen.reasoning, + calls: &parsed_gen.calls, + }, + &mut text_of, + ); + } + Ok((outcome, terminal)) +} + +fn last_delta(raw: &[u8], emit_limit: usize, piece_len: usize) -> Option<&[u8]> { + if emit_limit == 0 { + return None; + } + let start = raw.len().saturating_sub(piece_len); + if start >= emit_limit { + return None; + } + Some(&raw[start..emit_limit]) +} + +/// Tape engine for tests. Does not open a GGUF. +pub struct ScriptedDecode { + pub model_id: i32, + pub prompt_tokens: Vec, + pub steps: Vec, + pub idx: usize, + pub pos: i32, + pub ctx: i32, + pub generation: u64, + pub live: Vec, + pub suffix_tokens: Vec, +} + +#[derive(Debug, Clone)] +pub struct ScriptedStep { + pub token: i32, + pub piece: Vec, + pub stop: bool, +} + +impl ScriptedDecode { + pub fn from_pieces(pieces: &[&[u8]]) -> Self { + let steps = pieces + .iter() + .enumerate() + .map(|(i, p)| ScriptedStep { + token: (i as i32) + 1, + piece: p.to_vec(), + stop: false, + }) + .chain(std::iter::once(ScriptedStep { + token: 99, + piece: Vec::new(), + stop: true, + })) + .collect(); + Self { + model_id: 0, + prompt_tokens: vec![1], + steps, + idx: 0, + pos: 0, + ctx: 8192, + generation: 1, + live: Vec::new(), + suffix_tokens: Vec::new(), + } + } +} + +impl DecodeIo for ScriptedDecode { + fn model_id(&self) -> i32 { + self.model_id + } + + fn tokenize_text(&self, _text: &str) -> Result, GenerateError> { + Ok(self.prompt_tokens.clone()) + } + + fn tokenize_rendered_chat(&self, text: &[u8]) -> Result, GenerateError> { + if find_substr(text, b"Tool error:").is_some() { + if self.suffix_tokens.is_empty() { + Ok(vec![7]) + } else { + Ok(self.suffix_tokens.clone()) + } + } else { + Ok(self.prompt_tokens.clone()) + } + } + + fn tokenizes_control_literals(&self) -> bool { + false + } + + fn chat_system_token(&self) -> &'static str { + // A scripted V4.1 model implies a V4.1 tokenizer, which owns + // <|System|> (core_bpe.c:332); every other scripted model has none. + if self.model_id == ds4_core::Variant::DeepSeek41Flash as i32 { + crate::render::DSML_SYSTEM_TOKEN + } else { + "" + } + } + + fn token_text(&self, token: i32) -> Result, GenerateError> { + Ok(self + .steps + .iter() + .find(|s| s.token == token) + .map(|s| s.piece.clone()) + .unwrap_or_default()) + } + + fn token_is_stop(&self, token: i32) -> bool { + self.steps.iter().any(|s| s.token == token && s.stop) + } + + fn vision_probe(&self, _data: &[u8]) -> Result { + if matches!( + syntax_for_model_id(self.model_id), + ModelSyntax::Glm53 | ModelSyntax::Inkling + ) { + Ok(VisionProbe { token_count: 16 }) + } else { + Err(GenerateError::Unsupported("vision encoder is not loaded")) + } + } + + fn vision_tokens(&self, data: &[u8]) -> Result, GenerateError> { + if syntax_for_model_id(self.model_id) == ModelSyntax::Step37 { + return Ok([vec![128000], vec![128001; 169], vec![128002]].concat()); + } + if syntax_for_model_id(self.model_id) == ModelSyntax::Mimo2 { + let count = ds4_core::image_pad_count(data) + .map_err(|error| GenerateError::Engine(error.to_string()))?; + return Ok(vec![ds4_core::IMAGE_PAD; count as usize]); + } + let marker = if syntax_for_model_id(self.model_id) == ModelSyntax::Inkling { + 200054 + } else { + 154854 + }; + Ok(vec![marker; self.vision_probe(data)?.token_count as usize]) + } + + fn sync_vision_prompt( + &mut self, + tokens: &[i32], + _images: &[VisionPromptInput], + ) -> Result<(), GenerateError> { + self.sync(tokens) + } + + fn audio_probe(&self, data: &[u8]) -> Result { + if syntax_for_model_id(self.model_id) == ModelSyntax::Inkling { + return Ok(2); + } + if syntax_for_model_id(self.model_id) == ModelSyntax::Mimo2 { + return ds4_core::audio_pad_count(data) + .map_err(|error| GenerateError::Engine(error.to_string())); + } + Err(GenerateError::Unsupported("audio encoder is not loaded")) + } + + fn sync_media_prompt( + &mut self, + tokens: &[i32], + _images: &[VisionPromptInput], + _audios: &[AudioPromptInput], + ) -> Result<(), GenerateError> { + self.sync(tokens) + } + + fn sync(&mut self, tokens: &[i32]) -> Result<(), GenerateError> { + self.live = tokens.to_vec(); + self.pos = tokens.len() as i32; + Ok(()) + } + + fn eval(&mut self, _token: i32) -> Result<(), GenerateError> { + self.pos += 1; + Ok(()) + } + + fn sample( + &mut self, + _temperature: f32, + _top_k: i32, + _top_p: f32, + _min_p: f32, + _rng: &mut u64, + ) -> i32 { + if self.idx >= self.steps.len() { + return -1; + } + let t = self.steps[self.idx].token; + self.idx += 1; + t + } + + fn pos(&self) -> i32 { + self.pos + } + + fn ctx(&self) -> i32 { + self.ctx + } + + fn generation(&self) -> u64 { + self.generation + } + + fn session_tokens(&self) -> Vec { + if self.live.is_empty() { + self.prompt_tokens.clone() + } else { + self.live.clone() + } + } + + fn invalidate(&mut self) { + self.live.clear(); + self.pos = 0; + } +} + +#[cfg(feature = "native")] +struct NativeSerialKvIo<'s, 'm, 'v, 't> { + session: &'s mut ds4_core::Session<'m>, + vocab: &'v ds4_core::Vocab, + tool_memory: &'t ToolMemory, + prefill_checkpoints: bool, + sync_elapsed: Duration, + reuse: ReuseTaken, + miss: ReuseMiss, +} + +#[cfg(feature = "native")] +impl SerialKvIo for NativeSerialKvIo<'_, '_, '_, '_> { + fn ctx(&self) -> i32 { + self.session.ctx() + } + + fn note_reuse(&mut self, taken: ReuseTaken) { + self.reuse = taken; + } + + fn note_miss(&mut self, miss: ReuseMiss) { + if self.miss == ReuseMiss::None { + self.miss = miss; + } + } + + fn chat_token_ids(&self) -> (i32, i32) { + (self.vocab.user_id, self.vocab.assistant_id) + } + + fn live_len(&self) -> i32 { + self.session.host().live_len() + } + + fn sync_start(&self, tokens: &[i32]) -> i32 { + self.session.last_plan(tokens).start + } + + fn live_tokens(&self) -> Vec { + if self.session.host().valid { + self.session.host().tokens().to_vec() + } else { + Vec::new() + } + } + + fn render_tokens(&self, tokens: &[i32]) -> Result, GenerateError> { + let mut text = Vec::new(); + for &token in tokens { + text.extend(self.vocab.token_text(token)); + } + Ok(text) + } + + fn checkpoint_trailer(&self, text: &[u8]) -> Option> { + self.tool_memory.checkpoint(text) + } + + fn tokenize_suffix(&mut self, suffix: &[u8]) -> Result, GenerateError> { + Ok(self.vocab.encode_rendered_bytes(suffix)) + } + + fn sync(&mut self, tokens: &[i32]) -> Result<(), GenerateError> { + let tokens = ds4_core::TokenBuffer::from_tokens(tokens.to_vec()); + let started = Instant::now(); + let result = self + .session + .sync(&tokens) + .map_err(|error| GenerateError::Engine(error.to_string())); + self.sync_elapsed += started.elapsed(); + result + } + + fn sync_with_prefill_checkpoints( + &mut self, + tokens: &[i32], + store: &mut KvStore, + identity: (u8, u8, u32), + cached_floor: i32, + ) -> Result<(), GenerateError> { + if !self.prefill_checkpoints { + return self.sync(tokens); + } + let tokens = ds4_core::TokenBuffer::from_tokens(tokens.to_vec()); + let (model_id, quant_bits, ctx) = identity; + let vocab = self.vocab; + let tool_memory = self.tool_memory; + let started = Instant::now(); + let result = self + .session + .sync_progress(&tokens, |checkpoint| { + let current = i32::try_from(checkpoint.current()).unwrap_or(0); + if current <= cached_floor { + return; + } + let target = continued_target(store, current); + if target == 0 { + return; + } + let mut text = Vec::new(); + for &token in checkpoint.tokens() { + text.extend(vocab.token_text(token)); + } + let Some(trailer) = tool_memory.checkpoint(&text) else { + return; + }; + let Ok(token_count) = u32::try_from(checkpoint.current()) else { + return; + }; + let mut header = kv_header(model_id, quant_bits, ctx, token_count); + header.reason = KvReason::Continued; + match write_checkpoint(store, header, &text, &trailer, |path| { + checkpoint + .save_payload(path) + .map_err(|error| GenerateError::Engine(error.to_string())) + }) { + Ok(()) => { + store.continued_last_store_tokens = + store.continued_last_store_tokens.max(target); + } + Err(error) => { + eprintln!( + "ds4-server-rs: intermediate continued KV checkpoint failed: {error}" + ); + } + } + }) + .map_err(|error| GenerateError::Engine(error.to_string())); + self.sync_elapsed += started.elapsed(); + result + } + + fn save_payload(&mut self, path: &Path) -> Result<(), GenerateError> { + self.session + .save_payload(path) + .map_err(|error| GenerateError::Engine(error.to_string())) + } + + fn load_payload_range( + &mut self, + path: &Path, + offset: u64, + length: u64, + ) -> Result<(), GenerateError> { + self.session + .load_payload_range(path, offset, length) + .map_err(|error| GenerateError::Engine(error.to_string())) + } + + fn invalidate(&mut self) { + self.session.invalidate(); + } +} + +#[cfg(any(feature = "native", test))] +fn serial_mtp_ready(family: ds4_core::ModelFamily, sidecar: bool, dots3: bool, draft: i32) -> bool { + match family { + ds4_core::ModelFamily::Glm53 + | ds4_core::ModelFamily::Glm53Vq + | ds4_core::ModelFamily::Ling31Vq => ds4_core::check_mtp_draft(family, draft).is_ok(), + ds4_core::ModelFamily::IQuestQ1 => { + (2..=ds4_core::IQuestPlan::MAX_MTP_DRAFT as i32).contains(&draft) + } + ds4_core::ModelFamily::Dots3Note => dots3, + ds4_core::ModelFamily::Inkling + | ds4_core::ModelFamily::Step37 + | ds4_core::ModelFamily::NaiveN05 => sidecar, + // Beam's DFlash2 file binds natively (no sibling attach): an MTP-off + // open leaves the default one-token draft. + ds4_core::ModelFamily::Mimo2 | ds4_core::ModelFamily::BeamVq => draft > 1, + _ => false, + } +} + +#[test] +fn iquest_embedded_mtp_uses_serial_greedy_route() { + use ds4_core::ModelFamily::IQuestQ1; + assert!(serial_mtp_ready(IQuestQ1, false, false, 2)); + assert!(serial_mtp_ready(IQuestQ1, false, false, 7)); + assert!(!serial_mtp_ready(IQuestQ1, false, false, 1)); + assert!(!serial_mtp_ready(IQuestQ1, false, false, 8)); +} + +#[test] +fn glm_serial_mtp_gate() { + use ds4_core::ModelFamily::{Glm53, Glm53Vq}; + for family in [Glm53, Glm53Vq] { + assert!(serial_mtp_ready(family, false, false, 1)); + assert!(serial_mtp_ready(family, false, false, 3)); + assert!(!serial_mtp_ready(family, false, false, 0)); + assert!(!serial_mtp_ready(family, false, false, 4)); + } +} + +#[test] +fn dots3_embedded_mtp_needs_session_state() { + use ds4_core::ModelFamily::{Dots3Note, Inkling, Mimo2, Qwen4Exp, Step37}; + assert!(serial_mtp_ready(Dots3Note, false, true, 1)); + assert!(!serial_mtp_ready(Dots3Note, false, false, 1)); + assert!(!serial_mtp_ready(Dots3Note, true, false, 1)); + assert!(serial_mtp_ready(Inkling, true, false, 1)); + assert!(serial_mtp_ready(Step37, true, false, 1)); + assert!(!serial_mtp_ready(Qwen4Exp, true, false, 1)); + assert!(serial_mtp_ready(Mimo2, false, false, 3)); + assert!(!serial_mtp_ready(Mimo2, true, false, 1)); +} + +#[test] +fn naive_spec_needs_sidecar() { + use ds4_core::ModelFamily::NaiveN05; + assert!(serial_mtp_ready(NaiveN05, true, false, 6)); + assert!(!serial_mtp_ready(NaiveN05, false, true, 6)); +} + +#[cfg(feature = "native")] +pub struct NativeDecode<'a> { + model: &'a ds4_core::Model, + vocab: Option<&'a ds4_core::Vocab>, + session: Option>, + /// The KVC disk store, when `--kv-disk-dir` opened one. A RefCell so + /// the V4.1 route (a `&self` DecodeIo method, the callbacks hold the + /// engine shared for the run) can still drive it; the serial path uses + /// get_mut. + store: Option>, + session_disk_storable: bool, + thinking_visible: Option, + tool_memory: ToolMemory, + chat_history: Option, + prompt_sync_elapsed: Option, + ctx: i32, + prefix_reuse: ds4_core::ReuseKind, + speculated: bool, + reuse: ReuseTaken, + miss: ReuseMiss, + v41: Option, +} + +/// The V4.1 serial route's boot state: the artifact path the feed opens and +/// the run switches the server accepted (the engine's server flags, +/// cli_opts.c:258-395). Present only for a V4.1 model — the route has no +/// ds4_session, so it is the engine's own push entry or nothing. +#[cfg(feature = "native")] +#[derive(Clone, Debug)] +pub struct V41ServeRoute { + pub model_path: std::path::PathBuf, + pub engram_dir: Option, + pub no_engram: bool, + pub dspark: Option, + pub graph: Option, + pub verify_k: i32, + pub emit_trace: bool, + pub prof: bool, + /// `--no-kv-reuse` (U-B): disable the V4.1 KV prefix reuse + /// (`ds4_engine_v41_set_kv_reuse`); the route's default is on. + pub no_kv_reuse: bool, + /// `--zchain` / `--posttrain` (unit Z): the ②/③ sidecar directories + /// the Rust mount reads per run (the engine's per-state v41_amp_load, + /// core_v41_amp.c:227-270). + pub zchain: Option, + pub posttrain: Option, + /// The `--vision-gguf` sidecar's tower geometry and placeholder id + /// (core_validate_v41.c:48-66); None = text-only serving. + pub vision: Option, +} + +#[cfg(feature = "native")] +impl<'a> NativeDecode<'a> { + pub fn new(model: &'a ds4_core::Model, ctx: i32) -> Self { + Self { + model, + vocab: None, + session: None, + store: None, + session_disk_storable: false, + thinking_visible: None, + tool_memory: ToolMemory::default(), + chat_history: None, + prompt_sync_elapsed: None, + ctx, + prefix_reuse: ds4_core::ReuseKind::Exact, + speculated: false, + reuse: ReuseTaken::Cold, + miss: ReuseMiss::None, + v41: None, + } + } + + /// Load the V4.1 push route (the server boot calls this when the model + /// identified as ds41; the engine decides the same way from the open + /// engine, server_main.c:29). + pub fn with_v41_route(mut self, route: V41ServeRoute) -> Self { + self.v41 = Some(route); + self + } + + pub fn with_prefix_reuse(mut self, reuse: ds4_core::ReuseKind) -> Self { + self.prefix_reuse = reuse; + self + } + + /// The two exact-replay switches (`--disable-exact-dsml-tool-replay`, + /// `--tool-memory-max-ids`) and the forced-off state a test uses. The + /// reference keeps both on the server struct and checks them inside each + /// entry point (ds4_server.c:10635, :10654, :10666, :10682, :10976, :11041, + /// :11095), so the map is built once here at boot. + pub fn with_tool_memory(mut self, config: crate::tool_memory::ToolMemoryConfig) -> Self { + self.tool_memory = ToolMemory::from_config(config); + self + } + + pub fn with_vocab(mut self, vocab: &'a ds4_core::Vocab) -> Self { + self.vocab = Some(vocab); + self + } + + pub fn with_store(mut self, store: KvStore) -> Self { + self.store = Some(RefCell::new(store)); + self + } + + fn sync_prompt_inner( + &mut self, + prompt: &[u8], + tokens: &[i32], + disk_eligible: bool, + thinking_visible_eligible: bool, + tool_replay: bool, + ) -> Result { + self.prompt_sync_elapsed = None; + let model_id = self.model.model_id(); + let quant_bits = self.model.routed_quant_bits(); + let vocab = self.vocab.unwrap_or_else(|| self.model.vocab()); + let save_current = self.session_disk_storable; + let has_store = self.store.is_some(); + let prefill_checkpoints = intermediate_prefill_eligible( + self.model.family() == ds4_core::ModelFamily::DeepSeek4, + self.model.backend() == ds4_core::Backend::Cuda, + disk_eligible, + tool_replay, + ); + self.session()?; + let (session, mut store, checkpoint, tool_memory) = ( + &mut self.session, + self.store.as_mut().map(RefCell::get_mut), + &self.thinking_visible, + &self.tool_memory, + ); + let session = session + .as_mut() + .ok_or_else(|| GenerateError::Engine("native session was not created".into()))?; + let mut io = NativeSerialKvIo { + session, + vocab, + tool_memory, + prefill_checkpoints, + sync_elapsed: Duration::ZERO, + reuse: ReuseTaken::Cold, + miss: ReuseMiss::None, + }; + let policy = DiskSyncPolicy { + save_current, + load: disk_eligible, + }; + let reuse_off = self.prefix_reuse == ds4_core::ReuseKind::None; + let result = if reuse_off { + disk_sync_prompt_impl( + &mut io, + store.as_deref_mut(), + model_id, + quant_bits, + prompt, + tokens, + checkpoint.as_ref(), + thinking_visible_eligible, + policy, + false, + PromptReuse::Off, + ) + } else if self.model.chat_template().is_some() { + disk_sync_template( + &mut io, + store.as_deref_mut(), + model_id, + quant_bits, + prompt, + tokens, + policy, + ) + } else if tool_replay { + disk_sync_tool_replay( + &mut io, + store.as_deref_mut(), + model_id, + quant_bits, + prompt, + tokens, + policy, + ) + } else { + disk_sync_prompt( + &mut io, + store.as_deref_mut(), + model_id, + quant_bits, + prompt, + tokens, + checkpoint.as_ref(), + thinking_visible_eligible, + policy, + ) + }; + self.reuse = if result.is_ok() { + io.reuse + } else { + ReuseTaken::Cold + }; + // A request that reused something is not a miss. The mechanism is + // the answer; the reason says why nothing was taken. + self.miss = if self.reuse == ReuseTaken::Cold { + io.miss + } else { + ReuseMiss::None + }; + if result.is_ok() { + self.prompt_sync_elapsed = Some(io.sync_elapsed); + } + self.session_disk_storable = result.is_ok() && disk_eligible && has_store; + settle_thinking_visible_checkpoint(&mut self.thinking_visible, result.is_ok()); + result + } + + fn session(&mut self) -> Result<&mut ds4_core::Session<'a>, GenerateError> { + if self.session.is_none() { + let s = self + .model + .session(self.ctx) + .map_err(|e| GenerateError::Engine(e.to_string()))?; + self.session = Some(s); + } + Ok(self.session.as_mut().unwrap()) + } +} + +#[cfg(feature = "native")] +impl DecodeIo for NativeDecode<'_> { + fn model_id(&self) -> i32 { + self.model.model_id() + } + + fn is_v41(&self) -> bool { + self.v41.is_some() + } + + fn v41_generate( + &self, + prompt: &[i32], + n_predict: i32, + sampling: Option, + images: Option<&ds4_core::v41_media::V41Images>, + greedy: &mut dyn FnMut(*const i32, u32) -> bool, + emit: &mut dyn FnMut(i32) -> bool, + progress: &mut dyn FnMut(&str, i32, i32) -> bool, + ) -> Result<(), GenerateError> { + let route = self.v41.as_ref().ok_or(GenerateError::Unsupported( + "V4.1 generation is not loaded on this engine", + ))?; + // The disk half (item 4): the store, when --kv-disk-dir opened one. + // The RefCell borrow is held for the whole run; nothing else touches + // the store during it (the callbacks reach V41EmitState only). + // --no-kv-reuse disables both halves here, not only the native gate, + // so no find runs and no failure line can print. + let model_id = u8::try_from(self.model.model_id()).ok(); + let ctx = u32::try_from(self.ctx()).unwrap_or(0); + let mut store_guard; + let mut disk = match (self.store.as_ref(), model_id, route.no_kv_reuse) { + (Some(cell), Some(id), false) if ctx > 0 => { + store_guard = cell.borrow_mut(); + Some(crate::v41_disk::V41Disk::new(&mut store_guard, id, ctx)) + } + _ => None, + }; + let mut opts = ds4_core::V41RunOptions { + engram_dir: route.engram_dir.as_deref(), + no_engram: route.no_engram, + dspark: route.dspark, + graph: route.graph, + verify_k: route.verify_k, + emit_trace: route.emit_trace, + prof: route.prof, + kv_reuse: if route.no_kv_reuse { Some(false) } else { None }, + kv_disk: disk.as_mut().map(|d| d as &mut dyn ds4_core::V41KvDisk), + sampling, + greedy: Some(greedy), + zchain: route.zchain.as_deref(), + posttrain: route.posttrain.as_deref(), + images, + }; + self.model + .v41_generate( + &route.model_path, + prompt, + n_predict, + &mut opts, + emit, + progress, + ) + .map_err(|e| GenerateError::Engine(e.to_string())) + } + + fn v41_last_spec_stats(&self) -> (i32, i32, i32) { + ds4_core::v41_last_spec_stats() + } + + fn v41_kvreuse_last_reused(&self) -> i32 { + ds4_core::v41_kvreuse_last_reused() + } + + fn v41_vision(&self) -> Option { + self.v41.as_ref().and_then(|r| r.vision) + } + + fn template(&self) -> Option<&ds4_core::chat_template::Template> { + self.model.chat_template() + } + + fn chat_system_token(&self) -> &'static str { + self.model.vocab().chat_system_token() + } + + fn render_request(&self, parsed: &ParsedRequest) -> Result, GenerateError> { + crate::chat_input::render_model( + self.model.chat_template(), + self.model_id(), + parsed, + self.model.vocab().chat_system_token(), + ) + } + + fn restore_chat(&self, parsed: &mut ParsedRequest) -> Result<(), GenerateError> { + if self.model.chat_template().is_some() && !parsed.live_call_ids.is_empty() { + if let Some(history) = &self.chat_history { + history.restore(parsed)?; + } else if parsed.responses_requires_live_tool_state + || parsed.anthropic_requires_live_tool_state + { + return Err(GenerateError::Unsupported( + "retained chat history is unavailable", + )); + } + } + Ok(()) + } + + fn remember_chat(&mut self, parsed: &ParsedRequest, generated: &crate::tools::ParsedGenerated) { + if self.model.chat_template().is_some() { + self.chat_history = (!generated.calls.is_empty()) + .then(|| crate::chat_input::History::capture(parsed.clone(), generated)); + } + } + + fn kv_store_mut(&mut self) -> Option<&mut KvStore> { + self.store.as_mut().map(RefCell::get_mut) + } + + fn tokenize_text(&self, text: &str) -> Result, GenerateError> { + if let Some(v) = self.vocab { + return Ok(v.encode_text(text)); + } + self.model + .tokenize_text(text) + .map(|b| b.as_slice().to_vec()) + .map_err(|e| GenerateError::Engine(e.to_string())) + } + + fn tokenize_rendered_chat(&self, text: &[u8]) -> Result, GenerateError> { + if let Some(v) = self.vocab { + return Ok(v.encode_rendered_bytes(text)); + } + let s = std::str::from_utf8(text) + .map_err(|_| GenerateError::Engine("prompt not utf8".into()))?; + self.model + .tokenize_rendered_chat(s) + .map(|b| b.as_slice().to_vec()) + .map_err(|e| GenerateError::Engine(e.to_string())) + } + + fn token_text(&self, token: i32) -> Result, GenerateError> { + if let Some(v) = self.vocab { + return Ok(v.token_text(token)); + } + self.model + .token_text(token) + .map_err(|e| GenerateError::Engine(e.to_string())) + } + + fn token_is_stop(&self, token: i32) -> bool { + if let Some(v) = self.vocab { + return v.is_stop(token); + } + self.model.token_is_stop(token) + } + + fn eos_id(&self) -> i32 { + self.model.vocab().eos_id + } + + fn eot_id(&self) -> i32 { + self.model.vocab().eot_id + } + + fn vision_probe(&self, data: &[u8]) -> Result { + self.model + .vision_probe(data) + .map(|info| VisionProbe { + token_count: info.token_count, + }) + .map_err(|error| GenerateError::Engine(error.to_string())) + } + + fn vision_tokens(&self, data: &[u8]) -> Result, GenerateError> { + self.model + .vision_tokens(data) + .map_err(|error| GenerateError::Engine(error.to_string())) + } + + fn audio_probe(&self, data: &[u8]) -> Result { + self.model + .audio_probe(data) + .map_err(|error| GenerateError::Engine(error.to_string())) + } + + fn sync_vision_prompt( + &mut self, + tokens: &[i32], + images: &[VisionPromptInput], + ) -> Result<(), GenerateError> { + self.sync_media_prompt(tokens, images, &[]) + } + + fn sync_media_prompt( + &mut self, + tokens: &[i32], + images: &[VisionPromptInput], + audios: &[AudioPromptInput], + ) -> Result<(), GenerateError> { + self.sync_mimo_prompt(tokens, images, audios, &[]) + } + + fn sync_mimo_prompt( + &mut self, + tokens: &[i32], + images: &[VisionPromptInput], + audios: &[AudioPromptInput], + videos: &[VisionPromptInput], + ) -> Result<(), GenerateError> { + if !videos.is_empty() && syntax_for_model_id(self.model_id()) != ModelSyntax::Mimo2 { + return Err(GenerateError::Unsupported("video input requires MiMo")); + } + self.prompt_sync_elapsed = None; + self.session_disk_storable = false; + self.thinking_visible = None; + self.reuse = ReuseTaken::Cold; + let tokens = ds4_core::TokenBuffer::from_tokens(tokens.to_vec()); + let images = images + .iter() + .map(|image| ds4_core::VisionInput { + data: &image.data, + token_offset: image.token_offset, + }) + .collect::>(); + let audios = audios + .iter() + .map(|audio| ds4_core::AudioInput { + data: &audio.data, + token_offset: audio.token_offset, + }) + .collect::>(); + let videos_in = videos + .iter() + .map(|video| ds4_core::VisionInput { + data: &video.data, + token_offset: video.token_offset, + }) + .collect::>(); + let packed = videos + .iter() + .map(|video| video.frames.as_slice()) + .collect::>(); + let started = Instant::now(); + let session = self.session()?; + let result = if videos_in.is_empty() { + session.sync_media(&tokens, &images, &audios) + } else { + session.sync_mimo(&tokens, &images, &audios, &videos_in, &packed) + } + .map_err(|error| GenerateError::Engine(error.to_string())); + if result.is_ok() { + self.prompt_sync_elapsed = Some(started.elapsed()); + } + result + } + + fn native_graph_fit(&self, ctx: i32) -> Option { + // The quote sizes a session graph; v41 has no session at all + // (server_main.c:65 creates one only for non-v41), so there is no + // graph to quote and the unquoted margin applies. + if self.is_v41() { + return None; + } + let quote = self.model.session_graph_fit_quote(ctx)?; + Some(NativeGraphFit { + fits: quote.fits, + need_bytes: quote.need_bytes, + avail_bytes: quote.avail_bytes, + headroom_bytes: quote.headroom_bytes, + deficit_bytes: quote.deficit_bytes, + fail_open: quote.fail_open, + }) + } + + fn serial_session_probe(&self) -> Option { + // v41 has no serial session to probe or right-size + // (server_main.c:65); answer like an engine without a native + // session so ensure_serial_session_fit passes native. + if self.is_v41() { + return None; + } + Some(match &self.session { + Some(s) => SerialSessionProbe { + cur_ctx: s.ctx(), + graph_pending: s.graph_pending(), + }, + // No session yet: the C boot-shape lazy session (pending at -c). + None => SerialSessionProbe { + cur_ctx: self.ctx, + graph_pending: true, + }, + }) + } + + fn serial_session_rightsize(&mut self, target_ctx: i32) -> Result<(), GenerateError> { + // Free BEFORE creating so a committed right-sized graph's own GiBs + // count as available for its replacement (the C regrow case). + self.session = None; + self.session_disk_storable = false; + self.thinking_visible = None; + let s = self + .model + .session(target_ctx) + .map_err(|e| GenerateError::Engine(e.to_string()))?; + self.session = Some(s); + Ok(()) + } + + fn serial_session_reset(&mut self) { + self.session = None; + self.session_disk_storable = false; + self.thinking_visible = None; + } + + fn begin_trace(&mut self) { + self.reuse = ReuseTaken::Cold; + self.miss = ReuseMiss::None; + } + + fn sync(&mut self, tokens: &[i32]) -> Result<(), GenerateError> { + // A retry re-syncs mid-request: nothing was reused for what it + // produces, but why the first sync refused a candidate still holds. + self.reuse = ReuseTaken::Cold; + let buf = ds4_core::TokenBuffer::from_tokens(tokens.to_vec()); + self.session()? + .sync(&buf) + .map_err(|e| GenerateError::Engine(e.to_string())) + } + + fn sync_prompt( + &mut self, + prompt: &[u8], + tokens: &[i32], + disk_eligible: bool, + thinking_visible_eligible: bool, + ) -> Result { + self.sync_prompt_inner( + prompt, + tokens, + disk_eligible, + thinking_visible_eligible, + false, + ) + } + + fn prompt_sync_elapsed(&self) -> Option { + self.prompt_sync_elapsed + } + + fn restore_tool_replay(&mut self, messages: &mut [ChatMsg]) { + let Ok(model_id) = u8::try_from(self.model.model_id()) else { + return; + }; + if let Some(store) = &self.store { + self.tool_memory + .restore_store(&store.borrow(), model_id, messages); + } + self.tool_memory.attach(messages); + } + + fn sync_tool_replay_prompt( + &mut self, + prompt: &[u8], + tokens: &[i32], + ) -> Result { + self.sync_prompt_inner(prompt, tokens, true, false, true) + } + + fn remember_tool_replay(&mut self, calls: &[ToolCall], raw_dsml: &str) { + if self.tool_memory.remember(calls, raw_dsml) > 0 && self.store.is_some() { + self.session_disk_storable = true; + } + } + + fn maybe_store_continued(&mut self) -> Result<(), GenerateError> { + if !self.session_disk_storable { + return Ok(()); + } + let Some(identity) = kv_identity( + self.model.model_id(), + self.model.routed_quant_bits(), + self.ctx(), + ) else { + return Ok(()); + }; + let vocab = self.vocab.unwrap_or_else(|| self.model.vocab()); + let (Some(session), Some(store), tool_memory) = ( + &mut self.session, + self.store.as_mut().map(RefCell::get_mut), + &self.tool_memory, + ) else { + return Ok(()); + }; + let mut io = NativeSerialKvIo { + session, + vocab, + tool_memory, + prefill_checkpoints: false, + sync_elapsed: Duration::ZERO, + reuse: ReuseTaken::Cold, + miss: ReuseMiss::None, + }; + try_store_continued(&mut io, store, identity)?; + Ok(()) + } + + fn shutdown(&mut self) -> Result<(), GenerateError> { + let Some(identity) = kv_identity( + self.model.model_id(), + self.model.routed_quant_bits(), + self.ctx(), + ) else { + return Ok(()); + }; + let vocab = self.vocab.unwrap_or_else(|| self.model.vocab()); + let (Some(session), Some(store), checkpoint, tool_memory) = ( + &mut self.session, + self.store.as_mut().map(RefCell::get_mut), + &self.thinking_visible, + &self.tool_memory, + ) else { + return Ok(()); + }; + let mut io = NativeSerialKvIo { + session, + vocab, + tool_memory, + prefill_checkpoints: false, + sync_elapsed: Duration::ZERO, + reuse: ReuseTaken::Cold, + miss: ReuseMiss::None, + }; + let (model_id, quant_bits, ctx) = identity; + try_store_live( + &mut io, + store, + model_id, + quant_bits, + ctx, + KvReason::Shutdown, + checkpoint.as_ref(), + )?; + Ok(()) + } + + fn eval(&mut self, token: i32) -> Result<(), GenerateError> { + self.session()? + .eval(token) + .map(|_| ()) + .map_err(|e| GenerateError::Engine(e.to_string())) + } + + fn eval_greedy(&mut self, first: i32, budget: i32) -> Result, GenerateError> { + let family = self.model.family(); + let dots3 = family == ds4_core::ModelFamily::Dots3Note && self.session()?.has_dots3_mtp(); + if !serial_mtp_ready( + family, + self.model.mtp().is_some() || self.model.dspark().is_some(), + dots3, + self.model.mtp_draft_tokens(), + ) || std::env::var_os("DS4_MTP_SPEC_DISABLE").is_some() + { + self.speculated = false; + self.eval(first)?; + return Ok(vec![first]); + } + self.speculated = true; + let eos = self.model.token_eos(); + self.session()? + .eval_speculative_argmax(first, budget, eos) + .map_err(|e| GenerateError::Engine(e.to_string())) + } + + fn last_eval_speculated(&self) -> bool { + self.speculated + } + + fn trim_greedy(&mut self, pos: i32) -> Result<(), GenerateError> { + if !matches!( + self.model.family(), + ds4_core::ModelFamily::Glm53 | ds4_core::ModelFamily::Glm53Vq + ) { + self.invalidate(); + return Ok(()); + } + let session = self.session()?; + if pos <= 0 || pos > session.pos() { + session.invalidate(); + return Err(GenerateError::Engine("invalid GLM output frontier".into())); + } + // GLM retains each accepted lane's KDA, DSA and logits in its MTP + // journal. Keep the tool close, rather than losing its continuation. + session.rewind(pos); + if session.argmax() < 0 { + session.invalidate(); + return Err(GenerateError::Engine( + "GLM output frontier restore failed".into(), + )); + } + Ok(()) + } + + fn last_reuse(&self) -> ReuseTaken { + self.reuse + } + + fn last_miss(&self) -> ReuseMiss { + self.miss + } + + fn sample( + &mut self, + temperature: f32, + top_k: i32, + top_p: f32, + min_p: f32, + rng: &mut u64, + ) -> i32 { + match self.session() { + Ok(s) => s.sample(temperature, top_k, top_p, min_p, rng), + Err(_) => -1, + } + } + + fn sample_excluding( + &mut self, + temperature: f32, + top_k: i32, + top_p: f32, + min_p: f32, + rng: &mut u64, + excluded_id: i32, + ) -> i32 { + match self.session() { + Ok(s) => s.sample_excluding(temperature, top_k, top_p, min_p, rng, excluded_id), + Err(_) => -1, + } + } + + fn pos(&self) -> i32 { + self.session.as_ref().map(|s| s.pos()).unwrap_or(0) + } + + fn ctx(&self) -> i32 { + self.session.as_ref().map(|s| s.ctx()).unwrap_or(self.ctx) + } + + fn generation(&self) -> u64 { + self.session.as_ref().map(|s| s.generation()).unwrap_or(0) + } + + fn session_tokens(&self) -> Vec { + self.session + .as_ref() + .map(|s| s.host().tokens().to_vec()) + .unwrap_or_default() + } + + fn remember_thinking_visible_checkpoint(&mut self, text: Vec) { + self.thinking_visible = Some(ThinkingVisibleCheckpoint { + text, + frontier: self.pos(), + }); + } + + fn invalidate(&mut self) { + if let Some(session) = self.session.as_mut() { + session.invalidate(); + } + } +} + +#[cfg(test)] +mod disk_sync_tests { + use super::{ + continued_decode_allowed, discard_loaded, disk_sync_prompt, disk_sync_tool_replay, + intermediate_prefill_eligible, ordinary_disk_cache_eligible, + settle_thinking_visible_checkpoint, thinking_visible_cache_eligible, thinking_visible_key, + tool_replay_disk_cache_eligible, tool_replay_producer_eligible, try_store_continued, + try_store_live, DiskSyncPolicy, GenerateError, ReuseMiss, ReuseTaken, SerialKvIo, + ThinkingVisibleCheckpoint, + }; + use crate::parse::{parse_request, ChatMsg, ParseEnv, ToolCall}; + use crate::render::{render_motif3_chat_ex, ModelSyntax}; + use crate::route::{Api, ThinkMode, WireSurface}; + use crate::stream::ChatFormat; + use crate::tools::SemAccum; + use ds4_kv::{ + read_envelope, Header, Options, Reason, Record, Store, EXT_THINKING_VISIBLE, EXT_TOOL_MAP, + }; + use std::cell::Cell; + use std::fs; + use std::path::{Path, PathBuf}; + + #[derive(Debug)] + struct FakeSerial { + ctx: i32, + live: Vec, + rendered_live: Vec, + trailer: Option>, + suffix_tokens: Vec, + loaded_tokens: Vec, + fail_load: bool, + fail_sync: bool, + fail_sync_at: Option, + fail_save: bool, + fail_save_at: Option, + save_calls: usize, + saved_prefixes: Vec>, + progress_frontiers: Vec, + invalidations: usize, + syncs: Vec>, + suffixes: Vec>, + loads: Vec<(PathBuf, u64, u64)>, + events: Vec<&'static str>, + user_token_id: i32, + assistant_token_id: i32, + live_token_reads: Cell, + planned_start: Option, + reuse: ReuseTaken, + miss: ReuseMiss, + } + + impl FakeSerial { + fn new(live: &[i32], rendered_live: &[u8]) -> Self { + Self { + ctx: 4096, + live: live.to_vec(), + rendered_live: rendered_live.to_vec(), + trailer: Some(Vec::new()), + suffix_tokens: Vec::new(), + loaded_tokens: Vec::new(), + fail_load: false, + fail_sync: false, + fail_sync_at: None, + fail_save: false, + fail_save_at: None, + save_calls: 0, + saved_prefixes: Vec::new(), + progress_frontiers: Vec::new(), + invalidations: 0, + syncs: Vec::new(), + suffixes: Vec::new(), + loads: Vec::new(), + events: Vec::new(), + user_token_id: -1, + assistant_token_id: -1, + live_token_reads: Cell::new(0), + planned_start: None, + reuse: ReuseTaken::Cold, + miss: ReuseMiss::None, + } + } + } + + impl SerialKvIo for FakeSerial { + fn ctx(&self) -> i32 { + self.ctx + } + + fn note_reuse(&mut self, taken: ReuseTaken) { + self.reuse = taken; + } + + fn note_miss(&mut self, miss: ReuseMiss) { + if self.miss == ReuseMiss::None { + self.miss = miss; + } + } + + fn chat_token_ids(&self) -> (i32, i32) { + (self.user_token_id, self.assistant_token_id) + } + + fn live_len(&self) -> i32 { + i32::try_from(self.live.len()).unwrap_or(0) + } + + fn live_tokens(&self) -> Vec { + self.live_token_reads.set(self.live_token_reads.get() + 1); + self.live.clone() + } + + fn sync_start(&self, _tokens: &[i32]) -> i32 { + self.planned_start.unwrap_or_else(|| self.live_len()) + } + + fn render_tokens(&self, _tokens: &[i32]) -> Result, GenerateError> { + Ok(self.rendered_live.clone()) + } + + fn checkpoint_trailer(&self, _text: &[u8]) -> Option> { + self.trailer.clone() + } + + fn tokenize_suffix(&mut self, suffix: &[u8]) -> Result, GenerateError> { + self.suffixes.push(suffix.to_vec()); + Ok(self.suffix_tokens.clone()) + } + + fn sync(&mut self, tokens: &[i32]) -> Result<(), GenerateError> { + self.events.push("sync"); + self.syncs.push(tokens.to_vec()); + if self.fail_sync || self.fail_sync_at == Some(self.syncs.len()) { + return Err(GenerateError::Engine("injected suffix sync failure".into())); + } + self.live = tokens.to_vec(); + Ok(()) + } + + fn sync_with_prefill_checkpoints( + &mut self, + tokens: &[i32], + store: &mut Store, + identity: (u8, u8, u32), + cached_floor: i32, + ) -> Result<(), GenerateError> { + self.events.push("sync"); + self.syncs.push(tokens.to_vec()); + for frontier in self.progress_frontiers.clone() { + if frontier <= tokens.len() { + self.live = tokens[..frontier].to_vec(); + self.events.push("chunk"); + if i32::try_from(frontier).unwrap_or(0) > cached_floor { + let _ = try_store_continued(self, store, identity); + } + } + } + if self.fail_sync || self.fail_sync_at == Some(self.syncs.len()) { + return Err(GenerateError::Engine("injected suffix sync failure".into())); + } + self.live = tokens.to_vec(); + Ok(()) + } + + fn save_payload(&mut self, path: &Path) -> Result<(), GenerateError> { + self.events.push("save"); + self.save_calls += 1; + self.saved_prefixes.push(self.live.clone()); + if self.fail_save || self.fail_save_at == Some(self.save_calls) { + return Err(GenerateError::Engine( + "injected payload save failure".into(), + )); + } + fs::write(path, b"current-payload").map_err(|e| GenerateError::Engine(e.to_string())) + } + + fn load_payload_range( + &mut self, + path: &Path, + offset: u64, + length: u64, + ) -> Result<(), GenerateError> { + self.events.push("load"); + self.loads.push(( + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()), + offset, + length, + )); + if self.fail_load { + return Err(GenerateError::Engine( + "injected payload load failure".into(), + )); + } + self.live = self.loaded_tokens.clone(); + Ok(()) + } + + fn invalidate(&mut self) { + self.events.push("invalidate"); + self.invalidations += 1; + self.live.clear(); + } + } + + #[test] + fn template_rejects_text_prefix() { + // Identical prompt bytes do not prove identical tokens: a BPE merge + // can cross the old frontier. Only the complete tokenization is valid. + let mut io = FakeSerial::new(&[1, 2], b"prefix"); + io.suffix_tokens = vec![99]; + let cached = super::disk_sync_template( + &mut io, + None, + 6, + 2, + b"prefix suffix", + &[1, 3, 4], + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + assert_eq!(cached, 0); + assert_eq!(io.live, [1, 3, 4]); + assert!(io.suffixes.is_empty()); + } + + #[test] + fn template_keeps_token_prefix() { + let mut io = FakeSerial::new(&[1, 2], b"prefix"); + let cached = super::disk_sync_template( + &mut io, + None, + 6, + 2, + b"different text", + &[1, 2, 4], + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + assert_eq!(cached, 2); + assert_eq!(io.live, [1, 2, 4]); + assert!(io.suffixes.is_empty()); + } + + #[test] + fn template_checks_disk_tokens() { + for (name, loaded, expected) in [ + ("jinja-disk-hit", vec![1, 2], 2), + ("jinja-disk-miss", vec![1, 9], 0), + ] { + let (dir, mut store) = store(name); + candidate(&mut store, b"prefix", 2); + let mut io = FakeSerial::new(&[], b"prefix suffix"); + io.loaded_tokens = loaded; + io.suffix_tokens = vec![99]; + let cached = super::disk_sync_template( + &mut io, + Some(&mut store), + 0, + 2, + b"prefix suffix", + &[1, 2, 3], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(cached, expected, "{name}"); + assert_eq!(io.live, [1, 2, 3], "{name}"); + assert!(io.suffixes.is_empty(), "{name}"); + let _ = fs::remove_dir_all(dir); + } + } + + #[test] + fn step_restart_uses_history_kv() { + use ds4_core::chat_template::{ChatOptions, RenderClock, Template}; + use serde_json::json; + + let template = Template::compile( + include_str!("../../../tests/fixtures/step37/chat_template.jinja"), + RenderClock::Fixed(0), + ) + .unwrap(); + let options = ChatOptions::new(10, ds4_core::ChatThinkMode::None); + let first = template + .render_chat(&[json!({"role":"user", "content":"Hello"})], &[], options) + .unwrap(); + let follow = template + .render_chat( + &[ + json!({"role":"user", "content":"Hello"}), + json!({"role":"assistant", "content":"4"}), + json!({"role":"user", "content":"Again"}), + ], + &[], + options, + ) + .unwrap(); + let history = first + .strip_suffix("assistant\n\n\n") + .unwrap(); + let tokens = |text: &str| text.bytes().map(i32::from).collect::>(); + let (dir, mut store) = store("step-history-frontier"); + store.bind_identity([7; 32]); + let mut saving = FakeSerial::new(&[], first.as_bytes()); + saving.suffix_tokens = tokens(history); + super::disk_sync_template( + &mut saving, + Some(&mut store), + 10, + 2, + first.as_bytes(), + &tokens(&first), + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + let (_, envelope) = store + .text_prefix_candidate(follow.as_bytes(), 10, 2, 4096) + .unwrap() + .expect("next official render must find a saved history frontier"); + assert_eq!(envelope.text, history.as_bytes()); + assert_eq!(envelope.header.tokens as usize, history.len()); + assert!(saving.saved_prefixes.contains(&tokens(history))); + assert_eq!(saving.live, tokens(&first)); + + let options = store.opt.clone(); + drop(store); + let mut store = Store::open(&dir, 16, true, options).unwrap(); + store.bind_identity([7; 32]); + let mut loading = FakeSerial::new(&[], follow.as_bytes()); + loading.loaded_tokens = tokens(history); + loading.suffix_tokens = tokens( + follow + .strip_suffix("assistant\n\n\n") + .unwrap(), + ); + let cached = super::disk_sync_template( + &mut loading, + Some(&mut store), + 10, + 2, + follow.as_bytes(), + &tokens(&follow), + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(cached as usize, history.len()); + assert_eq!(loading.reuse, ReuseTaken::Exact); + assert_eq!(loading.live, tokens(&follow)); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn motif_restart_uses_history_kv() { + use ds4_core::chat_template::{ChatOptions, RenderClock, Template}; + use serde_json::json; + + let template = Template::compile( + include_str!("../../../tests/fixtures/chat-template/models/motif/chat_template.jinja"), + RenderClock::Fixed(0), + ) + .unwrap(); + let options = ChatOptions::new(3, ds4_core::ChatThinkMode::None); + let first = template + .render_chat( + &[json!({"role":"user", "content":"What is 2 + 2?"})], + &[], + options, + ) + .unwrap(); + let follow = template + .render_chat( + &[ + json!({"role":"user", "content":"What is 2 + 2?"}), + json!({"role":"assistant", "content":" 2 + 2 = 4.\n"}), + json!({"role":"user", "content":"What is 4 + 1?"}), + ], + &[], + options, + ) + .unwrap(); + let history = first.strip_suffix("").unwrap(); + assert!(follow.starts_with(history)); + assert!(!follow.starts_with(&first)); + let tokens = |text: &str| text.bytes().map(i32::from).collect::>(); + let (dir, mut store) = store("motif-history-frontier"); + store.bind_identity([7; 32]); + let mut saving = FakeSerial::new(&[], first.as_bytes()); + saving.suffix_tokens = tokens(history); + super::disk_sync_template( + &mut saving, + Some(&mut store), + 3, + 2, + first.as_bytes(), + &tokens(&first), + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + let (_, envelope) = store + .text_prefix_candidate(follow.as_bytes(), 3, 2, 4096) + .unwrap() + .expect("Motif must save the real history frontier"); + assert_eq!(envelope.text, history.as_bytes()); + assert_eq!(envelope.header.tokens as usize, history.len()); + assert!(saving.saved_prefixes.contains(&tokens(history))); + assert_eq!(saving.live, tokens(&first)); + + let options = store.opt.clone(); + drop(store); + let mut store = Store::open(&dir, 16, true, options).unwrap(); + store.bind_identity([7; 32]); + let mut loading = FakeSerial::new(&[], follow.as_bytes()); + loading.loaded_tokens = tokens(history); + loading.suffix_tokens = tokens(follow.strip_suffix("").unwrap()); + let cached = super::disk_sync_template( + &mut loading, + Some(&mut store), + 3, + 2, + follow.as_bytes(), + &tokens(&follow), + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(cached as usize, history.len()); + assert_eq!(loading.reuse, ReuseTaken::Exact); + assert_eq!(loading.live, tokens(&follow)); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn motif_history_needs_token_cut() { + let prompt = b"history<|startofturn|><|assistant|>"; + assert!(super::history_frontier(3, prompt, &[1, 2, 11, 12], |_| Ok(vec![1, 2])).is_some()); + assert!(super::history_frontier(3, prompt, &[1, 2, 11, 12], |_| Ok(vec![1, 9])).is_none()); + assert!(super::history_frontier(3, prompt, &[1, 2], |_| Ok(vec![1, 2])).is_none()); + for other in [b"user".as_slice(), b"<|assistant|>"] { + assert!( + super::history_frontier(3, other, &[1, 2], |_| panic!("not a history cut")) + .is_none() + ); + } + } + + #[test] + fn step_history_needs_token_cut() { + let prompt = b"history<|im_start|>assistant\n\n\n"; + assert!(super::history_frontier(10, prompt, &[1, 2, 3], |_| Ok(vec![1, 9])).is_none()); + assert!(super::history_frontier(10, prompt, &[1, 2, 3], |_| Ok(vec![1, 2, 3])).is_none()); + assert!( + super::history_frontier(6, prompt, &[1, 2, 3], |_| panic!("other family")).is_none() + ); + } + + #[test] + fn step_keeps_live_frontier() { + let (dir, mut store) = store("step-history-live"); + let mut io = FakeSerial::new( + &[1, 2, 3], + b"history<|im_start|>assistant\n\n\n", + ); + io.suffix_tokens = vec![1, 2]; + let cached = super::disk_sync_template( + &mut io, + Some(&mut store), + 10, + 2, + b"history<|im_start|>assistant\n\n\n", + &[1, 2, 3], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(cached, 3); + assert_eq!(io.syncs, [vec![1, 2, 3]]); + assert!(io.saved_prefixes.is_empty()); + let _ = fs::remove_dir_all(dir); + } + + /// Step's official follow-up render drops the empty think pair that the + /// stored KV still holds, so a text-prefix hit would continue from a + /// token sequence the client never sent. A record without a real + /// history-frontier payload must still fall back to a cold prefill. + #[test] + fn template_refuses_a_text_only_history_match() { + let (dir, mut store) = store("step-history"); + let history = + b"<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\n4<|im_end|>\n".to_vec(); + let mut saving = FakeSerial::new(&[41, 42], &history); + saving.trailer = Some(Vec::new()); + super::disk_sync_prompt( + &mut saving, + Some(&mut store), + 0, + 2, + &history, + &[41, 42], + None, + false, + DiskSyncPolicy { + save_current: true, + load: false, + }, + ) + .unwrap(); + + let follow = + b"<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\n4<|im_end|>\n<|im_start|>user\nAgain<|im_end|>\n<|im_start|>assistant\n\n\n"; + let mut loading = FakeSerial::new(&[], b""); + // The payload's KV still carries the generation-form think pair. + loading.loaded_tokens = vec![41, 42, 90]; + let cached = super::disk_sync_template( + &mut loading, + Some(&mut store), + 0, + 2, + follow, + &[41, 42, 3, 4], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(cached, 0); + assert_eq!(loading.reuse, ReuseTaken::Cold); + assert!(loading.syncs.last().unwrap().starts_with(&[41, 42, 3, 4])); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn appended_turn_records_exact_reuse() { + let mut io = FakeSerial::new(&[1, 2], b"prefix"); + let cached = super::disk_sync_template( + &mut io, + None, + 6, + 2, + b"prefix suffix", + &[1, 2, 4], + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + // Both counters are positive: cached prefix plus a prefilled turn. + assert_eq!(cached, 2); + assert_eq!(io.reuse, ReuseTaken::Exact); + } + + /// Candidate refusal reasons from `docs/serving-contract.md`, each from the + /// decision that produced it. + #[test] + fn a_refused_candidate_says_why() { + let (dir, mut store) = store("miss-reasons"); + candidate(&mut store, b"prefix", 2); + + // The template re-rendered the conversation: stored tokens are no + // longer a prefix of the prompt. + let mut io = FakeSerial::new(&[], b"prefix suffix"); + io.loaded_tokens = vec![41, 42]; + super::disk_sync_template( + &mut io, + Some(&mut store), + 0, + 2, + b"prefix suffix", + &[1, 2, 3], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(io.miss, ReuseMiss::RenderedPrefix); + + // Nothing stored under a text this prompt starts with. + let mut io = FakeSerial::new(&[], b""); + super::disk_sync_template( + &mut io, + Some(&mut store), + 0, + 2, + b"unrelated", + &[7], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(io.miss, ReuseMiss::NoCheckpoint); + + // The payload does not hold the token count its record claims. + let mut io = FakeSerial::new(&[], b"prefix suffix"); + io.loaded_tokens = vec![41]; + super::disk_sync_template( + &mut io, + Some(&mut store), + 0, + 2, + b"prefix suffix", + &[41, 42, 3], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(io.miss, ReuseMiss::PayloadMismatch); + + // The record was chosen and then could not be read back. + let mut io = FakeSerial::new(&[], b"prefix suffix"); + io.fail_load = true; + super::disk_sync_template( + &mut io, + Some(&mut store), + 0, + 2, + b"prefix suffix", + &[41, 42, 3], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(io.miss, ReuseMiss::PayloadMismatch); + + // A live session holds this conversation and the render moved: the + // text still leads here, the token sequence does not. + let mut io = FakeSerial::new(&[41, 42], b"prefix"); + super::disk_sync_template( + &mut io, + Some(&mut store), + 0, + 2, + b"prefix suffix", + &[7, 8, 9], + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + assert_eq!(io.miss, ReuseMiss::RenderedPrefix); + + // A live session about another conversation says nothing: its text + // does not lead to this prompt either. + let mut io = FakeSerial::new(&[41, 42], b"unrelated"); + super::disk_sync_template( + &mut io, + Some(&mut store), + 0, + 2, + b"prefix suffix", + &[7, 8, 9], + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + assert_eq!(io.miss, ReuseMiss::None); + + // A broader reason never masks the specific one that came first. + let mut io = FakeSerial::new(&[], b""); + io.note_miss(ReuseMiss::RenderedPrefix); + io.note_miss(ReuseMiss::BelowThreshold); + assert_eq!(io.miss, ReuseMiss::RenderedPrefix); + + // A conversation too short to have been stored says so, rather than + // reporting the absence that shortness caused. + let short_dir = std::env::temp_dir().join(format!( + "ds4-server-disk-sync-miss-threshold-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&short_dir); + let mut short_store = Store::open( + &short_dir, + 16, + true, + Options { + min_tokens: 8, + cold_max_tokens: 32, + continued_interval_tokens: 8, + boundary_trim_tokens: 0, + boundary_align_tokens: 0, + }, + ) + .unwrap(); + let mut io = FakeSerial::new(&[], b"hi"); + super::disk_sync_template( + &mut io, + Some(&mut short_store), + 0, + 2, + b"hi", + &[1], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(io.miss, ReuseMiss::BelowThreshold); + let _ = fs::remove_dir_all(short_dir); + + // A record for another model is a mismatch, not an absence: the + // prompt-keyed entry is there, its identity rules it out. + let mut io = FakeSerial::new(&[], b"prefix suffix"); + super::disk_sync_template( + &mut io, + Some(&mut store), + 9, + 2, + b"prefix suffix", + &[1, 2, 3], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(io.miss, ReuseMiss::PayloadMismatch); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn a_cold_prompt_records_cold_reuse() { + let mut io = FakeSerial::new(&[], b""); + let cached = super::disk_sync_template( + &mut io, + None, + 6, + 2, + b"prefix suffix", + &[1, 2, 4], + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + assert_eq!(cached, 0); + assert_eq!(io.reuse, ReuseTaken::Cold); + } + + #[test] + fn reuse_off_skips_live_prefix() { + let mut io = FakeSerial::new(&[1, 2], b"prefix"); + let cached = super::disk_sync_prompt_impl( + &mut io, + None, + 6, + 2, + b"prefix suffix", + &[1, 2, 4], + None, + false, + DiskSyncPolicy { + save_current: false, + load: false, + }, + false, + super::PromptReuse::Off, + ) + .unwrap(); + assert_eq!(cached, 0); + assert_eq!(io.live, [1, 2, 4]); + assert_eq!(io.reuse, ReuseTaken::Cold); + } + + fn store(tag: &str) -> (PathBuf, Store) { + let dir = + std::env::temp_dir().join(format!("ds4-server-disk-sync-{tag}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let options = Options { + min_tokens: 1, + cold_max_tokens: 32, + continued_interval_tokens: 8, + boundary_trim_tokens: 0, + boundary_align_tokens: 0, + }; + let store = Store::open(&dir, 16, true, options).unwrap(); + (dir, store) + } + + fn candidate(store: &mut Store, text: &[u8], tokens: u32) -> PathBuf { + store + .write(Record { + header: Header { + quant_bits: 2, + reason: Reason::Evict, + ext_flags: 0, + model_id: 0, + tokens, + hits: 0, + ctx_size: 4096, + created_at: 1, + last_used: 1, + payload_bytes: 0, + text_bytes: 0, + }, + text: text.to_vec(), + payload: b"candidate-payload".to_vec(), + trailer: Vec::new(), + }) + .unwrap() + } + + #[test] + fn live_reuse_counts_actual_start() { + // Dots3 MTP replays an unaligned append from zero; plain Dots3 + // replays its last partial chunk. Exact and aligned hits remain hits. + for (live_len, prompt_len, start, cached, taken) in [ + (600, 620, 0, 0, ReuseTaken::Cold), + (600, 620, 576, 576, ReuseTaken::Partial), + (600, 600, 600, 600, ReuseTaken::Exact), + (608, 620, 608, 608, ReuseTaken::Exact), + (600, 620, 620, 600, ReuseTaken::Exact), + (600, 620, -1, 0, ReuseTaken::Cold), + ] { + let tokens = vec![1; prompt_len]; + let mut io = FakeSerial::new(&tokens[..live_len], b"prefix"); + io.planned_start = Some(start); + let actual = super::disk_sync_template( + &mut io, + None, + 0, + 2, + b"prefix append", + &tokens, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!( + actual, cached, + "live={live_len}, prompt={prompt_len}, start={start}" + ); + assert_eq!(io.reuse, taken); + assert_eq!( + io.miss.as_str(), + if cached < live_len as i32 { + "session state requires prefix replay" + } else { + "none" + } + ); + assert_eq!(io.syncs, [tokens]); + } + } + + #[test] + fn disk_reuse_counts_actual_start() { + for (live_len, prompt_len, start, cached, taken) in [ + (600, 620, 0, 0, ReuseTaken::Cold), + (600, 620, 576, 576, ReuseTaken::Partial), + (600, 600, 600, 600, ReuseTaken::Exact), + (608, 620, 608, 608, ReuseTaken::Exact), + ] { + let (dir, mut store) = store("actual-start"); + candidate(&mut store, b"prefix", live_len as u32); + let tokens = vec![1; prompt_len]; + let mut io = FakeSerial::new(&[], b""); + io.loaded_tokens = tokens[..live_len].to_vec(); + io.planned_start = Some(start); + let actual = super::disk_sync_template( + &mut io, + Some(&mut store), + 0, + 2, + b"prefix append", + &tokens, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!( + actual, cached, + "loaded={live_len}, prompt={prompt_len}, start={start}" + ); + assert_eq!(io.reuse, taken); + assert_eq!( + io.miss.as_str(), + if cached < live_len as i32 { + "session state requires prefix replay" + } else { + "none" + } + ); + assert_eq!(io.loads.len(), 1); + assert_eq!(io.syncs, [tokens]); + let _ = fs::remove_dir_all(dir); + } + } + + #[test] + fn disk_sync_prompt_cold_miss_syncs_then_saves_full_prompt() { + let (dir, mut store) = store("cold"); + let mut io = FakeSerial::new(&[], b"cold prompt"); + + let cached = disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"cold prompt", + &[1, 2], + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(cached, 0); + assert_eq!(io.syncs, [vec![1, 2]]); + assert!(io.loads.is_empty()); + assert_eq!(io.events, ["sync", "save"]); + assert_eq!(store.entries().len(), 1); + assert_eq!(store.entries()[0].header.reason, Reason::Cold); + assert_eq!(store.entries()[0].header.tokens, 2); + assert_eq!(store.entries()[0].header.ext_flags, 0); + assert_eq!(store.continued_last_store_tokens, 2); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_sync_prompt_cold_miss_saves_stable_prefix_before_full_sync() { + let (dir, mut store) = store("cold-prefix"); + store.opt.boundary_trim_tokens = 1; + store.opt.boundary_align_tokens = 4; + let mut io = FakeSerial::new(&[], b"stable prefix"); + let prompt_tokens: Vec = (1..=10).collect(); + + let cached = disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"cold prompt", + &prompt_tokens, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(cached, 0); + assert_eq!(io.syncs, [prompt_tokens[..8].to_vec(), prompt_tokens]); + assert_eq!(io.events, ["sync", "save", "sync"]); + assert_eq!(store.entries().len(), 1); + assert_eq!(store.entries()[0].header.reason, Reason::Cold); + assert_eq!(store.entries()[0].header.tokens, 8); + assert_eq!(store.continued_last_store_tokens, 8); + let record = store.read(&store.entries()[0].path).unwrap(); + assert_eq!(record.text, b"stable prefix"); + assert_eq!(record.payload, b"current-payload"); + assert!(record.trailer.is_empty()); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_sync_prompt_cold_miss_prefers_chat_anchor() { + let (dir, mut store) = store("cold-anchor"); + store.opt.boundary_trim_tokens = 1; + store.opt.boundary_align_tokens = 4; + let mut io = FakeSerial::new(&[], b"chat anchor"); + io.user_token_id = 99; + io.assistant_token_id = 100; + let prompt_tokens = vec![10, 11, 99, 12, 13, 100, 14, 15, 16, 17]; + + disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"cold prompt", + &prompt_tokens, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(io.syncs, [prompt_tokens[..2].to_vec(), prompt_tokens]); + assert_eq!(store.entries()[0].header.tokens, 2); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_sync_prompt_cold_policy_respects_disable_limit_and_lane_gate() { + let prompt_tokens: Vec = (1..=10).collect(); + for (tag, min, cold_max, load) in [ + ("cold-below-min", 11, 32, true), + ("cold-disabled", 1, 0, true), + ("cold-above-max", 1, 4, true), + ("cold-ineligible", 1, 32, false), + ] { + let (dir, mut store) = store(tag); + store.opt.min_tokens = min; + store.opt.cold_max_tokens = cold_max; + let mut io = FakeSerial::new(&[], b"cold prompt"); + disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"cold prompt", + &prompt_tokens, + None, + false, + DiskSyncPolicy { + save_current: false, + load, + }, + ) + .unwrap(); + assert_eq!(io.syncs, [prompt_tokens.clone()], "{tag}"); + assert_eq!(io.events, ["sync"], "{tag}"); + assert!(store.entries().is_empty(), "{tag}"); + let _ = fs::remove_dir_all(dir); + } + } + + #[test] + fn disk_sync_prompt_cold_save_is_nonfatal_but_prefix_sync_failure_is_fatal() { + let prompt_tokens: Vec = (1..=10).collect(); + let (save_dir, mut save_store) = store("cold-save-fail"); + save_store.opt.boundary_trim_tokens = 1; + save_store.opt.boundary_align_tokens = 4; + let mut save_io = FakeSerial::new(&[], b"stable prefix"); + save_io.fail_save = true; + disk_sync_prompt( + &mut save_io, + Some(&mut save_store), + 0, + 2, + b"cold prompt", + &prompt_tokens, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!( + save_io.syncs, + [prompt_tokens[..8].to_vec(), prompt_tokens.clone()] + ); + assert_eq!(save_io.events, ["sync", "save", "sync"]); + assert!(save_store.entries().is_empty()); + assert_eq!(save_store.continued_last_store_tokens, 0); + + let (sync_dir, mut sync_store) = store("cold-sync-fail"); + sync_store.opt.boundary_trim_tokens = 1; + sync_store.opt.boundary_align_tokens = 4; + let mut sync_io = FakeSerial::new(&[], b"stable prefix"); + sync_io.fail_sync = true; + assert!(disk_sync_prompt( + &mut sync_io, + Some(&mut sync_store), + 0, + 2, + b"cold prompt", + &prompt_tokens, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .is_err()); + assert_eq!(sync_io.syncs, [prompt_tokens[..8].to_vec()]); + assert_eq!(sync_io.events, ["sync"]); + assert!(sync_store.entries().is_empty()); + + let (tail_dir, mut tail_store) = store("cold-tail-sync-fail"); + tail_store.opt.boundary_trim_tokens = 1; + tail_store.opt.boundary_align_tokens = 4; + let mut tail_io = FakeSerial::new(&[], b"stable prefix"); + tail_io.fail_sync_at = Some(2); + assert!(disk_sync_prompt( + &mut tail_io, + Some(&mut tail_store), + 0, + 2, + b"cold prompt", + &prompt_tokens, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .is_err()); + assert_eq!(tail_io.events, ["sync", "save", "sync"]); + assert_eq!(tail_store.entries().len(), 1); + assert_eq!(tail_store.entries()[0].header.reason, Reason::Cold); + assert_eq!(tail_store.continued_last_store_tokens, 8); + let _ = fs::remove_dir_all(save_dir); + let _ = fs::remove_dir_all(sync_dir); + let _ = fs::remove_dir_all(tail_dir); + } + + #[test] + fn continued_store_requires_an_exact_frontier_and_advances_only_on_success() { + let (dir, mut continued_store) = store("continued"); + let mut io = FakeSerial::new(&[1, 2, 3, 4, 5, 6, 7], b"continued"); + assert!(!try_store_continued(&mut io, &mut continued_store, (0, 2, 4096)).unwrap()); + assert!(io.events.is_empty()); + assert_eq!(io.live_token_reads.get(), 0); + assert_eq!(continued_store.continued_last_store_tokens, 0); + + io.live.push(8); + assert!(try_store_continued(&mut io, &mut continued_store, (0, 2, 4096)).unwrap()); + assert_eq!(io.events, ["save"]); + assert_eq!(io.live_token_reads.get(), 1); + assert_eq!(continued_store.continued_last_store_tokens, 8); + assert_eq!(continued_store.entries().len(), 1); + assert_eq!( + continued_store.entries()[0].header.reason, + Reason::Continued + ); + assert_eq!(continued_store.entries()[0].header.tokens, 8); + + continued_store.continued_last_store_tokens = 0; + io.events.clear(); + assert!(try_store_continued(&mut io, &mut continued_store, (0, 2, 4096)).unwrap()); + assert!(io.events.is_empty()); + assert_eq!(continued_store.continued_last_store_tokens, 8); + + let (fail_dir, mut fail_store) = store("continued-fail"); + let mut fail_io = FakeSerial::new(&[1, 2, 3, 4, 5, 6, 7, 8], b"continued-fail"); + fail_io.fail_save = true; + assert!(try_store_continued(&mut fail_io, &mut fail_store, (0, 2, 4096)).is_err()); + assert_eq!(fail_io.events, ["save"]); + assert_eq!(fail_store.continued_last_store_tokens, 0); + assert!(fail_store.entries().is_empty()); + + let _ = fs::remove_dir_all(dir); + let _ = fs::remove_dir_all(fail_dir); + } + + #[test] + fn intermediate_prefill_stores_due_frontier_from_one_sync() { + let (dir, mut store) = store("intermediate-one-sync"); + store.opt.cold_max_tokens = 0; + store.opt.continued_interval_tokens = 4; + let prompt: Vec = (1..=6).collect(); + let mut io = FakeSerial::new(&[], b"prefix-four"); + io.progress_frontiers = vec![4]; + + disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"full prompt", + &prompt, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(io.syncs, [prompt]); + assert_eq!(io.events, ["sync", "chunk", "save"]); + assert_eq!(store.entries().len(), 1); + assert_eq!(store.entries()[0].header.reason, Reason::Continued); + assert_eq!(store.entries()[0].header.tokens, 4); + assert_eq!(store.entries()[0].header.ext_flags, 0); + let record = store.read(&store.entries()[0].path).unwrap(); + assert_eq!(record.text, b"prefix-four"); + assert_eq!(record.payload, b"current-payload"); + assert!(record.trailer.is_empty()); + assert_eq!(store.continued_last_store_tokens, 4); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn intermediate_prefill_ignores_frontiers_at_or_below_cached_floor() { + let (dir, mut store) = store("intermediate-cached-floor"); + store.opt.cold_max_tokens = 0; + store.opt.continued_interval_tokens = 4; + let prompt: Vec = (1..=10).collect(); + let mut io = FakeSerial::new(&prompt[..6], b"prefix-eight"); + io.progress_frontiers = vec![4, 8]; + + disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"full prompt", + &prompt, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(io.syncs, [prompt]); + assert_eq!(io.save_calls, 1); + assert_eq!(store.entries().len(), 1); + assert_eq!(store.entries()[0].header.reason, Reason::Continued); + assert_eq!(store.entries()[0].header.tokens, 8); + assert_eq!(store.continued_last_store_tokens, 8); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn cold_prefill_suppresses_duplicate_continued_at_same_frontier() { + let (dir, mut store) = store("cold-suppresses-continued"); + store.opt.boundary_trim_tokens = 2; + store.opt.continued_interval_tokens = 4; + let prompt: Vec = (1..=6).collect(); + let mut io = FakeSerial::new(&[], b"prefix-four"); + io.progress_frontiers = vec![4]; + + disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"full prompt", + &prompt, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(io.syncs, [prompt[..4].to_vec(), prompt]); + assert_eq!(io.save_calls, 1); + assert_eq!(store.entries().len(), 1); + assert_eq!(store.entries()[0].header.reason, Reason::Cold); + assert_eq!(store.entries()[0].header.tokens, 4); + assert_eq!(store.continued_last_store_tokens, 4); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn cold_save_failure_restores_continued_at_resumed_start() { + let (dir, mut store) = store("cold-fail-restores-continued"); + store.opt.boundary_trim_tokens = 2; + store.opt.continued_interval_tokens = 4; + let prompt: Vec = (1..=6).collect(); + let mut io = FakeSerial::new(&[], b"prefix-four"); + io.progress_frontiers = vec![4]; + io.fail_save_at = Some(1); + + disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"full prompt", + &prompt, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(io.syncs, [prompt[..4].to_vec(), prompt]); + assert_eq!(io.save_calls, 2); + assert_eq!(store.entries().len(), 1); + assert_eq!(store.entries()[0].header.reason, Reason::Continued); + assert_eq!(store.entries()[0].header.tokens, 4); + assert_eq!(store.continued_last_store_tokens, 4); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn intermediate_save_failure_is_best_effort() { + let (dir, mut store) = store("intermediate-save-fail"); + store.opt.cold_max_tokens = 0; + store.opt.continued_interval_tokens = 4; + let prompt: Vec = (1..=6).collect(); + let mut io = FakeSerial::new(&[], b"prefix-four"); + io.progress_frontiers = vec![4]; + io.fail_save = true; + + disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"full prompt", + &prompt, + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(io.syncs, [prompt]); + assert!(store.entries().is_empty()); + assert_eq!(store.continued_last_store_tokens, 0); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn intermediate_prefill_scope_is_cuda_deepseek_ordinary_disk_only() { + assert!(intermediate_prefill_eligible(true, true, true, false)); + for case in [ + (false, true, true, false), + (true, false, true, false), + (true, true, false, false), + (true, true, true, true), + ] { + assert!(!intermediate_prefill_eligible( + case.0, case.1, case.2, case.3, + )); + } + } + + #[test] + fn continued_decode_gate_stops_after_tool_syntax_begins() { + let plain = SemAccum::init(true, false, false, ChatFormat::DeepSeek, b""); + assert!(continued_decode_allowed(&plain)); + + let mut tools = SemAccum::init(true, true, false, ChatFormat::DeepSeek, b""); + assert!(continued_decode_allowed(&tools)); + tools.feed("<|DSML|tool_calls>\n".as_bytes(), &[]); + assert!(tools.saw_tool_start); + assert!(!continued_decode_allowed(&tools)); + + let mut thinking = SemAccum::init(true, true, true, ChatFormat::DeepSeek, b""); + thinking.feed("<|DSML|tool_calls>\n".as_bytes(), &[]); + assert!(!thinking.saw_tool_start); + assert!(thinking.dsml_state().is_tool()); + assert!(!continued_decode_allowed(&thinking)); + } + + fn chat_msg(role: &str, content: &str) -> ChatMsg { + ChatMsg { + role: role.into(), + content: content.into(), + ..ChatMsg::default() + } + } + + #[test] + fn continued_prefix_hit_misses_motif_history_wrap() { + let (dir, mut store) = store("continued-motif-prefix"); + let user = "summarize this"; + let live_decode = "The model serves five families."; + let first = + render_motif3_chat_ex(&[chat_msg("user", user)], "", &[], ThinkMode::None).unwrap(); + let mut live = first.clone(); + live.extend_from_slice(live_decode.as_bytes()); + + store + .write(Record { + header: Header { + quant_bits: 2, + reason: Reason::Continued, + ext_flags: 0, + model_id: 0, + tokens: 8, + hits: 0, + ctx_size: 4096, + created_at: 1, + last_used: 1, + payload_bytes: 0, + text_bytes: 0, + }, + text: live.clone(), + payload: b"continued-payload".to_vec(), + trailer: Vec::new(), + }) + .unwrap(); + + let mut prefix_follow = live.clone(); + prefix_follow.extend_from_slice(b"\nReply with exactly RESTORED_OK."); + let (path, envelope) = store + .text_prefix_candidate(&prefix_follow, 0, 2, 4096) + .unwrap() + .expect("continued live text is a prefix of a completions follow-up"); + assert_eq!(envelope.header.reason, Reason::Continued); + assert_eq!(envelope.text, live); + + let mut hit = FakeSerial::new(&[], b""); + hit.loaded_tokens = vec![1, 2, 3, 4, 5, 6, 7, 8]; + hit.suffix_tokens = vec![9]; + let cached = disk_sync_prompt( + &mut hit, + Some(&mut store), + 0, + 2, + &prefix_follow, + &[90, 91], + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(cached, 8); + assert_eq!( + hit.suffixes, + [b"\nReply with exactly RESTORED_OK.".to_vec()] + ); + assert_eq!(hit.syncs, [vec![1, 2, 3, 4, 5, 6, 7, 8, 9]]); + assert_eq!( + hit.loads, + [(path, envelope.payload_offset, envelope.header.payload_bytes)] + ); + + let wrap = render_motif3_chat_ex( + &[ + chat_msg("user", user), + chat_msg("assistant", live_decode), + chat_msg("user", "Reply with exactly RESTORED_OK."), + ], + "", + &[], + ThinkMode::None, + ) + .unwrap(); + assert!( + !wrap.starts_with(&live), + "closed Motif assistant history is not live decode bytes" + ); + assert!( + live.windows(b"".len()) + .any(|window| window == b""), + "live frontier keeps the open-assistant think tags" + ); + assert!( + store + .text_prefix_candidate(&wrap, 0, 2, 4096) + .unwrap() + .is_none(), + "history wrap must miss the continued live-decode record" + ); + + let mut miss = FakeSerial::new(&[], b""); + miss.loaded_tokens = vec![1, 2, 3, 4, 5, 6, 7, 8]; + let cached = disk_sync_prompt( + &mut miss, + Some(&mut store), + 0, + 2, + &wrap, + &[90, 91], + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + assert_eq!(cached, 0); + assert!(miss.loads.is_empty()); + assert_eq!(miss.syncs, [vec![90, 91]]); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_sync_prompt_reuses_current_exact_and_byte_prefixes() { + let (dir, mut store) = store("current"); + let mut exact = FakeSerial::new(&[1, 2], b"unused"); + let cached = disk_sync_prompt( + &mut exact, + Some(&mut store), + 0, + 2, + b"hello", + &[1, 2, 3], + None, + false, + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + assert_eq!(cached, 2); + assert_eq!(exact.syncs, [vec![1, 2, 3]]); + assert!(exact.loads.is_empty()); + + let mut byte_prefix = FakeSerial::new(&[9], b"hello "); + byte_prefix.suffix_tokens = vec![10, 11]; + let cached = disk_sync_prompt( + &mut byte_prefix, + Some(&mut store), + 0, + 2, + b"hello world", + &[90, 91], + None, + false, + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + assert_eq!(cached, 1); + assert_eq!(byte_prefix.suffixes, [b"world".to_vec()]); + assert_eq!(byte_prefix.syncs, [vec![9, 10, 11]]); + assert!(byte_prefix.loads.is_empty()); + + let checkpoint = ThinkingVisibleCheckpoint { + text: b"canonical assistant".to_vec(), + frontier: 2, + }; + let mut visible = FakeSerial::new(&[41, 42], b"generation assistant"); + visible.suffix_tokens = vec![6, 43]; + let cached = disk_sync_prompt( + &mut visible, + None, + 0, + 2, + b"canonical assistant<|endofturn|>next", + &[90, 91], + Some(&checkpoint), + true, + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + assert_eq!(cached, 2); + assert_eq!(visible.suffixes, [b"<|endofturn|>next".to_vec()]); + assert_eq!(visible.syncs, [vec![41, 42, 6, 43]]); + + let mut ineligible = FakeSerial::new(&[41, 42], b"not a prompt prefix"); + let cached = disk_sync_prompt( + &mut ineligible, + None, + 0, + 2, + b"canonical assistant<|endofturn|>next", + &[90, 91], + Some(&checkpoint), + false, + DiskSyncPolicy { + save_current: false, + load: false, + }, + ) + .unwrap(); + assert_eq!(cached, 0); + assert_eq!(ineligible.syncs, [vec![90, 91]]); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_sync_prompt_hit_loads_exact_tokens_retokenizes_suffix_and_touches_hit() { + let (dir, mut store) = store("hit"); + let path = candidate(&mut store, b"hello ", 2); + let envelope = read_envelope(&path).unwrap(); + let mut io = FakeSerial::new(&[7], b"old conversation"); + io.loaded_tokens = vec![41, 42]; + io.suffix_tokens = vec![43]; + + let cached = disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"hello world", + &[90, 91], + None, + false, + DiskSyncPolicy { + save_current: true, + load: true, + }, + ) + .unwrap(); + + assert_eq!(cached, 2); + assert_eq!(io.suffixes, [b"world".to_vec()]); + assert_eq!(io.syncs, [vec![41, 42, 43]]); + assert_eq!( + io.loads, + [( + path.clone(), + envelope.payload_offset, + envelope.header.payload_bytes + )] + ); + let save = io + .events + .iter() + .position(|event| *event == "save") + .expect("live checkpoint save event"); + let load = io + .events + .iter() + .position(|event| *event == "load") + .expect("candidate load event"); + assert!( + save < load, + "live checkpoint must be staged before a disk restore: {:?}", + io.events + ); + let (_, old) = store + .text_prefix_candidate(b"old conversation", 0, 2, 4096) + .unwrap() + .expect("old conversation checkpoint"); + assert_eq!(old.header.reason, Reason::Evict); + assert_eq!(store.read(&path).unwrap().header.hits, 1); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_sync_prompt_token_count_mismatch_discards_then_cold_stores() { + let (dir, mut store) = store("count-mismatch"); + let path = candidate(&mut store, b"hello ", 2); + let mut io = FakeSerial::new(&[], b"hello world"); + io.loaded_tokens = vec![41]; + io.suffix_tokens = vec![43]; + + let cached = disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"hello world", + &[90, 91], + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(cached, 0); + assert_eq!(io.invalidations, 1); + assert_eq!(io.syncs, [vec![90, 91]]); + assert!(!path.exists()); + assert_eq!(store.entries().len(), 1); + assert_eq!(store.entries()[0].header.reason, Reason::Cold); + assert_eq!(store.entries()[0].header.tokens, 2); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_sync_prompt_load_failure_keeps_candidate_and_cold_syncs() { + let (dir, mut store) = store("load-failure"); + let path = candidate(&mut store, b"hello ", 2); + let mut io = FakeSerial::new(&[], b""); + io.fail_load = true; + + let cached = disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"hello world", + &[90, 91], + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(cached, 0); + assert_eq!(io.invalidations, 1); + assert_eq!(io.syncs, [vec![90, 91]]); + assert!(path.exists()); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_sync_prompt_suffix_sync_failure_discards_and_returns_error() { + let (dir, mut store) = store("sync-failure"); + let path = candidate(&mut store, b"hello ", 2); + let mut io = FakeSerial::new(&[], b""); + io.loaded_tokens = vec![41, 42]; + io.suffix_tokens = vec![43]; + io.fail_sync = true; + + let error = disk_sync_prompt( + &mut io, + Some(&mut store), + 0, + 2, + b"hello world", + &[90, 91], + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap_err(); + + assert!(error.to_string().contains("injected suffix sync failure")); + assert_eq!(io.invalidations, 1); + assert_eq!(io.syncs, [vec![41, 42, 43]]); + assert!(!path.exists()); + assert!(store.entries().is_empty()); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_sync_prompt_saves_and_restores_thinking_visible_checkpoint() { + let (dir, mut store) = store("thinking-visible"); + let checkpoint = ThinkingVisibleCheckpoint { + text: b"canonical assistant".to_vec(), + frontier: 2, + }; + let mut saving = FakeSerial::new(&[41, 42], b"generation assistant"); + saving.trailer = Some(b"must-not-mix".to_vec()); + + disk_sync_prompt( + &mut saving, + Some(&mut store), + 0, + 2, + b"unrelated prompt", + &[90], + Some(&checkpoint), + false, + DiskSyncPolicy { + save_current: true, + load: false, + }, + ) + .unwrap(); + + let prompt = b"canonical assistant<|endofturn|>next"; + let (path, envelope) = store + .text_prefix_candidate(prompt, 0, 2, 4096) + .unwrap() + .expect("canonical visible checkpoint"); + assert_eq!(envelope.header.ext_flags, EXT_THINKING_VISIBLE); + assert_eq!(envelope.trailer_bytes, 0); + assert_eq!(envelope.header.tokens, 2); + assert_eq!(envelope.text, checkpoint.text); + + let mut loading = FakeSerial::new(&[], b""); + loading.loaded_tokens = vec![41, 42]; + loading.suffix_tokens = vec![6, 43]; + let cached = disk_sync_prompt( + &mut loading, + Some(&mut store), + 0, + 2, + prompt, + &[90, 91], + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(); + + assert_eq!(cached, 2); + assert_eq!(loading.suffixes, [b"<|endofturn|>next".to_vec()]); + assert_eq!(loading.syncs, [vec![41, 42, 6, 43]]); + assert_eq!(store.read(&path).unwrap().header.hits, 1); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn live_store_writes_tool_map_trailer_only_for_plain_checkpoints() { + let (dir, mut store) = store("tool-map-write"); + let mut io = FakeSerial::new(&[41, 42], b"plain sampled tool block"); + io.trailer = Some(b"KTM\x01\0\0\0\0".to_vec()); + + try_store_live(&mut io, &mut store, 0, 2, 4096, Reason::Evict, None).unwrap(); + + assert_eq!(store.entries().len(), 1); + let record = store.read(&store.entries()[0].path).unwrap(); + assert_eq!(record.header.ext_flags, EXT_TOOL_MAP); + assert_eq!(Some(record.trailer), io.trailer); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn live_store_skips_checkpoint_when_tool_map_exceeds_its_bound() { + let (dir, mut store) = store("tool-map-overflow"); + let mut io = FakeSerial::new(&[1, 2, 3, 4, 5, 6, 7, 8], b"plain sampled tool block"); + io.trailer = None; + + assert!(!try_store_live(&mut io, &mut store, 0, 2, 4096, Reason::Evict, None,).unwrap()); + assert!(!try_store_continued(&mut io, &mut store, (0, 2, 4096)).unwrap()); + + assert!(store.entries().is_empty()); + assert_eq!(store.continued_last_store_tokens, 0); + assert!(!io.events.contains(&"save")); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn tool_replay_sync_is_the_only_lane_that_loads_tool_map_records() { + fn write_tool_candidate(store: &mut Store, ext_flags: u8, trailer: Vec) { + store + .write(Record { + header: Header { + quant_bits: 2, + reason: Reason::Evict, + ext_flags, + model_id: 0, + tokens: 2, + hits: 0, + ctx_size: 4096, + created_at: 1, + last_used: 1, + payload_bytes: 0, + text_bytes: 0, + }, + text: b"tool prompt".to_vec(), + payload: b"payload".to_vec(), + trailer, + }) + .unwrap(); + } + + let (ordinary_dir, mut ordinary_store) = store("tool-map-ordinary-reject"); + write_tool_candidate( + &mut ordinary_store, + EXT_TOOL_MAP, + b"KTM\x01\0\0\0\0".to_vec(), + ); + let mut ordinary = FakeSerial::new(&[], b""); + ordinary.loaded_tokens = vec![41, 42]; + assert_eq!( + disk_sync_prompt( + &mut ordinary, + Some(&mut ordinary_store), + 0, + 2, + b"tool prompt tail", + &[90, 91], + None, + false, + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(), + 0 + ); + assert!(ordinary.loads.is_empty()); + + let (tool_dir, mut tool_store) = store("tool-map-scoped-load"); + write_tool_candidate(&mut tool_store, EXT_TOOL_MAP, b"KTM\x01\0\0\0\0".to_vec()); + let mut tool = FakeSerial::new(&[], b""); + tool.loaded_tokens = vec![41, 42]; + tool.suffix_tokens = vec![43]; + assert_eq!( + disk_sync_tool_replay( + &mut tool, + Some(&mut tool_store), + 0, + 2, + b"tool prompt tail", + &[90, 91, 92], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(), + 2 + ); + assert_eq!(tool.loads.len(), 1); + + for (tag, flags, trailer) in [ + ("empty", EXT_TOOL_MAP, Vec::new()), + ( + "combined", + EXT_TOOL_MAP | EXT_THINKING_VISIBLE, + b"KTM\x01\0\0\0\0".to_vec(), + ), + ("unknown", 1 << 7, b"KTM\x01\0\0\0\0".to_vec()), + ] { + let (bad_dir, mut bad_store) = store(&format!("tool-map-{tag}-reject")); + write_tool_candidate(&mut bad_store, flags, trailer); + let mut bad = FakeSerial::new(&[], b""); + bad.loaded_tokens = vec![41, 42]; + assert_eq!( + disk_sync_tool_replay( + &mut bad, + Some(&mut bad_store), + 0, + 2, + b"tool prompt tail", + &[90, 91], + DiskSyncPolicy { + save_current: false, + load: true, + }, + ) + .unwrap(), + 0 + ); + assert!(bad.loads.is_empty()); + let _ = fs::remove_dir_all(bad_dir); + } + + let _ = fs::remove_dir_all(ordinary_dir); + let _ = fs::remove_dir_all(tool_dir); + } + + #[test] + fn thinking_visible_checkpoint_clears_only_after_successful_sync() { + let mut checkpoint = Some(ThinkingVisibleCheckpoint { + text: b"visible".to_vec(), + frontier: 2, + }); + + settle_thinking_visible_checkpoint(&mut checkpoint, false); + assert_eq!(checkpoint.as_ref().unwrap().frontier, 2); + + settle_thinking_visible_checkpoint(&mut checkpoint, true); + assert!(checkpoint.is_none()); + } + + #[test] + fn k2_thinking_visible_key_matches_adjacent_ifm_turns() { + let prompt = b"<|ifm|begin_of_text|><|ifm|im_start|>assistant\n\n"; + assert_eq!( + thinking_visible_key( + prompt, + b"answer", + ModelSyntax::K2Horizon, + ChatFormat::K2Horizon, + true, + ) + .unwrap(), + b"<|ifm|begin_of_text|><|ifm|im_start|>assistant\n\nanswer<|ifm|im_end|>" + ); + } + + #[test] + fn discard_failure_still_resets_marker_and_invalidates_session() { + let (dir, mut store) = store("discard-race"); + let mut io = FakeSerial::new(&[41, 42], b"hello"); + store.continued_last_store_tokens = 2; + + discard_loaded(&mut store, &mut io, &dir.join("missing.kvc")); + + assert_eq!(store.continued_last_store_tokens, 0); + assert_eq!(io.invalidations, 1); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn disk_cache_eligibility_excludes_protocol_state() { + let env = ParseEnv { + default_model: "ds4".into(), + default_tokens: 16, + default_effort: ThinkMode::None, + default_temp: 0.0, + live_ids: Vec::new(), + engine_defaults: false, + }; + let ordinary = parse_request( + WireSurface::OpenaiChat, + &env, + r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#, + ) + .unwrap(); + assert!(ordinary_disk_cache_eligible(&ordinary)); + assert!(thinking_visible_cache_eligible(&ordinary)); + + let mut anthropic = ordinary.clone(); + anthropic.api = Api::Anthropic; + assert!(thinking_visible_cache_eligible(&anthropic)); + + let mut completion = ordinary.clone(); + completion.kind = crate::route::ReqKind::Completion; + assert!(!thinking_visible_cache_eligible(&completion)); + + let mut responses = ordinary.clone(); + responses.api = Api::Responses; + assert!(!thinking_visible_cache_eligible(&responses)); + + let mut cases = Vec::new(); + let mut request = ordinary.clone(); + request.api = Api::Anthropic; + cases.push(request); + let mut request = ordinary.clone(); + request.api = Api::Responses; + cases.push(request); + let mut request = ordinary.clone(); + request.think_mode = ThinkMode::Low; + cases.push(request); + let mut request = ordinary.clone(); + request.has_tools = true; + cases.push(request); + let mut request = ordinary.clone(); + request.has_tool_results = true; + cases.push(request); + let mut request = ordinary; + request.live_call_ids.push("call_1".into()); + cases.push(request); + + assert!(cases + .iter() + .all(|request| !ordinary_disk_cache_eligible(request))); + + let mut replay = cases[4].clone(); + replay.api = Api::Openai; + replay.think_mode = ThinkMode::None; + replay.live_call_ids.clear(); + replay.has_tools = true; + replay.has_tool_results = false; + replay.messages = vec![ChatMsg { + role: "assistant".into(), + calls: vec![ToolCall { + id: "call_history".into(), + ..Default::default() + }], + ..Default::default() + }]; + // V4.1 follows the reference, which attaches in all three chat-shaped + // parsers and refuses nothing but its own disable switch + // (ds4_server.c:4272, :4498, :5529, :10682): another surface, + // thinking mode and live call ids all take the lane. + let v41 = ds4_core::Variant::DeepSeek41Flash as i32; + let v4 = ds4_core::Variant::Flash as i32; + assert!(tool_replay_disk_cache_eligible(&replay, v41)); + assert!(tool_replay_producer_eligible(&replay, v41)); + assert!(tool_replay_disk_cache_eligible(&replay, v4)); + for mutate in 0..3 { + let mut accepted = replay.clone(); + match mutate { + 0 => accepted.api = Api::Anthropic, + 1 => accepted.think_mode = ThinkMode::Low, + _ => accepted.live_call_ids.push("call_live".into()), + } + assert!( + tool_replay_disk_cache_eligible(&accepted, v41), + "V4.1 scope rejected mutation {mutate}" + ); + // Every other family keeps the narrow lane. + assert!( + !tool_replay_disk_cache_eligible(&accepted, v4), + "V4 scope widened by mutation {mutate}" + ); + } + + // The one request the reference cannot serve: the legacy completion + // surface carries no messages, and the reference has no such parser. + let mut completion = replay.clone(); + completion.kind = crate::route::ReqKind::Completion; + assert!(!tool_replay_disk_cache_eligible(&completion, v41)); + assert!(!tool_replay_producer_eligible(&completion, v41)); + } +} + +#[cfg(test)] +mod mimo_prepare { + use super::{prepare_media, ScriptedDecode}; + use crate::parse::{parse_chat_request, ParseEnv}; + + fn engine() -> ScriptedDecode { + ScriptedDecode { + model_id: 12, + prompt_tokens: vec![77], + steps: Vec::new(), + idx: 0, + pos: 0, + ctx: 8192, + generation: 1, + live: Vec::new(), + suffix_tokens: Vec::new(), + } + } + + fn pads(tokens: &[i32], pad: i32) -> usize { + tokens.iter().filter(|token| **token == pad).count() + } + + fn tiny_png() -> Vec { + let path = std::env::temp_dir().join(format!("ds4-mimo-prep-{}.png", std::process::id())); + let status = std::process::Command::new("ffmpeg") + .args([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=red:s=64x64", + "-frames:v", + "1", + ]) + .arg(&path) + .status() + .expect("ffmpeg"); + assert!(status.success()); + let bytes = std::fs::read(&path).unwrap(); + let _ = std::fs::remove_file(&path); + bytes + } + + #[test] + fn still_image_expands_the_single_pad() { + let png = tiny_png(); + let body = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"image_url","image_url":{{"url":"data:image/png;base64,{}"}}}}]}}]}}"#, + b64(&png) + ); + let parsed = parse_chat_request(&ParseEnv::default(), &body).unwrap(); + let count = ds4_core::image_pad_count(&parsed.images[0].data).unwrap() as usize; + let stub = vec![ + ds4_core::VISION_START, + ds4_core::IMAGE_PAD, + ds4_core::VISION_END, + ]; + let media = prepare_media(&engine(), &parsed, stub).unwrap(); + assert_eq!(pads(&media.tokens, ds4_core::IMAGE_PAD), count); + assert!(count > 1); + assert_eq!(media.vision.len(), 1); + assert!(media.videos.is_empty()); + } + + fn tiny_mp4() -> Vec { + let path = std::env::temp_dir().join(format!("ds4-mimo-prep-{}.mp4", std::process::id())); + let status = std::process::Command::new("ffmpeg") + .args([ + "-y", + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=red:s=64x64:d=0.4", + "-r", + "2", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + ]) + .arg(&path) + .status() + .expect("ffmpeg"); + assert!(status.success()); + let bytes = std::fs::read(&path).unwrap(); + let _ = std::fs::remove_file(&path); + bytes + } + + fn b64(data: &[u8]) -> String { + const TABLE: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::new(); + let mut index = 0; + while index < data.len() { + let b0 = data[index]; + let b1 = if index + 1 < data.len() { + data[index + 1] + } else { + 0 + }; + let b2 = if index + 2 < data.len() { + data[index + 2] + } else { + 0 + }; + let n = ((b0 as u32) << 16) | ((b1 as u32) << 8) | b2 as u32; + out.push(TABLE[((n >> 18) & 63) as usize] as char); + out.push(TABLE[((n >> 12) & 63) as usize] as char); + if index + 1 < data.len() { + out.push(TABLE[((n >> 6) & 63) as usize] as char); + } else { + out.push('='); + } + if index + 2 < data.len() { + out.push(TABLE[(n & 63) as usize] as char); + } else { + out.push('='); + } + index += 3; + } + out + } + + fn wav_silence(samples: usize) -> Vec { + let data_bytes = samples * 2; + let mut out = Vec::new(); + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&((36 + data_bytes) as u32).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16u32.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&24_000u32.to_le_bytes()); + out.extend_from_slice(&(48_000u32).to_le_bytes()); + out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&(data_bytes as u32).to_le_bytes()); + out.extend(std::iter::repeat(0).take(data_bytes)); + out + } + + #[test] + fn video_and_joint_audio_replace_the_jinja_stubs() { + let mp4 = tiny_mp4(); + let (_duration, packed) = ds4_core::load_video(&mp4).unwrap(); + let visual: u32 = packed.iter().map(|pair| pair.tokens).sum(); + let body = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"video_url","video_url":{{"url":"data:video/mp4;base64,{}"}}}}]}}]}}"#, + b64(&mp4) + ); + let parsed = parse_chat_request(&ParseEnv::default(), &body).unwrap(); + let stub = vec![ + ds4_core::VISION_START, + ds4_core::VIDEO_PAD, + ds4_core::VISION_END, + ]; + let media = prepare_media(&engine(), &parsed, stub).unwrap(); + assert_eq!(media.tokens.first().copied(), Some(ds4_core::VIDEO_START)); + assert_eq!(media.tokens.last().copied(), Some(ds4_core::VIDEO_END)); + assert_eq!(pads(&media.tokens, ds4_core::VIDEO_PAD), visual as usize); + assert_eq!(pads(&media.tokens, 77), packed.len()); + assert_eq!(media.videos.len(), 1); + assert!(!media.videos[0].frames.is_empty()); + assert!(media.audios.is_empty()); + + let wav = wav_silence(24_000); + let audio_len = ds4_core::audio_pad_count(&wav).unwrap() as usize; + let joint_body = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"video_url","video_url":{{"url":"data:video/mp4;base64,{}"}}}},{{"type":"input_audio","input_audio":{{"format":"wav","data":"{}"}}}}]}}]}}"#, + b64(&mp4), + b64(&wav) + ); + let joint = parse_chat_request(&ParseEnv::default(), &joint_body).unwrap(); + let joint_stub = vec![ + ds4_core::VISION_START, + ds4_core::VIDEO_PAD, + ds4_core::VISION_END, + ds4_core::AUDIO_START, + ds4_core::AUDIO_PAD, + ds4_core::AUDIO_END, + ]; + let joint_media = prepare_media(&engine(), &joint, joint_stub.clone()).unwrap(); + assert_eq!(pads(&joint_media.tokens, ds4_core::AUDIO_PAD), audio_len); + assert_eq!(joint_media.audios.len(), 1); + assert_eq!(joint_media.videos.len(), 1); + + let doubled = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"video_url","video_url":{{"url":"data:video/mp4;base64,{}"}}}},{{"type":"input_audio","input_audio":{{"format":"wav","data":"{}"}}}},{{"type":"input_audio","input_audio":{{"format":"wav","data":"{}"}}}}]}}]}}"#, + b64(&mp4), + b64(&wav), + b64(&wav) + ); + let doubled = parse_chat_request(&ParseEnv::default(), &doubled).unwrap(); + let mut doubled_stub = joint_stub; + doubled_stub.extend_from_slice(&[ + ds4_core::AUDIO_START, + ds4_core::AUDIO_PAD, + ds4_core::AUDIO_END, + ]); + let err = match prepare_media(&engine(), &doubled, doubled_stub) { + Err(err) => err, + Ok(_) => panic!("second audio was accepted"), + }; + assert!(err.to_string().contains("second audio"), "{err}"); + + let split = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"video_url","video_url":{{"url":"data:video/mp4;base64,{}"}}}}]}},{{"role":"user","content":[{{"type":"input_audio","input_audio":{{"format":"wav","data":"{}"}}}}]}}]}}"#, + b64(&mp4), + b64(&wav) + ); + let split = parse_chat_request(&ParseEnv::default(), &split).unwrap(); + let split_stub = vec![ + ds4_core::VISION_START, + ds4_core::VIDEO_PAD, + ds4_core::VISION_END, + 198, + ds4_core::AUDIO_START, + ds4_core::AUDIO_PAD, + ds4_core::AUDIO_END, + ]; + let split_media = prepare_media(&engine(), &split, split_stub).unwrap(); + assert_eq!(split_media.videos.len(), 1); + assert_eq!(split_media.audios.len(), 1); + assert_eq!(pads(&split_media.tokens, ds4_core::AUDIO_PAD), audio_len); + } +} + +/// Exact sampled DSML replay (docs/ds41-dsml-conformance.md TP-15, TP-16). +/// +/// The reference keeps the sampled block per tool-call id and puts those bytes +/// back into the messages before rendering (`tool_memory_attach_to_messages`, +/// ds4_server.c:10679), attached at all three chat-shaped parsers (:4272 chat, +/// :4498 Anthropic, :5529 Responses); its own tests assert the message bytes +/// (`test_tool_memory_replays_sampled_dsml` :19933, +/// `test_anthropic_tool_memory_replays_sampled_dsml` :19996). These assert the +/// rendered prompt instead, which is what the model reads. +#[cfg(test)] +mod dsml_replay { + use super::{render_prompt_ex, tool_replay_disk_cache_eligible}; + use crate::parse::{parse_request, ParseEnv}; + use crate::render::dsml_head; + use crate::route::WireSurface; + use crate::tool_memory::{ExactReplay, Source, ToolMemory, ToolMemoryConfig}; + use crate::tools::{ + DSML_INVOKE_END, DSML_INVOKE_START, DSML_PARAM_END, DSML_PARAM_START, DSML_TOOL_CALLS_END, + DSML_TOOL_CALLS_START, + }; + + /// The ds41 model id. Its canonical render uses the spaced spelling + /// (`render.rs:1387`), which is what makes the plain spelling below + /// unmistakable: only a replayed block can put it in the prompt. + const MODEL_ID: i32 = ds4_core::Variant::DeepSeek41Flash as i32; + + const CHAT_ID: &str = "call_echoed"; + const TOOLU_ID: &str = "toolu_echoed"; + const CALL_ID_PLACEHOLDER: &str = "__ID__"; + + fn sampled() -> String { + format!( + "{DSML_TOOL_CALLS_START}\n{DSML_INVOKE_START} name=\"read_file\">\n\ + {DSML_PARAM_START} name=\"target_file\" string=\"true\">/tmp/x.txt{DSML_PARAM_END}\n\ + {DSML_INVOKE_END}\n{DSML_TOOL_CALLS_END}" + ) + } + + /// The client's own echo of the call, respaced the way a re-serializing + /// client sends it (`{"target_file": "/tmp/x.txt"}` against the sampled + /// `{"target_file":"/tmp/x.txt"}`). + const OPENAI_BODY: &str = r#"{"model":"t","max_tokens":16,"messages":[ + {"role":"user","content":"Read the file /tmp/x.txt"}, + {"role":"assistant","content":"","tool_calls":[{"id":"__ID__","type":"function","function":{"name":"read_file","arguments":"{\"target_file\": \"/tmp/x.txt\"}"}}]}, + {"role":"tool","tool_call_id":"__ID__","content":"file body"}, + {"role":"user","content":"and the next file?"}], + "tools":[{"type":"function","function":{"name":"read_file","description":"Read a file","parameters":{"type":"object","properties":{"target_file":{"type":"string"}},"required":["target_file"]}}}]}"#; + + const ANTHROPIC_BODY: &str = r#"{"model":"t","max_tokens":16,"tools":[{"name":"read_file","description":"Read a file","input_schema":{"type":"object","properties":{"target_file":{"type":"string"}},"required":["target_file"]}}],"messages":[ + {"role":"user","content":"Read the file /tmp/x.txt"}, + {"role":"assistant","content":[{"type":"tool_use","id":"__ID__","name":"read_file","input":{"target_file":"/tmp/x.txt"}}]}, + {"role":"user","content":[{"type":"tool_result","tool_use_id":"__ID__","content":"file body"}]}]}"#; + + const RESPONSES_BODY: &str = r#"{"model":"t","max_output_tokens":16,"tools":[{"type":"function","function":{"name":"read_file","description":"Read a file","parameters":{"type":"object","properties":{"target_file":{"type":"string"}},"required":["target_file"]}}}],"input":[ + {"role":"user","content":"Read the file /tmp/x.txt"}, + {"type":"function_call","call_id":"__ID__","name":"read_file","arguments":"{\"target_file\": \"/tmp/x.txt\"}"}, + {"type":"function_call_output","call_id":"__ID__","output":"file body"}]}"#; + + fn body(template: &str, id: &str) -> String { + template.replace(CALL_ID_PLACEHOLDER, id) + } + + fn memory(replay: ExactReplay, id: &str) -> ToolMemory { + let mut memory = ToolMemory::from_config(ToolMemoryConfig { + max_ids: 1000, + replay, + }); + memory.put(id, &sampled(), Source::Ram); + memory + } + + fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) + } + + /// TP-15. Each surface parses its own echo of the call, and the replayed + /// bytes must survive into the rendered prompt. + #[test] + fn sampled_bytes_reach_the_prompt_on_every_chat_surface() { + let cases = [ + (WireSurface::OpenaiChat, OPENAI_BODY, CHAT_ID), + (WireSurface::Anthropic, ANTHROPIC_BODY, TOOLU_ID), + (WireSurface::Responses, RESPONSES_BODY, CHAT_ID), + ]; + let sampled = sampled(); + for (surface, template, id) in cases { + let mut parsed = parse_request(surface, &ParseEnv::default(), &body(template, id)) + .unwrap_or_else(|error| panic!("{surface:?} did not parse: {error}")); + assert!( + tool_replay_disk_cache_eligible(&parsed, MODEL_ID), + "{surface:?} did not take the replay lane" + ); + let head = dsml_head(MODEL_ID, "sys"); + let before = render_prompt_ex(&parsed, MODEL_ID, head).unwrap(); + assert!( + !contains(&before, sampled.as_bytes()), + "{surface:?} rendered the sampled block with no replay" + ); + // The call is rendered canonically in that arm: the spaced invoke + // marker is what the ds41 renderer emits. + assert!( + contains(&before, crate::tools::DSML_INVOKE_START_SPACED.as_bytes()), + "{surface:?} did not render the call at all, so the check above is vacuous" + ); + + let mut memory = memory(ExactReplay::On, id); + assert_eq!(memory.attach(&mut parsed.messages), 1, "{surface:?}"); + let after = render_prompt_ex(&parsed, MODEL_ID, head).unwrap(); + assert!( + contains(&after, sampled.as_bytes()), + "{surface:?} did not replay the sampled block into the prompt" + ); + } + } + + /// TP-16. `--disable-exact-dsml-tool-replay` (ds4_help.c:378) leaves the + /// canonical render in place. + #[test] + fn the_disable_switch_leaves_the_canonical_render() { + let mut parsed = parse_request( + WireSurface::OpenaiChat, + &ParseEnv::default(), + &body(OPENAI_BODY, CHAT_ID), + ) + .unwrap(); + let head = dsml_head(MODEL_ID, "sys"); + let mut memory = memory(ExactReplay::Off, CHAT_ID); + assert_eq!(memory.attach(&mut parsed.messages), 0); + let prompt = render_prompt_ex(&parsed, MODEL_ID, head).unwrap(); + assert!(!contains(&prompt, sampled().as_bytes())); + assert!( + contains(&prompt, crate::tools::DSML_INVOKE_START_SPACED.as_bytes()), + "the call must still render canonically" + ); + } +} diff --git a/crates/ds4-server/src/http.rs b/crates/ds4-server/src/http.rs new file mode 100644 index 0000000000000000000000000000000000000000..c0d2b0c121bba595b8e8d7ecfe084aee341fbe36 --- /dev/null +++ b/crates/ds4-server/src/http.rs @@ -0,0 +1,340 @@ +//! Blocking HTTP/1.1 reader. Content-Length + chunked (v0.6.3). No Tokio. + +use std::io::{self, Read}; + +use crate::route::WireSurface; + +pub const MAX_HEADER: usize = 64 * 1024; +pub const MAX_BODY: usize = 64 * 1024 * 1024; + +#[derive(Debug, Clone)] +pub struct HttpRequest { + pub method: String, + pub path: String, + pub body: Vec, + pub accept_json: bool, +} + +pub fn chunked_enabled() -> bool { + std::env::var("DS4_SERVER_CHUNKED") + .map(|v| v != "0") + .unwrap_or(true) +} + +pub fn header_end(p: &[u8]) -> Option { + for i in 3..p.len() { + if p[i - 3] == b'\r' && p[i - 2] == b'\n' && p[i - 1] == b'\r' && p[i] == b'\n' { + return Some(i + 1); + } + } + for i in 1..p.len() { + if p[i - 1] == b'\n' && p[i] == b'\n' { + return Some(i + 1); + } + } + None +} + +fn header_ci_eq(line: &[u8], name: &[u8]) -> bool { + line.len() >= name.len() && line[..name.len()].eq_ignore_ascii_case(name) +} + +pub fn header_accepts_json(h: &[u8]) -> bool { + let mut i = 0; + while i < h.len() { + let start = i; + while i < h.len() && h[i] != b'\n' { + i += 1; + } + let mut len = i - start; + if len > 0 && h[start + len - 1] == b'\r' { + len -= 1; + } + let line = &h[start..start + len]; + if header_ci_eq(line, b"Accept:") { + let mut j = 7; + while j + 16 <= line.len() { + if line[j..j + 16].eq_ignore_ascii_case(b"application/json") { + return true; + } + j += 1; + } + } + if i < h.len() { + i += 1; + } + } + false +} + +pub fn content_length(h: &[u8]) -> i64 { + let mut i = 0; + while i < h.len() { + let start = i; + while i < h.len() && h[i] != b'\n' { + i += 1; + } + let mut len = i - start; + if len > 0 && h[start + len - 1] == b'\r' { + len -= 1; + } + let line = &h[start..start + len]; + if header_ci_eq(line, b"Content-Length:") { + let mut v = &line[15..]; + while !v.is_empty() && v[0].is_ascii_whitespace() { + v = &v[1..]; + } + return parse_i64_c_strtol(v); + } + if i < h.len() { + i += 1; + } + } + 0 +} + +pub fn header_chunked(h: &[u8]) -> bool { + let mut i = 0; + while i < h.len() { + let start = i; + while i < h.len() && h[i] != b'\n' { + i += 1; + } + let mut len = i - start; + if len > 0 && h[start + len - 1] == b'\r' { + len -= 1; + } + let line = &h[start..start + len]; + if header_ci_eq(line, b"Transfer-Encoding:") { + let mut j = 18; + while j + 7 <= line.len() { + if line[j..j + 7].eq_ignore_ascii_case(b"chunked") { + return true; + } + j += 1; + } + } + if i < h.len() { + i += 1; + } + } + false +} + +fn recv_more(r: &mut R, buf: &mut Vec, tmp_len: usize) -> io::Result { + let mut tmp = vec![0u8; tmp_len]; + loop { + match r.read(&mut tmp) { + Ok(0) => return Ok(0), + Ok(n) => { + buf.extend_from_slice(&tmp[..n]); + return Ok(n); + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + } +} + +fn chunk_line_end( + r: &mut R, + buf: &mut Vec, + pos: usize, + max_line: usize, +) -> Option { + let mut scan = pos; + loop { + while scan < buf.len() { + if buf[scan] == b'\n' { + return Some(scan + 1); + } + scan += 1; + } + if buf.len() - pos > max_line { + return None; + } + match recv_more(r, buf, 8192) { + Ok(0) | Err(_) => return None, + Ok(_) => {} + } + } +} + +fn read_chunked_body( + r: &mut R, + buf: &mut Vec, + mut pos: usize, + max_body: usize, +) -> Option> { + let mut body = Vec::new(); + loop { + if pos > 0 { + buf.drain(..pos); + pos = 0; + } + let le = chunk_line_end(r, buf, pos, 8192)?; + if pos >= buf.len() || !buf[pos].is_ascii_hexdigit() { + return None; + } + let line = &buf[pos..le]; + let mut endp = 0; + while endp < line.len() && line[endp].is_ascii_hexdigit() { + endp += 1; + } + let hex = std::str::from_utf8(&line[..endp]).ok()?; + let sz = usize::from_str_radix(hex, 16).ok()?; + let mut rest = &line[endp..]; + while !rest.is_empty() && (rest[0] == b' ' || rest[0] == b'\t') { + rest = &rest[1..]; + } + if rest.is_empty() || (rest[0] != b';' && rest[0] != b'\r' && rest[0] != b'\n') { + return None; + } + pos = le; + if sz == 0 { + break; + } + if sz > max_body || body.len() + sz > max_body { + return None; + } + while buf.len() < pos + sz { + if recv_more(r, buf, 8192).ok()? == 0 { + return None; + } + } + body.extend_from_slice(&buf[pos..pos + sz]); + pos += sz; + let crlf = chunk_line_end(r, buf, pos, 2)?; + let l2 = crlf - pos; + if !(l2 == 1 || (l2 == 2 && buf[pos] == b'\r')) { + return None; + } + pos = crlf; + } + let trailer_start = pos; + loop { + let le = chunk_line_end(r, buf, pos, 8192)?; + if le - trailer_start > MAX_HEADER { + return None; + } + let linelen = le - pos; + let blank = linelen == 1 || (linelen == 2 && buf[pos] == b'\r'); + pos = le; + if blank { + break; + } + } + Some(body) +} + +pub fn read_http_request(r: &mut R, chunked_on: bool) -> Option { + let mut b = Vec::new(); + let mut hend = None; + while hend.is_none() && b.len() < MAX_HEADER { + if recv_more(r, &mut b, 4096).ok()? == 0 { + return None; + } + hend = header_end(&b); + } + let hend = hend?; + let mut i = 0; + let mut line = [0u8; 512]; + while i < b.len() && b[i] != b'\n' && i + 1 < line.len() { + line[i] = b[i]; + i += 1; + } + let line = std::str::from_utf8(&line[..i]).ok()?; + /* sscanf("%7s %255s"): stop at whitespace or the width. */ + let (method, rest) = scan_token(line, 7)?; + let (path, _) = scan_token(rest, 255)?; + let method = method.to_string(); + let mut path = path.to_string(); + if let Some(q) = path.find('?') { + path.truncate(q); + } + let clen = content_length(&b[..hend]); + if clen < 0 || clen as usize > MAX_BODY { + return None; + } + let accept_json = header_accepts_json(&b[..hend]); + if header_chunked(&b[..hend]) && chunked_on { + let body = read_chunked_body(r, &mut b, hend, MAX_BODY)?; + return Some(HttpRequest { + method, + path, + body, + accept_json, + }); + } + while b.len() < hend + clen as usize { + if recv_more(r, &mut b, 8192).ok()? == 0 { + return None; + } + } + let body = b[hend..hend + clen as usize].to_vec(); + Some(HttpRequest { + method, + path, + body, + accept_json, + }) +} + +pub fn shed_surface_for_path(path: &str) -> Option { + match path { + "/v1/messages" => Some(WireSurface::Anthropic), + "/v1/chat/completions" => Some(WireSurface::OpenaiChat), + "/v1/responses" => Some(WireSurface::Responses), + "/v1/completions" => Some(WireSurface::OpenaiCompletion), + _ => None, + } +} + +pub fn parse_surface_for_path(path: &str) -> Option { + shed_surface_for_path(path) +} + +fn parse_i64_c_strtol(v: &[u8]) -> i64 { + let mut i = 0; + while i < v.len() && matches!(v[i], b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c) { + i += 1; + } + if i >= v.len() { + return 0; + } + let mut neg = false; + if v[i] == b'+' { + i += 1; + } else if v[i] == b'-' { + neg = true; + i += 1; + } + if i >= v.len() || !v[i].is_ascii_digit() { + return 0; + } + let mut n: i64 = 0; + while i < v.len() && v[i].is_ascii_digit() { + n = n.saturating_mul(10).saturating_add((v[i] - b'0') as i64); + i += 1; + } + if neg { + -n + } else { + n + } +} + +fn scan_token(s: &str, max: usize) -> Option<(&str, &str)> { + let s = s.trim_start_matches(|c: char| c == ' ' || c == '\t'); + if s.is_empty() { + return None; + } + let n = s + .find(|c: char| c == ' ' || c == '\t' || c == '\r') + .unwrap_or(s.len()) + .min(max); + if n == 0 { + return None; + } + Some((&s[..n], &s[n..])) +} diff --git a/crates/ds4-server/src/json.rs b/crates/ds4-server/src/json.rs new file mode 100644 index 0000000000000000000000000000000000000000..5f9f1be91d6dfcd4a36a5c491ab63b3666e3e39b --- /dev/null +++ b/crates/ds4-server/src/json.rs @@ -0,0 +1,492 @@ +//! Request JSON walker copied from `ds4_server.c`. Ceiling 256 on skip nest. + +pub const JSON_MAX_NESTING: i32 = 256; + +#[derive(Debug)] +pub struct Json<'a> { + s: &'a [u8], + pub i: usize, +} + +impl<'a> Json<'a> { + pub fn new(s: &'a str) -> Self { + Self { + s: s.as_bytes(), + i: 0, + } + } + + pub fn from_bytes(s: &'a [u8]) -> Self { + Self { s, i: 0 } + } + + pub fn remaining(&self) -> &[u8] { + &self.s[self.i..] + } + + pub fn peek(&self) -> Option { + self.s.get(self.i).copied() + } + + pub fn bump(&mut self) -> Option { + let c = self.peek()?; + self.i += 1; + Some(c) + } + + pub fn ws(&mut self) { + while matches!(self.peek(), Some(c) if is_c_space(c)) { + self.i += 1; + } + } + + pub fn lit(&mut self, lit: &str) -> bool { + let b = lit.as_bytes(); + if self.s.get(self.i..self.i + b.len()) == Some(b) { + self.i += b.len(); + true + } else { + false + } + } +} + +fn is_c_space(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c) +} + +fn hex(c: u8) -> Option { + match c { + b'0'..=b'9' => Some((c - b'0') as u32), + b'a'..=b'f' => Some(10 + (c - b'a') as u32), + b'A'..=b'F' => Some(10 + (c - b'A') as u32), + _ => None, + } +} + +fn utf8_put(out: &mut Vec, cp: u32) { + if cp <= 0x7f { + out.push(cp as u8); + } else if cp <= 0x7ff { + out.push((0xc0 | (cp >> 6)) as u8); + out.push((0x80 | (cp & 0x3f)) as u8); + } else if cp <= 0xffff { + out.push((0xe0 | (cp >> 12)) as u8); + out.push((0x80 | ((cp >> 6) & 0x3f)) as u8); + out.push((0x80 | (cp & 0x3f)) as u8); + } else { + out.push((0xf0 | (cp >> 18)) as u8); + out.push((0x80 | ((cp >> 12) & 0x3f)) as u8); + out.push((0x80 | ((cp >> 6) & 0x3f)) as u8); + out.push((0x80 | (cp & 0x3f)) as u8); + } +} + +fn json_u16(p: &mut Json<'_>) -> Option { + if p.peek() != Some(b'\\') { + return None; + } + if p.s.get(p.i + 1) != Some(&b'u') { + return None; + } + let mut cp = 0u32; + for k in 0..4 { + let h = hex(*p.s.get(p.i + 2 + k)?)?; + cp = (cp << 4) | h; + } + p.i += 6; + Some(cp) +} + +pub fn json_string(p: &mut Json<'_>) -> Option { + p.ws(); + if p.bump() != Some(b'"') { + return None; + } + let mut b = Vec::new(); + while let Some(c) = p.peek() { + if c == b'"' { + break; + } + p.i += 1; + if c != b'\\' { + b.push(c); + continue; + } + let e = p.bump()?; + match e { + b'"' | b'\\' | b'/' => b.push(e), + b'b' => b.push(b'\x08'), + b'f' => b.push(b'\x0c'), + b'n' => b.push(b'\n'), + b'r' => b.push(b'\r'), + b't' => b.push(b'\t'), + b'u' => { + p.i -= 2; + let mut cp = json_u16(p)?; + if (0xd800..=0xdbff).contains(&cp) { + let save = p.i; + if let Some(lo) = json_u16(p) { + if (0xdc00..=0xdfff).contains(&lo) { + cp = 0x10000 + ((cp - 0xd800) << 10) + (lo - 0xdc00); + } else { + p.i = save; + } + } + } + utf8_put(&mut b, cp); + } + _ => return None, + } + } + if p.bump() != Some(b'"') { + return None; + } + String::from_utf8(b).ok() +} + +pub fn json_number(p: &mut Json<'_>) -> Option { + p.ws(); + let start = p.i; + if matches!(p.peek(), Some(b'+' | b'-')) { + p.i += 1; + } + let after_sign = p.i; + while matches!(p.peek(), Some(c) if c.is_ascii_digit()) { + p.i += 1; + } + if p.peek() == Some(b'.') { + p.i += 1; + while matches!(p.peek(), Some(c) if c.is_ascii_digit()) { + p.i += 1; + } + } + if matches!(p.peek(), Some(b'e' | b'E')) { + p.i += 1; + if matches!(p.peek(), Some(b'+' | b'-')) { + p.i += 1; + } + while matches!(p.peek(), Some(c) if c.is_ascii_digit()) { + p.i += 1; + } + } + if p.i == after_sign { + p.i = start; + return None; + } + let s = std::str::from_utf8(&p.s[start..p.i]).ok()?; + s.parse().ok() +} + +pub fn json_int(p: &mut Json<'_>) -> Option { + let mut v = json_number(p)?; + if v < 0.0 { + v = 0.0; + } + if v > i32::MAX as f64 { + v = i32::MAX as f64; + } + Some(v as i32) +} + +pub fn json_bool(p: &mut Json<'_>) -> Option { + p.ws(); + if p.lit("true") { + return Some(true); + } + if p.lit("false") { + return Some(false); + } + None +} + +fn skip_value_depth(p: &mut Json<'_>, depth: i32) -> bool { + p.ws(); + match p.peek() { + Some(b'"') => json_string(p).is_some(), + Some(b'{') => skip_object_depth(p, depth), + Some(b'[') => skip_array_depth(p, depth), + Some(_) if p.lit("true") || p.lit("false") || p.lit("null") => true, + Some(_) => json_number(p).is_some(), + None => false, + } +} + +fn skip_array_depth(p: &mut Json<'_>, depth: i32) -> bool { + if depth >= JSON_MAX_NESTING { + return false; + } + p.ws(); + if p.bump() != Some(b'[') { + return false; + } + p.ws(); + if p.peek() == Some(b']') { + p.i += 1; + return true; + } + loop { + if !skip_value_depth(p, depth + 1) { + return false; + } + p.ws(); + if p.peek() == Some(b']') { + p.i += 1; + return true; + } + if p.bump() != Some(b',') { + return false; + } + } +} + +fn skip_object_depth(p: &mut Json<'_>, depth: i32) -> bool { + if depth >= JSON_MAX_NESTING { + return false; + } + p.ws(); + if p.bump() != Some(b'{') { + return false; + } + p.ws(); + if p.peek() == Some(b'}') { + p.i += 1; + return true; + } + loop { + if json_string(p).is_none() { + return false; + } + p.ws(); + if p.bump() != Some(b':') { + return false; + } + if !skip_value_depth(p, depth + 1) { + return false; + } + p.ws(); + if p.peek() == Some(b'}') { + p.i += 1; + return true; + } + if p.bump() != Some(b',') { + return false; + } + } +} + +pub fn json_skip_value(p: &mut Json<'_>) -> bool { + skip_value_depth(p, 0) +} + +pub fn json_raw_value(p: &mut Json<'_>) -> Option { + p.ws(); + let start = p.i; + if !json_skip_value(p) { + return None; + } + String::from_utf8(p.s[start..p.i].to_vec()).ok() +} + +pub fn json_content(p: &mut Json<'_>) -> Option { + p.ws(); + if p.peek() == Some(b'"') { + return json_string(p); + } + if p.lit("null") { + return Some(String::new()); + } + if p.peek() != Some(b'[') { + if !json_skip_value(p) { + return None; + } + return Some(String::new()); + } + p.i += 1; + let mut b = String::new(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.peek() == Some(b'"') { + b.push_str(&json_string(p)?); + } else if p.peek() == Some(b'{') { + p.i += 1; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "text" { + b.push_str(&json_string(p)?); + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + Some(b) +} + +/// Surrounding quotes included, matching C `json_escape`. Walks raw bytes +/// so a held-back UTF-8 fragment can be escaped the same way as C. +pub fn json_escape_bytes(s: &[u8]) -> Vec { + // Model text can hold bytes that form no character (a sampled lead byte, + // then ASCII). JSON text is UTF-8 and openai-python decodes each SSE line + // strictly, so one such delta ended the whole stream: they become U+FFFD. + // Streams hold back a character cut between tokens (utf8_stream_safe_len), + // so a valid one never arrives here in pieces. The C reference passes + // the bytes through raw. + let s = String::from_utf8_lossy(s); + let mut out = vec![b'"']; + for &c in s.as_bytes() { + match c { + b'"' | b'\\' => { + out.push(b'\\'); + out.push(c); + } + b'\n' => out.extend_from_slice(b"\\n"), + b'\r' => out.extend_from_slice(b"\\r"), + b'\t' => out.extend_from_slice(b"\\t"), + c if c < 0x20 => { + let esc = format!("\\u{c:04x}"); + out.extend_from_slice(esc.as_bytes()); + } + c => out.push(c), + } + } + out.push(b'"'); + out +} + +/// Surrounding quotes included, matching `json_escape`. +pub fn json_escape(s: &str) -> String { + String::from_utf8(json_escape_bytes(s.as_bytes())).expect("json_escape preserves UTF-8") +} + +/// C `json_minify_raw_value`: drop insignificant whitespace outside strings. +pub fn json_minify_raw_value(json: &str) -> String { + let src = if json.is_empty() { "null" } else { json }; + let mut p = Json::new(src); + p.ws(); + let start = p.i; + if !json_skip_value(&mut p) { + return src.to_string(); + } + let end = p.i; + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut in_string = false; + let mut escape = false; + for &c in &bytes[start..end] { + if in_string { + out.push(c); + if escape { + escape = false; + } else if c == b'\\' { + escape = true; + } else if c == b'"' { + in_string = false; + } + } else if c == b'"' { + in_string = true; + out.push(c); + } else if !is_c_space(c) { + out.push(c); + } + } + String::from_utf8(out).unwrap_or_else(|_| src.to_string()) +} + +#[derive(Debug, Clone)] +pub struct JsonArg { + pub key: String, + pub value: String, + pub is_string: bool, + pub used: bool, +} + +/// C `json_args_parse`. Does not consume the closing `}`. +pub fn json_args_parse(json: &str) -> Option> { + let mut p = Json::new(json); + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + p.ws(); + let mut args = Vec::new(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(&mut p) else { + return None; + }; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + p.ws(); + let (is_string, value) = if p.peek() == Some(b'"') { + let Some(v) = json_string(&mut p) else { + return None; + }; + (true, v) + } else { + let Some(raw) = json_raw_value(&mut p) else { + return None; + }; + (false, json_minify_raw_value(&raw)) + }; + args.push(JsonArg { + key, + value, + is_string, + used: false, + }); + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.peek() != Some(b'}') { + return None; + } + Some(args) +} + +/// The contents of `src` when it is exactly one JSON string literal, and None +/// otherwise. +/// +/// The V4.1 oracle tolerates a stringified `arguments` value: it decodes the +/// field up to twice before it gives up (encoding.py:528-537), so a replay +/// whose arguments were encoded one level too deep still renders that object's +/// parameters. On the wire `arguments` is already a string +/// (docs/ds41-dsml-conformance.md requirement 2.3.5), so that extra decode is +/// this unwrap. +pub fn json_string_contents(src: &str) -> Option { + let mut p = Json::new(src); + p.ws(); + let text = json_string(&mut p)?; + p.ws(); + if p.peek().is_some() { + return None; + } + Some(text) +} diff --git a/crates/ds4-server/src/kv_cli.rs b/crates/ds4-server/src/kv_cli.rs new file mode 100644 index 0000000000000000000000000000000000000000..3c54327a3780e0d6ac0b098c003441b0db763c53 --- /dev/null +++ b/crates/ds4-server/src/kv_cli.rs @@ -0,0 +1,348 @@ +use ds4_kv::{Options, Store}; +use std::path::PathBuf; + +#[derive(Debug)] +pub struct DiskKvArgs { + dir: Option, + space_mb: u64, + options: Options, + reject_different_quant: bool, +} + +impl Default for DiskKvArgs { + fn default() -> Self { + Self { + dir: None, + space_mb: 0, + options: Options::default(), + reject_different_quant: false, + } + } +} + +fn require_value(option: &str, value: Option) -> Result { + value.ok_or_else(|| format!("ds4-server-rs: missing value for {option}")) +} + +fn positive_i32(option: &str, value: &str) -> Result { + value + .parse::() + .ok() + .filter(|value| *value > 0) + .ok_or_else(|| format!("ds4-server-rs: invalid value for {option}: {value}")) +} + +fn nonnegative_i32(option: &str, value: &str) -> Result { + value + .parse::() + .ok() + .filter(|value| *value >= 0) + .ok_or_else(|| format!("ds4-server-rs: invalid value for {option}: {value}")) +} + +impl DiskKvArgs { + pub fn parse_arg( + &mut self, + option: &str, + args: &mut impl Iterator, + ) -> Result { + let mut value = || require_value(option, args.next()); + match option { + "--kv-disk-dir" => self.dir = Some(value()?.into()), + "--kv-disk-space-mb" => self.set_space_mb(&value()?)?, + "--kv-disk-space" => { + self.space_mb = ds4_core::parse_disk_space(&value()?)?; + } + "--kv-cache-min-tokens" => self.set_min_tokens(&value()?)?, + "--kv-cache-cold-max-tokens" => { + self.options.cold_max_tokens = nonnegative_i32(option, &value()?)? + } + "--kv-cache-continued-interval-tokens" => { + self.options.continued_interval_tokens = nonnegative_i32(option, &value()?)? + } + "--kv-cache-boundary-trim-tokens" => { + self.options.boundary_trim_tokens = nonnegative_i32(option, &value()?)? + } + "--kv-cache-boundary-align-tokens" => { + self.options.boundary_align_tokens = nonnegative_i32(option, &value()?)? + } + "--kv-cache-reject-different-quant" => self.reject_different_quant = true, + _ => return Ok(false), + } + Ok(true) + } + + fn set_space_mb(&mut self, value: &str) -> Result<(), String> { + self.space_mb = positive_i32("--kv-disk-space-mb", value)? as u64; + Ok(()) + } + + fn set_min_tokens(&mut self, value: &str) -> Result<(), String> { + self.options.min_tokens = positive_i32("--kv-cache-min-tokens", value)?; + Ok(()) + } + + pub fn dir(&self) -> Option<&std::path::Path> { + self.dir.as_deref() + } + + pub fn space_mb(&self) -> u64 { + self.space_mb + } + + pub fn min_tokens(&self) -> i32 { + self.options.min_tokens + } + + pub fn validate(&self) -> Result<(), String> { + if self.options.cold_max_tokens > 0 + && self.options.cold_max_tokens < self.options.min_tokens + { + return Err( + "ds4-server-rs: --kv-cache-cold-max-tokens must be 0 or >= --kv-cache-min-tokens" + .into(), + ); + } + Ok(()) + } + + pub fn open(&self) -> Option { + let dir = self.dir.as_ref()?; + match Store::open( + dir, + self.space_mb, + self.reject_different_quant, + self.options, + ) { + Ok(store) => { + eprintln!( + "ds4-server-rs: KV disk cache {} (budget={} MiB, cross-quant={}, min={}, cold_max={}, continued={}, trim={}, align={}; ordinary serial cold/continued/evict/load)", + dir.display(), + store.budget_bytes / (1024 * 1024), + if self.reject_different_quant { "reject" } else { "accept" }, + self.options.min_tokens, + self.options.cold_max_tokens, + self.options.continued_interval_tokens, + self.options.boundary_trim_tokens, + self.options.boundary_align_tokens, + ); + Some(store) + } + Err(error) => { + eprintln!( + "ds4-server-rs: failed to create KV cache directory {}: {error}", + dir.display() + ); + None + } + } + } +} + +#[cfg(test)] +mod tests { + use super::DiskKvArgs; + use ds4_kv::Options; + use std::fs; + + fn temp_path(tag: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("ds4-server-kv-cli-{tag}-{}", std::process::id())) + } + + #[test] + fn defaults_disable_disk_cache_and_keep_tuning_inert() { + let mut kv = DiskKvArgs::default(); + assert!(kv.dir.is_none()); + assert_eq!(kv.space_mb, 0); + assert_eq!(kv.options.min_tokens, Options::default().min_tokens); + assert!(!kv.reject_different_quant); + + kv.set_space_mb("64").unwrap(); + kv.set_min_tokens("1024").unwrap(); + kv.reject_different_quant = true; + assert!(kv.open().is_none()); + } + + #[test] + fn ordinary_serial_checkpoint_policy_flags_parse() { + let mut kv = DiskKvArgs::default(); + let cases = [ + ("--kv-disk-dir", "cache"), + ("--kv-disk-space-mb", "64"), + ("--kv-cache-min-tokens", "1024"), + ("--kv-cache-cold-max-tokens", "4096"), + ("--kv-cache-continued-interval-tokens", "2048"), + ("--kv-cache-boundary-trim-tokens", "16"), + ("--kv-cache-boundary-align-tokens", "512"), + ]; + for (option, value) in cases { + let mut values = [value.to_string()].into_iter(); + assert!(kv.parse_arg(option, &mut values).unwrap()); + } + let mut empty = std::iter::empty(); + assert!(kv + .parse_arg("--kv-cache-reject-different-quant", &mut empty) + .unwrap()); + assert_eq!(kv.dir.as_deref(), Some(std::path::Path::new("cache"))); + assert_eq!(kv.space_mb, 64); + assert_eq!(kv.options.min_tokens, 1024); + assert_eq!(kv.options.cold_max_tokens, 4096); + assert_eq!(kv.options.continued_interval_tokens, 2048); + assert_eq!(kv.options.boundary_trim_tokens, 16); + assert_eq!(kv.options.boundary_align_tokens, 512); + assert!(kv.reject_different_quant); + } + + #[test] + fn cold_policy_allows_zero_and_validates_against_minimum() { + let mut kv = DiskKvArgs::default(); + for option in [ + "--kv-cache-cold-max-tokens", + "--kv-cache-continued-interval-tokens", + "--kv-cache-boundary-trim-tokens", + "--kv-cache-boundary-align-tokens", + ] { + let mut zero = ["0".to_string()].into_iter(); + assert!(kv.parse_arg(option, &mut zero).unwrap()); + } + assert_eq!(kv.options.cold_max_tokens, 0); + assert_eq!(kv.options.continued_interval_tokens, 0); + assert_eq!(kv.options.boundary_trim_tokens, 0); + assert_eq!(kv.options.boundary_align_tokens, 0); + kv.validate().unwrap(); + + let mut enabled = DiskKvArgs::default(); + enabled.options.min_tokens = 1024; + enabled.options.cold_max_tokens = 512; + assert_eq!( + enabled.validate().unwrap_err(), + "ds4-server-rs: --kv-cache-cold-max-tokens must be 0 or >= --kv-cache-min-tokens" + ); + } + + #[test] + fn positive_values_and_int_max_are_accepted() { + let mut kv = DiskKvArgs::default(); + kv.set_space_mb("2147483647").unwrap(); + kv.set_min_tokens("2147483647").unwrap(); + for option in [ + "--kv-cache-cold-max-tokens", + "--kv-cache-continued-interval-tokens", + "--kv-cache-boundary-trim-tokens", + "--kv-cache-boundary-align-tokens", + ] { + let mut max = ["2147483647".to_string()].into_iter(); + assert!(kv.parse_arg(option, &mut max).unwrap()); + } + assert_eq!(kv.space_mb, i32::MAX as u64); + assert_eq!(kv.options.min_tokens, i32::MAX); + assert_eq!(kv.options.cold_max_tokens, i32::MAX); + assert_eq!(kv.options.continued_interval_tokens, i32::MAX); + assert_eq!(kv.options.boundary_trim_tokens, i32::MAX); + assert_eq!(kv.options.boundary_align_tokens, i32::MAX); + } + + #[test] + fn invalid_numeric_values_match_the_c_contract() { + for value in ["", "0", "-1", "junk", "2147483648"] { + let mut kv = DiskKvArgs::default(); + assert_eq!( + kv.set_space_mb(value).unwrap_err(), + format!("ds4-server-rs: invalid value for --kv-disk-space-mb: {value}") + ); + assert_eq!( + kv.set_min_tokens(value).unwrap_err(), + format!("ds4-server-rs: invalid value for --kv-cache-min-tokens: {value}") + ); + } + for option in [ + "--kv-cache-cold-max-tokens", + "--kv-cache-continued-interval-tokens", + "--kv-cache-boundary-trim-tokens", + "--kv-cache-boundary-align-tokens", + ] { + for value in ["", "-1", "junk", "2147483648"] { + let mut kv = DiskKvArgs::default(); + let mut values = [value.to_string()].into_iter(); + assert_eq!( + kv.parse_arg(option, &mut values).unwrap_err(), + format!("ds4-server-rs: invalid value for {option}: {value}") + ); + } + } + } + + #[test] + fn missing_value_is_reported() { + let mut missing = DiskKvArgs::default(); + for option in [ + "--kv-disk-dir", + "--kv-cache-cold-max-tokens", + "--kv-cache-continued-interval-tokens", + "--kv-cache-boundary-trim-tokens", + "--kv-cache-boundary-align-tokens", + ] { + let mut empty = std::iter::empty(); + assert_eq!( + missing.parse_arg(option, &mut empty).unwrap_err(), + format!("ds4-server-rs: missing value for {option}") + ); + } + } + + #[test] + fn omitted_budget_opens_with_the_c_default() { + let dir = temp_path("default-budget"); + let _ = fs::remove_dir_all(&dir); + let mut kv = DiskKvArgs::default(); + kv.dir = Some(dir.clone()); + + let store = kv.open().unwrap(); + + assert_eq!(store.budget_bytes, 4096 * 1024 * 1024); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn directory_open_failure_is_returned_for_nonfatal_disable() { + let parent = temp_path("open-failure"); + let _ = fs::remove_file(&parent); + let _ = fs::remove_dir_all(&parent); + fs::write(&parent, b"not a directory").unwrap(); + let mut kv = DiskKvArgs::default(); + kv.dir = Some(parent.join("child")); + + assert!(kv.open().is_none()); + + let _ = fs::remove_file(parent); + } + + #[test] + fn disk_space_alias_accepts_gib() { + let mut kv = DiskKvArgs::default(); + assert!(kv.dir().is_none()); + let min_tokens = kv.min_tokens(); + let mut values = ["32G".to_string()].into_iter(); + assert!(kv.parse_arg("--kv-disk-space", &mut values).unwrap()); + assert_eq!(kv.space_mb(), 32 * 1024); + assert_eq!(kv.min_tokens(), min_tokens); + } + + #[test] + fn server_bin_parses_serving_flags() { + let src = include_str!("bin/ds4-server-rs.rs"); + for flag in [ + "--prefix-reuse", + "--mtp-mode", + "--max-seqs", + "--print-plan", + "--check-config", + "--prefill-chunk", + "--prefill-chunk-live", + "--mem-floor-gb", + "--kv-disk-space", + ] { + assert!(src.contains(flag), "ds4-server-rs must parse {flag}"); + } + } +} diff --git a/crates/ds4-server/src/lib.rs b/crates/ds4-server/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..061236612aacd8720776da3c9ab9a1b87dfbbb33 --- /dev/null +++ b/crates/ds4-server/src/lib.rs @@ -0,0 +1,161 @@ +//! Server host. Phase 7 ports surfaces by feature; no API redesign. + +pub mod admit; +pub mod cache_identity; +pub mod chat_input; +pub mod cont; +pub mod dist_cli; +pub mod dsml; +pub mod error; +pub mod expected_plan; +pub mod format; +pub mod generate; +pub mod http; +pub mod json; +pub mod kv_cli; +pub mod metrics; +pub mod models; +pub mod parse; +pub mod render; +pub mod retry; +pub mod route; +pub mod serve; +#[cfg(test)] +mod serve_503_body_test; +pub mod serve_cont; +#[cfg(any(feature = "native", test))] +mod serve_cont_evict; +mod serve_cont_prefill; +#[cfg(test)] +mod serve_cont_prefill_test; +mod serve_cont_roll; +mod serve_serial_fit; +mod serve_serial_reclaim; +#[cfg(test)] +mod serve_serial_reclaim_test; +pub mod serve_static; +pub mod stream; +#[cfg(any(feature = "native", test))] +mod tool_memory; +#[cfg(any(feature = "native", test))] +pub use tool_memory::{ExactReplay, ToolMemoryConfig}; +pub mod tool_stream; +pub mod tools; +pub mod trace; +#[cfg(any(feature = "native", test))] +mod v41_disk; +pub mod worker; +#[cfg(feature = "native")] +pub mod worker_run; + +pub use admit::{ + enqueue, enqueue_release, enqueue_shed_error, mint_job_id, next_job_id, preparse_shed, + queue_unlink_head, AdmitState, EnqVerdict, SHED_CLIENTS, SHED_CONT_HOLD, SHED_NAMES, + SHED_QUEUE_AGE, SHED_QUEUE_BYTES, SHED_QUEUE_DEPTH, SHED_REASONS, SHED_SLOW_READER, +}; +pub use cont::{ + dump_script, live_tool_result_ids, place_bank_continuation, BankContConflict, ContOwner, + ContRecord, ContRegistry, ContState, CONT_GRACE_S, CONT_HOLD_SHED_S, CONT_PIN_DEADLINE_S, + CONT_REGISTRY_MAX_DEFAULT, CONT_TTL_S, +}; +pub use dist_cli::DistArgs; +pub use dsml::{ + agent_turn_reasoning_cap, dsml_decode_state_for_text, dump_script as dsml_dump_script, + DsmlDecodeState, DsmlDecodeTracker, SampleOverride, SamplePolicy, ToolSyntax, + SAMPLE_OVERRIDE_GREEDY, SAMPLE_OVERRIDE_NONE, +}; +pub use error::{ + anthropic_error_body, anthropic_error_type, http_head, http_reason, http_response_bytes, + openai_error_body, openai_error_type, retry_after_header, wire_error_body, + wire_http_error_bytes, +}; +pub use format::{ + output_format_type_supported, parse_output_config_effort, parse_output_config_format, + parse_output_format_value, parse_reasoning_effort_value, parse_responses_text_value, +}; +#[cfg(feature = "native")] +pub use generate::NativeDecode; +#[cfg(feature = "native")] +pub use generate::V41ServeRoute; +pub use generate::{ + generate_and_write, generation_blocked, render_prompt, stop_list_find_from, DecodeIo, + GenerateError, GenerateOutcome, ScriptedDecode, ScriptedStep, +}; +pub use http::{ + chunked_enabled, content_length, header_accepts_json, header_chunked, header_end, + parse_surface_for_path, read_http_request, shed_surface_for_path, HttpRequest, +}; +pub use json::{ + json_args_parse, json_bool, json_content, json_escape, json_escape_bytes, json_int, + json_minify_raw_value, json_number, json_raw_value, json_skip_value, json_string, Json, + JsonArg, +}; +pub use metrics::{ + dump_memgov_names, gov_modes_from_env, render_memgov_metrics, render_metrics, + render_metrics_fragment, render_metrics_prefix, render_metrics_runtime, render_stats_json, + render_stats_json_ex, render_stats_json_plan, render_stats_memgov_json, MemgovSnap, + ReconcileSnap, RouteMetrics, RuntimeMetrics, GOV_CMP_NAMES, GOV_CONSUMER_NAMES, + GOV_STATUS_NAMES, MEM_CLASS_NAMES, MEM_DOMAIN_NAMES, RECLAIM_STATUS_NAMES, REJECT_REASON_NAMES, + REJLANE_NAMES, ROUTE_LANE_NAMES, ROUTE_SURFACE_NAMES, THINK_MODE_NAMES, +}; +pub use models::{ + append_model_json_values, json_models_array_dup, model_alias_disables_thinking, + model_alias_enables_thinking, model_alias_known, model_id_from_gguf_path, model_id_known, + model_one_json, models_list_json, +}; +pub use parse::{ + default_temperature, parse_anthropic_request, parse_chat_request, parse_completion_request, + parse_request, parse_responses_request, ChatMsg, ChatPart, ImageMime, ParseEnv, ParsedRequest, + RequestImage, ToolCall, ToolChoice, ToolSchemaOrder, DEFAULT_MIN_P, DEFAULT_TEMPERATURE, + DEFAULT_TOP_P, +}; +pub use render::{ + append_tool_result_text, render_chat, render_chat_choice, render_dots3_chat, render_dsml_chat, + render_dsml_chat_choice, render_dsml_chat_choice_ex, render_exaone_chat, render_live_tool_tail, + render_motif3_chat, render_motif3_chat_ex, render_qwen_chat_ex, render_solar_chat, + render_solar_chat_ex, responses_visible_assistant_suffix, role_is_system, role_is_user_like, + syntax_for_model_id, think_effort_prefix, tool_start_marker, DsmlHead, ModelSyntax, + RenderError, DSML_ASSISTANT, DSML_BOS, DSML_EOS, DSML_SYSTEM_TOKEN, DSML_THINK_END, + DSML_THINK_START, DSML_USER, THINK_HIGH_PREFIX, THINK_MAX_PREFIX, V41_THINK_HIGH_PREFIX, + V41_THINK_MAX_PREFIX, +}; +pub use retry::dump_script as retry_dump_script; +pub use route::{ + compute_needs, decode_budget, mode_budget, parse_reasoning_effort_name, route_decide, + think_mode_enabled, think_mode_from_enabled, wire_surface_for, Api, NeedInput, ReqKind, + RequestedEffort, RouteDecision, RouteEnv, ThinkMode, WireSurface, LANE_CONTINUOUS, LANE_NONE, + LANE_SERIAL, LANE_STATIC, NEED_BANK_FRONTIER, NEED_CONTINUATION_PUBLISH, + NEED_CORRECTIVE_RECOVERY, NEED_DURABLE_RESPONSE, NEED_IMAGE, NEED_LIVE_FRONTIER, + NEED_PER_ROW_SAMPLING, NEED_PREFILL_ONLY, NEED_STOP_SCAN, NEED_STREAMING, NEED_THINKING, + NEED_TOKEN_IDS, NEED_TOOL_SCAN, REASON_COALESCE_OFF, REASON_CONT, REASON_CONT_BANK, + REASON_CONT_UNAVAILABLE, REASON_NAMES, REASON_NEED_CONTINUATION_PUBLISH, + REASON_NEED_CORRECTIVE_RECOVERY, REASON_NEED_DURABLE, REASON_NEED_LIVE_FRONTIER, + REASON_NEED_PREFILL_ONLY, REASON_STATIC_NO_CONT, REASON_STATIC_PROMPT_BOUNDS, REASON_SURFACE, + REASON_TOKEN_IDS_PROJECTION, REASON_TOOLS_COMPLETION, +}; +pub use serve::{ + accept_loop, accept_loop_with_engine, accept_loop_with_engine_cont, handle_client, + handle_client_inner, listen, listen_if_allowed, ServerConfig, ServerInner, +}; +#[cfg(feature = "native")] +pub use serve_cont::ContLane; +pub use serve_cont::{cont_prompt_tokens, ContExec, ContStep, ContStepper}; +pub use stream::{ + anthropic_final_response, anthropic_sse_finish_live, anthropic_sse_start_live, + anthropic_sse_stream_update, append_tool_call_deltas_json, append_tool_calls_json, + final_response, openai_sse_finish_live, openai_sse_stream_update, openai_stream_start, + project_anthropic_thinking, project_openai_chat_thinking, project_openai_chat_utf8, + project_openai_completion, project_responses_thinking, responses_final_response, + responses_sse_created, responses_sse_finish_live, responses_sse_stream_update, + responses_stream_init, sse_chunk, sse_done, sse_headers, think_end, think_start, unix_now, + utf8_stream_safe_len, ChatFormat, ReqTimings, StreamReq, ThinkBlock, Writer, CREATED_TEST, + TAPE_PLAIN, TAPE_THINKING, TAPE_UTF8, TEST_MSG_ID, TEST_RESP_ID, TEST_RS_ID, +}; +pub use tool_stream::{dump_script as tool_stream_dump_script, DsmlToolState, DsmlToolStream}; +pub use tools::{ + assign_tool_ids, parse_dsml_tolerant, parse_generated_for_model_id, + parse_generated_for_response, parse_generated_message, ParsedGenerated, SemAccum, SemFeed, +}; +pub use worker::{dist_weight_slice, server_launch, ServerLaunch, WORKER_REQUIRES_MODEL}; +#[cfg(feature = "native")] +pub use worker_run::run_assembled_worker; diff --git a/crates/ds4-server/src/metrics.rs b/crates/ds4-server/src/metrics.rs new file mode 100644 index 0000000000000000000000000000000000000000..48e6f02466c5d42459f006ae288c930b9e3df0d2 --- /dev/null +++ b/crates/ds4-server/src/metrics.rs @@ -0,0 +1,971 @@ +//! Host-owned `/metrics` / `/v1/stats` porcelain, including the memgov +//! census format from `ds4_server.c` at v0.6.3-dfm. +//! +//! Live CUDA/census numbers stay native; this crate owns the wire text. +//! `census_supported = false` follows C: the cell family is absent, not zero. + +use crate::admit::{AdmitState, SHED_NAMES, SHED_REASONS}; +use crate::route::{ + ThinkMode, WireSurface, LANE_CONTINUOUS, LANE_SERIAL, LANE_STATIC, REASON_NAMES, +}; + +pub const ROUTE_SURFACES: usize = 4; +pub const ROUTE_LANES: usize = 3; +pub const ROUTE_REASONS: usize = 14; +pub const THINK_MODES: usize = 4; + +pub const ROUTE_SURFACE_NAMES: [&str; ROUTE_SURFACES] = [ + "openai_chat", + "openai_completion", + "anthropic_messages", + "openai_responses", +]; +pub const ROUTE_LANE_NAMES: [&str; ROUTE_LANES] = ["serial", "continuous", "static"]; +pub const THINK_MODE_NAMES: [&str; THINK_MODES] = ["none", "low", "high", "max"]; + +#[derive(Debug, Clone, Default)] +pub struct RouteMetrics { + pub route_requests: [[u64; ROUTE_LANES]; ROUTE_SURFACES], + pub route_decisions: [u64; ROUTE_REASONS], + pub shed: [u64; SHED_REASONS], + pub think: [u64; THINK_MODES], +} + +pub fn surface_index(surf: WireSurface) -> usize { + surf as usize +} + +pub fn lane_index(lane: u8) -> usize { + match lane { + LANE_SERIAL => 0, + LANE_CONTINUOUS => 1, + LANE_STATIC => 2, + _ => 0, + } +} + +impl RouteMetrics { + pub fn record_route(&mut self, surf: WireSurface, lane: u8, reason: u8, think: ThinkMode) { + let si = surface_index(surf); + let li = lane_index(lane); + self.route_requests[si][li] += 1; + if (reason as usize) < ROUTE_REASONS { + self.route_decisions[reason as usize] += 1; + } + let ti = think as usize; + if ti < THINK_MODES { + self.think[ti] += 1; + } + } + + /// C `route_metrics_record`: one lane ENTRY tick alone (no decision or + /// think tick). Lane-entry counters fire where a lane takes the job and + /// can fire more than once per request — C's `cont_admit` success ticks + /// continuous before the engine's funding verdict, and the stranded + /// fallback's `generate_job` entry then ticks serial as well. + pub fn record_lane_entry(&mut self, surf: WireSurface, lane: u8) { + self.route_requests[surface_index(surf)][lane_index(lane)] += 1; + } + + pub fn record_shed(&mut self, reason: u8) { + if (reason as usize) < SHED_REASONS { + self.shed[reason as usize] += 1; + } + } +} + +/// Host-owned `/metrics` fragment (route / shed / think / queue gauges). +pub fn render_metrics_fragment(m: &RouteMetrics, admit: &AdmitState) -> String { + let mut b = String::new(); + b.push_str("# TYPE ds4_route_requests_total counter\n"); + for si in 0..ROUTE_SURFACES { + for li in 0..ROUTE_LANES { + b.push_str(&format!( + "ds4_route_requests_total{{surface=\"{}\",lane=\"{}\"}} {}\n", + ROUTE_SURFACE_NAMES[si], ROUTE_LANE_NAMES[li], m.route_requests[si][li] + )); + } + } + b.push_str("# TYPE ds4_route_decisions_total counter\n"); + for ri in 0..ROUTE_REASONS { + b.push_str(&format!( + "ds4_route_decisions_total{{reason=\"{}\"}} {}\n", + REASON_NAMES[ri], m.route_decisions[ri] + )); + } + b.push_str("# TYPE ds4_requests_shed_total counter\n"); + for ri in 0..SHED_REASONS { + b.push_str(&format!( + "ds4_requests_shed_total{{reason=\"{}\"}} {}\n", + SHED_NAMES[ri], m.shed[ri] + )); + } + b.push_str("# TYPE ds4_requests_think_total counter\n"); + for ti in 0..THINK_MODES { + b.push_str(&format!( + "ds4_requests_think_total{{mode=\"{}\"}} {}\n", + THINK_MODE_NAMES[ti], m.think[ti] + )); + } + b.push_str(&format!( + "# TYPE ds4_clients_connected gauge\nds4_clients_connected {}\n\ + # TYPE ds4_queue_depth gauge\nds4_queue_depth {}\n\ + # TYPE ds4_inflight_body_bytes gauge\nds4_inflight_body_bytes {}\n", + admit.clients, admit.queued, admit.inflight_body_bytes + )); + b +} + +/// Host-owned `/v1/stats` JSON: route observation plus C memory/governor siblings. +pub fn render_stats_json(m: &RouteMetrics, admit: &AdmitState) -> String { + render_stats_json_ex(m, admit, &RuntimeMetrics::default()) +} + +pub fn render_stats_json_ex(m: &RouteMetrics, admit: &AdmitState, rt: &RuntimeMetrics) -> String { + render_stats_json_plan(m, admit, rt, None, None) +} + +/// GET /v1/stats body; plan and last_request are omitted when absent. +pub fn render_stats_json_plan( + m: &RouteMetrics, + admit: &AdmitState, + rt: &RuntimeMetrics, + serving: Option<&str>, + last_request: Option<&str>, +) -> String { + let mut b = render_stats_observation(m, admit); + debug_assert!(b.ends_with("}\n")); + b.truncate(b.len() - 2); + b.push(','); + b.push_str(&render_stats_memgov_json(&rt.memgov)); + if let Some(serving) = serving { + b.push_str(",\"serving\":"); + b.push_str(serving); + } + if let Some(last_request) = last_request { + b.push_str(",\"last_request\":"); + b.push_str(last_request); + } + b.push_str("}\n"); + b +} + +fn render_stats_observation(m: &RouteMetrics, admit: &AdmitState) -> String { + let mut b = String::from("{\"routes\":{"); + let mut first = true; + for si in 0..ROUTE_SURFACES { + for li in 0..ROUTE_LANES { + if !first { + b.push(','); + } + first = false; + b.push_str(&format!( + "\"{}_{}\":{}", + ROUTE_SURFACE_NAMES[si], ROUTE_LANE_NAMES[li], m.route_requests[si][li] + )); + } + } + b.push_str("},\"route_decisions\":{"); + for ri in 0..ROUTE_REASONS { + if ri > 0 { + b.push(','); + } + b.push_str(&format!( + "\"{}\":{}", + REASON_NAMES[ri], m.route_decisions[ri] + )); + } + b.push_str("},\"think_modes\":{"); + for ti in 0..THINK_MODES { + if ti > 0 { + b.push(','); + } + b.push_str(&format!("\"{}\":{}", THINK_MODE_NAMES[ti], m.think[ti])); + } + b.push_str("},\"sheds\":{"); + for ri in 0..SHED_REASONS { + if ri > 0 { + b.push(','); + } + b.push_str(&format!("\"{}\":{}", SHED_NAMES[ri], m.shed[ri])); + } + b.push_str(&format!( + "}},\"clients\":{},\"queue_depth\":{},\"inflight_body_bytes\":{}}}\n", + admit.clients, admit.queued, admit.inflight_body_bytes + )); + b +} + +pub const MEM_CLASS_NAMES: [&str; 17] = [ + "engine_other", + "weight_arena", + "weight_span", + "weight_whole", + "weight_derived", + "weight_import", + "weight_artifact", + "weight_host_pin", + "stage_pin", + "scalars_mirror", + "session_tensors", + "kv_primary", + "batch_bank", + "scratch_sticky", + "kernel_partials", + "graph_exec", + "diag", +]; +pub const MEM_DOMAIN_NAMES: [&str; 2] = ["unified_device", "pinned_host"]; +pub const MEM_OBS_STATUS_NAMES: [&str; 3] = ["ok", "unsupported", "query_error"]; +pub const MEM_OBS_SOURCE_NAMES: [&str; 3] = ["none", "cuda_free", "meminfo_available"]; +pub const GOV_CONSUMER_NAMES: [&str; 5] = [ + "engine_boot", + "prewarm", + "batch_bank_plan", + "serial_session", + "static_batch", +]; +pub const GOV_STATUS_NAMES: [&str; 6] = [ + "admit", + "refuse_class", + "refuse_live", + "retry_obs", + "unsupported", + "fault", +]; +pub const GOV_CMP_NAMES: [&str; 6] = [ + "agree", + "live_stricter", + "shadow_stricter", + "verdict_class", + "obs_policy", + "fault", +]; +pub const REJLANE_NAMES: [&str; 3] = ["continuous", "serial", "static"]; +pub const REJECT_REASON_NAMES: [&str; 7] = [ + "class_budget", + "live_headroom", + "obs_retry", + "unsupported", + "fault", + "deep_policy", + "lane_disabled", +]; +pub const RECLAIM_STATUS_NAMES: [&str; 6] = [ + "ok", + "partial", + "stale_plan", + "busy", + "unsupported", + "device_error", +]; +pub const RESUNIT_STATE_NAMES: [&str; 8] = [ + "populated", + "already_ready", + "satisfied_import", + "host_mapped_by_policy", + "lazy_deferred", + "expert_cold_by_policy", + "waived_optional", + "failed", +]; +pub const RESSTAGE_NAMES: [&str; 2] = ["boot", "lazy"]; +pub const RESIDENCY_POLICY_NAMES: [&str; 4] = ["eager", "lazy", "mapped", "unattributed"]; +pub const MODEL_ROLE_NAMES: [&str; 3] = ["primary", "auxiliary", "drafter"]; + +#[derive(Debug, Clone, Copy, Default)] +pub struct MemCell { + pub requested: u64, + pub committed: u64, + pub freed_requested: u64, + pub freed_committed: u64, +} + +impl MemCell { + pub fn live_requested(self) -> u64 { + self.requested.wrapping_sub(self.freed_requested) + } + pub fn live(self) -> u64 { + self.committed.wrapping_sub(self.freed_committed) + } + pub fn slack(self) -> u64 { + let com = self.live(); + let req = self.live_requested(); + if com > req { + com - req + } else { + 0 + } + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct GovLease { + pub intent: u64, + pub resident: u64, + pub reservation: u64, +} + +#[derive(Debug, Clone, Copy)] +pub struct ReconcileSnap { + pub residual: i64, + pub onetime: u64, +} + +#[derive(Debug, Clone)] +pub struct MemgovSnap { + pub census_supported: bool, + pub cells: [[MemCell; 2]; 17], + pub census_faults: u64, + pub torn_fallbacks: u64, + pub census_epoch: u64, + pub substrate_outstanding: u64, + pub emit_substrate: bool, + pub obs_status: u8, + pub obs_source: u8, + pub obs_free: u64, + pub obs_total: u64, + pub obs_cuda_free: u64, + pub obs_meminfo: u64, + pub memobs_calls: [u64; 3], + pub memobs_errors: u64, + pub own_trim_calls: u64, + pub own_trim_recovered: u64, + pub reconcile: Option, + pub reconcile_flagged: u64, + pub gov_epoch: u64, + pub leases: [GovLease; 5], + pub decisions: [[[u64; 6]; 6]; 5], + pub deficit: [u64; 5], + pub faults: u64, + pub gov_modes: [&'static str; 5], + pub residency_supported: bool, + pub residency_units: [[u64; 8]; 4], + pub residency_failures: [[u64; 2]; 3], +} + +impl Default for MemgovSnap { + fn default() -> Self { + Self { + census_supported: false, + cells: [[MemCell::default(); 2]; 17], + census_faults: 0, + torn_fallbacks: 0, + census_epoch: 0, + substrate_outstanding: 0, + emit_substrate: true, + obs_status: 1, + obs_source: 0, + obs_free: 0, + obs_total: 0, + obs_cuda_free: 0, + obs_meminfo: 0, + memobs_calls: [0; 3], + memobs_errors: 0, + own_trim_calls: 0, + own_trim_recovered: 0, + reconcile: None, + reconcile_flagged: 0, + gov_epoch: 0, + leases: [GovLease::default(); 5], + decisions: [[[0; 6]; 6]; 5], + deficit: [0; 5], + faults: 0, + gov_modes: ["enforce"; 5], + residency_supported: false, + residency_units: [[0; 8]; 4], + residency_failures: [[0; 2]; 3], + } + } +} + +pub fn parse_gov_mode(s: &str) -> Option<&'static str> { + match s { + "off" => Some("off"), + "observe" => Some("observe"), + "enforce" => Some("enforce"), + _ => None, + } +} + +/// C `ds4_gov_modes_init` name board (defaults enforce; `DS4_MEMGOV` then per-family knobs). +pub fn gov_modes_from_env() -> [&'static str; 5] { + const KNOBS: [&str; 5] = [ + "DS4_MEMGOV_BOOT", + "DS4_MEMGOV_PREWARM", + "DS4_MEMGOV_BANK", + "DS4_MEMGOV_SERIAL", + "DS4_MEMGOV_STATIC", + ]; + let gdef = std::env::var("DS4_MEMGOV") + .ok() + .filter(|v| !v.is_empty()) + .and_then(|v| parse_gov_mode(&v)); + let mut out = ["enforce"; 5]; + for (i, knob) in KNOBS.iter().enumerate() { + let mut m = gdef.unwrap_or("enforce"); + if let Ok(v) = std::env::var(knob) { + if !v.is_empty() { + if let Some(pm) = parse_gov_mode(&v) { + m = pm; + } + } + } + out[i] = m; + } + out +} + +#[derive(Debug, Clone, Default)] +pub struct RuntimeMetrics { + pub uptime_seconds: u64, + pub requests_started: u64, + pub requests_completed: u64, + pub requests_failed: u64, + pub requests_canceled: u64, + pub requests_refused_deep_serial: u64, + pub requests_inflight: u64, + pub requests_serial: u64, + pub out_backlog_bytes: u64, + pub cont_admit_rejects: u64, + pub cont_credit_ext_granted: u64, + pub cont_credit_ext_refused: u64, + pub have_batch_ctx: bool, + pub cont_commit_observed: u64, + pub cont_commit_packed: u64, + pub cont_admit_band_x1024: u32, + pub serial_idle_reaps: u64, + pub cont_batch_failures: u64, + pub graph_fit_refusals: u64, + pub rejected: [[u64; 7]; 3], + pub reclaim_banks: [u64; 6], + pub reclaim_bytes: [u64; 6], + pub admits_cold: u64, + pub admits_warm: u64, + pub admits_fork: u64, + pub admits_partial_fork: u64, + pub admits_partial_truncate: u64, + pub tokens_prefilled_computed: u64, + pub tokens_prefilled_cached: u64, + pub tokens_decoded: u64, + pub decode_steps: u64, + pub spec_drafts: u64, + pub spec_hits: u64, + pub spec_quench: u64, + pub decode_tok_s: f64, + pub prefill_tok_s: f64, + pub tok_per_step: f64, + pub banks_live: u64, + pub banks_total: u64, + pub warm_records: u64, + pub kv_pages_resident: u64, + pub cont_ontoken_ns: u64, + pub cont_ontoken_tokens: u64, + pub batch_genmu_wait_ns: u64, + pub batch_genmu_waits: u64, + pub creg_published: u64, + pub creg_resolved: u64, + pub creg_missed: u64, + pub creg_demoted: u64, + pub creg_records_live: u64, + pub artifact_source: u64, + pub derived_artifacts: u64, + pub derived_artifact_bytes: u64, + pub memgov: MemgovSnap, +} + +pub fn artifact_source_name(source: u64) -> &'static str { + match source { + 3 => "mixed", + 2 => "built", + 1 => "imported", + _ => "none", + } +} + +fn spec_accept_ratio(drafts: u64, hits: u64) -> f64 { + if drafts == 0 { + 0.0 + } else { + hits as f64 / drafts as f64 + } +} + +/// C `/metrics` lines before the route/shed/think fragment. +pub fn render_metrics_prefix(rt: &RuntimeMetrics) -> String { + format!( + "# TYPE ds4_uptime_seconds gauge\n\ + ds4_uptime_seconds {}\n\ + # TYPE ds4_requests_started_total counter\n\ + ds4_requests_started_total {}\n\ + # TYPE ds4_requests_total counter\n\ + ds4_requests_total{{outcome=\"completed\"}} {}\n\ + ds4_requests_total{{outcome=\"failed\"}} {}\n\ + ds4_requests_total{{outcome=\"canceled\"}} {}\n\ + ds4_requests_total{{outcome=\"refused_deep_serial\"}} {}\n\ + # TYPE ds4_requests_inflight gauge\n\ + ds4_requests_inflight {}\n\ + # TYPE ds4_requests_serial_total counter\n\ + ds4_requests_serial_total {}\n", + rt.uptime_seconds, + rt.requests_started, + rt.requests_completed, + rt.requests_failed, + rt.requests_canceled, + rt.requests_refused_deep_serial, + rt.requests_inflight, + rt.requests_serial + ) +} + +/// C `/metrics` lines after the queue gauges, before memgov census. +pub fn render_metrics_runtime(rt: &RuntimeMetrics) -> String { + let mut b = String::new(); + b.push_str(&format!( + "# TYPE ds4_out_backlog_bytes gauge\n\ + ds4_out_backlog_bytes {}\n\ + # TYPE ds4_cont_admit_rejects_total counter\n\ + ds4_cont_admit_rejects_total {}\n\ + # TYPE ds4_cont_credit_extension_granted_total counter\n\ + ds4_cont_credit_extension_granted_total {}\n\ + # TYPE ds4_cont_credit_extension_refused_total counter\n\ + ds4_cont_credit_extension_refused_total {}\n", + rt.out_backlog_bytes, + rt.cont_admit_rejects, + rt.cont_credit_ext_granted, + rt.cont_credit_ext_refused + )); + if rt.have_batch_ctx { + b.push_str(&format!( + "# TYPE ds4_cont_commit_bytes_per_token gauge\n\ + ds4_cont_commit_bytes_per_token{{kind=\"observed\"}} {}\n\ + ds4_cont_commit_bytes_per_token{{kind=\"packed\"}} {}\n\ + # TYPE ds4_cont_admit_band_x1024 gauge\n\ + ds4_cont_admit_band_x1024 {}\n", + rt.cont_commit_observed, rt.cont_commit_packed, rt.cont_admit_band_x1024 + )); + } + b.push_str(&format!( + "# TYPE ds4_serial_idle_reaps_total counter\n\ + ds4_serial_idle_reaps_total {}\n\ + # TYPE ds4_cont_batch_failures_total counter\n\ + ds4_cont_batch_failures_total {}\n\ + # TYPE ds4_graph_fit_refusals_total counter\n\ + ds4_graph_fit_refusals_total {}\n", + rt.serial_idle_reaps, rt.cont_batch_failures, rt.graph_fit_refusals + )); + b.push_str("# TYPE ds4_requests_rejected_total counter\n"); + for (ln, lane) in REJLANE_NAMES.iter().enumerate() { + for (rr, reason) in REJECT_REASON_NAMES.iter().enumerate() { + b.push_str(&format!( + "ds4_requests_rejected_total{{lane=\"{lane}\",reason=\"{reason}\"}} {}\n", + rt.rejected[ln][rr] + )); + } + } + b.push_str("# TYPE ds4_reclaim_banks_total counter\n"); + for (rs, name) in RECLAIM_STATUS_NAMES.iter().enumerate() { + b.push_str(&format!( + "ds4_reclaim_banks_total{{result=\"{name}\"}} {}\n", + rt.reclaim_banks[rs] + )); + } + b.push_str("# TYPE ds4_reclaim_bytes_total counter\n"); + for (rs, name) in RECLAIM_STATUS_NAMES.iter().enumerate() { + b.push_str(&format!( + "ds4_reclaim_bytes_total{{result=\"{name}\"}} {}\n", + rt.reclaim_bytes[rs] + )); + } + let ratio = spec_accept_ratio(rt.spec_drafts, rt.spec_hits); + b.push_str(&format!( + "# TYPE ds4_admits_total counter\n\ + ds4_admits_total{{kind=\"cold\"}} {}\n\ + ds4_admits_total{{kind=\"warm\"}} {}\n\ + ds4_admits_total{{kind=\"fork\"}} {}\n\ + ds4_admits_total{{kind=\"partial_fork\"}} {}\n\ + ds4_admits_total{{kind=\"partial_truncate\"}} {}\n\ + # TYPE ds4_tokens_prefilled_total counter\n\ + ds4_tokens_prefilled_total{{kind=\"computed\"}} {}\n\ + ds4_tokens_prefilled_total{{kind=\"cached\"}} {}\n\ + # TYPE ds4_tokens_decoded_total counter\n\ + ds4_tokens_decoded_total {}\n\ + # TYPE ds4_decode_steps_total counter\n\ + ds4_decode_steps_total {}\n\ + # TYPE ds4_spec_drafts_total counter\n\ + ds4_spec_drafts_total {}\n\ + # TYPE ds4_spec_hits_total counter\n\ + ds4_spec_hits_total {}\n\ + # TYPE ds4_spec_quench_total counter\n\ + ds4_spec_quench_total {}\n\ + # TYPE ds4_spec_accept_ratio gauge\n\ + ds4_spec_accept_ratio {:.4}\n\ + # TYPE ds4_decode_tok_s gauge\n\ + ds4_decode_tok_s {:.2}\n\ + # TYPE ds4_prefill_tok_s gauge\n\ + ds4_prefill_tok_s {:.2}\n\ + # TYPE ds4_tok_per_step gauge\n\ + ds4_tok_per_step {:.3}\n\ + # TYPE ds4_banks_live gauge\n\ + ds4_banks_live {}\n\ + # TYPE ds4_banks_total gauge\n\ + ds4_banks_total {}\n\ + # TYPE ds4_warm_records gauge\n\ + ds4_warm_records {}\n\ + # TYPE ds4_kv_pages_resident gauge\n\ + ds4_kv_pages_resident {}\n\ + # TYPE ds4_cont_ontoken_ns_total counter\n\ + ds4_cont_ontoken_ns_total {}\n\ + # TYPE ds4_cont_ontoken_tokens_total counter\n\ + ds4_cont_ontoken_tokens_total {}\n\ + # TYPE ds4_batch_genmu_wait_ns_total counter\n\ + ds4_batch_genmu_wait_ns_total {}\n\ + # TYPE ds4_batch_genmu_waits_total counter\n\ + ds4_batch_genmu_waits_total {}\n\ + # TYPE ds4_continuation_records_published_total counter\n\ + ds4_continuation_records_published_total {}\n\ + # TYPE ds4_continuation_resolved_total counter\n\ + ds4_continuation_resolved_total {}\n\ + # TYPE ds4_continuation_missed_total counter\n\ + ds4_continuation_missed_total {}\n\ + # TYPE ds4_continuation_demoted_total counter\n\ + ds4_continuation_demoted_total {}\n\ + # TYPE ds4_continuation_records_live gauge\n\ + ds4_continuation_records_live {}\n\ + # TYPE ds4_derived_artifacts gauge\n\ + ds4_derived_artifacts{{source=\"{}\"}} {}\n\ + # TYPE ds4_derived_artifact_bytes gauge\n\ + ds4_derived_artifact_bytes {}\n", + rt.admits_cold, + rt.admits_warm, + rt.admits_fork, + rt.admits_partial_fork, + rt.admits_partial_truncate, + rt.tokens_prefilled_computed, + rt.tokens_prefilled_cached, + rt.tokens_decoded, + rt.decode_steps, + rt.spec_drafts, + rt.spec_hits, + rt.spec_quench, + ratio, + rt.decode_tok_s, + rt.prefill_tok_s, + rt.tok_per_step, + rt.banks_live, + rt.banks_total, + rt.warm_records, + rt.kv_pages_resident, + rt.cont_ontoken_ns, + rt.cont_ontoken_tokens, + rt.batch_genmu_wait_ns, + rt.batch_genmu_waits, + rt.creg_published, + rt.creg_resolved, + rt.creg_missed, + rt.creg_demoted, + rt.creg_records_live, + artifact_source_name(rt.artifact_source), + rt.derived_artifacts, + rt.derived_artifact_bytes + )); + b +} + +/// C memgov D0a/D0b/D5 `/metrics` family. +pub fn render_memgov_metrics(g: &MemgovSnap) -> String { + let mut b = String::new(); + b.push_str(&format!( + "# TYPE ds4_memory_census_supported gauge\n\ + ds4_memory_census_supported {}\n", + u32::from(g.census_supported) + )); + if g.census_supported { + b.push_str("# TYPE ds4_memory_bytes gauge\n"); + for (d, domain) in MEM_DOMAIN_NAMES.iter().enumerate() { + for (c, class) in MEM_CLASS_NAMES.iter().enumerate() { + let cell = g.cells[c][d]; + b.push_str(&format!( + "ds4_memory_bytes{{domain=\"{domain}\",class=\"{class}\",state=\"requested\"}} {}\n\ + ds4_memory_bytes{{domain=\"{domain}\",class=\"{class}\",state=\"allocated\"}} {}\n\ + ds4_memory_bytes{{domain=\"{domain}\",class=\"{class}\",state=\"slack\"}} {}\n", + cell.live_requested(), + cell.live(), + cell.slack() + )); + } + } + b.push_str(&format!( + "# TYPE ds4_memory_census_faults_total counter\n\ + ds4_memory_census_faults_total {}\n", + g.census_faults + )); + } + b.push_str(&format!( + "# TYPE ds4_memory_census_torn_fallbacks_total counter\n\ + ds4_memory_census_torn_fallbacks_total {}\n\ + # TYPE ds4_memory_census_epoch gauge\n\ + ds4_memory_census_epoch {}\n", + g.torn_fallbacks, g.census_epoch + )); + if g.emit_substrate { + b.push_str(&format!( + "# TYPE ds4_memory_substrate_outstanding_bytes gauge\n\ + ds4_memory_substrate_outstanding_bytes {}\n", + g.substrate_outstanding + )); + } + b.push_str("# TYPE ds4_memory_observation_info gauge\n"); + for (st, status) in MEM_OBS_STATUS_NAMES.iter().enumerate() { + for (so, source) in MEM_OBS_SOURCE_NAMES.iter().enumerate() { + let one = i32::from(g.obs_status as usize == st && g.obs_source as usize == so); + b.push_str(&format!( + "ds4_memory_observation_info{{status=\"{status}\",source=\"{source}\"}} {one}\n" + )); + } + } + if g.obs_status == 0 { + b.push_str(&format!( + "# TYPE ds4_memory_observation_bytes gauge\n\ + ds4_memory_observation_bytes{{kind=\"free\"}} {}\n\ + ds4_memory_observation_bytes{{kind=\"total\"}} {}\n\ + ds4_memory_observation_bytes{{kind=\"cuda_free\"}} {}\n\ + ds4_memory_observation_bytes{{kind=\"meminfo_available\"}} {}\n", + g.obs_free, g.obs_total, g.obs_cuda_free, g.obs_meminfo + )); + } + b.push_str("# TYPE ds4_memory_observation_calls_total counter\n"); + for (so, source) in MEM_OBS_SOURCE_NAMES.iter().enumerate() { + b.push_str(&format!( + "ds4_memory_observation_calls_total{{source=\"{source}\"}} {}\n", + g.memobs_calls[so] + )); + } + b.push_str(&format!( + "# TYPE ds4_memory_observation_errors_total counter\n\ + ds4_memory_observation_errors_total {}\n\ + # TYPE ds4_mem_own_trim_calls_total counter\n\ + ds4_mem_own_trim_calls_total {}\n\ + # TYPE ds4_mem_own_trim_recovered_bytes_total counter\n\ + ds4_mem_own_trim_recovered_bytes_total {}\n", + g.memobs_errors, g.own_trim_calls, g.own_trim_recovered + )); + if let Some(rr) = g.reconcile { + b.push_str(&format!( + "# TYPE ds4_mem_reconcile_residual_bytes gauge\n\ + ds4_mem_reconcile_residual_bytes {}\n\ + # TYPE ds4_mem_reconcile_onetime_bytes gauge\n\ + ds4_mem_reconcile_onetime_bytes {}\n", + rr.residual, rr.onetime + )); + } + b.push_str(&format!( + "# TYPE ds4_mem_reconcile_flagged_total counter\n\ + ds4_mem_reconcile_flagged_total {}\n\ + # TYPE ds4_memory_governor_epoch gauge\n\ + ds4_memory_governor_epoch {}\n", + g.reconcile_flagged, g.gov_epoch + )); + b.push_str("# TYPE ds4_memory_lease_bytes gauge\n"); + for (c, consumer) in GOV_CONSUMER_NAMES.iter().enumerate() { + let l = g.leases[c]; + b.push_str(&format!( + "ds4_memory_lease_bytes{{consumer=\"{consumer}\",field=\"intent\"}} {}\n\ + ds4_memory_lease_bytes{{consumer=\"{consumer}\",field=\"resident\"}} {}\n\ + ds4_memory_lease_bytes{{consumer=\"{consumer}\",field=\"reservation\"}} {}\n", + l.intent, l.resident, l.reservation + )); + } + b.push_str("# TYPE ds4_memory_decisions_total counter\n"); + for (c, consumer) in GOV_CONSUMER_NAMES.iter().enumerate() { + for (st, result) in GOV_STATUS_NAMES.iter().enumerate() { + for (r, reason) in GOV_CMP_NAMES.iter().enumerate() { + b.push_str(&format!( + "ds4_memory_decisions_total{{consumer=\"{consumer}\",result=\"{result}\",reason=\"{reason}\"}} {}\n", + g.decisions[c][st][r] + )); + } + } + } + b.push_str("# TYPE ds4_memory_decision_deficit_bytes gauge\n"); + for (c, consumer) in GOV_CONSUMER_NAMES.iter().enumerate() { + b.push_str(&format!( + "ds4_memory_decision_deficit_bytes{{consumer=\"{consumer}\"}} {}\n", + g.deficit[c] + )); + } + b.push_str(&format!( + "# TYPE ds4_memory_governor_faults_total counter\n\ + ds4_memory_governor_faults_total {}\n", + g.faults + )); + if g.residency_supported { + b.push_str("# TYPE ds4_residency_units counter\n"); + for (p, policy) in RESIDENCY_POLICY_NAMES.iter().enumerate() { + for (st, state) in RESUNIT_STATE_NAMES.iter().enumerate() { + b.push_str(&format!( + "ds4_residency_units{{policy=\"{policy}\",state=\"{state}\"}} {}\n", + g.residency_units[p][st] + )); + } + } + b.push_str("# TYPE ds4_residency_failures_total counter\n"); + for (r, role) in MODEL_ROLE_NAMES.iter().enumerate() { + for (sg, stage) in RESSTAGE_NAMES.iter().enumerate() { + b.push_str(&format!( + "ds4_residency_failures_total{{model_role=\"{role}\",stage=\"{stage}\"}} {}\n", + g.residency_failures[r][sg] + )); + } + } + } + b +} + +/// Full C `/metrics` body (prefix + route fragment + runtime + memgov). +pub fn render_metrics(m: &RouteMetrics, admit: &AdmitState, rt: &RuntimeMetrics) -> String { + let mut b = render_metrics_prefix(rt); + b.push_str(&render_metrics_fragment(m, admit)); + b.push_str(&render_metrics_runtime(rt)); + b.push_str(&render_memgov_metrics(&rt.memgov)); + b +} + +pub fn render_stats_memgov_json(g: &MemgovSnap) -> String { + let mut b = format!( + "\"memory\":{{\"census_supported\":{},\"census_epoch\":{}", + if g.census_supported { "true" } else { "false" }, + g.census_epoch + ); + if g.census_supported { + b.push_str(",\"classes\":{"); + let mut tl = [0u64; 2]; + let mut ts = [0u64; 2]; + for (c, class) in MEM_CLASS_NAMES.iter().enumerate() { + if c > 0 { + b.push(','); + } + b.push_str(&format!("\"{class}\":{{")); + for (d, domain) in MEM_DOMAIN_NAMES.iter().enumerate() { + if d > 0 { + b.push(','); + } + let lv = g.cells[c][d].live(); + let sl = g.cells[c][d].slack(); + tl[d] += lv; + ts[d] += sl; + b.push_str(&format!("\"{domain}\":{{\"live\":{lv},\"slack\":{sl}}}")); + } + b.push('}'); + } + b.push_str(&format!( + "}},\"totals\":{{\"device_live\":{},\"device_slack\":{},\"host_pin_live\":{},\"host_pin_slack\":{}}},\"census_faults\":{}", + tl[0], ts[0], tl[1], ts[1], g.census_faults + )); + if g.emit_substrate { + b.push_str(&format!( + ",\"substrate_outstanding\":{}", + g.substrate_outstanding + )); + } + } + let st = MEM_OBS_STATUS_NAMES[g.obs_status.min(2) as usize]; + let so = MEM_OBS_SOURCE_NAMES[g.obs_source.min(2) as usize]; + b.push_str(&format!( + ",\"observation\":{{\"status\":\"{st}\",\"source\":\"{so}\"" + )); + if g.obs_status == 0 { + b.push_str(&format!( + ",\"free_bytes\":{},\"total_bytes\":{},\"cuda_free_bytes\":{},\"meminfo_available_bytes\":{}", + g.obs_free, g.obs_total, g.obs_cuda_free, g.obs_meminfo + )); + } + b.push_str(&format!( + ",\"calls_cuda_free\":{},\"calls_meminfo_available\":{},\"errors\":{}}}", + g.memobs_calls[1], g.memobs_calls[2], g.memobs_errors + )); + b.push_str(",\"reconcile\":{\"supported\":false}}"); + b.push_str(&format!( + ",\"governor\":{{\"shadow\":true,\"epoch\":{},\"faults\":{},\"modes\":{{", + g.gov_epoch, g.faults + )); + for (c, consumer) in GOV_CONSUMER_NAMES.iter().enumerate() { + if c > 0 { + b.push(','); + } + b.push_str(&format!("\"{consumer}\":\"{}\"", g.gov_modes[c])); + } + b.push_str("},\"leases\":{"); + for (c, consumer) in GOV_CONSUMER_NAMES.iter().enumerate() { + if c > 0 { + b.push(','); + } + let l = g.leases[c]; + b.push_str(&format!( + "\"{consumer}\":{{\"intent\":{},\"resident\":{},\"reservation\":{}}}", + l.intent, l.resident, l.reservation + )); + } + b.push_str("},\"decisions\":{"); + for (r, reason) in GOV_CMP_NAMES.iter().enumerate() { + if r > 0 { + b.push(','); + } + let mut n = 0u64; + for c in 0..5 { + for st in 0..6 { + n += g.decisions[c][st][r]; + } + } + b.push_str(&format!("\"{reason}\":{n}")); + } + b.push_str("}}"); + b +} + +pub fn dump_memgov_names() -> String { + let mut b = String::new(); + for n in MEM_CLASS_NAMES { + b.push_str(&format!("CLASS {n}\n")); + } + for n in MEM_DOMAIN_NAMES { + b.push_str(&format!("DOMAIN {n}\n")); + } + for n in GOV_CONSUMER_NAMES { + b.push_str(&format!("GOV {n}\n")); + } + for n in GOV_STATUS_NAMES { + b.push_str(&format!("STATUS {n}\n")); + } + for n in GOV_CMP_NAMES { + b.push_str(&format!("CMP {n}\n")); + } + for n in REJLANE_NAMES { + b.push_str(&format!("REJLANE {n}\n")); + } + for n in REJECT_REASON_NAMES { + b.push_str(&format!("REJECT {n}\n")); + } + for n in RECLAIM_STATUS_NAMES { + b.push_str(&format!("RECLAIM {n}\n")); + } + for n in MEM_OBS_STATUS_NAMES { + b.push_str(&format!("OBS_STATUS {n}\n")); + } + for n in MEM_OBS_SOURCE_NAMES { + b.push_str(&format!("OBS_SOURCE {n}\n")); + } + for n in RESIDENCY_POLICY_NAMES { + b.push_str(&format!("POLICY {n}\n")); + } + for n in RESUNIT_STATE_NAMES { + b.push_str(&format!("RESUNIT {n}\n")); + } + for n in RESSTAGE_NAMES { + b.push_str(&format!("RESSTAGE {n}\n")); + } + for n in MODEL_ROLE_NAMES { + b.push_str(&format!("ROLE {n}\n")); + } + b +} diff --git a/crates/ds4-server/src/models.rs b/crates/ds4-server/src/models.rs new file mode 100644 index 0000000000000000000000000000000000000000..60f546f277864f544b470b7969a5289d9e74114e --- /dev/null +++ b/crates/ds4-server/src/models.rs @@ -0,0 +1,225 @@ +//! /v1/models id from the GGUF path, and the OpenAI list JSON. + +use crate::json::json_escape; + +pub fn model_alias_disables_thinking(model: &str) -> bool { + model == "deepseek-chat" || model == "k-exaone-236b-a23b-chat" +} + +pub fn model_alias_enables_thinking(model: &str) -> bool { + model == "deepseek-reasoner" +} + +pub fn model_alias_known(id: &str) -> bool { + matches!( + id, + "deepseek-v4-flash" + | "deepseek-v4-pro" + | "solar-open2-250b" + | "motif-3" + | "Motif-Technologies/Motif-3" + | "Motif-3-Mixed-Quant-GGUF" + | "k-exaone-236b-a23b" + | "k-exaone-236b-a23b-chat" + | "K-EXAONE-236B-A23B" + | "LGAI-EXAONE/K-EXAONE-236B-A23B" + | "dots3-note-prev" + | "dots-studio/dots3-note-prev" + | "dots3-note-prev-Mixed-Quant-GGUF" + // Prism Bonsai 2 27B (qwen35); the names the sibling C server + // answers to. The advertised id stays the GGUF stem, as it does + // for every family here. + | "prism-bonsai-2-27b" + | "prism-bonsai-2-27b-chat" + | "prism-bonsai-2-27b-no-think" + | "prism-bonsai-2-27b-nothink" + | "prism-bonsai-2-27b-reasoner" + | "prism/bonsai-2-27b" + ) +} + +pub fn model_id_known(advertised: &str, id: &str) -> bool { + model_alias_known(id) || id == advertised +} + +pub fn parent_is_generic_dir(parent: &str) -> bool { + parent.is_empty() + || parent == "." + || parent == ".." + || parent == "gguf" + || parent == "GGUF" + || parent == "models" + || parent == "model" + || parent == "weights" + || parent == "artifacts" + || parent == "tmp" + || parent == "temp" + || parent == "scratch" + || parent == "data" +} + +pub fn parent_is_gguf_artifact_dir(parent: &str) -> bool { + if parent_is_generic_dir(parent) { + return false; + } + if parent.contains("Mixed-Quant") { + return true; + } + let n = parent.len(); + n > 4 + && parent.as_bytes()[n - 4].eq_ignore_ascii_case(&b'G') + && parent.as_bytes()[n - 3].eq_ignore_ascii_case(&b'G') + && parent.as_bytes()[n - 2].eq_ignore_ascii_case(&b'U') + && parent.as_bytes()[n - 1].eq_ignore_ascii_case(&b'F') +} + +fn strip_gguf_ext(name: &mut String) { + let n = name.len(); + if n >= 5 && name.as_bytes()[n - 5] == b'.' { + let ext = &name[n - 4..]; + if ext.eq_ignore_ascii_case("gguf") { + name.truncate(n - 5); + } + } +} + +fn strip_gguf_shard_suffix(name: &mut String) { + let bytes = name.as_bytes(); + let n = bytes.len(); + let mut i = n; + while i > 0 && bytes[i - 1].is_ascii_digit() { + i -= 1; + } + if i == n || i < 4 || &bytes[i - 4..i] != b"-of-" { + return; + } + let of = i - 4; + let mut d = of; + while d > 0 && bytes[d - 1].is_ascii_digit() { + d -= 1; + } + if d == of || d == 0 || bytes[d - 1] != b'-' { + return; + } + name.truncate(d - 1); +} + +pub fn model_id_from_gguf_path(path: &str) -> Option { + if path.is_empty() { + return None; + } + let mut n = path.len(); + while n > 1 && path.as_bytes()[n - 1] == b'/' { + n -= 1; + } + let path = &path[..n]; + let base = path.rfind('/').map(|i| i + 1).unwrap_or(0); + let mut stem = path[base..].to_string(); + if stem.is_empty() || stem.len() >= 256 { + return None; + } + strip_gguf_ext(&mut stem); + strip_gguf_shard_suffix(&mut stem); + if stem.is_empty() { + return None; + } + if base > 0 { + let parent_end = base - 1; + let parent_start = path[..parent_end].rfind('/').map(|i| i + 1).unwrap_or(0); + let parent = &path[parent_start..parent_end]; + if !parent.is_empty() && parent.len() < 256 && parent_is_gguf_artifact_dir(parent) { + return Some(parent.to_string()); + } + } + if stem.len() >= 256 { + return None; + } + Some(stem) +} + +pub fn append_model_json_values(id: &str, name: &str, ctx: i32, default_tokens: i32) -> String { + let max_completion = if default_tokens < ctx { + default_tokens + } else { + ctx + }; + format!( + "{{\"id\":{},\"object\":\"model\",\"created\":1767225600,\"owned_by\":\"ds4.c\",\"name\":{},\ + \"context_length\":{ctx},\"top_provider\":{{\"context_length\":{ctx},\ + \"max_completion_tokens\":{max_completion},\"is_moderated\":false}},\ + \"supported_parameters\":[\"tools\",\"tool_choice\",\"max_tokens\",\"temperature\",\ + \"top_p\",\"top_k\",\"min_p\",\"stop\",\"seed\",\"stream\",\"reasoning_effort\"]}}", + json_escape(id), + json_escape(name) + ) +} + +pub fn models_list_json( + id: &str, + name: &str, + ctx: i32, + default_tokens: i32, + codex: Option<&str>, +) -> String { + let mut b = String::from("{\"object\":\"list\",\"data\":["); + b.push_str(&append_model_json_values(id, name, ctx, default_tokens)); + b.push(']'); + if let Some(arr) = codex { + b.push_str(",\"models\":"); + b.push_str(arr); + } + b.push_str("}\n"); + b +} + +pub fn model_one_json(id: &str, name: &str, ctx: i32, default_tokens: i32) -> String { + let mut b = append_model_json_values(id, name, ctx, default_tokens); + b.push('\n'); + b +} + +pub fn json_models_array_dup(text: &str) -> Option { + let mut search = text; + while let Some(at) = search.find("\"models\"") { + let after = &search[at + 8..]; + let q = after.trim_start_matches(|c: char| c.is_ascii_whitespace()); + if !q.starts_with(':') { + search = &search[at + 8..]; + continue; + } + let q = q[1..].trim_start_matches(|c: char| c.is_ascii_whitespace()); + if !q.starts_with('[') { + search = &search[at + 8..]; + continue; + } + let start_off = text.len() - q.len(); + let bytes = q.as_bytes(); + let mut depth = 0i32; + let mut in_str = false; + let mut esc = false; + for (i, &c) in bytes.iter().enumerate() { + if in_str { + if esc { + esc = false; + } else if c == b'\\' { + esc = true; + } else if c == b'"' { + in_str = false; + } + continue; + } + if c == b'"' { + in_str = true; + } else if c == b'[' { + depth += 1; + } else if c == b']' { + depth -= 1; + if depth == 0 { + return Some(text[start_off..start_off + i + 1].to_string()); + } + } + } + return None; + } + None +} diff --git a/crates/ds4-server/src/parse.rs b/crates/ds4-server/src/parse.rs new file mode 100644 index 0000000000000000000000000000000000000000..d7c2df1992ba41e93882670d8ac75b5254594c2a --- /dev/null +++ b/crates/ds4-server/src/parse.rs @@ -0,0 +1,4042 @@ +//! Four surface parsers from `ds4_server.c`. No prompt tokenize / generation. + +use crate::format::{ + parse_output_config_effort, parse_output_format_value, parse_reasoning_effort_value, + parse_responses_text_value, +}; +use crate::json::{ + json_bool, json_content, json_escape, json_int, json_number, json_raw_value, json_skip_value, + json_string, Json, +}; +use crate::models::{model_alias_disables_thinking, model_alias_enables_thinking}; +use crate::render::{ModelSyntax, DSML_IMAGE}; +use crate::route::{ + compute_needs, think_mode_from_enabled, Api, NeedInput, ReqKind, RequestedEffort, ThinkMode, + WireSurface, +}; +use std::sync::Arc; + +pub const DEFAULT_TEMPERATURE: f32 = 1.0; +pub const DEFAULT_TOP_P: f32 = 1.0; +pub const DEFAULT_MIN_P: f32 = 0.05; + +/// Thinking requests sample with the fixed tuple above, whatever the client +/// sent (the C server's rule for its DeepSeek family). IQuest, GLM-5.3 and +/// Beam keep the request's sampler instead: their client must be able to +/// lower the temperature, and their drafts only run at temperature 0 (Beam +/// always writes its analysis channel). An omitted value still takes the +/// defaults above, so such a request samples as before. +pub fn fixed_thinking_sampling(syntax: ModelSyntax) -> bool { + !matches!( + syntax, + ModelSyntax::IQuestQ1 | ModelSyntax::Glm53 | ModelSyntax::BeamVq + ) +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum EosPolicy { + #[default] + Default, + Reasoning, + Global, +} + +impl EosPolicy { + pub(crate) fn excluded(self, eos: i32, in_reasoning: bool) -> Option { + (eos >= 0 + && match self { + Self::Default => false, + Self::Reasoning => in_reasoning, + Self::Global => true, + }) + .then_some(eos) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ToolChoice { + Auto = 0, + None = 1, + Required = 2, +} + +#[derive(Debug, Clone, Default)] +pub struct ToolCall { + pub id: String, + pub name: String, + pub arguments: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ChatPart { + Text(String), + Image(usize), + Audio(usize), + Video(usize), + ToolResult { id: String, content: String }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImageMime { + Png, + Jpeg, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RequestImage { + pub mime: ImageMime, + pub data: Arc<[u8]>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RequestAudio { + pub data: Arc<[u8]>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RequestVideo { + pub data: Arc<[u8]>, +} + +#[derive(Debug, Clone, Default)] +pub struct ChatMsg { + pub role: String, + pub name: String, + pub content: String, + pub reasoning: String, + pub tool_call_id: String, + pub tool_call_ids: Vec, + pub calls: Vec, + pub raw_dsml: String, + pub raw_tool_text: String, + pub parts: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct ToolSchemaOrder { + pub name: String, + pub wire_name: String, + pub namespace: String, + pub responses_tool_search: bool, + pub prop: Vec, + pub prop_type: Vec, + /// The leading non-null member of a `type` array. A client that renders an + /// optional field the way schemars does -- `Option` goes out as + /// `{"type":["integer","null"]}` -- writes no scalar type at all, so + /// `prop_type` is empty for it. Only the GLM path reads this, because + /// GLM's template writes a string verbatim and every other value as JSON + /// (chat_template.jinja:50) and the declared type is the only way to tell + /// the two apart; the tolerant paths keep reading `prop_type` alone. + pub prop_type_first: Vec, +} + +#[derive(Debug, Clone)] +pub struct ParseEnv { + pub default_model: String, + pub default_tokens: i32, + pub default_effort: ThinkMode, + pub default_temp: f32, + /// LIVE call ids for this surface at parse time (C `*_live_has_call_id`). + pub live_ids: Vec, + /// The engine's request-omitted defaults for its own model set (unit H): + /// thinking OFF unless asked (server_msgs.c request_init), reasoning + /// effort HIGH when a bare thinking flag asks for it, min_p 0.0 + /// (ds4.h:54-56) and the engine's effort-name collapse + /// (`think_mode_from_enabled`, server_msgs.c:186-189: everything non-zero + /// below max is HIGH). True only for the V4.1 (ds41) family in this + /// port: the engine's parse is shared across its DeepSeek models, but the + /// other families keep the port's own wire vocabulary (their references + /// are their own stacks, not this engine). + pub engine_defaults: bool, +} + +impl Default for ParseEnv { + fn default() -> Self { + Self { + default_model: "deepseek-v4-flash".into(), + default_tokens: 393216, + default_effort: ThinkMode::Low, + default_temp: default_temperature(), + live_ids: Vec::new(), + engine_defaults: false, + } + } +} + +pub fn default_temperature() -> f32 { + std::env::var("DS4_SERVER_DEFAULT_TEMP") + .ok() + .filter(|s| !s.is_empty()) + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_TEMPERATURE) +} + +#[derive(Debug, Clone)] +pub struct ParsedRequest { + pub kind: ReqKind, + pub api: Api, + pub model: String, + pub model_from_request: bool, + pub max_tokens: i32, + pub max_tokens_set: bool, + pub top_k: i32, + pub temperature: f32, + pub top_p: f32, + pub min_p: f32, + pub seed: u64, + /// OpenAI frequency / presence penalties (the engine parses both, + /// server_parse_chat.c:91-104 and server_parse_responses.c:328-334); any + /// non-zero value takes the V4.1 host penalty route. + pub frequency_penalty: f32, + pub presence_penalty: f32, + /// DRY (llama.cpp's sequence repetition penalty; the engine parses the + /// three keys, server_parse_chat.c:119-127): dry_set = dry_multiplier + /// appeared; base / allowed_length fall back to 1.75 / 2 at face build + /// (server_generate_v41.c:363-365). + pub dry_multiplier: f32, + pub dry_base: f32, + pub dry_allowed_length: i32, + pub dry_set: bool, + pub stream: bool, + pub stream_include_usage: bool, + pub return_token_ids: bool, + pub think_mode: ThinkMode, + /// The effort the request asked for, kept beside the collapsed + /// `think_mode`: the V4.1 head renders this budget (encoding.py:456-473) + /// while the engine keeps its own mode. + pub effort: RequestedEffort, + pub eos_policy: EosPolicy, + pub has_tools: bool, + pub has_tool_results: bool, + pub tool_choice: ToolChoice, + /// The function name of the named tool_choice form. The OpenAI wire + /// defines `{"type":"function","function":{"name":"x"}}` as "forces the + /// model to call that tool"; the request's tool set is narrowed to that + /// one function (prepare_tool_choice) so the forced call is the only one + /// the prompt offers. + pub forced_tool: Option, + pub required_tool_prefix: Vec, + pub required_think_end_prefix: Vec, + pub stops: Vec, + pub reasoning_summary_emit: bool, + pub responses_requires_live_tool_state: bool, + pub responses_requires_live_reasoning: bool, + pub anthropic_requires_live_tool_state: bool, + pub live_state_bank_owned: bool, + pub directed_bank: Option, + pub live_call_ids: Vec, + pub messages: Vec, + pub images: Vec, + pub audios: Vec, + pub videos: Vec, + pub tool_schemas: String, + pub tool_orders: Vec, + pub prompt_text: Option, + pub needs: u32, +} + +impl ParsedRequest { + fn init(kind: ReqKind, env: &ParseEnv) -> Self { + Self { + kind, + api: Api::Openai, + model: env.default_model.clone(), + model_from_request: false, + max_tokens: env.default_tokens, + max_tokens_set: false, + top_k: 0, + temperature: env.default_temp, + top_p: DEFAULT_TOP_P, + min_p: if env.engine_defaults { + 0.0 + } else { + DEFAULT_MIN_P + }, + seed: 0, + frequency_penalty: 0.0, + presence_penalty: 0.0, + dry_multiplier: 0.0, + dry_base: 0.0, + dry_allowed_length: 0, + dry_set: false, + stream: false, + stream_include_usage: false, + return_token_ids: false, + think_mode: ThinkMode::Low, + effort: RequestedEffort::named(ThinkMode::Low), + eos_policy: EosPolicy::Default, + has_tools: false, + has_tool_results: false, + tool_choice: ToolChoice::Auto, + forced_tool: None, + required_tool_prefix: Vec::new(), + required_think_end_prefix: Vec::new(), + stops: Vec::new(), + reasoning_summary_emit: false, + responses_requires_live_tool_state: false, + responses_requires_live_reasoning: false, + anthropic_requires_live_tool_state: false, + live_state_bank_owned: false, + directed_bank: None, + live_call_ids: Vec::new(), + messages: Vec::new(), + images: Vec::new(), + audios: Vec::new(), + videos: Vec::new(), + tool_schemas: String::new(), + tool_orders: Vec::new(), + prompt_text: None, + needs: 0, + } + } + + pub(crate) fn finish_needs(&mut self) { + self.needs = compute_needs(&NeedInput { + api: self.api, + kind: self.kind, + stream: self.stream, + temperature: self.temperature, + think: crate::route::think_mode_enabled(self.think_mode), + stop_count: self.stops.len() as u32, + has_tools: self.has_tools, + return_token_ids: self.return_token_ids, + responses_requires_live_tool_state: self.responses_requires_live_tool_state, + responses_requires_live_reasoning: self.responses_requires_live_reasoning, + anthropic_requires_live_tool_state: self.anthropic_requires_live_tool_state, + live_state_bank_owned: self.live_state_bank_owned, + max_tokens_set: self.max_tokens_set, + max_tokens: self.max_tokens, + has_images: !self.images.is_empty(), + }); + if !self.audios.is_empty() || !self.videos.is_empty() { + self.needs |= crate::route::NEED_AUDIO; + } + if self.eos_policy != EosPolicy::Default { + self.needs |= crate::route::NEED_EOS_POLICY; + } + } +} + +const CHAT_IMAGE_MAX_COUNT: usize = 4; +const CHAT_IMAGE_MAX_BYTES: usize = 10 * 1024 * 1024; +const CHAT_IMAGE_TOTAL_MAX: usize = 20 * 1024 * 1024; + +struct ImageInputs { + items: Vec, + bytes: usize, + count_limit: Option, +} + +impl ImageInputs { + fn new(env: &ParseEnv) -> Self { + Self { + items: Vec::new(), + bytes: 0, + // YoungAi server_parse_tools.c:289 grows the image list; ds41 + // checks expanded context and memory. Other families keep four. + count_limit: (!env.engine_defaults).then_some(CHAT_IMAGE_MAX_COUNT), + } + } +} + +impl std::ops::Deref for ImageInputs { + type Target = [RequestImage]; + + fn deref(&self) -> &Self::Target { + &self.items + } +} + +fn base64_value(c: u8) -> Option { + match c { + b'A'..=b'Z' => Some((c - b'A') as u32), + b'a'..=b'z' => Some((26 + c - b'a') as u32), + b'0'..=b'9' => Some((52 + c - b'0') as u32), + b'+' => Some(62), + b'/' => Some(63), + _ => None, + } +} + +fn decode_media_base64(src: &str, request_remaining: usize) -> Result, String> { + let src = src.as_bytes(); + if src.is_empty() || src.len() & 3 != 0 { + return Err("invalid media base64 length".into()); + } + let padding = usize::from(src[src.len() - 1] == b'=') + usize::from(src[src.len() - 2] == b'='); + let decoded = src.len() / 4 * 3 - padding; + if decoded == 0 || decoded > CHAT_IMAGE_MAX_BYTES { + return Err("media exceeds 10 MiB decoded limit".into()); + } + if decoded > request_remaining { + return Err("media exceeds 20 MiB request limit".into()); + } + let mut out = Vec::with_capacity(decoded); + for (i, chunk) in src.chunks_exact(4).enumerate() { + let last = i + 1 == src.len() / 4; + let a = base64_value(chunk[0]); + let b = base64_value(chunk[1]); + let c = (chunk[2] != b'=').then(|| base64_value(chunk[2])).flatten(); + let d = (chunk[3] != b'=').then(|| base64_value(chunk[3])).flatten(); + if a.is_none() + || b.is_none() + || (chunk[2] != b'=' && c.is_none()) + || (chunk[3] != b'=' && d.is_none()) + || (!last && (c.is_none() || d.is_none())) + || (chunk[2] == b'=' && chunk[3] != b'=') + || (chunk[2] == b'=' && b.unwrap() & 15 != 0) + || (chunk[3] == b'=' && c.is_some_and(|v| v & 3 != 0)) + { + return Err("invalid media base64 data".into()); + } + let v = a.unwrap() << 18 | b.unwrap() << 12 | c.unwrap_or(0) << 6 | d.unwrap_or(0); + if out.len() < decoded { + out.push((v >> 16) as u8); + } + if out.len() < decoded { + out.push((v >> 8) as u8); + } + if out.len() < decoded { + out.push(v as u8); + } + } + if out.len() != decoded { + return Err("invalid media base64 padding".into()); + } + Ok(out) +} + +fn media_count(images: &[RequestImage], audios: &[RequestAudio], videos: &[RequestVideo]) -> usize { + images.len() + audios.len() + videos.len() +} + +fn media_bytes(images: &[RequestImage], audios: &[RequestAudio], videos: &[RequestVideo]) -> usize { + images.iter().map(|v| v.data.len()).sum::() + + audios.iter().map(|v| v.data.len()).sum::() + + videos.iter().map(|v| v.data.len()).sum::() +} + +fn add_image_base64( + images: &mut ImageInputs, + audios: &[RequestAudio], + videos: &[RequestVideo], + mime_type: &str, + payload: &str, +) -> Result { + let mime = match mime_type { + "image/png" => ImageMime::Png, + "image/jpeg" => ImageMime::Jpeg, + _ => return Err(format!("unsupported image media type: {mime_type}")), + }; + let limit = if audios.is_empty() && videos.is_empty() { + images.count_limit + } else { + Some(CHAT_IMAGE_MAX_COUNT) + }; + if limit.is_some_and(|n| media_count(images, audios, videos) >= n) { + return Err("at most 4 media inputs are supported".into()); + } + let total = images.bytes + media_bytes(&[], audios, videos); + if total > CHAT_IMAGE_TOTAL_MAX { + return Err("media exceeds 20 MiB request limit".into()); + } + let data = decode_media_base64(payload, CHAT_IMAGE_TOTAL_MAX - total)?; + let magic_matches = match mime { + ImageMime::Png => data.starts_with(b"\x89PNG\r\n\x1a\n"), + ImageMime::Jpeg => data.starts_with(&[0xff, 0xd8, 0xff]), + }; + if !magic_matches { + return Err("image bytes do not match declared media type".into()); + } + let index = images.len(); + images.bytes += data.len(); + images.items.push(RequestImage { + mime, + data: data.into(), + }); + Ok(index) +} + +fn add_image_data_uri( + images: &mut ImageInputs, + audios: &[RequestAudio], + videos: &[RequestVideo], + uri: &str, +) -> Result { + let Some(rest) = uri.strip_prefix("data:") else { + return Err("image_url must be a base64 data URI".into()); + }; + let Some((mime, payload)) = rest.split_once(";base64,") else { + return Err("image_url must be a base64 data URI".into()); + }; + if mime.is_empty() { + return Err("image_url must be a base64 data URI".into()); + } + add_image_base64(images, audios, videos, mime, payload) +} + +fn add_video( + videos: &mut Vec, + images: &[RequestImage], + audios: &[RequestAudio], + payload: &str, +) -> Result { + if media_count(images, audios, videos) >= CHAT_IMAGE_MAX_COUNT { + return Err("at most 4 media inputs are supported".into()); + } + let total = media_bytes(images, audios, videos); + if total > CHAT_IMAGE_TOTAL_MAX { + return Err("media exceeds 20 MiB request limit".into()); + } + let data = decode_media_base64(payload, CHAT_IMAGE_TOTAL_MAX - total)?; + let ftyp = data.len() >= 12 && data[4..8] == *b"ftyp"; + if !ftyp { + return Err("video bytes do not match MP4".into()); + } + let index = videos.len(); + videos.push(RequestVideo { data: data.into() }); + Ok(index) +} + +fn add_audio( + audios: &mut Vec, + images: &[RequestImage], + videos: &[RequestVideo], + format: &str, + payload: &str, +) -> Result { + if format != "wav" { + return Err("audio input currently requires WAV format".into()); + } + if media_count(images, audios, videos) >= CHAT_IMAGE_MAX_COUNT { + return Err("at most 4 media inputs are supported".into()); + } + let total = media_bytes(images, audios, videos); + if total > CHAT_IMAGE_TOTAL_MAX { + return Err("media exceeds 20 MiB request limit".into()); + } + let data = decode_media_base64(payload, CHAT_IMAGE_TOTAL_MAX - total)?; + if !data.starts_with(b"RIFF") || data.get(8..12) != Some(b"WAVE") { + return Err("audio bytes do not match WAV format".into()); + } + let index = audios.len(); + audios.push(RequestAudio { data: data.into() }); + Ok(index) +} + +fn validate_audio_refs( + msgs: &[ChatMsg], + images: &[RequestImage], + audios: &[RequestAudio], + videos: &[RequestVideo], +) -> Result<(), String> { + if ((!audios.is_empty() || !videos.is_empty()) + && media_count(images, audios, videos) > CHAT_IMAGE_MAX_COUNT) + || media_bytes(images, audios, videos) > CHAT_IMAGE_TOTAL_MAX + { + return Err("media exceeds the 4 input or 20 MiB request limit".into()); + } + let mut seen = vec![false; audios.len()]; + for msg in msgs { + for part in &msg.parts { + let ChatPart::Audio(index) = part else { + continue; + }; + if msg.role != "user" { + return Err("audio is allowed only in user messages".into()); + } + let Some(slot) = seen.get_mut(*index) else { + return Err("invalid audio reference".into()); + }; + if *slot { + return Err("duplicate audio reference".into()); + } + *slot = true; + } + } + if seen.iter().any(|v| !v) { + return Err("unreferenced audio payload".into()); + } + Ok(()) +} + +/// The oracle refuses a user-supplied image placeholder in a textual field +/// (encoding.py:355-367; its text-block half is :346-352, nested tool-result +/// text included): an image is an image content block, never the literal +/// token in text. A stray literal tokenizes to the placeholder id +/// (crates/ds4-core/src/tok.rs:2603) and a request that carries no image never +/// reaches the V4.1 expansion which counts placeholders +/// (crates/ds4-server/src/generate.rs:3107), so it would reach the model as a +/// lone image token at a text position. +fn reject_stray_image_in_content(text: &str) -> Result<(), String> { + if !text.contains(DSML_IMAGE) { + return Ok(()); + } + Err(format!( + "Message content contains image special token '{DSML_IMAGE}'. Images should be provided as image content blocks." + )) +} + +/// The oracle's other half (encoding.py:363-366): the same literal in +/// `reasoning_content`. +fn reject_stray_image_in_reasoning(text: &str) -> Result<(), String> { + if !text.contains(DSML_IMAGE) { + return Ok(()); + } + Err(format!( + "reasoning_content contains image special token '{DSML_IMAGE}'" + )) +} + +fn append_text_part(msg: &mut ChatMsg, text: String, err: &mut String) -> bool { + if let Err(e) = reject_stray_image_in_content(&text) { + *err = e; + return false; + } + msg.content.push_str(&text); + msg.parts.push(ChatPart::Text(text)); + true +} + +fn validate_image_references(msgs: &[ChatMsg], images: &[RequestImage]) -> Result<(), String> { + if images.is_empty() { + return Ok(()); + } + let mut seen = vec![false; images.len()]; + for msg in msgs { + for part in &msg.parts { + let ChatPart::Image(index) = part else { + continue; + }; + // The engine's render treats user, tool and function as the same + // user-like body (server_dsml_render.c:218-221 role_is_user_like), + // so a V4.1 tool result may carry pixels -- the MCP screenshot + // loop. A head that admits images but cannot place one inside a + // tool result is refused by name in generation_blocked; a + // system/assistant image stays a parse error. + if !matches!(msg.role.as_str(), "user" | "tool" | "function") { + return Err("images are allowed only in user messages or tool results".into()); + } + let Some(slot) = seen.get_mut(*index) else { + return Err("invalid image reference".into()); + }; + *slot = true; + } + } + if seen.iter().any(|seen| !seen) { + return Err("unreferenced image payload".into()); + } + Ok(()) +} + +/// A tool/function message is rendered by its content, wrapped in the +/// `` envelope, so an image block has to leave its placeholder in +/// that text, in client block order. The engine writes it into the same +/// buffer at parse time (`server_parse_tools.c:289` through +/// `mm_image_source_to_text`; the Chat Completions content sink at +/// `server_buf.c:232`). A user message gets its placeholder from the parts at +/// render time instead, so the text a non-DSML head renders never carries the +/// token. A tool_result that already rendered its envelope (the Anthropic +/// path writes both the envelope and the placeholder into the content) is left +/// alone. +fn write_tool_image_placeholders(msg: &mut ChatMsg) { + if !matches!(msg.role.as_str(), "tool" | "function") { + return; + } + if msg + .parts + .iter() + .any(|part| matches!(part, ChatPart::ToolResult { .. })) + { + return; + } + if !msg + .parts + .iter() + .any(|part| matches!(part, ChatPart::Image(_))) + { + return; + } + let mut rendered = String::with_capacity(msg.content.len() + DSML_IMAGE.len()); + for part in &msg.parts { + match part { + ChatPart::Text(text) => rendered.push_str(text), + ChatPart::Image(_) => rendered.push_str(DSML_IMAGE), + _ => {} + } + } + msg.content = rendered; +} + +fn validate_video_refs(msgs: &[ChatMsg], videos: &[RequestVideo]) -> Result<(), String> { + if videos.is_empty() { + return Ok(()); + } + let mut seen = vec![false; videos.len()]; + for msg in msgs { + for part in &msg.parts { + let ChatPart::Video(index) = part else { + continue; + }; + if msg.role != "user" { + return Err("video is allowed only in user messages".into()); + } + let Some(slot) = seen.get_mut(*index) else { + return Err("invalid video reference".into()); + }; + if *slot { + return Err("duplicate video reference".into()); + } + *slot = true; + } + } + if seen.iter().any(|seen| !seen) { + return Err("unreferenced video payload".into()); + } + Ok(()) +} + +fn bad(err: &mut String, fallback: &str) -> Result { + if err.is_empty() { + *err = fallback.into(); + } + Err(err.clone()) +} + +fn json_skip_ok(p: &mut Json<'_>) -> Option<()> { + if json_skip_value(p) { + Some(()) + } else { + None + } +} + +fn parse_budget(p: &mut Json<'_>, key: &str, r: &mut ParsedRequest, err: &mut String) -> bool { + let Some(budget) = json_number(p) else { + return false; + }; + if budget < 0.0 { + *err = format!("{key} must be >= 0"); + return false; + } + r.max_tokens = if budget > i32::MAX as f64 { + i32::MAX + } else { + budget as i32 + }; + r.max_tokens_set = true; + true +} + +fn parse_stop(p: &mut Json<'_>) -> Option> { + p.ws(); + let mut out = Vec::new(); + if p.peek() == Some(b'"') { + let s = json_string(p)?; + if !s.is_empty() { + out.push(s); + } + return Some(out); + } + if p.peek() != Some(b'[') { + json_skip_ok(p)?; + return Some(out); + } + p.i += 1; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.peek() == Some(b'"') { + let s = json_string(p)?; + if !s.is_empty() { + out.push(s); + } + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + Some(out) +} + +fn parse_stream_options(p: &mut Json<'_>) -> Option { + p.ws(); + if p.peek() != Some(b'{') { + json_skip_ok(p)?; + return Some(false); + } + p.i += 1; + p.ws(); + let mut include = false; + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "include_usage" { + include = json_bool(p)?; + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + Some(include) +} + +fn parse_parallel_tool_calls(p: &mut Json<'_>, err: &mut String) -> bool { + match json_bool(p) { + Some(true) => true, + Some(false) => { + *err = "parallel_tool_calls=false is not supported".into(); + false + } + None => false, + } +} + +pub fn parse_thinking_control_value(p: &mut Json<'_>) -> Option> { + p.ws(); + if p.lit("null") { + return Some(None); + } + if matches!(p.peek(), Some(b't' | b'f')) { + return Some(Some(json_bool(p)?)); + } + if p.peek() != Some(b'{') { + json_skip_ok(p)?; + return Some(None); + } + p.i += 1; + p.ws(); + let mut enabled = None; + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "type" { + let typ = json_string(p)?; + if typ == "enabled" { + enabled = Some(true); + } else if typ == "disabled" { + enabled = Some(false); + } + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + Some(enabled) +} + +/// The OpenAI chat tool_choice value: the three modes, or the named-function +/// object `{"type":"function","function":{"name":"x"}}`, which the wire +/// defines as "forces the model to call that tool". The named form returns +/// its function name beside `ToolChoice::Required`; prepare_tool_choice then +/// narrows the request's tool set to it. The other object forms of the +/// surface -- the newer `allowed_tools` and `custom` -- stay refused by name. +fn parse_openai_tool_choice( + p: &mut Json<'_>, + err: &mut String, +) -> Option<(ToolChoice, Option)> { + p.ws(); + if p.peek() == Some(b'{') { + return parse_openai_named_tool_choice(p, err); + } + if p.peek() != Some(b'"') { + *err = "invalid tool_choice".into(); + return None; + } + let choice = json_string(p)?; + match choice.as_str() { + "auto" => Some((ToolChoice::Auto, None)), + "none" => Some((ToolChoice::None, None)), + "required" => Some((ToolChoice::Required, None)), + other => { + *err = format!("tool_choice={other} not supported"); + None + } + } +} + +fn parse_openai_named_tool_choice( + p: &mut Json<'_>, + err: &mut String, +) -> Option<(ToolChoice, Option)> { + p.i += 1; + p.ws(); + let mut typ: Option = None; + let mut name: Option = None; + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + *err = "invalid tool_choice".into(); + return None; + } + if key == "type" { + typ = Some(json_string(p)?); + } else if key == "function" { + name = parse_named_tool_function(p); + } else if key == "name" { + // The Responses form names the function at the top level: + // {"type":"function","name":"x"}. + name = Some(json_string(p)?); + } else if !json_skip_value(p) { + *err = "invalid tool_choice".into(); + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + *err = "invalid tool_choice".into(); + return None; + } + match typ.as_deref() { + Some("function") => match name { + Some(n) if !n.is_empty() => Some((ToolChoice::Required, Some(n))), + _ => { + *err = "invalid tool_choice".into(); + None + } + }, + Some(t) => { + *err = format!("tool_choice type={t} not supported"); + None + } + None => { + *err = "invalid tool_choice".into(); + None + } + } +} + +/// The named tool choice's `function` member, `{"name":"x"}`. A missing or +/// non-string name leaves None and the caller refuses the choice as invalid. +fn parse_named_tool_function(p: &mut Json<'_>) -> Option { + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + p.ws(); + let mut name = None; + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "name" { + name = json_string(p); + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + name +} + +fn parse_anthropic_tool_choice(p: &mut Json<'_>, err: &mut String) -> Option { + p.ws(); + if p.peek() != Some(b'{') { + *err = "invalid tool_choice".into(); + return None; + } + p.i += 1; + p.ws(); + let mut typ = None; + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + if err.is_empty() { + *err = "invalid tool_choice".into(); + } + return None; + } + if key == "type" { + typ = Some(json_string(p)?); + } else if !json_skip_value(p) { + if err.is_empty() { + *err = "invalid tool_choice".into(); + } + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + if err.is_empty() { + *err = "invalid tool_choice".into(); + } + return None; + } + match typ.as_deref() { + Some("auto") => Some(ToolChoice::Auto), + Some("none") => Some(ToolChoice::None), + Some("any") => Some(ToolChoice::Required), + Some("tool") => { + *err = "forced tool_choice not supported".into(); + None + } + Some(t) => { + *err = format!("tool_choice type={t} not supported"); + None + } + None => { + if err.is_empty() { + *err = "invalid tool_choice".into(); + } + None + } + } +} + +fn add_tool_call_id(msg: &mut ChatMsg, id: &str) { + if id.is_empty() { + return; + } + if msg.tool_call_id.is_empty() { + msg.tool_call_id = id.to_string(); + } + if !msg.tool_call_ids.iter().any(|x| x == id) { + msg.tool_call_ids.push(id.to_string()); + } +} + +fn parse_function_call(p: &mut Json<'_>, tc: &mut ToolCall) -> bool { + p.ws(); + if p.bump() != Some(b'{') { + return false; + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(p) else { + return false; + }; + p.ws(); + if p.bump() != Some(b':') { + return false; + } + if key == "name" { + let Some(n) = json_string(p) else { + return false; + }; + tc.name = n; + } else if key == "arguments" { + p.ws(); + if p.peek() == Some(b'"') { + let Some(a) = json_string(p) else { + return false; + }; + tc.arguments = a; + } else { + let Some(a) = json_raw_value(p) else { + return false; + }; + tc.arguments = a; + } + } else if !json_skip_value(p) { + return false; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + p.bump() == Some(b'}') +} + +fn parse_tool_calls_value(p: &mut Json<'_>) -> Option> { + p.ws(); + if p.lit("null") { + return Some(Vec::new()); + } + if p.bump() != Some(b'[') { + return None; + } + let mut calls = Vec::new(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.bump() != Some(b'{') { + return None; + } + let mut tc = ToolCall::default(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "id" { + tc.id = json_string(p)?; + } else if key == "function" { + if !parse_function_call(p, &mut tc) { + return None; + } + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + if !tc.name.is_empty() && !tc.arguments.is_empty() { + calls.push(tc); + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + Some(calls) +} + +fn parse_chat_image_url(p: &mut Json<'_>) -> Option { + p.ws(); + if p.peek() == Some(b'"') { + return json_string(p); + } + if p.bump() != Some(b'{') { + return None; + } + let mut url = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "url" { + url = Some(json_string(p)?); + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + url +} + +fn parse_openai_message_content( + p: &mut Json<'_>, + msg: &mut ChatMsg, + images: &mut ImageInputs, + audios: &mut Vec, + videos: &mut Vec, + err: &mut String, +) -> bool { + p.ws(); + if p.peek() == Some(b'"') { + let Some(s) = json_string(p) else { + return false; + }; + if let Err(e) = reject_stray_image_in_content(&s) { + *err = e; + return false; + } + msg.content = s; + return true; + } + if p.lit("null") { + msg.content.clear(); + return true; + } + if p.bump() != Some(b'[') { + return false; + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.peek() == Some(b'"') { + let Some(text) = json_string(p) else { + return false; + }; + if !append_text_part(msg, text, err) { + return false; + } + } else if p.bump() == Some(b'{') { + let mut typ = None; + let mut text = None; + let mut image_url = None; + let mut input_audio = None; + let mut video_url = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(p) else { + return false; + }; + p.ws(); + if p.bump() != Some(b':') { + return false; + } + if key == "type" { + typ = json_string(p); + if typ.is_none() { + return false; + } + } else if key == "text" { + text = json_string(p); + if text.is_none() { + return false; + } + } else if key == "input_audio" { + input_audio = json_raw_value(p); + if input_audio.is_none() { + return false; + } + } else if key == "image_url" { + image_url = parse_chat_image_url(p); + if image_url.is_none() { + return false; + } + } else if key == "video_url" { + video_url = parse_chat_image_url(p); + if video_url.is_none() { + return false; + } + } else if !json_skip_value(p) { + return false; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return false; + } + match (typ.as_deref(), text, image_url) { + (Some("text"), Some(text), _) => { + if !append_text_part(msg, text, err) { + return false; + } + } + (Some("image_url"), _, Some(url)) => { + match add_image_data_uri(images, audios, videos, &url) { + Ok(index) => msg.parts.push(ChatPart::Image(index)), + Err(error) => { + *err = error; + return false; + } + } + } + (Some("video_url") | Some("video"), _, _) => { + let Some(url) = video_url.as_deref() else { + *err = "video_url must be a base64 data URI".into(); + return false; + }; + let Some(rest) = url.strip_prefix("data:") else { + *err = "video_url must be a base64 data URI".into(); + return false; + }; + let Some((mime, payload)) = rest.split_once(";base64,") else { + *err = "video_url must be a base64 data URI".into(); + return false; + }; + if mime != "video/mp4" { + *err = "video input currently requires video/mp4".into(); + return false; + } + match add_video(videos, images, audios, payload) { + Ok(index) => msg.parts.push(ChatPart::Video(index)), + Err(error) => { + *err = error; + return false; + } + } + } + (Some("input_audio"), _, _) => { + let value = input_audio + .as_deref() + .and_then(|raw| serde_json::from_str::(raw).ok()); + let format = value.as_ref().and_then(|v| v["format"].as_str()); + let data = value.as_ref().and_then(|v| v["data"].as_str()); + let (Some(format), Some(data)) = (format, data) else { + *err = "input_audio requires format and base64 data".into(); + return false; + }; + match add_audio(audios, images, videos, format, data) { + Ok(index) => msg.parts.push(ChatPart::Audio(index)), + Err(error) => { + *err = error; + return false; + } + } + } + _ => { + *err = "unsupported chat content block".into(); + return false; + } + } + } else { + return false; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + p.bump() == Some(b']') +} + +fn parse_messages( + p: &mut Json<'_>, + images: &mut ImageInputs, + audios: &mut Vec, + videos: &mut Vec, + err: &mut String, +) -> Option> { + p.ws(); + if p.bump() != Some(b'[') { + return None; + } + let mut msgs = Vec::new(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.bump() != Some(b'{') { + return None; + } + let mut msg = ChatMsg::default(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "role" { + msg.role = json_string(p)?; + } else if key == "name" { + msg.name = json_string(p)?; + } else if key == "content" { + msg.content.clear(); + msg.parts.clear(); + if !parse_openai_message_content(p, &mut msg, images, audios, videos, err) { + if err.is_empty() { + *err = "invalid chat content".into(); + } + return None; + } + } else if key == "reasoning_content" { + let Some(r) = json_content(p) else { + return None; + }; + if let Err(e) = reject_stray_image_in_reasoning(&r) { + *err = e; + return None; + } + msg.reasoning = r; + } else if key == "tool_call_id" { + let id = json_string(p)?; + add_tool_call_id(&mut msg, &id); + } else if key == "tool_calls" { + msg.calls = parse_tool_calls_value(p)?; + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + if msg.role.is_empty() { + msg.role = "user".into(); + } + write_tool_image_placeholders(&mut msg); + msgs.push(msg); + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + Some(msgs) +} + +fn append_tool_result_text(out: &mut String, s: &str) { + let end = ""; + let mut rest = s; + while !rest.is_empty() { + if rest.starts_with(end) { + out.push_str("<"); + rest = &rest[1..]; + } else { + let ch = rest.chars().next().unwrap(); + out.push(ch); + rest = &rest[ch.len_utf8()..]; + } + } +} + +fn parse_anthropic_image_source(p: &mut Json<'_>) -> Option<(String, String, String)> { + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + let mut source_type = None; + let mut media_type = None; + let mut data = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "type" { + source_type = Some(json_string(p)?); + } else if key == "media_type" { + media_type = Some(json_string(p)?); + } else if key == "data" { + data = Some(json_string(p)?); + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + Some(( + source_type.unwrap_or_default(), + media_type.unwrap_or_default(), + data.unwrap_or_default(), + )) +} + +/// Anthropic `tool_result.content`: a bare string, null, or an array of content +/// blocks. The engine's `json_tool_result_content` (server_parse_tools.c:330): +/// text blocks concatenate exactly like the legacy string path, so rendered +/// bytes stay stable and existing disk-KV prefixes keep matching; image blocks +/// -- browser/screenshot MCP tools -- register through the same path +/// user-message images use, with the image placeholder left where the block +/// sits in the client's order (the engine's text-level render, +/// server_parse_tools.c:290). Only pixels fail closed: an undeliverable image +/// rejects the request, never silently vanishes; text-bearing blocks of other +/// types keep their text. +/// +/// Returns (rendered, plain). The DSML render writes `rendered`, the text with +/// the placeholders it can place; every other head reads the ToolResult part, +/// which carries `plain`: they have no way to place a V4.1 placeholder, and a +/// leaked DSML token would be worse than the image staying out of their text +/// (their tool results keep rendering exactly as before). +fn json_tool_result_content( + p: &mut Json<'_>, + msg: &mut ChatMsg, + images: &mut ImageInputs, + err: &mut String, +) -> Option<(String, String)> { + p.ws(); + if p.peek() == Some(b'"') { + let text = json_string(p)?; + if let Err(e) = reject_stray_image_in_content(&text) { + *err = e; + return None; + } + return Some((text.clone(), text)); + } + if p.lit("null") { + return Some((String::new(), String::new())); + } + if p.peek() != Some(b'[') { + if !json_skip_value(p) { + return None; + } + return Some((String::new(), String::new())); + } + p.i += 1; + let mut rendered = String::new(); + let mut plain = String::new(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.peek() == Some(b'"') { + let text = json_string(p)?; + rendered.push_str(&text); + plain.push_str(&text); + } else if p.peek() == Some(b'{') { + p.i += 1; + let mut typ = None; + let mut text = None; + let mut source = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "type" { + typ = json_string(p); + if typ.is_none() { + return None; + } + } else if key == "text" { + text = json_string(p); + if text.is_none() { + return None; + } + } else if key == "source" { + source = parse_anthropic_image_source(p); + if source.is_none() { + return None; + } + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + if typ.as_deref() == Some("image") { + let Some((source_type, media_type, data)) = source else { + *err = "Anthropic image source must use base64 data".into(); + return None; + }; + if source_type != "base64" || media_type.is_empty() || data.is_empty() { + *err = "Anthropic image source must use base64 data".into(); + return None; + } + match add_image_base64(images, &[], &[], &media_type, &data) { + Ok(index) => { + msg.parts.push(ChatPart::Image(index)); + rendered.push_str(DSML_IMAGE); + } + Err(error) => { + *err = error; + return None; + } + } + } else if let Some(t) = text { + if let Err(e) = reject_stray_image_in_content(&t) { + *err = e; + return None; + } + rendered.push_str(&t); + plain.push_str(&t); + } + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + Some((rendered, plain)) +} + +fn parse_anthropic_content_block( + p: &mut Json<'_>, + msg: &mut ChatMsg, + images: &mut ImageInputs, + err: &mut String, +) -> bool { + if p.bump() != Some(b'{') { + return false; + } + let mut typ = None; + let mut text = None; + let mut thinking = None; + let mut id = None; + let mut name = None; + let mut input = None; + let mut tool_result = None; + let mut tool_result_plain = None; + let mut result_is_error = false; + let mut source = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(p) else { + return false; + }; + p.ws(); + if p.bump() != Some(b':') { + return false; + } + if key == "type" { + typ = json_string(p); + if typ.is_none() { + return false; + } + } else if key == "text" { + text = json_content(p); + if text.is_none() { + return false; + } + } else if key == "thinking" { + thinking = json_content(p); + if thinking.is_none() { + return false; + } + } else if key == "id" || key == "tool_use_id" { + id = json_string(p); + if id.is_none() { + return false; + } + } else if key == "name" { + name = json_string(p); + if name.is_none() { + return false; + } + } else if key == "input" { + input = json_raw_value(p); + if input.is_none() { + return false; + } + } else if key == "content" { + match json_tool_result_content(p, msg, images, err) { + Some((rendered, plain)) => { + tool_result = Some(rendered); + tool_result_plain = Some(plain); + } + None => return false, + } + } else if key == "is_error" { + match json_bool(p) { + Some(value) => result_is_error = value, + None => return false, + } + } else if key == "source" { + source = parse_anthropic_image_source(p); + if source.is_none() { + return false; + } + } else if !json_skip_value(p) { + return false; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return false; + } + match typ.as_deref() { + Some("image") => { + let Some((source_type, media_type, data)) = source else { + *err = "Anthropic image source must use base64 data".into(); + return false; + }; + if source_type != "base64" || media_type.is_empty() || data.is_empty() { + *err = "Anthropic image source must use base64 data".into(); + return false; + } + match add_image_base64(images, &[], &[], &media_type, &data) { + Ok(index) => msg.parts.push(ChatPart::Image(index)), + Err(error) => { + *err = error; + return false; + } + } + } + Some("tool_use") => { + msg.calls.push(ToolCall { + id: id.unwrap_or_default(), + name: name.unwrap_or_default(), + arguments: input.unwrap_or_else(|| "{}".into()), + }); + } + Some("tool_result") => { + let text = tool_result.unwrap_or_default(); + // Keep original bytes before the legacy encoder escapes XML. The + // part carries the plain text (no V4.1 placeholder): it is what the + // non-DSML heads render. + msg.parts.push(ChatPart::ToolResult { + id: id.clone().unwrap_or_default(), + content: tool_result_plain.take().unwrap_or_default(), + }); + if let Some(ref i) = id { + add_tool_call_id(msg, i); + } + let mut b = msg.content.clone(); + b.push_str(""); + // A failed tool's payload is often empty or short; the flag itself + // is the signal (the engine's [tool_error] marker, log.md 2026-07-12). + if result_is_error { + b.push_str("[tool_error] "); + } + append_tool_result_text(&mut b, &text); + b.push_str(""); + msg.content = b; + } + _ => { + if let Some(t) = text { + if !append_text_part(msg, t, err) { + return false; + } + } + if let Some(t) = thinking { + if let Err(e) = reject_stray_image_in_reasoning(&t) { + *err = e; + return false; + } + msg.reasoning.push_str(&t); + } + } + } + true +} + +fn parse_anthropic_content( + p: &mut Json<'_>, + msg: &mut ChatMsg, + images: &mut ImageInputs, + err: &mut String, +) -> bool { + p.ws(); + if p.peek() == Some(b'"') { + let Some(s) = json_string(p) else { + return false; + }; + if let Err(e) = reject_stray_image_in_content(&s) { + *err = e; + return false; + } + msg.content = s; + return true; + } + if p.lit("null") { + msg.content = String::new(); + return true; + } + if p.peek() != Some(b'[') { + return json_skip_value(p); + } + p.i += 1; + let mut image_offsets = Vec::new(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.peek() == Some(b'"') { + if let Some(s) = json_string(p) { + if !append_text_part(msg, s, err) { + return false; + } + } else { + return false; + } + } else if p.peek() == Some(b'{') { + let offset = msg.content.len(); + let parts_before = msg.parts.len(); + if !parse_anthropic_content_block(p, msg, images, err) { + return false; + } + if matches!(&msg.parts[parts_before..], [ChatPart::Image(_)]) { + image_offsets.push(offset); + } + } else if !json_skip_value(p) { + return false; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return false; + } + // Mixed tool messages use flat DSML content. Preserve top-level images + // there too, in block order (YoungAi server_parse_tools.c:281-290). + if image_offsets.is_empty() + || !msg + .parts + .iter() + .any(|p| matches!(p, ChatPart::ToolResult { .. })) + { + return true; + } + let mut rendered = + String::with_capacity(msg.content.len() + image_offsets.len() * DSML_IMAGE.len()); + let mut from = 0; + for offset in image_offsets { + rendered.push_str(&msg.content[from..offset]); + rendered.push_str(DSML_IMAGE); + from = offset; + } + rendered.push_str(&msg.content[from..]); + msg.content = rendered; + true +} + +fn parse_anthropic_messages( + p: &mut Json<'_>, + images: &mut ImageInputs, + err: &mut String, +) -> Option> { + p.ws(); + if p.bump() != Some(b'[') { + return None; + } + let mut msgs = Vec::new(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.bump() != Some(b'{') { + return None; + } + let mut msg = ChatMsg::default(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "role" { + msg.role = json_string(p)?; + } else if key == "content" { + msg.content.clear(); + msg.parts.clear(); + if !parse_anthropic_content(p, &mut msg, images, err) { + return None; + } + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + if msg.role.is_empty() { + msg.role = "user".into(); + } + write_tool_image_placeholders(&mut msg); + msgs.push(msg); + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + Some(msgs) +} + +fn anthropic_system_part_is_private(s: &str) -> bool { + s.starts_with("x-anthropic-") +} + +fn append_anthropic_system_part(b: &mut String, s: &str) { + if s.is_empty() || anthropic_system_part_is_private(s) { + return; + } + if !b.is_empty() && !b.ends_with('\n') { + b.push('\n'); + } + b.push_str(s); +} + +fn parse_anthropic_system_object(p: &mut Json<'_>, out: &mut String) -> bool { + if p.bump() != Some(b'{') { + return false; + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(p) else { + return false; + }; + p.ws(); + if p.bump() != Some(b':') { + return false; + } + if key == "text" { + let Some(text) = json_string(p) else { + return false; + }; + append_anthropic_system_part(out, &text); + } else if !json_skip_value(p) { + return false; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + p.bump() == Some(b'}') +} + +fn parse_anthropic_system(p: &mut Json<'_>) -> Option { + p.ws(); + let mut b = String::new(); + if p.peek() == Some(b'"') { + let text = json_string(p)?; + append_anthropic_system_part(&mut b, &text); + return Some(b); + } + if p.lit("null") { + return Some(String::new()); + } + if p.peek() != Some(b'[') { + json_skip_ok(p)?; + return Some(String::new()); + } + p.i += 1; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.peek() == Some(b'"') { + let text = json_string(p)?; + append_anthropic_system_part(&mut b, &text); + } else if p.peek() == Some(b'{') { + if !parse_anthropic_system_object(p, &mut b) { + return None; + } + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + Some(b) +} + +fn parse_prompt(p: &mut Json<'_>) -> Option { + p.ws(); + if p.peek() == Some(b'"') { + return json_string(p); + } + if p.peek() != Some(b'[') { + json_skip_ok(p)?; + return Some(String::new()); + } + p.i += 1; + p.ws(); + let out = if p.peek() == Some(b'"') { + json_string(p)? + } else { + if p.peek().is_some() && p.peek() != Some(b']') && !json_skip_value(p) { + return None; + } + String::new() + }; + while p.peek().is_some() && p.peek() != Some(b']') { + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + if !json_skip_value(p) { + return None; + } + } else { + break; + } + } + if p.bump() != Some(b']') { + return None; + } + Some(out) +} + +fn openai_function_schema_from_tool(raw: &str) -> Option { + let mut p = Json::new(raw); + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(&mut p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "function" { + return json_raw_value(&mut p); + } + json_skip_ok(&mut p)?; + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + None +} + +fn responses_special_schema_from_tool(raw: &str) -> Option { + let mut p = Json::new(raw); + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + let mut typ = None; + let mut description = None; + let mut parameters = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(&mut p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "type" { + typ = Some(json_string(&mut p)?); + } else if key == "description" { + description = Some(json_string(&mut p)?); + } else if key == "parameters" { + parameters = Some(json_raw_value(&mut p)?); + } else if !json_skip_value(&mut p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if typ.as_deref() == Some("tool_search") { + let desc = description.unwrap_or_else(|| "Search available tools.".into()); + let params = parameters.unwrap_or_else(|| "{\"type\":\"object\",\"properties\":{}}".into()); + return Some(format!( + "{{\"name\":\"tool_search\",\"description\":{},\"parameters\":{params}}}", + json_escape(&desc) + )); + } + None +} + +fn schema_property_type(json: &str) -> Option { + let mut p = Json::new(json); + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(&mut p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "type" { + return json_string(&mut p); + } + json_skip_ok(&mut p)?; + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + None +} + +/// The head of a `type` array, skipping "null": schemars renders `Option` +/// as `{"type":["integer","null"]}`, and `schema_property_type` reads only the +/// scalar spelling, so the GLM path needs this to learn the declared type. +fn schema_property_type_first(json: &str) -> Option { + let mut p = Json::new(json); + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(&mut p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "type" { + return json_type_list_first(&mut p); + } + json_skip_ok(&mut p)?; + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + None +} + +/// Reads a `type` value written as an array and returns its first member that +/// is not "null" (the first member when every entry is "null"); a scalar type +/// or anything unreadable yields None. +fn json_type_list_first(p: &mut Json) -> Option { + p.ws(); + if p.peek() != Some(b'[') { + return None; + } + p.i += 1; + let mut head: Option = None; + loop { + p.ws(); + if p.peek() != Some(b'"') { + return head; + } + let member = json_string(p)?; + if head.is_none() { + head = Some(member.clone()); + } + if member != "null" { + return Some(member); + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + } +} + +fn parse_schema_properties(json: &str, order: &mut ToolSchemaOrder) -> bool { + let mut p = Json::new(json); + p.ws(); + if p.bump() != Some(b'{') { + return false; + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(&mut p) else { + return false; + }; + p.ws(); + if p.bump() != Some(b':') { + return false; + } + if key == "properties" { + p.ws(); + if p.bump() != Some(b'{') { + return false; + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(prop) = json_string(&mut p) else { + return false; + }; + p.ws(); + if p.bump() != Some(b':') { + return false; + } + let Some(property) = json_raw_value(&mut p) else { + return false; + }; + let typ = schema_property_type(&property).unwrap_or_default(); + let typ_first = schema_property_type_first(&property).unwrap_or_default(); + order.prop.push(prop); + order.prop_type.push(typ); + order.prop_type_first.push(typ_first); + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return false; + } + } else if !json_skip_value(&mut p) { + return false; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + p.peek() == Some(b'}') +} + +fn tool_schema_orders_add_json_wire( + orders: &mut Vec, + json: &str, + namespace: Option<&str>, + wire_name: Option<&str>, + responses_tool_search: bool, +) { + let mut p = Json::new(json); + p.ws(); + if p.bump() != Some(b'{') { + return; + } + let mut order = ToolSchemaOrder { + responses_tool_search, + ..Default::default() + }; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(&mut p) else { + return; + }; + p.ws(); + if p.bump() != Some(b':') { + return; + } + if key == "name" { + if let Some(n) = json_string(&mut p) { + order.name = n; + } else { + return; + } + } else if key == "input_schema" || key == "parameters" { + if let Some(schema) = json_raw_value(&mut p) { + parse_schema_properties(&schema, &mut order); + } else { + return; + } + } else if !json_skip_value(&mut p) { + return; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if !order.name.is_empty() { + if let Some(ns) = namespace { + if !ns.is_empty() { + order.namespace = ns.into(); + } + } + if let Some(w) = wire_name { + if !w.is_empty() { + order.wire_name = w.into(); + } + } + orders.push(order); + } +} + +fn responses_namespace_function_schema(raw: &str, namespace: &str) -> Option<(String, String)> { + let mut p = Json::new(raw); + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + let mut typ = None; + let mut name = None; + let mut description = None; + let mut parameters = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(&mut p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "type" { + typ = Some(json_string(&mut p)?); + } else if key == "name" { + name = Some(json_string(&mut p)?); + } else if key == "description" { + description = Some(json_string(&mut p)?); + } else if key == "parameters" || key == "input_schema" { + parameters = Some(json_raw_value(&mut p)?); + } else if !json_skip_value(&mut p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + let name = name?; + if name.is_empty() { + return None; + } + if typ.as_deref().is_some_and(|t| t != "function") { + return None; + } + let prompt_name = format!("{namespace}{name}"); + let desc = description.unwrap_or_default(); + let params = parameters.unwrap_or_else(|| "{\"type\":\"object\",\"properties\":{}}".into()); + Some(( + format!( + "{{\"name\":{},\"description\":{},\"parameters\":{params}}}", + json_escape(&prompt_name), + json_escape(&desc) + ), + name, + )) +} + +fn append_responses_namespace_tool_schemas( + schemas: &mut String, + orders: &mut Vec, + raw: &str, +) -> bool { + let mut p = Json::new(raw); + p.ws(); + if p.bump() != Some(b'{') { + return false; + } + let mut typ = None; + let mut name = None; + let mut tools = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(&mut p) else { + return false; + }; + p.ws(); + if p.bump() != Some(b':') { + return false; + } + if key == "type" { + typ = json_string(&mut p); + } else if key == "name" { + name = json_string(&mut p); + } else if key == "tools" { + tools = json_raw_value(&mut p); + } else if !json_skip_value(&mut p) { + return false; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if typ.as_deref() != Some("namespace") || name.is_none() || tools.is_none() { + return false; + } + let name = name.unwrap(); + let mut tp = Json::new(tools.as_ref().unwrap()); + tp.ws(); + if tp.bump() != Some(b'[') { + return false; + } + tp.ws(); + let mut appended = false; + while tp.peek().is_some() && tp.peek() != Some(b']') { + let Some(tool_raw) = json_raw_value(&mut tp) else { + return false; + }; + if let Some((schema, wire)) = responses_namespace_function_schema(&tool_raw, &name) { + if !schemas.is_empty() { + schemas.push('\n'); + } + schemas.push_str(&schema); + tool_schema_orders_add_json_wire(orders, &schema, Some(&name), Some(&wire), false); + appended = true; + } + tp.ws(); + if tp.peek() == Some(b',') { + tp.i += 1; + } + tp.ws(); + } + appended +} + +fn parse_tools_value(p: &mut Json<'_>) -> Option<(String, Vec)> { + p.ws(); + if p.lit("null") { + return Some((String::new(), Vec::new())); + } + if p.bump() != Some(b'[') { + return None; + } + let mut schemas = String::new(); + let mut orders = Vec::new(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + let raw = json_raw_value(p)?; + if let Some(function) = openai_function_schema_from_tool(&raw) { + if !schemas.is_empty() { + schemas.push('\n'); + } + schemas.push_str(&function); + tool_schema_orders_add_json_wire(&mut orders, &function, None, None, false); + } else if !append_responses_namespace_tool_schemas(&mut schemas, &mut orders, &raw) { + if let Some(special) = responses_special_schema_from_tool(&raw) { + if !schemas.is_empty() { + schemas.push('\n'); + } + schemas.push_str(&special); + tool_schema_orders_add_json_wire(&mut orders, &special, None, None, true); + } else { + if !schemas.is_empty() { + schemas.push('\n'); + } + schemas.push_str(&raw); + tool_schema_orders_add_json_wire(&mut orders, &raw, None, None, false); + } + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + Some((schemas, orders)) +} + +fn chat_msg_is_model_tool_result(m: &ChatMsg) -> bool { + m.role == "tool" + || m.role == "function" + || (m.role == "user" && (!m.tool_call_id.is_empty() || !m.tool_call_ids.is_empty())) +} + +fn chat_history_has_pending_tool_results(msgs: &[ChatMsg]) -> bool { + let mut saw = false; + for m in msgs.iter().rev() { + if m.role == "system" || m.role == "developer" { + continue; + } + if chat_msg_is_model_tool_result(m) { + saw = true; + continue; + } + if m.role == "assistant" { + return saw; + } + } + saw +} + +fn msg_has_call_id(m: &ChatMsg, id: &str) -> bool { + m.role == "assistant" && m.calls.iter().any(|c| c.id == id) +} + +fn find_prior_call<'a>(msgs: &'a [ChatMsg], before: usize, id: &str) -> Option<&'a ChatMsg> { + if id.is_empty() { + return None; + } + let end = before.min(msgs.len()); + msgs[..end].iter().rev().find(|m| msg_has_call_id(m, id)) +} + +fn collect_tool_call_ids(m: &ChatMsg) -> Vec { + let mut ids = Vec::new(); + if !m.tool_call_id.is_empty() { + ids.push(m.tool_call_id.clone()); + } + for id in &m.tool_call_ids { + if !ids.iter().any(|x| x == id) { + ids.push(id.clone()); + } + } + ids +} + +fn prepare_tool_choice( + r: &mut ParsedRequest, + schemas: &mut String, + orders: &mut Vec, + err: &mut String, +) -> bool { + if r.tool_choice == ToolChoice::None { + schemas.clear(); + orders.clear(); + } + // The named form forces one function. OpenAI's own constraint for that + // intent is allowed_tools ("constrains the tools available to the model + // to a pre-defined set"), so the forced name is served the same way: the + // prompt carries that function alone, and the call is then required. + if let Some(name) = r.forced_tool.clone() { + if !narrow_tool_schemas(schemas, orders, &name) { + *err = format!("tool_choice function \"{name}\" is not among the request tools"); + return false; + } + } + r.has_tools = !schemas.is_empty(); + if r.tool_choice == ToolChoice::Required && !r.has_tools { + *err = "tool_choice=required requires at least one tool".into(); + return false; + } + true +} + +/// Keeps the forced function alone: the schema lines whose own `name` is one +/// of the matched orders' prompt names, and those orders. The line and the +/// order of one tool are built together and read the same `name` key +/// (tool_schema_orders_add_json_wire), so the two filters agree. A client +/// names the wire name, which differs from the prompt name only for a +/// namespaced Responses tool. False when the name is in no order. +fn narrow_tool_schemas( + schemas: &mut String, + orders: &mut Vec, + name: &str, +) -> bool { + let keep: Vec = orders + .iter() + .filter(|o| o.name == name || o.wire_name == name) + .map(|o| o.name.clone()) + .collect(); + if keep.is_empty() { + return false; + } + // The schemas are the client's raw values joined by newlines, and a + // pretty-printed tool spans many lines: walk JSON values, as the + // renderer does, not lines. + let mut kept = String::new(); + let mut p = Json::new(schemas); + loop { + p.ws(); + if p.peek().is_none() { + break; + } + let Some(one) = json_raw_value(&mut p) else { + break; + }; + let Some(one_name) = schema_line_name(&one) else { + continue; + }; + if !keep.iter().any(|k| *k == one_name) { + continue; + } + if !kept.is_empty() { + kept.push('\n'); + } + kept.push_str(&one); + } + *schemas = kept; + orders.retain(|o| keep.iter().any(|k| *k == o.name)); + true +} + +/// The schema line's own `name` member: the key tool_schema_orders_add_json_wire +/// builds the order from, so both filters select the same tool. +fn schema_line_name(json: &str) -> Option { + let mut p = Json::new(json); + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(&mut p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "name" { + return json_string(&mut p); + } + if !json_skip_value(&mut p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + None +} + +fn apply_think_aliases(r: &ParsedRequest, got_thinking: bool, enabled: bool) -> bool { + let mut e = enabled; + if !got_thinking && model_alias_disables_thinking(&r.model) { + e = false; + } + if !got_thinking && model_alias_enables_thinking(&r.model) { + e = true; + } + e +} + +fn id_is_live(live: &[String], id: &str) -> bool { + live.iter().any(|x| x == id) +} + +fn anthropic_validate_tool_results( + msgs: &[ChatMsg], + live: &[String], + err: &mut String, +) -> Result { + let mut requires_live = false; + for (i, m) in msgs.iter().enumerate() { + if m.role != "user" || (m.tool_call_id.is_empty() && m.tool_call_ids.is_empty()) { + continue; + } + for id in collect_tool_call_ids(m) { + let prior = find_prior_call(msgs, i, &id); + if prior.is_none() && !id_is_live(live, &id) { + *err = format!( + "Anthropic continuation state is not available for tool_use_id {id}; retry by replaying the full messages history" + ); + return Err(()); + } + if prior.is_none() { + requires_live = true; + } + } + } + Ok(requires_live) +} + +fn responses_validate_tool_outputs( + msgs: &[ChatMsg], + think: ThinkMode, + live: &[String], + err: &mut String, +) -> Result<(bool, bool), ()> { + let mut live_tool = false; + let mut live_reason = false; + let needs_reasoning = crate::route::think_mode_enabled(think); + for (i, m) in msgs.iter().enumerate() { + if m.role != "tool" && m.role != "function" { + continue; + } + for id in collect_tool_call_ids(m) { + let prior = find_prior_call(msgs, i, &id); + if prior.is_none() && !id_is_live(live, &id) { + *err = format!( + "Responses continuation state is not available for call_id {id}; retry by replaying the full input history" + ); + return Err(()); + } + if prior.is_none() { + live_tool = true; + continue; + } + if needs_reasoning && prior.unwrap().reasoning.is_empty() { + live_reason = true; + } + } + } + Ok((live_tool, live_reason)) +} + +fn parse_responses_content_array_mm( + p: &mut Json<'_>, + mut images: Option<&mut ImageInputs>, + err: &mut String, +) -> Option<(String, Vec)> { + p.ws(); + if p.peek() == Some(b'"') { + let text = json_string(p)?; + if let Err(e) = reject_stray_image_in_content(&text) { + *err = e; + return None; + } + return Some((text.clone(), vec![ChatPart::Text(text)])); + } + if p.lit("null") { + return Some((String::new(), Vec::new())); + } + if p.peek() != Some(b'[') { + return None; + } + p.i += 1; + let mut b = String::new(); + let mut parts = Vec::new(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.peek() == Some(b'"') { + let text = json_string(p)?; + b.push_str(&text); + parts.push(ChatPart::Text(text)); + } else if p.peek() == Some(b'{') { + p.i += 1; + let mut typ = None; + let mut text = None; + let mut image_url = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "type" { + typ = Some(json_string(p)?); + } else if key == "text" { + p.ws(); + if p.lit("null") { + text = Some(String::new()); + } else { + text = Some(json_string(p)?); + } + } else if key == "image_url" { + image_url = Some(json_string(p)?); + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + let is_text = matches!( + typ.as_deref(), + Some("input_text" | "output_text" | "text" | "summary_text" | "reasoning_text") + ); + if is_text { + let text = text?; + if let Err(e) = reject_stray_image_in_content(&text) { + *err = e; + return None; + } + b.push_str(&text); + parts.push(ChatPart::Text(text)); + } else if typ.as_deref() == Some("input_image") { + let Some(images) = images.as_deref_mut() else { + *err = "unsupported Responses content block".into(); + return None; + }; + let Some(image_url) = image_url else { + *err = "unsupported Responses content block".into(); + return None; + }; + match add_image_data_uri(images, &[], &[], &image_url) { + Ok(index) => parts.push(ChatPart::Image(index)), + Err(error) => { + *err = error; + return None; + } + } + } else { + *err = "unsupported Responses content block".into(); + return None; + } + } else { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + Some((b, parts)) +} + +fn parse_responses_content_array(p: &mut Json<'_>, err: &mut String) -> Option { + parse_responses_content_array_mm(p, None, err).map(|(text, _)| text) +} + +fn parse_responses_reasoning( + p: &mut Json<'_>, + effort: &mut RequestedEffort, + summary_opted: &mut bool, +) -> Option { + p.ws(); + if p.lit("null") { + return Some(false); + } + if p.peek() != Some(b'{') { + json_skip_ok(p)?; + return Some(false); + } + p.i += 1; + p.ws(); + let mut effort_seen = false; + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "effort" { + p.ws(); + if p.lit("null") { + } else { + *effort = parse_reasoning_effort_value(p).ok()??; + effort_seen = true; + } + } else if key == "summary" { + p.ws(); + if p.lit("null") { + } else if p.peek() == Some(b'"') { + let mode = json_string(p)?; + if matches!(mode.as_str(), "auto" | "concise" | "detailed") { + *summary_opted = true; + } + } else if !json_skip_value(p) { + return None; + } + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + Some(effort_seen) +} + +fn parse_responses_input( + p: &mut Json<'_>, + loaded: &mut String, + orders: &mut Vec, + images: &mut ImageInputs, + err: &mut String, +) -> Option> { + p.ws(); + if p.bump() != Some(b'[') { + return None; + } + let mut msgs = Vec::new(); + let mut pending = String::new(); + p.ws(); + while p.peek().is_some() && p.peek() != Some(b']') { + if p.bump() != Some(b'{') { + return None; + } + let mut typ = None; + let mut role = None; + let mut content = None; + let mut content_parts = Vec::new(); + let mut name = None; + let mut namespace = None; + let mut call_id = None; + let mut item_id = None; + let mut arguments = None; + let mut output = None; + let mut output_parts = Vec::new(); + let mut input_str = None; + let mut summary = None; + let mut action = None; + let mut result = None; + let mut tools_json = None; + let mut status_str = None; + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "type" { + typ = Some(json_string(p)?); + } else if key == "role" { + role = Some(json_string(p)?); + } else if key == "content" { + let parsed = parse_responses_content_array_mm(p, Some(images), err)?; + content = Some(parsed.0); + content_parts = parsed.1; + } else if key == "name" { + name = Some(json_string(p)?); + } else if key == "namespace" { + namespace = Some(json_string(p)?); + } else if key == "call_id" { + call_id = Some(json_string(p)?); + } else if key == "id" { + item_id = Some(json_string(p)?); + } else if key == "arguments" { + p.ws(); + arguments = if p.peek() == Some(b'"') { + Some(json_string(p)?) + } else { + Some(json_raw_value(p)?) + }; + } else if key == "output" { + p.ws(); + output = if p.peek() == Some(b'[') { + let (text, parts) = parse_responses_content_array_mm(p, Some(images), err)?; + output_parts = parts; + Some(text) + } else if p.peek() == Some(b'"') { + Some(json_string(p)?) + } else { + Some(json_raw_value(p)?) + }; + } else if key == "input" { + p.ws(); + input_str = if p.peek() == Some(b'"') { + Some(json_string(p)?) + } else { + Some(json_raw_value(p)?) + }; + } else if key == "summary" { + summary = Some(parse_responses_content_array(p, err)?); + } else if key == "action" { + action = Some(json_raw_value(p)?); + } else if key == "result" { + p.ws(); + result = if p.peek() == Some(b'"') { + Some(json_string(p)?) + } else { + Some(json_raw_value(p)?) + }; + } else if key == "status" { + status_str = Some(json_string(p)?); + } else if key == "tools" { + tools_json = Some(json_raw_value(p)?); + } else if !json_skip_value(p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + if let Some(st) = status_str.as_deref() { + if !st.is_empty() && st != "completed" { + return None; + } + } + let t = typ.as_deref().unwrap_or("message"); + let consumes = (t == "message" && role.as_deref() == Some("assistant")) + || matches!( + t, + "function_call" + | "custom_tool_call" + | "local_shell_call" + | "web_search_call" + | "tool_search_call" + | "image_generation_call" + ); + let bookkeeping = t == "compaction" || t == "context_compaction"; + if !consumes && !bookkeeping && !pending.is_empty() { + msgs.push(ChatMsg { + role: "assistant".into(), + content: String::new(), + reasoning: std::mem::take(&mut pending), + ..Default::default() + }); + } + if t == "message" { + let mut msg = ChatMsg { + role: role.unwrap_or_else(|| "user".into()), + content: content.take().unwrap_or_default(), + parts: std::mem::take(&mut content_parts), + ..Default::default() + }; + if msg.role == "assistant" && !pending.is_empty() { + msg.reasoning = std::mem::take(&mut pending); + } + write_tool_image_placeholders(&mut msg); + msgs.push(msg); + } else if t == "function_call" || t == "custom_tool_call" { + let args = arguments + .as_deref() + .or(input_str.as_deref()) + .unwrap_or("{}"); + let tc_name = if t != "custom_tool_call" + && namespace.as_ref().is_some_and(|n| !n.is_empty()) + && name.as_ref().is_some_and(|n| !n.is_empty()) + { + format!("{}{}", namespace.as_ref().unwrap(), name.as_ref().unwrap()) + } else { + name.clone().unwrap_or_default() + }; + let tc = ToolCall { + id: call_id.clone().or(item_id.clone()).unwrap_or_default(), + name: tc_name, + arguments: args.to_string(), + }; + if let Some(last) = msgs.last_mut().filter(|m| m.role == "assistant") { + if !pending.is_empty() && last.reasoning.is_empty() { + last.reasoning = std::mem::take(&mut pending); + } + last.calls.push(tc); + } else { + let mut msg = ChatMsg { + role: "assistant".into(), + ..Default::default() + }; + if !pending.is_empty() { + msg.reasoning = std::mem::take(&mut pending); + } + msg.calls.push(tc); + msgs.push(msg); + } + } else if t == "function_call_output" || t == "custom_tool_call_output" { + let mut msg = ChatMsg { + role: "tool".into(), + content: output.take().unwrap_or_default(), + parts: output_parts, + ..Default::default() + }; + if let Some(id) = call_id.as_deref().or(item_id.as_deref()) { + add_tool_call_id(&mut msg, id); + } + write_tool_image_placeholders(&mut msg); + msgs.push(msg); + } else if t == "reasoning" { + if let Some(s) = summary.as_deref() { + if !s.is_empty() { + if !pending.is_empty() { + pending.push('\n'); + } + pending.push_str(s); + } + } + if let Some(s) = content.as_deref() { + if !s.is_empty() { + if !pending.is_empty() { + pending.push('\n'); + } + pending.push_str(s); + } + } + } else if matches!( + t, + "local_shell_call" | "web_search_call" | "tool_search_call" | "image_generation_call" + ) { + let tc = ToolCall { + id: call_id.clone().or(item_id.clone()).unwrap_or_default(), + name: match t { + "tool_search_call" => "tool_search".into(), + "local_shell_call" => "local_shell".into(), + other => other.into(), + }, + arguments: action + .as_deref() + .or(arguments.as_deref()) + .or(input_str.as_deref()) + .unwrap_or("{}") + .to_string(), + }; + if let Some(last) = msgs.last_mut().filter(|m| m.role == "assistant") { + if !pending.is_empty() && last.reasoning.is_empty() { + last.reasoning = std::mem::take(&mut pending); + } + last.calls.push(tc); + } else { + let mut msg = ChatMsg { + role: "assistant".into(), + ..Default::default() + }; + if !pending.is_empty() { + msg.reasoning = std::mem::take(&mut pending); + } + msg.calls.push(tc); + msgs.push(msg); + } + } else if matches!( + t, + "local_shell_call_output" + | "web_search_call_output" + | "tool_search_output" + | "tool_search_call_output" + | "image_generation_call_output" + ) { + if t == "tool_search_output" { + if let Some(ref tj) = tools_json { + let mut tp = Json::new(tj); + if let Some((schemas, ords)) = parse_tools_value(&mut tp) { + if !schemas.is_empty() { + if !loaded.is_empty() { + loaded.push('\n'); + } + loaded.push_str(&schemas); + } + orders.extend(ords); + } else { + return None; + } + } + } + let body = output + .as_deref() + .or(result.as_deref()) + .or(tools_json.as_deref()) + .unwrap_or(""); + let mut msg = ChatMsg { + role: "tool".into(), + content: body.to_string(), + ..Default::default() + }; + if let Some(id) = call_id.as_deref().or(item_id.as_deref()) { + add_tool_call_id(&mut msg, id); + } + msgs.push(msg); + } else if !bookkeeping { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b']') { + return None; + } + if !pending.is_empty() { + msgs.push(ChatMsg { + role: "assistant".into(), + reasoning: pending, + ..Default::default() + }); + } + Some(msgs) +} + +fn apply_think( + r: &mut ParsedRequest, + got: bool, + mut enabled: bool, + effort: RequestedEffort, + engine_defaults: bool, +) -> Result<(), String> { + // Only the V4.1 head renders a numeric budget; every other family would + // silently read it as HIGH, so the number is refused there. + if effort.budget.is_some() && !engine_defaults { + return Err("numeric reasoning_effort requires DeepSeek V4.1".into()); + } + enabled = apply_think_aliases(r, got, enabled); + r.think_mode = if engine_defaults { + engine_think_mode(enabled, effort.mode) + } else { + think_mode_from_enabled(enabled, effort.mode) + }; + r.effort = effort; + Ok(()) +} + +/// The engine's `think_mode_from_enabled` (server_msgs.c:186-189): nothing +/// below max survives above zero -- DS4 exposes only HIGH and MAX, so low / +/// medium / minimal collapse to HIGH ("callers that need *no* reasoning must +/// use none instead", the engine's own comment) and a bare thinking flag +/// with the engine's HIGH default means HIGH. +fn engine_think_mode(enabled: bool, effort: ThinkMode) -> ThinkMode { + if !enabled || effort == ThinkMode::None { + ThinkMode::None + } else if effort == ThinkMode::Max { + ThinkMode::Max + } else { + ThinkMode::High + } +} + +pub fn parse_chat_request(env: &ParseEnv, body: &str) -> Result { + let mut r = ParsedRequest::init(ReqKind::Chat, env); + let mut err = String::new(); + let mut p = Json::new(body); + let mut got_messages = false; + let mut got_thinking = false; + // The engine's request-omitted defaults (server_parse_chat.c:16-17 and + // the other parsers' same pair): thinking OFF unless a flag asks, effort + // HIGH. Only the engine's own model set takes them (unit H, + // ParseEnv::engine_defaults); other families keep the port's defaults. + let mut thinking_enabled = !env.engine_defaults; + let mut reasoning_effort = RequestedEffort::named(if env.engine_defaults { + ThinkMode::High + } else { + env.default_effort + }); + let mut msgs = Vec::new(); + let mut images = ImageInputs::new(env); + let mut audios = Vec::new(); + let mut videos = Vec::new(); + let mut tool_schemas = String::new(); + let mut orders = Vec::new(); + + p.ws(); + if p.bump() != Some(b'{') { + return bad(&mut err, "invalid JSON request"); + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(&mut p) else { + return bad(&mut err, "invalid JSON request"); + }; + p.ws(); + if p.bump() != Some(b':') { + return bad(&mut err, "invalid JSON request"); + } + let ok = if key == "messages" { + match parse_messages(&mut p, &mut images, &mut audios, &mut videos, &mut err) { + Some(m) => { + msgs = m; + got_messages = true; + true + } + None => false, + } + } else if key == "tools" { + match parse_tools_value(&mut p) { + Some((s, o)) => { + tool_schemas = s; + orders = o; + true + } + None => false, + } + } else if key == "tool_choice" { + match parse_openai_tool_choice(&mut p, &mut err) { + Some((c, forced)) => { + r.tool_choice = c; + r.forced_tool = forced; + true + } + None => false, + } + } else if key == "parallel_tool_calls" { + parse_parallel_tool_calls(&mut p, &mut err) + } else if key == "model" { + match json_string(&mut p) { + Some(m) => { + r.model = m; + r.model_from_request = true; + true + } + None => false, + } + } else if key == "max_tokens" || key == "max_completion_tokens" { + parse_budget(&mut p, &key, &mut r, &mut err) + } else if key == "temperature" { + json_number(&mut p) + .map(|v| r.temperature = v as f32) + .is_some() + } else if key == "top_p" { + json_number(&mut p).map(|v| r.top_p = v as f32).is_some() + } else if key == "min_p" { + json_number(&mut p).map(|v| r.min_p = v as f32).is_some() + } else if key == "frequency_penalty" { + json_number(&mut p) + .map(|v| r.frequency_penalty = v as f32) + .is_some() + } else if key == "presence_penalty" { + json_number(&mut p) + .map(|v| r.presence_penalty = v as f32) + .is_some() + } else if key == "dry_multiplier" || key == "dry_base" || key == "dry_allowed_length" { + // DRY, llama.cpp's names (server_parse_chat.c:119-127): the + // multiplier appearing marks dry_set; base / allowed_length fall + // back to 1.75 / 2 at face build (server_generate_v41.c:363-365). + json_number(&mut p) + .map(|v| { + if key == "dry_multiplier" { + r.dry_multiplier = v as f32; + r.dry_set = true; + } else if key == "dry_base" { + r.dry_base = v as f32; + } else { + r.dry_allowed_length = v as i32; + } + }) + .is_some() + } else if key == "top_k" { + json_int(&mut p).map(|v| r.top_k = v).is_some() + } else if key == "seed" { + json_number(&mut p) + .map(|v| r.seed = if v > 0.0 { v as u64 } else { 0 }) + .is_some() + } else if key == "stream" { + json_bool(&mut p).map(|v| r.stream = v).is_some() + } else if key == "stream_options" { + parse_stream_options(&mut p) + .map(|v| r.stream_include_usage = v) + .is_some() + } else if key == "return_token_ids" { + json_bool(&mut p).map(|v| r.return_token_ids = v).is_some() + } else if key == "thinking" { + match parse_thinking_control_value(&mut p) { + Some(v) => { + if let Some(e) = v { + thinking_enabled = e; + } + got_thinking = true; + true + } + None => false, + } + } else if key == "reasoning_effort" { + match parse_reasoning_effort_value(&mut p) { + Ok(Some(e)) => { + reasoning_effort = e; + // The engine (server_parse_chat.c:157-165): the OpenAI + // effort field alone is the client asking for thinking -- + // got_thinking set, enabled = effort != none. + got_thinking = true; + thinking_enabled = e.mode != ThinkMode::None; + true + } + Ok(None) => { + // "null": the effort stays at its default and the key + // still counts as an explicit ask (the engine's + // parse_reasoning_effort_value returns true unchanged). + got_thinking = true; + thinking_enabled = reasoning_effort.mode != ThinkMode::None; + true + } + Err(_) => false, + } + } else if key == "think" || key == "enable_thinking" { + match json_bool(&mut p) { + Some(v) => { + thinking_enabled = v; + got_thinking = true; + true + } + None => false, + } + } else if key == "stop" { + match parse_stop(&mut p) { + Some(s) => { + r.stops = s; + true + } + None => false, + } + } else if key == "response_format" { + parse_output_format_value(&mut p, "response_format") + .map_err(|e| err = e) + .is_ok() + } else { + json_skip_value(&mut p) + }; + if !ok { + return bad(&mut err, "invalid JSON request"); + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.peek() != Some(b'}') { + return bad(&mut err, "invalid JSON request"); + } + if !got_messages { + return Err("missing messages".into()); + } + validate_image_references(&msgs, &images)?; + validate_audio_refs(&msgs, &images, &audios, &videos)?; + validate_video_refs(&msgs, &videos)?; + r.has_tool_results = chat_history_has_pending_tool_results(&msgs); + if !prepare_tool_choice(&mut r, &mut tool_schemas, &mut orders, &mut err) { + return Err(err); + } + apply_think( + &mut r, + got_thinking, + thinking_enabled, + reasoning_effort, + env.engine_defaults, + )?; + r.messages = msgs; + r.images = images.items; + r.audios = audios; + r.videos = videos; + r.tool_schemas = tool_schemas; + r.tool_orders = orders; + r.finish_needs(); + Ok(r) +} + +pub fn parse_completion_request(env: &ParseEnv, body: &str) -> Result { + let mut r = ParsedRequest::init(ReqKind::Completion, env); + let mut err = String::new(); + let mut p = Json::new(body); + let mut prompt = None; + let mut got_thinking = false; + // The engine's request-omitted defaults (server_parse_chat.c:16-17 and + // the other parsers' same pair): thinking OFF unless a flag asks, effort + // HIGH. Only the engine's own model set takes them (unit H, + // ParseEnv::engine_defaults); other families keep the port's defaults. + let mut thinking_enabled = !env.engine_defaults; + let mut reasoning_effort = RequestedEffort::named(if env.engine_defaults { + ThinkMode::High + } else { + env.default_effort + }); + + p.ws(); + if p.bump() != Some(b'{') { + return bad(&mut err, "invalid JSON request"); + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(&mut p) else { + return bad(&mut err, "invalid JSON request"); + }; + p.ws(); + if p.bump() != Some(b':') { + return bad(&mut err, "invalid JSON request"); + } + let ok = if key == "prompt" { + match parse_prompt(&mut p) { + Some(s) => { + prompt = Some(s); + true + } + None => false, + } + } else if key == "model" { + match json_string(&mut p) { + Some(m) => { + r.model = m; + r.model_from_request = true; + true + } + None => false, + } + } else if key == "max_tokens" { + parse_budget(&mut p, &key, &mut r, &mut err) + } else if key == "temperature" { + json_number(&mut p) + .map(|v| r.temperature = v as f32) + .is_some() + } else if key == "top_p" { + json_number(&mut p).map(|v| r.top_p = v as f32).is_some() + } else if key == "min_p" { + json_number(&mut p).map(|v| r.min_p = v as f32).is_some() + } else if key == "frequency_penalty" || key == "presence_penalty" { + // The engine's completions wire has both, same meaning as the chat + // wire (server_parse_completion.c:108-115: "以前这里没解析, 传了被 + // 静默丢掉" -- unparsed meant silently dropped; fixed in a8e878a, + // the KV-reuse commit whose gate sends frequency_penalty 1e-30 to + // take the host penalty route). + let is_freq = key == "frequency_penalty"; + json_number(&mut p) + .map(|v| { + if is_freq { + r.frequency_penalty = v as f32; + } else { + r.presence_penalty = v as f32; + } + }) + .is_some() + } else if key == "top_k" { + json_int(&mut p).map(|v| r.top_k = v).is_some() + } else if key == "seed" { + json_number(&mut p) + .map(|v| r.seed = if v > 0.0 { v as u64 } else { 0 }) + .is_some() + } else if key == "stream" { + json_bool(&mut p).map(|v| r.stream = v).is_some() + } else if key == "stream_options" { + parse_stream_options(&mut p) + .map(|v| r.stream_include_usage = v) + .is_some() + } else if key == "return_token_ids" { + json_bool(&mut p).map(|v| r.return_token_ids = v).is_some() + } else if key == "thinking" { + match parse_thinking_control_value(&mut p) { + Some(v) => { + if let Some(e) = v { + thinking_enabled = e; + } + got_thinking = true; + true + } + None => false, + } + } else if key == "reasoning_effort" { + match parse_reasoning_effort_value(&mut p) { + Ok(Some(e)) => { + reasoning_effort = e; + true + } + Ok(None) => true, + Err(_) => false, + } + } else if key == "think" || key == "enable_thinking" { + match json_bool(&mut p) { + Some(v) => { + thinking_enabled = v; + got_thinking = true; + true + } + None => false, + } + } else if key == "stop" { + match parse_stop(&mut p) { + Some(s) => { + r.stops = s; + true + } + None => false, + } + } else if key == "response_format" { + parse_output_format_value(&mut p, "response_format") + .map_err(|e| err = e) + .is_ok() + } else { + json_skip_value(&mut p) + }; + if !ok { + return bad(&mut err, "invalid JSON request"); + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.peek() != Some(b'}') { + return bad(&mut err, "invalid JSON request"); + } + let Some(prompt) = prompt else { + return Err("missing prompt".into()); + }; + apply_think( + &mut r, + got_thinking, + thinking_enabled, + reasoning_effort, + env.engine_defaults, + )?; + r.prompt_text = Some(prompt); + r.finish_needs(); + Ok(r) +} + +pub fn parse_anthropic_request(env: &ParseEnv, body: &str) -> Result { + let mut r = ParsedRequest::init(ReqKind::Chat, env); + r.api = Api::Anthropic; + let mut err = String::new(); + let mut p = Json::new(body); + let mut got_messages = false; + let mut got_thinking = false; + // The engine's request-omitted defaults (server_parse_chat.c:16-17 and + // the other parsers' same pair): thinking OFF unless a flag asks, effort + // HIGH. Only the engine's own model set takes them (unit H, + // ParseEnv::engine_defaults); other families keep the port's defaults. + let mut thinking_enabled = !env.engine_defaults; + let mut reasoning_effort = RequestedEffort::named(if env.engine_defaults { + ThinkMode::High + } else { + env.default_effort + }); + let mut msgs = Vec::new(); + let mut images = ImageInputs::new(env); + let mut system = None; + let mut tool_schemas = String::new(); + let mut orders = Vec::new(); + + p.ws(); + if p.bump() != Some(b'{') { + return bad(&mut err, "invalid JSON request"); + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(&mut p) else { + return bad(&mut err, "invalid JSON request"); + }; + p.ws(); + if p.bump() != Some(b':') { + return bad(&mut err, "invalid JSON request"); + } + let ok = if key == "messages" { + match parse_anthropic_messages(&mut p, &mut images, &mut err) { + Some(m) => { + msgs = m; + got_messages = true; + true + } + None => false, + } + } else if key == "system" { + match parse_anthropic_system(&mut p) { + Some(s) => { + system = Some(s); + true + } + None => false, + } + } else if key == "tools" { + match parse_tools_value(&mut p) { + Some((s, o)) => { + tool_schemas = s; + orders = o; + true + } + None => false, + } + } else if key == "tool_choice" { + match parse_anthropic_tool_choice(&mut p, &mut err) { + Some(c) => { + r.tool_choice = c; + true + } + None => false, + } + } else if key == "model" { + match json_string(&mut p) { + Some(m) => { + r.model = m; + r.model_from_request = true; + true + } + None => false, + } + } else if key == "max_tokens" { + parse_budget(&mut p, &key, &mut r, &mut err) + } else if key == "temperature" { + json_number(&mut p) + .map(|v| r.temperature = v as f32) + .is_some() + } else if key == "top_p" { + json_number(&mut p).map(|v| r.top_p = v as f32).is_some() + } else if key == "top_k" { + json_int(&mut p).map(|v| r.top_k = v).is_some() + } else if key == "stream" { + json_bool(&mut p).map(|v| r.stream = v).is_some() + } else if key == "stop_sequences" { + match parse_stop(&mut p) { + Some(s) => { + r.stops = s; + true + } + None => false, + } + } else if key == "thinking" { + match parse_thinking_control_value(&mut p) { + Some(v) => { + if let Some(e) = v { + thinking_enabled = e; + } + got_thinking = true; + true + } + None => false, + } + } else if key == "output_config" { + match parse_output_config_effort(&mut p) { + Ok(e) => { + if let Some(v) = e { + reasoning_effort = v; + // The engine (server_parse_anthropic_req.c:178-183): + // an effort-only ask turns thinking on (high/max) or + // off (none); an explicit thinking field still wins. + if !got_thinking { + thinking_enabled = reasoning_effort.mode != ThinkMode::None; + } + } + true + } + Err(e) => { + err = e; + false + } + } + } else if key == "output_format" { + parse_output_format_value(&mut p, "output_format") + .map_err(|e| err = e) + .is_ok() + } else if key == "reasoning_effort" { + match parse_reasoning_effort_value(&mut p) { + Ok(Some(e)) => { + reasoning_effort = e; + if !got_thinking { + thinking_enabled = reasoning_effort.mode != ThinkMode::None; + } + true + } + Ok(None) => true, + Err(_) => false, + } + } else { + json_skip_value(&mut p) + }; + if !ok { + return bad(&mut err, "invalid JSON request"); + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.peek() != Some(b'}') { + return bad(&mut err, "invalid JSON request"); + } + if !got_messages { + return Err("missing messages".into()); + } + if let Some(sys) = system { + if !sys.is_empty() { + msgs.push(ChatMsg { + role: "system".into(), + content: sys, + ..Default::default() + }); + } + } + validate_image_references(&msgs, &images)?; + r.has_tool_results = chat_history_has_pending_tool_results(&msgs); + if !prepare_tool_choice(&mut r, &mut tool_schemas, &mut orders, &mut err) { + return Err(err); + } + apply_think( + &mut r, + got_thinking, + thinking_enabled, + reasoning_effort, + env.engine_defaults, + )?; + match anthropic_validate_tool_results(&msgs, &env.live_ids, &mut err) { + Ok(live) => r.anthropic_requires_live_tool_state = live, + Err(()) => return Err(err), + } + r.live_call_ids = crate::cont::live_tool_result_ids(Api::Anthropic, &msgs); + r.messages = msgs; + r.images = images.items; + r.tool_schemas = tool_schemas; + r.tool_orders = orders; + r.finish_needs(); + Ok(r) +} + +pub fn parse_responses_request(env: &ParseEnv, body: &str) -> Result { + let mut r = ParsedRequest::init(ReqKind::Chat, env); + r.api = Api::Responses; + let mut err = String::new(); + let mut p = Json::new(body); + let mut got_input = false; + let mut got_thinking = false; + // The engine's request-omitted defaults (server_parse_chat.c:16-17 and + // the other parsers' same pair): thinking OFF unless a flag asks, effort + // HIGH. Only the engine's own model set takes them (unit H, + // ParseEnv::engine_defaults); other families keep the port's defaults. + let mut thinking_enabled = !env.engine_defaults; + let mut reasoning_effort = RequestedEffort::named(if env.engine_defaults { + ThinkMode::High + } else { + env.default_effort + }); + let mut msgs = Vec::new(); + let mut images = ImageInputs::new(env); + let mut loaded = String::new(); + let mut orders = Vec::new(); + let mut instructions = None; + let mut tool_schemas = String::new(); + + p.ws(); + if p.bump() != Some(b'{') { + return bad(&mut err, "invalid JSON request"); + } + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = json_string(&mut p) else { + return bad(&mut err, "invalid JSON request"); + }; + p.ws(); + if p.bump() != Some(b':') { + return bad(&mut err, "invalid JSON request"); + } + let ok = if key == "input" { + p.ws(); + if p.peek() == Some(b'"') { + match json_string(&mut p) { + Some(plain) => { + msgs = vec![ChatMsg { + role: "user".into(), + content: plain, + ..Default::default() + }]; + got_input = true; + true + } + None => false, + } + } else { + match parse_responses_input(&mut p, &mut loaded, &mut orders, &mut images, &mut err) + { + Some(m) => { + msgs = m; + got_input = true; + true + } + None => false, + } + } + } else if key == "instructions" { + p.ws(); + if p.lit("null") { + instructions = Some(String::new()); + true + } else { + match json_string(&mut p) { + Some(s) => { + instructions = Some(s); + true + } + None => false, + } + } + } else if key == "tools" { + match parse_tools_value(&mut p) { + Some((s, o)) => { + tool_schemas = s; + orders.extend(o); + true + } + None => false, + } + } else if key == "tool_choice" { + match parse_openai_tool_choice(&mut p, &mut err) { + Some((c, forced)) => { + r.tool_choice = c; + r.forced_tool = forced; + true + } + None => false, + } + } else if key == "parallel_tool_calls" { + parse_parallel_tool_calls(&mut p, &mut err) + } else if key == "model" { + match json_string(&mut p) { + Some(m) => { + r.model = m; + r.model_from_request = true; + true + } + None => false, + } + } else if key == "max_output_tokens" || key == "max_tokens" { + parse_budget(&mut p, &key, &mut r, &mut err) + } else if key == "temperature" { + json_number(&mut p) + .map(|v| r.temperature = v as f32) + .is_some() + } else if key == "top_p" { + json_number(&mut p).map(|v| r.top_p = v as f32).is_some() + } else if key == "frequency_penalty" { + json_number(&mut p) + .map(|v| r.frequency_penalty = v as f32) + .is_some() + } else if key == "presence_penalty" { + json_number(&mut p) + .map(|v| r.presence_penalty = v as f32) + .is_some() + } else if key == "stream" { + json_bool(&mut p).map(|v| r.stream = v).is_some() + } else if key == "reasoning" { + match parse_responses_reasoning( + &mut p, + &mut reasoning_effort, + &mut r.reasoning_summary_emit, + ) { + Some(seen) => { + if seen { + // The engine (server_parse_responses.c:363-370): an + // explicit effort is the client opting into thinking + // control -- got_thinking set, enabled = effort != none. + got_thinking = true; + thinking_enabled = reasoning_effort.mode != ThinkMode::None; + } + true + } + None => false, + } + } else if key == "text" { + parse_responses_text_value(&mut p) + .map_err(|e| err = e) + .is_ok() + } else if key == "previous_response_id" || key == "conversation" { + p.ws(); + if p.lit("null") { + true + } else { + err = format!("{key} is not supported; replay full input instead"); + false + } + } else { + json_skip_value(&mut p) + }; + if !ok { + return bad(&mut err, "invalid JSON request"); + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.peek() != Some(b'}') { + return bad(&mut err, "invalid JSON request"); + } + if !got_input { + return Err("missing input".into()); + } + if let Some(ins) = instructions { + if !ins.is_empty() { + msgs.push(ChatMsg { + role: "system".into(), + content: ins, + ..Default::default() + }); + if msgs.len() > 1 { + let last = msgs.pop().unwrap(); + msgs.insert(0, last); + } + } + } + validate_image_references(&msgs, &images)?; + let mut combined = tool_schemas.clone(); + if !loaded.is_empty() { + if !combined.is_empty() { + combined.push('\n'); + } + combined.push_str(&loaded); + } + r.has_tool_results = chat_history_has_pending_tool_results(&msgs); + if !prepare_tool_choice(&mut r, &mut combined, &mut orders, &mut err) { + return Err(err); + } + apply_think( + &mut r, + got_thinking, + thinking_enabled, + reasoning_effort, + env.engine_defaults, + )?; + match responses_validate_tool_outputs(&msgs, r.think_mode, &env.live_ids, &mut err) { + Ok((t, reason)) => { + r.responses_requires_live_tool_state = t; + r.responses_requires_live_reasoning = reason; + } + Err(()) => return Err(err), + } + r.live_call_ids = crate::cont::live_tool_result_ids(Api::Responses, &msgs); + r.messages = msgs; + r.images = images.items; + r.tool_schemas = combined; + r.tool_orders = orders; + r.finish_needs(); + Ok(r) +} + +pub fn parse_request( + surface: WireSurface, + env: &ParseEnv, + body: &str, +) -> Result { + match surface { + WireSurface::OpenaiChat => parse_chat_request(env, body), + WireSurface::OpenaiCompletion => parse_completion_request(env, body), + WireSurface::Anthropic => parse_anthropic_request(env, body), + WireSurface::Responses => parse_responses_request(env, body), + } +} + +#[cfg(test)] +mod eos_policy_tests { + use super::EosPolicy; + use crate::stream::ChatFormat; + use crate::tools::SemAccum; + + #[test] + fn reasoning_policy_tracks_think_close_across_pieces() { + const EOS: i32 = 248_046; + let mut acc = SemAccum::init(true, false, true, ChatFormat::Qwen4Exp, b""); + + assert_eq!( + EosPolicy::Default.excluded(EOS, acc.thinking_inside()), + None + ); + assert_eq!( + EosPolicy::Reasoning.excluded(EOS, acc.thinking_inside()), + Some(EOS) + ); + assert_eq!( + EosPolicy::Global.excluded(EOS, acc.thinking_inside()), + Some(EOS) + ); + + acc.feed(b"", &[]); + assert_eq!( + EosPolicy::Reasoning.excluded(EOS, acc.thinking_inside()), + None + ); + assert_eq!( + EosPolicy::Global.excluded(EOS, acc.thinking_inside()), + Some(EOS) + ); + } +} diff --git a/crates/ds4-server/src/render.rs b/crates/ds4-server/src/render.rs new file mode 100644 index 0000000000000000000000000000000000000000..ad4be2ee053ea98f65693bd9d1ecf8acab543c0d --- /dev/null +++ b/crates/ds4-server/src/render.rs @@ -0,0 +1,2829 @@ +//! Family chat render from `ds4_server.c` at v0.6.5-dfm, including +//! tool-schema prompts and invoke reconstruct. Live tool stream +//! machines are host-owned (`tool_stream`). + +use crate::json::{ + json_args_parse, json_escape_bytes, json_minify_raw_value, json_raw_value, + json_string_contents, Json, +}; +use crate::parse::{ChatMsg, ChatPart, ToolCall, ToolChoice, ToolSchemaOrder}; +use crate::route::{mode_budget, think_mode_enabled, Api, ThinkMode}; +use crate::tools::{ + DSML_PARAM_END, DSML_PARAM_END_SPACED, DSML_PARAM_START, DSML_PARAM_START_SPACED, + DSML_TOOL_CALLS_END, DSML_TOOL_CALLS_END_SPACED, DSML_TOOL_CALLS_START, + DSML_TOOL_CALLS_START_SPACED, +}; + +pub(crate) mod inkling; + +/// Copied from `ds4.c` `DS4_REASONING_EFFORT_HIGH_PREFIX`. +pub const THINK_HIGH_PREFIX: &str = concat!( + "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n", + "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n", + "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n", +); + +/// Copied from `ds4.c` `DS4_REASONING_EFFORT_MAX_PREFIX`. +pub const THINK_MAX_PREFIX: &str = concat!( + "Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n", + "You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n", + "Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n", +); + +/// The engine's official V4.1 encoding.py effort lines (`core_globals.c:14-17`, +/// `DS4_REASONING_EFFORT_HIGH_PREFIX` / `DS4_REASONING_EFFORT_MAX_PREFIX`). +/// The `THINK_*_PREFIX` pair above is the port's pre-2026-09-21 V4-era text +/// and stays for the V4 head only. +pub const V41_THINK_HIGH_PREFIX: &str = + "Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning)\n\n"; +pub const V41_THINK_MAX_PREFIX: &str = "Reasoning Effort: 100 (range 1-100, the higher the value, the more thorough the reasoning)\n\n"; + +/// The oracle's numeric effort line (`REASONING_EFFORT_TEMPLATE`, +/// encoding.py:443-446): the requested budget verbatim, at index 0 and in +/// thinking mode only. The 75/100 constants above are its two pinned cases; +/// every other budget in [1,100] renders through this template. +pub const V41_EFFORT_TEMPLATE: &str = "Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n"; + +/// The DSML prompt head a model writes. `V4` is the port's own C lineage +/// (`THINK_*_PREFIX`, no system token). `V41` is the engine's official +/// encoding.py head: the effort line (core_globals.c:14-17, encoding.py:443-473) +/// and the <|System|> token written before it when the effort or the system +/// body is non-empty (server_dsml_render.c:375-383), gated on the loaded +/// tokenizer owning it (core_bpe.c:332-333, core_engine_api.c:94-96). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DsmlHead { + V4, + /// `effort` is the budget the request asked for; `None` leaves the number to + /// the thinking mode (75 high, 100 max, 50 low -- encoding.py:448-453). + V41 { + system_token: bool, + effort: Option, + }, +} + +impl DsmlHead { + /// The requested effort budget this head writes. A V4 head has no numeric + /// effort line, so it ignores the budget and keeps its own text. + pub fn with_effort(self, effort: Option) -> Self { + match self { + DsmlHead::V41 { system_token, .. } => DsmlHead::V41 { + system_token, + effort, + }, + DsmlHead::V4 => DsmlHead::V4, + } + } +} + +/// C `ds4_chat_system_token()` (core_engine_api.c:94-96). +pub const DSML_SYSTEM_TOKEN: &str = "<|System|>"; + +/// The head for a model id: only `Variant::DeepSeek41Flash` takes the engine's +/// V4.1 encoding.py head; every other DSML model keeps the port's own. +pub fn dsml_head(model_id: i32, system_token: &str) -> DsmlHead { + if model_id == ds4_core::Variant::DeepSeek41Flash as i32 { + DsmlHead::V41 { + system_token: !system_token.is_empty(), + effort: None, + } + } else { + DsmlHead::V4 + } +} + +/// The effort line for one head. The V4 head keeps the port's own effort text; +/// the V4.1 head writes the oracle's number, taken from the request when it gave +/// one and from the mode otherwise (encoding.py:448-473), and writes nothing at +/// all without thinking mode. +fn head_effort_prefix(head: DsmlHead, mode: ThinkMode) -> String { + match head { + DsmlHead::V4 => think_effort_prefix(mode).to_string(), + DsmlHead::V41 { effort, .. } => v41_effort_prefix(mode, effort), + } +} + +fn v41_effort_prefix(mode: ThinkMode, effort: Option) -> String { + if mode == ThinkMode::None { + return String::new(); + } + let Some(budget) = effort.or(mode_budget(mode)) else { + return String::new(); + }; + match budget { + 75 => V41_THINK_HIGH_PREFIX.to_string(), + 100 => V41_THINK_MAX_PREFIX.to_string(), + n => V41_EFFORT_TEMPLATE.replace("{budget}", &n.to_string()), + } +} + +pub const DSML_BOS: &str = "<|begin▁of▁sentence|>"; +pub const DSML_USER: &str = "<|User|>"; +pub const DSML_ASSISTANT: &str = "<|Assistant|>"; +pub const DSML_EOS: &str = "<|end▁of▁sentence|>"; + +/// The DSML think tags (the engine's literal in server_dsml_render.c: an ASCII +/// `` / `` pair). Written with \u escapes so the string +/// survives any pipeline that parses bare angle brackets as markup. +pub const DSML_THINK_START: &str = "\u{3c}think\u{3e}"; +pub const DSML_THINK_END: &str = "\u{3c}/think\u{3e}"; +pub const DSML_TOOL_CALLS: &str = "<|DSML|tool_calls>"; +/// The V4.1 image placeholder the server writes per image (`core_bpe.c:365` +/// image token, id `deepseek4.image_token_id` 129264); the prefill expands it +/// into the span (`ds4_v41_images_prepare`). +pub const DSML_IMAGE: &str = "<|deepseek_image|>"; +pub const MOTIF_TOOL_CALLS: &str = ""; +pub const EXAONE_TOOL_CALLS: &str = ""; +pub const DOTS3_TOOL_CALLS: &str = ""; +pub const SOLAR_TOOL_CALLS: &str = "<|tool_call:start|>"; +pub const SOLAR_TOOL_START: &str = "<|tool:start|>"; +pub const SOLAR_TOOL_END: &str = "<|tool:end|>"; +pub const SOLAR_TOOL_CALL_END: &str = "<|tool_call:end|>"; +pub const SOLAR_TOOL_ARG_START: &str = "<|tool_arg:start|>"; +pub const SOLAR_TOOL_ARG_VALUE: &str = "<|tool_arg:value|>"; +pub const SOLAR_TOOL_ARG_END: &str = "<|tool_arg:end|>"; +pub const SOLAR_IM_START: &str = "<|im:start|>"; +pub const SOLAR_IM_CONTENT: &str = "<|im:content|>"; +pub const SOLAR_IM_END: &str = "<|im:end|>"; +pub const SOLAR_THINK_START: &str = "<|think:start|>"; +pub const SOLAR_THINK_END: &str = "<|think:end|>"; +pub const SOLAR_TOOL_RESPONSE_START: &str = "<|tool_response:start|>"; +pub const SOLAR_TOOL_RESPONSE_END: &str = "<|tool_response:end|>"; +pub const QWEN_IM_START: &str = "<|im_start|>"; +pub const QWEN_IM_END: &str = "<|im_end|>"; +pub const QWEN_TOOL_CALL_START: &str = ""; +pub const QWEN_TOOL_CALL_END: &str = ""; +pub const QWEN_TOOL_RESPONSE_START: &str = ""; +pub const QWEN_TOOL_RESPONSE_END: &str = ""; +pub const QWEN_VISION_START: &str = "<|vision_start|>"; +pub const QWEN_IMAGE_PAD: &str = "<|image_pad|>"; +pub const QWEN_VISION_END: &str = "<|vision_end|>"; +pub const GLM_BOS: &str = "[gMASK]"; +pub const GLM_TOOL_CALL_START: &str = ""; +pub const GLM_TOOL_CALL_END: &str = ""; +pub const IQUEST_TOOL_CALL_START: &str = ""; +pub const IQUEST_TOOL_CALL_END: &str = ""; +pub const GLM_VISION_START: &str = "<|begin_of_image|>"; +pub const GLM_IMAGE: &str = "<|image|>"; +pub const GLM_VISION_END: &str = "<|end_of_image|>"; +pub const K2_BOS: &str = "<|ifm|begin_of_text|>"; +pub const K2_IM_START: &str = "<|ifm|im_start|>"; +pub const K2_IM_END: &str = "<|ifm|im_end|>"; +pub const K2_THINK_START: &str = ""; +pub const K2_THINK_END: &str = ""; +pub const K2_TOOL_CALLS_START: &str = ""; +pub const K2_TOOL_CALLS_END: &str = ""; +pub const K2_TOOL_CALL_START: &str = ""; +pub const K2_TOOL_CALL_END: &str = ""; +pub const K2_ARG_KEY_START: &str = ""; +pub const K2_ARG_KEY_END: &str = ""; +pub const K2_ARG_VALUE_START: &str = ""; +pub const K2_ARG_VALUE_END: &str = ""; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ModelSyntax { + DeepSeek = 0, + SolarOpen2 = 2, + Motif3 = 3, + Exaone = 4, + Dots3 = 5, + Qwen4Exp = 6, + Glm53 = 7, + K2Horizon = 8, + Inkling = 9, + Step37 = 10, + Ling3Vl = 11, + Mimo2 = 12, + Qwen35 = 14, + NaiveN05 = 13, + IQuestQ1 = 15, + BeamVq = 19, +} + +/// C `server_model_syntax_for_engine`. +pub fn syntax_for_model_id(model_id: i32) -> ModelSyntax { + match model_id { + 2 => ModelSyntax::SolarOpen2, + 3 => ModelSyntax::Motif3, + 4 => ModelSyntax::Exaone, + 5 => ModelSyntax::Dots3, + 6 => ModelSyntax::Qwen4Exp, + // 17: GLM VQ keeps GLM-5.3's template, tool markers and parser. + 7 | 17 => ModelSyntax::Glm53, + 8 => ModelSyntax::K2Horizon, + 9 => ModelSyntax::Inkling, + 10 => ModelSyntax::Step37, + // 18: Ling-3.1 keeps Bailing V3's template, tool markers and parser. + 11 | 18 => ModelSyntax::Ling3Vl, + 12 => ModelSyntax::Mimo2, + 14 => ModelSyntax::Qwen35, + 13 => ModelSyntax::NaiveN05, + 15 => ModelSyntax::IQuestQ1, + 19 => ModelSyntax::BeamVq, + _ => ModelSyntax::DeepSeek, + } +} + +pub fn tool_start_marker(syntax: ModelSyntax) -> &'static str { + match syntax { + ModelSyntax::SolarOpen2 => SOLAR_TOOL_CALLS, + ModelSyntax::Motif3 | ModelSyntax::Exaone => MOTIF_TOOL_CALLS, + ModelSyntax::Dots3 => DOTS3_TOOL_CALLS, + ModelSyntax::Qwen4Exp + | ModelSyntax::Step37 + | ModelSyntax::Mimo2 + | ModelSyntax::Qwen35 + | ModelSyntax::NaiveN05 => QWEN_TOOL_CALL_START, + ModelSyntax::Glm53 | ModelSyntax::Ling3Vl => GLM_TOOL_CALL_START, + ModelSyntax::K2Horizon => K2_TOOL_CALLS_START, + ModelSyntax::Inkling => inkling::INVOKE, + ModelSyntax::IQuestQ1 => IQUEST_TOOL_CALL_START, + // Harmony opens a call as the next assistant message, after analysis; + // `=` merges into the recipient's first token. + ModelSyntax::BeamVq => "<|start|>assistant to", + ModelSyntax::DeepSeek => DSML_TOOL_CALLS, + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RenderError(pub &'static str); + +impl std::fmt::Display for RenderError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0) + } +} + +pub fn think_effort_prefix(mode: ThinkMode) -> &'static str { + match mode { + ThinkMode::High => THINK_HIGH_PREFIX, + ThinkMode::Max => THINK_MAX_PREFIX, + ThinkMode::None | ThinkMode::Low => "", + } +} + +pub fn role_is_system(role: &str) -> bool { + role == "system" || role == "developer" +} + +pub fn role_is_user_like(role: &str) -> bool { + role == "user" || role == "tool" || role == "function" +} + +fn chat_history_uses_tool_context(msgs: &[ChatMsg], tool_schemas: &str) -> bool { + if !tool_schemas.is_empty() { + return true; + } + msgs.iter().any(|m| { + (m.role == "assistant" && !m.calls.is_empty()) || m.role == "tool" || m.role == "function" + }) +} + +fn put(out: &mut Vec, s: &str) { + out.extend_from_slice(s.as_bytes()); +} + +fn tool_schema_orders_find<'a>( + orders: &'a [ToolSchemaOrder], + name: &str, +) -> Option<&'a ToolSchemaOrder> { + orders.iter().find(|o| o.name == name) +} + +fn json_args_find_unused(args: &[crate::json::JsonArg], key: &str) -> Option { + args.iter().position(|a| !a.used && a.key == key) +} + +fn append_sentinel_escape(out: &mut Vec, s: &[u8], end: &[u8], repl: &[u8]) { + let mut i = 0; + while i < s.len() { + if s[i..].starts_with(end) { + out.extend_from_slice(repl); + i += 1; + } else { + out.push(s[i]); + i += 1; + } + } +} + +fn append_dsml_attr_escaped(out: &mut Vec, s: &str) { + for &c in s.as_bytes() { + match c { + b'&' => out.extend_from_slice(b"&"), + b'<' => out.extend_from_slice(b"<"), + b'>' => out.extend_from_slice(b">"), + b'"' => out.extend_from_slice(b"""), + c => out.push(c), + } + } +} + +fn append_dsml_parameter_text(out: &mut Vec, param_end: &[u8], s: &str) { + append_sentinel_escape(out, s.as_bytes(), param_end, b"<"); +} + +fn append_dsml_json_literal(out: &mut Vec, param_end: &[u8], s: &str) { + append_sentinel_escape(out, s.as_bytes(), param_end, b"\\u003c"); +} + +/// The engine's `json_pyfmt_raw_value` (server_buf.c pyjson_value / +/// pyjson_string): the official generation serializes tool schemas and +/// non-string parameters the way Python's json.dumps(ensure_ascii=False) +/// does -- ", " and ": " separators, non-ASCII raw, only Python's escapes. +/// Scalars keep their source digits (serde_json's arbitrary precision). +fn pyjson_raw_value(json: &str) -> Option { + let v: serde_json::Value = serde_json::from_str(json).ok()?; + let mut out = String::new(); + pyjson_write(&v, &mut out); + Some(out) +} + +fn pyjson_write(v: &serde_json::Value, out: &mut String) { + match v { + serde_json::Value::Object(map) => { + out.push('{'); + for (i, (k, val)) in map.iter().enumerate() { + if i > 0 { + out.push_str(", "); + } + pyjson_write_str(k, out); + out.push_str(": "); + pyjson_write(val, out); + } + out.push('}'); + } + serde_json::Value::Array(arr) => { + out.push('['); + for (i, val) in arr.iter().enumerate() { + if i > 0 { + out.push_str(", "); + } + pyjson_write(val, out); + } + out.push(']'); + } + serde_json::Value::String(s) => pyjson_write_str(s, out), + other => out.push_str(&other.to_string()), + } +} + +/// server_buf.c pyjson_string: Python's json.dumps string shape. +fn pyjson_write_str(s: &str, out: &mut String) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{8}' => out.push_str("\\b"), + '\u{c}' => out.push_str("\\f"), + c if (c as u32) < 0x20 => { + out.push_str(&format!("\\u{:04x}", c as u32)); + } + c => out.push(c), + } + } + out.push('"'); +} + +/// The engine's append_dsml_arg (server_dsml_render.c:121-141): the spaced +/// generation (4294136) writes raw string values, serializes non-string +/// parameters the way Python's json.dumps does (json_pyfmt_raw_value), and +/// ends with the closing tag -- the newline joins belong to the caller. +fn append_dsml_arg(out: &mut Vec, spaced: bool, arg: &crate::json::JsonArg) { + let (param_start, param_end): (&[u8], &[u8]) = if spaced { + ( + DSML_PARAM_START_SPACED.as_bytes(), + DSML_PARAM_END_SPACED.as_bytes(), + ) + } else { + (DSML_PARAM_START.as_bytes(), DSML_PARAM_END.as_bytes()) + }; + out.extend_from_slice(param_start); + put(out, " name=\""); + append_dsml_attr_escaped(out, &arg.key); + put(out, "\" string=\""); + put(out, if arg.is_string { "true" } else { "false" }); + put(out, "\">"); + if arg.is_string { + append_dsml_parameter_text(out, param_end, &arg.value); + } else if spaced { + let py = pyjson_raw_value(&arg.value).unwrap_or_else(|| arg.value.clone()); + append_dsml_json_literal(out, param_end, &py); + } else { + append_dsml_json_literal(out, param_end, &arg.value); + } + out.extend_from_slice(param_end); +} + +fn append_dsml_arguments_from_json( + out: &mut Vec, + json: &str, + order: Option<&ToolSchemaOrder>, + spaced: bool, +) -> i32 { + let Some(mut args) = json_args_parse(json) else { + return -1; + }; + if let Some(order) = order { + for prop in &order.prop { + if let Some(idx) = json_args_find_unused(&args, prop) { + append_dsml_arg(out, spaced, &args[idx]); + args[idx].used = true; + } + } + } + let mut first = order + .map(|o| { + o.prop + .iter() + .any(|p| args.iter().any(|a| a.used && a.key == *p)) + }) + .unwrap_or(false); + for arg in &args { + if arg.used { + continue; + } + if first { + out.push(b'\n'); + } + first = true; + append_dsml_arg(out, spaced, arg); + } + args.len() as i32 +} + +fn append_dsml_tools_prompt_text( + out: &mut Vec, + tool_schemas: &str, + tool_required: bool, + spaced: bool, +) { + if tool_schemas.is_empty() { + return; + } + // The engine's dispatch (server_knowledge.c append_tools_prompt_text, + // 4294136): the official spaced generation takes its own template. + if spaced { + append_dsml_tools_prompt_spaced(out, tool_schemas, tool_required); + return; + } + put( + out, + "## Tools\n\n\ +You have access to a set of tools to help answer the user question. \ +You can invoke tools by writing a \"<|DSML|tool_calls>\" block like the following:\n\n\ +<|DSML|tool_calls>\n\ +<|DSML|invoke name=\"$TOOL_NAME\">\n\ +<|DSML|parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n\ +...\n\ +\n\ +<|DSML|invoke name=\"$TOOL_NAME2\">\n\ +...\n\ +\n\ +\n\n\ +String parameters should be specified as raw text and set `string=\"true\"`. \ +Preserve characters such as `>`, `&`, and `&&` exactly; never replace normal string characters with XML or HTML entity escapes. \ +Only if a string value itself contains the exact closing parameter tag ``, write that tag as `</|DSML|parameter>` inside the value. \ +For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\n\ +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\n\ +Otherwise, output directly after with tool calls or final response.\n\n\ +### Available Tool Schemas\n\n", + ); + put(out, tool_schemas); + put( + out, + "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. \ +Use the exact parameter names from the schemas.", + ); + if tool_required { + put( + out, + "\n\n### Required Tool Use\n\n\ +You MUST call at least one available tool in this turn. \ +Do not end the turn with only reasoning or a final answer. \ +After any reasoning, emit a valid <|DSML|tool_calls> block.", + ); + } +} + +/// The engine's append_dsml_tool_calls_text (server_dsml_render.c:180-207): +/// the spaced generation joins parameters and invokes with "\n" (the official +/// encoding.py's "\n".join) and closes the block with "\n" + the tag; the +/// old generations keep their per-line newlines byte for byte. +/// The engine's append_tools_prompt_spaced (server_knowledge.c:106-150, +/// 4294136): the official encoding.py TOOLS_TEMPLATE verbatim. Every +/// difference from the old template is training-distribution: "user's +/// question", "specified as is", no entity-escape note, the schema in +/// json.dumps shape, a single trailing "\n". +fn append_dsml_tools_prompt_spaced(out: &mut Vec, tool_schemas: &str, tool_required: bool) { + put( + out, + "## Tools\n\n\ +You have access to a set of tools to help answer the user's question. \ +You can invoke tools by writing a \"<|DSML| calls>\" block like the following:\n\n\ +<|DSML| calls>\n\ +<|DSML| invoke name=\"$TOOL_NAME\">\n\ +<|DSML| parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n\ +...\n\ +\n\ +<|DSML| invoke name=\"$TOOL_NAME2\">\n\ +...\n\ +\n\ +\n\n\ +String parameters should be specified as is and set `string=\"true\"`. \ +For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\n\ +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\n\ +Otherwise, output directly after with tool calls or final response.\n\n\ +### Available Tool Schemas\n\n", + ); + // tool_schemas is one client JSON value per line; each is reformatted + // to the json.dumps shape (server_knowledge.c append_tools_prompt_spaced). + let mut p = Json::new(tool_schemas); + let mut first = true; + loop { + p.ws(); + if p.peek().is_none() { + break; + } + let before = p.i; + if let Some(one) = json_raw_value(&mut p) { + if !first { + put(out, "\n"); + } + first = false; + let py = pyjson_raw_value(&one).unwrap_or(one); + put(out, &py); + } else { + if !first { + put(out, "\n"); + } + out.extend_from_slice(&tool_schemas.as_bytes()[before..]); + break; + } + } + put( + out, + "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n", + ); + if tool_required { + put( + out, + "\n\n### Required Tool Use\n\nYou MUST call at least one available tool in this turn. \ +Do not end the turn with only reasoning or a final answer. \ +After any reasoning, emit a valid <|DSML| calls> block.", + ); + } +} + +fn append_dsml_tool_calls_text(out: &mut Vec, m: &ChatMsg, spaced: bool) { + if m.calls.is_empty() { + return; + } + if !m.raw_dsml.is_empty() { + put(out, &m.raw_dsml); + return; + } + let (calls_start, calls_end, invoke_start, invoke_end, param_start, param_end): ( + &[u8], + &[u8], + &[u8], + &[u8], + &[u8], + &[u8], + ) = if spaced { + ( + DSML_TOOL_CALLS_START_SPACED.as_bytes(), + DSML_TOOL_CALLS_END_SPACED.as_bytes(), + b"<\xef\xbd\x9cDSML\xef\xbd\x9c invoke", + b"", + DSML_PARAM_START_SPACED.as_bytes(), + DSML_PARAM_END_SPACED.as_bytes(), + ) + } else { + ( + DSML_TOOL_CALLS_START.as_bytes(), + DSML_TOOL_CALLS_END.as_bytes(), + b"<\xef\xbd\x9cDSML\xef\xbd\x9cinvoke", + b"", + DSML_PARAM_START.as_bytes(), + DSML_PARAM_END.as_bytes(), + ) + }; + put(out, "\n\n"); + out.extend_from_slice(calls_start); + out.push(b'\n'); + for (i, tc) in m.calls.iter().enumerate() { + if spaced && i > 0 { + out.push(b'\n'); + } + out.extend_from_slice(invoke_start); + put(out, " name=\""); + append_dsml_attr_escaped(out, &tc.name); + put(out, "\">\n"); + let mut n = append_dsml_arguments_from_json(out, &tc.arguments, None, spaced); + if n < 0 && spaced { + // A stringified `arguments` value still renders its parameters on + // the V4.1 head: the oracle decodes the field up to twice before + // it gives up (encoding.py:528-537). The C engine has no such + // pass (ds4_server.c:3443-3463) and writes the single `arguments` + // parameter below instead, which the V4 head keeps; enc-args-double + // pins the difference. + if let Some(inner) = json_string_contents(&tc.arguments) { + n = append_dsml_arguments_from_json(out, &inner, None, spaced); + } + } + if n < 0 { + out.extend_from_slice(param_start); + put(out, " name=\"arguments\" string=\"true\">"); + append_dsml_parameter_text(out, param_end, &tc.arguments); + out.extend_from_slice(param_end); + n = 1; + } + if spaced || n > 0 { + out.push(b'\n'); + } + out.extend_from_slice(invoke_end); + if !spaced { + out.push(b'\n'); + } + } + if spaced { + out.push(b'\n'); + } + out.extend_from_slice(calls_end); +} + +fn append_solar_tool_schema_blocks(out: &mut Vec, tool_schemas: &str) { + let mut p = Json::new(tool_schemas); + while p.peek().is_some() { + p.ws(); + if p.peek().is_none() { + break; + } + let before = p.i; + if let Some(schema) = json_raw_value(&mut p) { + put(out, SOLAR_TOOL_START); + put(out, &schema); + put(out, SOLAR_TOOL_END); + out.push(b'\n'); + } else { + put(out, SOLAR_TOOL_START); + out.extend_from_slice(&tool_schemas.as_bytes()[before..]); + put(out, SOLAR_TOOL_END); + out.push(b'\n'); + return; + } + } +} + +fn append_solar_tools_prompt_text(out: &mut Vec, tool_schemas: &str) { + if tool_schemas.is_empty() { + return; + } + put( + out, + "## Tools\n\ +- You may invoke one or more tools to assist with the user's query.\n\n\ +### Available Tools\n", + ); + append_solar_tool_schema_blocks(out, tool_schemas); + put(out, "\n### Tool Call Instruction\n"); + put( + out, + "- If using a tool, any reasoning must strictly precede the call. Do not append any text after the tool call.\n\ +- If no tool is required, answer directly from your knowledge without ever mentioning the availability or absence of tools.\n\ +- Each tool call MUST use this following format: ", + ); + put(out, SOLAR_TOOL_CALLS); + put(out, "{example-tool-name}\n"); + put(out, SOLAR_TOOL_ARG_START); + put(out, "{example-key-name-1}"); + put(out, SOLAR_TOOL_ARG_VALUE); + put(out, "{example-value-1}"); + put(out, SOLAR_TOOL_ARG_END); + put(out, "\n"); + put(out, SOLAR_TOOL_ARG_START); + put(out, "{example-key-name-2}"); + put(out, SOLAR_TOOL_ARG_VALUE); + put(out, "{example-value-2}"); + put(out, SOLAR_TOOL_ARG_END); + put(out, "\n"); + put(out, SOLAR_TOOL_CALL_END); + put(out, "\n"); +} + +fn append_solar_arg(out: &mut Vec, arg: &crate::json::JsonArg) { + put(out, SOLAR_TOOL_ARG_START); + put(out, &arg.key); + put(out, SOLAR_TOOL_ARG_VALUE); + put(out, &arg.value); + put(out, SOLAR_TOOL_ARG_END); + out.push(b'\n'); +} + +fn append_solar_arguments_from_json( + out: &mut Vec, + json: &str, + order: Option<&ToolSchemaOrder>, +) -> bool { + let Some(mut args) = json_args_parse(json) else { + return false; + }; + if let Some(order) = order { + for prop in &order.prop { + if let Some(idx) = json_args_find_unused(&args, prop) { + append_solar_arg(out, &args[idx]); + args[idx].used = true; + } + } + } + for arg in &args { + if !arg.used { + append_solar_arg(out, arg); + } + } + true +} + +fn append_solar_tool_calls_text(out: &mut Vec, m: &ChatMsg, orders: &[ToolSchemaOrder]) { + if m.calls.is_empty() { + return; + } + if !m.raw_dsml.is_empty() { + put(out, &m.raw_dsml); + return; + } + for (i, tc) in m.calls.iter().enumerate() { + if i > 0 { + out.push(b'\n'); + } + put(out, SOLAR_TOOL_CALLS); + put(out, &tc.name); + out.push(b'\n'); + let order = tool_schema_orders_find(orders, &tc.name); + if !append_solar_arguments_from_json(out, &tc.arguments, order) { + put(out, SOLAR_TOOL_ARG_START); + put(out, "arguments"); + put(out, SOLAR_TOOL_ARG_VALUE); + put( + out, + if tc.arguments.is_empty() { + "{}" + } else { + &tc.arguments + }, + ); + put(out, SOLAR_TOOL_ARG_END); + out.push(b'\n'); + } + put(out, SOLAR_TOOL_CALL_END); + } +} + +fn append_qwen_arg(out: &mut Vec, arg: &crate::json::JsonArg) { + put(out, "\n"); + put(out, &arg.value); + put(out, "\n\n"); +} + +fn append_qwen_arguments_from_json( + out: &mut Vec, + json: &str, + order: Option<&ToolSchemaOrder>, +) -> bool { + let Some(mut args) = json_args_parse(json) else { + return false; + }; + if let Some(order) = order { + for prop in &order.prop { + if let Some(idx) = json_args_find_unused(&args, prop) { + append_qwen_arg(out, &args[idx]); + args[idx].used = true; + } + } + } + for arg in &args { + if !arg.used { + append_qwen_arg(out, arg); + } + } + true +} + +fn append_qwen_tool_calls_text(out: &mut Vec, m: &ChatMsg, orders: &[ToolSchemaOrder]) { + if m.calls.is_empty() { + return; + } + if !m.raw_dsml.is_empty() { + put(out, &m.raw_dsml); + return; + } + for (i, tc) in m.calls.iter().enumerate() { + if i > 0 { + out.push(b'\n'); + } + put(out, QWEN_TOOL_CALL_START); + put(out, "\n\n"); + let order = tool_schema_orders_find(orders, &tc.name); + if !append_qwen_arguments_from_json(out, &tc.arguments, order) { + put(out, "\n"); + put( + out, + if tc.arguments.is_empty() { + "{}" + } else { + &tc.arguments + }, + ); + put(out, "\n\n"); + } + put(out, "\n"); + put(out, QWEN_TOOL_CALL_END); + } +} + +fn append_glm_tag_body(out: &mut Vec, text: &[u8], end: &str) { + append_sentinel_escape(out, text, end.as_bytes(), b"<"); +} + +fn append_glm_arg(out: &mut Vec, arg: &crate::json::JsonArg) { + put(out, ""); + append_glm_tag_body(out, arg.key.as_bytes(), ""); + put(out, ""); + append_glm_tag_body(out, arg.value.as_bytes(), ""); + put(out, ""); +} + +fn append_glm_arguments_from_json( + out: &mut Vec, + json: &str, + order: Option<&ToolSchemaOrder>, +) -> bool { + let Some(mut args) = json_args_parse(json) else { + return false; + }; + if let Some(order) = order { + for prop in &order.prop { + if let Some(idx) = json_args_find_unused(&args, prop) { + append_glm_arg(out, &args[idx]); + args[idx].used = true; + } + } + } + for arg in &args { + if !arg.used { + append_glm_arg(out, arg); + } + } + true +} + +fn append_glm_tool_calls_text(out: &mut Vec, m: &ChatMsg, orders: &[ToolSchemaOrder]) { + if m.calls.is_empty() { + return; + } + // The sampled block goes back verbatim (ds4_server.c:2979-2983), so a + // replayed turn is byte-identical to what the model produced. + if !m.raw_tool_text.is_empty() { + put(out, &m.raw_tool_text); + return; + } + // GLM's template writes the block inline, with no newline before or after + // it (ds4_server.c:2984-2986); the reconstruction must not add one either. + for tc in &m.calls { + put(out, GLM_TOOL_CALL_START); + put(out, &tc.name); + let order = tool_schema_orders_find(orders, &tc.name); + if !append_glm_arguments_from_json(out, &tc.arguments, order) { + put(out, "arguments"); + append_glm_tag_body(out, tc.arguments.as_bytes(), ""); + put(out, ""); + } + put(out, GLM_TOOL_CALL_END); + } +} + +fn append_glm_tool_schema_json(out: &mut Vec, schema: &str) -> Option { + let args = json_args_parse(schema)?; + if args + .iter() + .any(|arg| arg.key == "defer_loading" && !arg.is_string && arg.value == "true") + { + return Some(false); + } + out.push(b'{'); + let mut wrote = false; + for arg in args + .iter() + .filter(|arg| arg.key != "defer_loading" && arg.key != "strict") + { + if wrote { + put(out, ", "); + } + out.extend(json_escape_bytes(arg.key.as_bytes())); + put(out, ": "); + if arg.is_string { + out.extend(json_escape_bytes(arg.value.as_bytes())); + } else { + put(out, &arg.value); + } + wrote = true; + } + out.push(b'}'); + Some(true) +} + +fn append_glm_tools_prompt_text(out: &mut Vec, tool_schemas: &str) { + if tool_schemas.is_empty() { + return; + } + put( + out, + "\n# Tools\n\n\ +You may call one or more functions to assist with the user query.\n\n\ +You are provided with function signatures within XML tags:\n\ +\n\n", + ); + let mut p = Json::new(tool_schemas); + let mut any = false; + loop { + p.ws(); + if p.peek().is_none() { + break; + } + let Some(schema) = json_raw_value(&mut p) else { + break; + }; + match append_glm_tool_schema_json(out, &schema) { + Some(true) => { + put(out, "\n\n\n"); + any = true; + } + Some(false) => {} + None => { + put(out, &schema); + put(out, "\n\n\n"); + any = true; + } + } + } + if !any { + out.push(b'\n'); + } + put( + out, + "\n\n\ +For each function call, output the function name and arguments within the following XML format:\n\ +{function-name}{arg-key-1}\ +{arg-value-1}{arg-key-2}\ +{arg-value-2}...", + ); +} + +fn append_glm_message_content(out: &mut Vec, m: &ChatMsg) -> Result<(), RenderError> { + if m.parts.is_empty() { + put(out, &m.content); + return Ok(()); + } + for part in &m.parts { + match part { + ChatPart::Text(text) => put(out, text), + ChatPart::Image(_) => { + put(out, GLM_VISION_START); + put(out, GLM_IMAGE); + put(out, GLM_VISION_END); + } + ChatPart::Audio(_) => return Err(RenderError("audio input requires Inkling")), + ChatPart::Video(_) => return Err(RenderError("video input requires MiMo")), + ChatPart::ToolResult { .. } => {} + } + } + Ok(()) +} + +fn append_glm_tool_result_message(out: &mut Vec, m: &ChatMsg) { + let mut views = Vec::new(); + collect_tool_result_message(m, &mut views); + for view in views { + put(out, ""); + append_glm_tag_body(out, view.text, ""); + put(out, ""); + } +} + +fn append_glm_assistant_prefix(out: &mut Vec, m: &ChatMsg, preserve_reasoning: bool) { + if m.content.starts_with("") || m.content.starts_with("") { + return; + } + put(out, ""); + if preserve_reasoning { + put(out, &m.reasoning); + } + put(out, ""); +} + +/// The level line the artifact's template writes into its first system turn +/// (`<|system|>Reasoning Effort: X`, chat_template.jinja:3). The template +/// honours only `low` and `high` by name and reads every other level as max +/// (chat_template.jinja:2), so the two named levels say themselves and +/// everything else is max. +fn glm_effort_text(mode: ThinkMode) -> &'static str { + match mode { + ThinkMode::Low => "Reasoning Effort: Low", + ThinkMode::High => "Reasoning Effort: High", + // The template's else branch; None never reaches here (thinking off + // is rendered without the line at all). + ThinkMode::Max | ThinkMode::None => "Reasoning Effort: Max", + } +} + +pub fn render_glm_chat_ex( + msgs: &[ChatMsg], + tool_schemas: &str, + tool_orders: &[ToolSchemaOrder], + think_mode: ThinkMode, +) -> Result, RenderError> { + let think = think_mode_enabled(think_mode); + let tool_context = chat_history_uses_tool_context(msgs, tool_schemas); + let last_user_idx = msgs + .iter() + .rposition(|m| role_is_user_like(&m.role)) + .map(|i| i as i32) + .unwrap_or(-1); + let mut out = GLM_BOS.as_bytes().to_vec(); + if think { + put(&mut out, "<|system|>"); + put(&mut out, glm_effort_text(think_mode)); + } + if !tool_schemas.is_empty() { + let mut tools = Vec::new(); + append_glm_tools_prompt_text(&mut tools, tool_schemas); + if !tools.is_empty() { + put(&mut out, "<|system|>"); + out.extend(tools); + } + } + for m in msgs.iter().filter(|m| role_is_system(&m.role)) { + put(&mut out, "<|system|>"); + put(&mut out, &m.content); + } + + let mut pending_assistant = false; + let mut observation_open = false; + for (i, m) in msgs.iter().enumerate() { + if role_is_system(&m.role) { + observation_open = false; + } else if chat_msg_is_model_tool_result(m) { + if !observation_open { + put(&mut out, "<|observation|>"); + } + append_glm_tool_result_message(&mut out, m); + observation_open = true; + pending_assistant = true; + } else if m.role == "user" { + observation_open = false; + put(&mut out, "<|user|>"); + append_glm_message_content(&mut out, m)?; + pending_assistant = true; + } else if m.role == "assistant" { + observation_open = false; + put(&mut out, "<|assistant|>"); + append_glm_assistant_prefix( + &mut out, + m, + think && (tool_context || i as i32 > last_user_idx), + ); + put_trimmed(&mut out, &m.content); + append_glm_tool_calls_text(&mut out, m, tool_orders); + pending_assistant = false; + } + } + if pending_assistant { + put(&mut out, "<|assistant|>"); + put(&mut out, if think { "" } else { "" }); + } + Ok(out) +} + +fn append_motif3_tool_calls_text(out: &mut Vec, m: &ChatMsg, include_ids: bool) { + if m.calls.is_empty() { + return; + } + if !m.raw_tool_text.is_empty() { + put(out, &m.raw_tool_text); + return; + } + for tc in &m.calls { + put(out, "\n{\"name\": "); + out.extend(json_escape_bytes(tc.name.as_bytes())); + put(out, ", \"arguments\": "); + if tc.arguments.is_empty() { + put(out, "null"); + } else { + put(out, &tc.arguments); + } + if include_ids && !tc.id.is_empty() { + put(out, ", \"id\": "); + out.extend(json_escape_bytes(tc.id.as_bytes())); + } + put(out, "}"); + } +} + +fn append_exaone_tool_calls_text(out: &mut Vec, m: &ChatMsg, content_before: bool) { + if m.calls.is_empty() { + return; + } + if !m.raw_tool_text.is_empty() { + let raw = m.raw_tool_text.as_bytes(); + let mut start = 0; + while start < raw.len() && is_c_space(raw[start]) { + start += 1; + } + if content_before { + out.push(b'\n'); + } + out.extend_from_slice(&raw[start..]); + return; + } + for (i, tc) in m.calls.iter().enumerate() { + if content_before || i > 0 { + out.push(b'\n'); + } + put(out, "{\"name\": "); + out.extend(json_escape_bytes(tc.name.as_bytes())); + put(out, ", \"arguments\": "); + if tc.arguments.is_empty() { + put(out, "null"); + } else { + put(out, &tc.arguments); + } + put(out, "}"); + } +} + +fn append_exaone_tools_declaration(out: &mut Vec, tool_schemas: &str) { + if tool_schemas.is_empty() { + return; + } + put(out, "<|tool_declare|>\n# Tools\n"); + let mut p = Json::new(tool_schemas); + while p.peek().is_some() { + p.ws(); + if p.peek().is_none() { + break; + } + let rest = p.i; + if let Some(raw) = json_raw_value(&mut p) { + let schema = json_minify_raw_value(&raw); + put(out, ""); + put(out, &schema); + put(out, "\n"); + } else { + put(out, ""); + out.extend_from_slice(&tool_schemas.as_bytes()[rest..]); + put(out, "\n"); + break; + } + } + put(out, "<|endofturn|>\n"); +} + +fn append_motif3_tools_system_text( + out: &mut Vec, + tool_schemas: &str, + tool_orders: &[ToolSchemaOrder], +) { + put( + out, + "# Tools\n\n\ +You may call one or more functions to assist with the user query.\n\n\ +You are provided with function signatures within XML tags:\n\n\ +", + ); + let bytes = tool_schemas.as_bytes(); + let mut i = 0; + while i < bytes.len() { + out.push(b'\n'); + if let Some(rel) = bytes[i..].iter().position(|&c| c == b'\n') { + out.extend_from_slice(&bytes[i..i + rel]); + i += rel + 1; + } else { + out.extend_from_slice(&bytes[i..]); + break; + } + } + put( + out, + "\n\n\nFor each function call, output in JSON within tags:\n", + ); + for order in tool_orders { + put(out, "\n{\"name\": "); + out.extend(json_escape_bytes(order.name.as_bytes())); + put(out, ", \"arguments\": {"); + for (j, prop) in order.prop.iter().enumerate() { + if j > 0 { + put(out, ", "); + } + out.extend(json_escape_bytes(prop.as_bytes())); + put(out, ": <"); + put(out, prop); + out.push(b'>'); + } + put(out, "}}"); + } +} + +fn dots3_pyspace_json(out: &mut Vec, raw: &str) { + let bytes = raw.as_bytes(); + let mut i = 0; + let mut in_string = false; + while i < bytes.len() { + let c = bytes[i]; + if in_string { + out.push(c); + if c == b'\\' && i + 1 < bytes.len() { + out.push(bytes[i + 1]); + i += 2; + continue; + } + if c == b'"' { + in_string = false; + } + i += 1; + continue; + } + if is_c_space(c) { + i += 1; + continue; + } + out.push(c); + if c == b'"' { + in_string = true; + } + if c == b',' || c == b':' { + out.push(b' '); + } + i += 1; + } +} + +fn append_dots3_tool_call_text(out: &mut Vec, tc: &ToolCall) { + put(out, "\n\n"); + let mut p = Json::new(&tc.arguments); + p.ws(); + if p.bump() == Some(b'{') { + p.ws(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let Some(key) = crate::json::json_string(&mut p) else { + break; + }; + p.ws(); + if p.bump() != Some(b':') { + break; + } + p.ws(); + let is_string = p.peek() == Some(b'"'); + if is_string { + let Some(plain) = crate::json::json_string(&mut p) else { + break; + }; + put(out, "\n\n"); + put(out, &plain); + put(out, "\n"); + } else if let Some(value) = json_raw_value(&mut p) { + put(out, "\n\n"); + dots3_pyspace_json(out, &value); + put(out, "\n"); + } else { + break; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + } + put(out, "\n\n"); +} + +fn append_dots3_tool_calls_text(out: &mut Vec, m: &ChatMsg) { + for tc in &m.calls { + append_dots3_tool_call_text(out, tc); + } +} + +fn append_dots3_tools_system_text(out: &mut Vec, tool_schemas: &str) { + put( + out, + "\n\n# Tools\n\n\ +You may call one or more functions to assist with the user query.\n\n\ +You are provided with function signatures within \ +XML tags:\n", + ); + let bytes = tool_schemas.as_bytes(); + let mut i = 0; + while i < bytes.len() { + let rel = bytes[i..].iter().position(|&c| c == b'\n'); + let n = rel.unwrap_or(bytes.len() - i); + let line = &tool_schemas[i..i + n]; + put(out, "\n{\"type\": \"function\", \"function\": "); + dots3_pyspace_json(out, line); + out.push(b'}'); + i += n; + if i < bytes.len() && bytes[i] == b'\n' { + i += 1; + } + } + put( + out, + "\n\n\n\ +When making tool calls, use XML format to invoke tools and pass \ +parameters:\n\n\ +\n\ +\n\ +\n\ +param-value-1\n\ +\n\ +\n\ +param-value-2\n\ +\n\ +...\n\ +\n\ +", + ); +} + +/// Protect `` inside tool output. C writes `<` and advances +/// one byte, so a match becomes `</tool_result>`. +pub fn append_tool_result_text(out: &mut Vec, s: &[u8]) { + let end = b""; + let mut i = 0; + while i < s.len() { + if s[i..].starts_with(end) { + out.extend_from_slice(b"<"); + i += 1; + } else { + out.push(s[i]); + i += 1; + } + } +} + +/// Byte-for-byte `render_dsml_chat_prompt_text_choice`. +/// The DSML user body. The V4.1 head lays one `<|deepseek_image|>` per image +/// part where the image sits in the message (the engine's text-level render, +/// server_parse_tools.c:290); the prefill expands each into a span. Every +/// other case writes the flat content byte for byte, so text-only prompts are +/// unchanged (images never reach the V4 head: they are refused earlier). +fn append_dsml_user_content(out: &mut Vec, m: &ChatMsg, head: DsmlHead) { + let has_image = matches!(head, DsmlHead::V41 { .. }) + && m.parts.iter().any(|p| matches!(p, ChatPart::Image(_))); + // A tool-result message keeps its content: the envelope and the image + // placeholders were written into it at parse time in the client's block + // order (the engine's text-level render, server_parse_tools.c:330), so + // the parts loop must not run for it. + let has_tool_result = m + .parts + .iter() + .any(|p| matches!(p, ChatPart::ToolResult { .. })); + if !has_image || has_tool_result { + put(out, &m.content); + return; + } + for part in &m.parts { + match part { + ChatPart::Text(text) => put(out, text), + ChatPart::Image(_) => put(out, DSML_IMAGE), + _ => {} + } + } +} + +/// A tool-result message: its body renders as one `` block. +fn is_tool_msg(m: &ChatMsg) -> bool { + m.role == "tool" || m.role == "function" +} + +/// The call ids of the assistant message that precedes a tool-result run, in +/// declared order. `None` when there is no such assistant or it declared no +/// ids, which is the oracle's own guard before it sorts (encoding.py:233-266 +/// sort_tool_results_by_call_order). +fn preceding_call_order(msgs: &[ChatMsg], before: usize) -> Option> { + let assistant = msgs[..before] + .iter() + .rev() + .find(|m| m.role == "assistant" && !m.calls.is_empty())?; + let ids: Vec<&str> = assistant + .calls + .iter() + .map(|c| c.id.as_str()) + .filter(|id| !id.is_empty()) + .collect(); + (!ids.is_empty()).then_some(ids) +} + +/// A run of consecutive tool-result messages in the order the turn must render +/// them. The run is one user-side block, and the V4.1 generation orders its +/// bodies by the preceding assistant's call order, not by arrival, so a client +/// that sends results in completion order still produces one canonical prompt +/// (encoding.py:233-266). A result whose id the assistant never declared keeps +/// the oracle's default key 0, and the sort is stable, so it stays where it +/// arrived. +fn tool_run_in_call_order(msgs: &[ChatMsg], start: usize) -> Vec { + let end = msgs[start..] + .iter() + .position(|m| !is_tool_msg(m)) + .map_or(msgs.len(), |n| start + n); + let mut run: Vec = (start..end).collect(); + if run.len() < 2 { + return run; + } + let Some(order) = preceding_call_order(msgs, start) else { + return run; + }; + run.sort_by_key(|&i| { + let id = msgs[i].tool_call_id.as_str(); + order.iter().position(|x| *x == id).unwrap_or(0) + }); + run +} + +pub fn render_dsml_chat( + msgs: &[ChatMsg], + tool_schemas: &str, + think_mode: ThinkMode, +) -> Result, RenderError> { + render_dsml_chat_choice_ex( + msgs, + tool_schemas, + think_mode, + ToolChoice::Auto, + DsmlHead::V4, + ) +} + +pub fn render_dsml_chat_choice( + msgs: &[ChatMsg], + tool_schemas: &str, + think_mode: ThinkMode, + tool_choice: ToolChoice, +) -> Result, RenderError> { + render_dsml_chat_choice_ex(msgs, tool_schemas, think_mode, tool_choice, DsmlHead::V4) +} + +pub fn render_dsml_chat_choice_ex( + msgs: &[ChatMsg], + tool_schemas: &str, + think_mode: ThinkMode, + tool_choice: ToolChoice, + head: DsmlHead, +) -> Result, RenderError> { + let think = think_mode_enabled(think_mode); + let tool_context = chat_history_uses_tool_context(msgs, tool_schemas); + // The spaced generation (the official encoding.py, 4294136): tool + // instructions follow the system content (separated by "\n\n"), and + // adjacent user-side blocks share one <|User|> -- the old generations keep + // the schemas first and a fresh user turn per block. + let spaced = matches!( + head, + DsmlHead::V41 { + system_token: true, + .. + } + ); + let mut last_user_idx = -1i32; + let mut system = Vec::new(); + if !tool_schemas.is_empty() && !spaced { + append_dsml_tools_prompt_text( + &mut system, + tool_schemas, + tool_choice == ToolChoice::Required, + spaced, + ); + } + for (i, m) in msgs.iter().enumerate() { + if !role_is_system(&m.role) { + continue; + } + // The head carries the leading system message. The V4.1 generation + // also supports a mid-conversation system message, and that one + // renders in place with its own <|System|> instead of joining the head + // (encoding.py:621-627). + if spaced && i > 0 { + continue; + } + if !system.is_empty() { + put(&mut system, "\n\n"); + } + put(&mut system, &m.content); + } + if !tool_schemas.is_empty() && spaced { + if !system.is_empty() { + put(&mut system, "\n\n"); + } + append_dsml_tools_prompt_text( + &mut system, + tool_schemas, + tool_choice == ToolChoice::Required, + spaced, + ); + } + // A mid-conversation system message counts as user-like for the assistant + // header that follows it (encoding.py:562-582 find_last_user_index). + for (i, m) in msgs.iter().enumerate() { + if role_is_user_like(&m.role) || (spaced && i > 0 && role_is_system(&m.role)) { + last_user_idx = i as i32; + } + } + + let mut out = Vec::new(); + put(&mut out, DSML_BOS); + // The engine's one V4.1 branch (server_dsml_render.c:375-383): a non-empty + // system body or effort prefix writes <|System|> once, before the effort + // prefix. The token is "" for the V4 head (the V4 tokenizer has none). + let effort = head_effort_prefix(head, think_mode); + if !effort.is_empty() || !system.is_empty() { + if let DsmlHead::V41 { + system_token: true, .. + } = head + { + put(&mut out, DSML_SYSTEM_TOKEN); + } + } + put(&mut out, &effort); + out.extend_from_slice(&system); + + let mut pending_assistant = false; + let mut pending_tool_result = false; + // Spaced only: the previous message is a user-side block that a following + // user message merges into -- one shared <|User|> with the bodies joined by + // a blank line. A mid-conversation system message is not one of those, so + // it clears the flag and the next user message opens its own turn + // (encoding.py:184-231 merge_tool_messages). + let mut pending_merge = false; + for (i, m) in msgs.iter().enumerate() { + if role_is_system(&m.role) { + // A mid-conversation system message introduces its own turn: its + // own token, at its position, and the assistant header that + // follows it (encoding.py:621-627, :733-739). + if spaced && i > 0 { + put(&mut out, DSML_SYSTEM_TOKEN); + put(&mut out, &m.content); + pending_assistant = true; + pending_merge = false; + } + continue; + } else if m.role == "user" { + put( + &mut out, + if spaced && pending_merge { + "\n\n" + } else { + DSML_USER + }, + ); + append_dsml_user_content(&mut out, m, head); + pending_assistant = true; + pending_merge = true; + pending_tool_result = false; + } else if is_tool_msg(m) { + // The run renders as one user-side block, so it is emitted here at + // its first message and the later ones are skipped. + if spaced && i > 0 && is_tool_msg(&msgs[i - 1]) { + continue; + } + if spaced && pending_merge { + put(&mut out, "\n\n"); + } else if !pending_tool_result { + put(&mut out, DSML_USER); + } + let run: Vec = if spaced { + tool_run_in_call_order(msgs, i) + } else { + vec![i] + }; + for (n, &k) in run.iter().enumerate() { + if n > 0 { + put(&mut out, "\n\n"); + } + put(&mut out, ""); + append_tool_result_text(&mut out, msgs[k].content.as_bytes()); + put(&mut out, ""); + } + pending_assistant = true; + pending_merge = true; + pending_tool_result = true; + } else if m.role == "assistant" { + // The oracle renders an assistant as [transition token] + + // `{reasoning}` + content + tool calls + EOS + // (encoding.py render_message:678-712). The <|Assistant|> and the + // thinking token after it belong to the preceding message, so a + // second assistant in a row has no header to open its thinking; + // the V4.1 body still carries the reasoning and its closer, exactly + // as the oracle writes it (measured by enc-think-two-assistants). + let keep_reasoning = think && (tool_context || (i as i32) > last_user_idx); + let wrote_header = if pending_assistant { + put(&mut out, DSML_ASSISTANT); + put( + &mut out, + if keep_reasoning { + DSML_THINK_START + } else { + DSML_THINK_END + }, + ); + true + } else { + false + }; + if keep_reasoning && (wrote_header || matches!(head, DsmlHead::V41 { .. })) { + put(&mut out, &m.reasoning); + put(&mut out, DSML_THINK_END); + } + put(&mut out, &m.content); + append_dsml_tool_calls_text(&mut out, m, spaced); + put(&mut out, DSML_EOS); + pending_assistant = false; + pending_merge = false; + pending_tool_result = false; + } + } + + if pending_assistant { + put(&mut out, DSML_ASSISTANT); + put(&mut out, if think { "" } else { "" }); + } + Ok(out) +} + +/// The visible assistant tail a Responses client can replay (C +/// `build_responses_visible_assistant_suffix`, ds4_server.c:12821-12855). +/// +/// This mirrors what the client can echo, not every token in KV: hidden +/// reasoning joins the remembered prefix only when the turn ended in tool calls +/// and the request asked for reasoning summaries, because pi replays summaries +/// for tool-call turns but not for final answers, and Codex asks for none. A +/// client that does replay a final answer's reasoning simply misses this +/// visible shortcut and still has the exact token-prefix replay. GLM writes no +/// EOS (the reference's own twist) and every other syntax writes the EOS. The +/// reference writes the literal `` for whatever syntax it serves, and so +/// does this. +pub fn responses_visible_assistant_suffix( + syntax: ModelSyntax, + spaced: bool, + think_mode: ThinkMode, + summary_emit: bool, + content: &[u8], + reasoning: &[u8], + calls: &[ToolCall], + orders: &[ToolSchemaOrder], +) -> Vec { + let mut out = Vec::new(); + if think_mode_enabled(think_mode) { + if summary_emit && !calls.is_empty() { + out.extend_from_slice(reasoning); + } + out.extend_from_slice(DSML_THINK_END.as_bytes()); + } + out.extend_from_slice(content); + let m = ChatMsg { + calls: calls.to_vec(), + ..Default::default() + }; + match syntax { + ModelSyntax::Glm53 => append_glm_tool_calls_text(&mut out, &m, orders), + _ => append_dsml_tool_calls_text(&mut out, &m, spaced), + } + if syntax != ModelSyntax::Glm53 { + out.extend_from_slice(DSML_EOS.as_bytes()); + } + out +} + +fn is_c_space(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c) +} + +fn put_trimmed(out: &mut Vec, text: &str) -> bool { + let b = text.as_bytes(); + let mut start = 0; + while start < b.len() && is_c_space(b[start]) { + start += 1; + } + let mut end = b.len(); + while end > start && is_c_space(b[end - 1]) { + end -= 1; + } + out.extend_from_slice(&b[start..end]); + end > start +} + +fn chat_msg_is_model_tool_result(m: &ChatMsg) -> bool { + m.role == "tool" + || m.role == "function" + || (m.role == "user" && (!m.tool_call_id.is_empty() || !m.tool_call_ids.is_empty())) +} + +fn tool_result_id_at(m: &ChatMsg, index: usize) -> &str { + if index < m.tool_call_ids.len() { + &m.tool_call_ids[index] + } else if index == 0 { + &m.tool_call_id + } else { + "" + } +} + +struct ToolResultView<'a> { + text: &'a [u8], + id: &'a str, + used: bool, +} + +fn collect_tool_result_message<'a>(m: &'a ChatMsg, views: &mut Vec>) { + if m.role == "user" && (!m.tool_call_id.is_empty() || !m.tool_call_ids.is_empty()) { + let before = views.len(); + let bytes = m.content.as_bytes(); + let mut i = 0; + let mut exact = true; + let mut result_index = 0usize; + while i < bytes.len() { + while i < bytes.len() && is_c_space(bytes[i]) { + i += 1; + } + if i >= bytes.len() { + break; + } + let open = b""; + let close = b""; + if !bytes[i..].starts_with(open) { + exact = false; + break; + } + i += open.len(); + if let Some(rel) = bytes[i..].windows(close.len()).position(|w| w == close) { + views.push(ToolResultView { + text: &bytes[i..i + rel], + id: tool_result_id_at(m, result_index), + used: false, + }); + result_index += 1; + i += rel + close.len(); + } else { + exact = false; + break; + } + } + if exact && views.len() > before { + return; + } + views.truncate(before); + } + views.push(ToolResultView { + text: m.content.as_bytes(), + id: tool_result_id_at(m, 0), + used: false, + }); +} + +/// Official Motif-3 path (`render_motif3_chat_prompt_text`). +pub fn render_motif3_chat( + msgs: &[ChatMsg], + tool_schemas: &str, + think_mode: ThinkMode, +) -> Result, RenderError> { + render_motif3_chat_ex(msgs, tool_schemas, &[], think_mode) +} + +pub fn render_motif3_chat_ex( + msgs: &[ChatMsg], + tool_schemas: &str, + tool_orders: &[ToolSchemaOrder], + think_mode: ThinkMode, +) -> Result, RenderError> { + let think = think_mode_enabled(think_mode); + let have_tools = !tool_schemas.is_empty(); + let first_system = msgs.first().is_some_and(|m| role_is_system(&m.role)); + let last_assistant = msgs.iter().rposition(|m| m.role == "assistant"); + + let mut out = Vec::new(); + put(&mut out, "<|beginoftext|>"); + if have_tools { + put(&mut out, "<|startofturn|><|system|>"); + append_motif3_tools_system_text(&mut out, tool_schemas, tool_orders); + if first_system { + put(&mut out, "\n\n"); + put(&mut out, &msgs[0].content); + } + put(&mut out, "<|endofturn|>"); + } else if first_system { + put(&mut out, "<|startofturn|><|system|>"); + put(&mut out, &msgs[0].content); + put(&mut out, "<|endofturn|>"); + } + for (i, m) in msgs.iter().enumerate() { + if i == 0 && first_system { + continue; + } else if role_is_system(&m.role) { + put(&mut out, "<|startofturn|><|system|>"); + put(&mut out, &m.content); + put(&mut out, "<|endofturn|>"); + } else if m.role == "user" { + put(&mut out, "<|startofturn|><|user|>"); + put(&mut out, &m.content); + put(&mut out, "<|endofturn|>"); + } else if m.role == "assistant" { + put(&mut out, "<|startofturn|><|assistant|>"); + if !m.reasoning.is_empty() && (have_tools || last_assistant == Some(i)) { + put(&mut out, ""); + put_trimmed(&mut out, &m.reasoning); + put(&mut out, ""); + } + put_trimmed(&mut out, &m.content); + append_motif3_tool_calls_text(&mut out, m, true); + put(&mut out, "<|endofturn|>"); + } else if m.role == "tool" || m.role == "function" { + let group_start = + i == 0 || (msgs[i - 1].role != "tool" && msgs[i - 1].role != "function"); + let group_end = i + 1 == msgs.len() + || (msgs[i + 1].role != "tool" && msgs[i + 1].role != "function"); + if group_start { + put(&mut out, "<|startofturn|><|tool|>"); + } + put(&mut out, "{\"tool_call_id\": "); + out.extend(json_escape_bytes(m.tool_call_id.as_bytes())); + put(&mut out, ", \"content\": "); + out.extend(json_escape_bytes(m.content.as_bytes())); + put(&mut out, "}"); + if group_end { + put(&mut out, "<|endofturn|>"); + } + } + } + put(&mut out, "<|startofturn|><|assistant|>"); + if !think { + put(&mut out, ""); + } + Ok(out) +} + +fn dots3_reason_from_content(content: &str) -> (String, String) { + if let Some(close) = content.find("") { + let head = &content[..close]; + let open = head.rfind(""); + let mut rb = if let Some(o) = open { o + 7 } else { 0 }; + let mut re = close; + let b = content.as_bytes(); + while re > rb && b[re - 1] == b'\n' { + re -= 1; + } + while rb < re && b[rb] == b'\n' { + rb += 1; + } + let mut cb = close + 8; + while cb < content.len() && b[cb] == b'\n' { + cb += 1; + } + (content[rb..re].to_string(), content[cb..].to_string()) + } else { + (String::new(), content.to_string()) + } +} + +fn trim_ws_range(s: &str) -> &str { + let b = s.as_bytes(); + let mut start = 0; + while start < b.len() && is_c_space(b[start]) { + start += 1; + } + let mut end = b.len(); + while end > start && is_c_space(b[end - 1]) { + end -= 1; + } + &s[start..end] +} + +/// Official dots3-note path (`render_dots3_chat_prompt_text`). +pub fn render_dots3_chat( + msgs: &[ChatMsg], + tool_schemas: &str, + think_mode: ThinkMode, +) -> Result, RenderError> { + let think = think_mode_enabled(think_mode); + let have_tools = !tool_schemas.is_empty(); + let first_system = msgs.first().is_some_and(|m| role_is_system(&m.role)); + let mut out = Vec::new(); + put(&mut out, "<|system|>"); + if first_system { + put(&mut out, &msgs[0].content); + } else { + put(&mut out, "You are a helpful assistant."); + } + if have_tools { + append_dots3_tools_system_text(&mut out, tool_schemas); + } + put(&mut out, "<|endofsystem|>"); + for (i, m) in msgs.iter().enumerate() { + if i == 0 && first_system { + continue; + } + let is_user = m.role == "user"; + if is_user || role_is_system(&m.role) { + put(&mut out, "<|user|>"); + put(&mut out, &m.content); + if is_user && !think && !m.content.ends_with("") { + put(&mut out, ""); + } + put(&mut out, "<|endofuser|>"); + } else if m.role == "assistant" { + let (reason, content) = if m.reasoning.is_empty() { + dots3_reason_from_content(&m.content) + } else { + (m.reasoning.clone(), m.content.clone()) + }; + let trimmed = trim_ws_range(&reason); + put(&mut out, "<|assistant|>"); + if !think { + put(&mut out, "\n\n\n\n"); + put(&mut out, &content); + } else if !trimmed.is_empty() { + put(&mut out, "\n"); + put(&mut out, trimmed); + put(&mut out, "\n\n\n"); + put(&mut out, &content); + } else { + put(&mut out, &content); + } + append_dots3_tool_calls_text(&mut out, m); + put(&mut out, "<|endofassistant|>"); + } else if m.role == "tool" || m.role == "function" { + let group_start = + i == 0 || (msgs[i - 1].role != "tool" && msgs[i - 1].role != "function"); + let group_end = i + 1 == msgs.len() + || (msgs[i + 1].role != "tool" && msgs[i + 1].role != "function"); + if group_start { + put(&mut out, "<|user|>"); + } + put(&mut out, "\n\n"); + put(&mut out, &m.content); + put(&mut out, "\n"); + if group_end { + put(&mut out, "<|endofuser|>"); + } + } + } + put(&mut out, "<|assistant|>"); + if !think { + put(&mut out, "\n\n\n\n"); + } + Ok(out) +} + +/// K-EXAONE path (`render_exaone_chat_prompt_text`). +pub fn render_exaone_chat( + msgs: &[ChatMsg], + tool_schemas: &str, + think_mode: ThinkMode, +) -> Result, RenderError> { + let mut last_user_idx = -1i32; + for (i, m) in msgs.iter().enumerate() { + if m.role == "user" && !chat_msg_is_model_tool_result(m) { + last_user_idx = i as i32; + } + } + let mut out = Vec::new(); + append_exaone_tools_declaration(&mut out, tool_schemas); + let mut i = 0; + while i < msgs.len() { + let m = &msgs[i]; + if chat_msg_is_model_tool_result(m) { + let mut views = Vec::new(); + while i < msgs.len() && chat_msg_is_model_tool_result(&msgs[i]) { + collect_tool_result_message(&msgs[i], &mut views); + i += 1; + } + put(&mut out, "<|tool|>\n"); + for (k, v) in views.iter().enumerate() { + if k > 0 { + out.push(b'\n'); + } + put(&mut out, ""); + out.extend_from_slice(v.text); + put(&mut out, ""); + } + put(&mut out, "<|endofturn|>\n"); + continue; + } else if role_is_system(&m.role) { + put(&mut out, "<|system|>\n"); + put(&mut out, &m.content); + put(&mut out, "<|endofturn|>\n"); + } else if m.role == "user" { + put(&mut out, "<|user|>\n"); + put(&mut out, &m.content); + put(&mut out, "<|endofturn|>\n"); + } else if m.role == "assistant" { + put(&mut out, "<|assistant|>\n\n"); + if !m.reasoning.is_empty() && (i as i32) > last_user_idx { + put_trimmed(&mut out, &m.reasoning); + } + put(&mut out, "\n\n\n"); + let have_content = put_trimmed(&mut out, &m.content); + append_exaone_tool_calls_text(&mut out, m, have_content); + put(&mut out, "<|endofturn|>\n"); + } + i += 1; + } + put(&mut out, "<|assistant|>\n\n"); + if !think_mode_enabled(think_mode) { + put(&mut out, "\n\n\n"); + } + Ok(out) +} + +pub(crate) fn solar_role_open(out: &mut Vec, role: &str) { + put(out, SOLAR_IM_START); + put(out, role); + put(out, SOLAR_IM_CONTENT); +} + +pub(crate) fn append_solar_tool_response_text(out: &mut Vec, s: &[u8]) { + let end = SOLAR_TOOL_RESPONSE_END.as_bytes(); + let mut i = 0; + while i < s.len() { + if s[i..].starts_with(end) { + out.extend_from_slice(b"<"); + i += 1; + } else { + out.push(s[i]); + i += 1; + } + } +} + +/// Solar-Open2 path (`render_solar_chat_prompt_text_choice`). +pub fn render_solar_chat( + msgs: &[ChatMsg], + tool_schemas: &str, + think_mode: ThinkMode, +) -> Result, RenderError> { + render_solar_chat_ex(msgs, tool_schemas, &[], think_mode) +} + +pub fn render_solar_chat_ex( + msgs: &[ChatMsg], + tool_schemas: &str, + tool_orders: &[ToolSchemaOrder], + think_mode: ThinkMode, +) -> Result, RenderError> { + let mut last_user_idx = -1i32; + for (i, m) in msgs.iter().enumerate() { + if m.role == "user" && !chat_msg_is_model_tool_result(m) { + last_user_idx = i as i32; + } + } + let mut system = Vec::new(); + let mut have_system = false; + for m in msgs { + if !role_is_system(&m.role) { + continue; + } + if !have_system { + put(&mut system, "## System Prompt"); + have_system = true; + } + put(&mut system, "\n\n"); + put(&mut system, &m.content); + } + if !tool_schemas.is_empty() { + if !system.is_empty() { + put(&mut system, "\n\n"); + } + append_solar_tools_prompt_text(&mut system, tool_schemas); + } + let mut out = Vec::new(); + if !system.is_empty() { + solar_role_open(&mut out, "system"); + out.extend_from_slice(&system); + put(&mut out, SOLAR_IM_END); + out.push(b'\n'); + } + let mut i = 0; + while i < msgs.len() { + let m = &msgs[i]; + if role_is_system(&m.role) { + i += 1; + continue; + } else if m.role == "user" && !chat_msg_is_model_tool_result(m) { + solar_role_open(&mut out, "user"); + put(&mut out, &m.content); + put(&mut out, SOLAR_IM_END); + out.push(b'\n'); + i += 1; + } else if chat_msg_is_model_tool_result(m) { + let prior = if i > 0 { Some(&msgs[i - 1]) } else { None }; + let mut views = Vec::new(); + while i < msgs.len() && chat_msg_is_model_tool_result(&msgs[i]) { + collect_tool_result_message(&msgs[i], &mut views); + i += 1; + } + solar_role_open(&mut out, "tool"); + let mut first = true; + if let Some(prior) = prior { + if prior.role == "assistant" { + for tc in &prior.calls { + if tc.id.is_empty() { + continue; + } + if let Some(v) = views.iter_mut().find(|v| !v.used && v.id == tc.id) { + if !first { + out.push(b'\n'); + } + first = false; + put(&mut out, SOLAR_TOOL_RESPONSE_START); + append_solar_tool_response_text(&mut out, v.text); + put(&mut out, SOLAR_TOOL_RESPONSE_END); + v.used = true; + } + } + } + } + for v in &views { + if v.used { + continue; + } + if !first { + out.push(b'\n'); + } + first = false; + put(&mut out, SOLAR_TOOL_RESPONSE_START); + append_solar_tool_response_text(&mut out, v.text); + put(&mut out, SOLAR_TOOL_RESPONSE_END); + } + put(&mut out, "\n"); + put(&mut out, SOLAR_IM_END); + out.push(b'\n'); + } else if m.role == "assistant" { + solar_role_open(&mut out, "assistant"); + put(&mut out, SOLAR_THINK_START); + if !m.reasoning.is_empty() && (i as i32) > last_user_idx { + put(&mut out, &m.reasoning); + } + put(&mut out, SOLAR_THINK_END); + put(&mut out, &m.content); + append_solar_tool_calls_text(&mut out, m, tool_orders); + if !m.calls.is_empty() { + out.push(b'\n'); + } + put(&mut out, SOLAR_IM_END); + out.push(b'\n'); + i += 1; + } else { + i += 1; + } + } + solar_role_open(&mut out, "assistant"); + put(&mut out, SOLAR_THINK_START); + if !think_mode_enabled(think_mode) { + put(&mut out, SOLAR_THINK_END); + } + Ok(out) +} + +fn qwen_reasoning_instruction(mode: ThinkMode) -> &'static str { + match mode { + ThinkMode::Low => concat!( + "Reasoning effort is set to low. Keep your thinking brief and focused, ", + "moving directly to the conclusion without unnecessary elaboration." + ), + ThinkMode::High | ThinkMode::Max => concat!( + "Reasoning effort is set to xhigh. Please think carefully through the task, ", + "validate key assumptions, consider plausible alternatives, and prioritize ", + "correctness, consistency, and clarity in the final answer." + ), + ThinkMode::None => "", + } +} + +fn append_qwen_tools_prompt_text(out: &mut Vec, tool_schemas: &str) { + put( + out, + "# Tools\n\nYou have access to the following functions:\n\n", + ); + let mut p = Json::new(tool_schemas); + while p.peek().is_some() { + p.ws(); + if p.peek().is_none() { + break; + } + let Some(schema) = json_raw_value(&mut p) else { + break; + }; + put(out, "\n{\"type\": \"function\", \"function\": "); + dots3_pyspace_json(out, &schema); + out.push(b'}'); + } + put( + out, + concat!( + "\n", + "\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:", + "\n\n\n", + "\n\nvalue_1\n", + "\n\nThis is the value for the second parameter", + "\nthat can span\nmultiple lines\n\n\n", + "\n\n\nReminder:", + "\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags", + "\n- Required parameters MUST be specified", + "\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after", + "\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls", + "\n" + ), + ); +} + +pub(crate) fn append_qwen_tool_response_text(out: &mut Vec, s: &[u8]) { + let end = QWEN_TOOL_RESPONSE_END.as_bytes(); + let mut i = 0; + while i < s.len() { + if s[i..].starts_with(end) { + out.extend_from_slice(b"<"); + i += 1; + } else { + out.push(s[i]); + i += 1; + } + } +} + +pub fn render_qwen_chat_ex( + msgs: &[ChatMsg], + tool_schemas: &str, + tool_orders: &[ToolSchemaOrder], + think_mode: ThinkMode, +) -> Result, RenderError> { + let mut system = Vec::new(); + for m in msgs.iter().filter(|m| role_is_system(&m.role)) { + if !system.is_empty() { + put(&mut system, "\n\n"); + } + put_trimmed(&mut system, &m.content); + } + + let instruction = qwen_reasoning_instruction(think_mode); + let mut out = Vec::new(); + if !tool_schemas.is_empty() { + put(&mut out, QWEN_IM_START); + put(&mut out, "system\n"); + if !instruction.is_empty() { + put(&mut out, instruction); + put(&mut out, "\n\n"); + } + append_qwen_tools_prompt_text(&mut out, tool_schemas); + if !system.is_empty() { + put(&mut out, "\n\n"); + out.extend_from_slice(&system); + } + put(&mut out, QWEN_IM_END); + out.push(b'\n'); + } else if !system.is_empty() || !instruction.is_empty() { + put(&mut out, QWEN_IM_START); + put(&mut out, "system\n"); + if !instruction.is_empty() { + put(&mut out, instruction); + if !system.is_empty() { + put(&mut out, "\n\n"); + } + } + out.extend_from_slice(&system); + put(&mut out, QWEN_IM_END); + out.push(b'\n'); + } + + let mut i = 0; + while i < msgs.len() { + let m = &msgs[i]; + if role_is_system(&m.role) { + i += 1; + continue; + } + if m.role == "user" && !chat_msg_is_model_tool_result(m) { + put(&mut out, QWEN_IM_START); + put(&mut out, "user\n"); + if m.parts.is_empty() { + put_trimmed(&mut out, &m.content); + } else { + let mut content = String::new(); + for part in &m.parts { + match part { + ChatPart::Text(text) => content.push_str(text), + ChatPart::Image(_) => { + content.push_str(QWEN_VISION_START); + content.push_str(QWEN_IMAGE_PAD); + content.push_str(QWEN_VISION_END); + } + ChatPart::Audio(_) => { + return Err(RenderError("audio input requires Inkling")) + } + ChatPart::Video(_) => return Err(RenderError("video input requires MiMo")), + ChatPart::ToolResult { .. } => {} + } + } + put_trimmed(&mut out, &content); + } + put(&mut out, QWEN_IM_END); + out.push(b'\n'); + i += 1; + continue; + } + if chat_msg_is_model_tool_result(m) { + let mut views = Vec::new(); + while i < msgs.len() && chat_msg_is_model_tool_result(&msgs[i]) { + collect_tool_result_message(&msgs[i], &mut views); + i += 1; + } + if !views.is_empty() { + put(&mut out, QWEN_IM_START); + put(&mut out, "user"); + for v in views { + put(&mut out, "\n"); + put(&mut out, QWEN_TOOL_RESPONSE_START); + put(&mut out, "\n"); + append_qwen_tool_response_text(&mut out, v.text); + put(&mut out, "\n"); + put(&mut out, QWEN_TOOL_RESPONSE_END); + } + put(&mut out, QWEN_IM_END); + out.push(b'\n'); + } + continue; + } + if m.role == "assistant" { + put(&mut out, QWEN_IM_START); + put(&mut out, "assistant\n\n"); + put_trimmed(&mut out, &m.reasoning); + put(&mut out, "\n\n\n"); + let content_start = out.len(); + put_trimmed(&mut out, &m.content); + if !m.calls.is_empty() && out.len() > content_start { + put(&mut out, "\n\n"); + } + append_qwen_tool_calls_text(&mut out, m, tool_orders); + put(&mut out, QWEN_IM_END); + out.push(b'\n'); + } + i += 1; + } + + put(&mut out, QWEN_IM_START); + put(&mut out, "assistant\n\n"); + if !think_mode_enabled(think_mode) { + put(&mut out, "\n\n\n"); + } + Ok(out) +} + +pub fn render_chat( + syntax: ModelSyntax, + msgs: &[ChatMsg], + tool_schemas: &str, + think_mode: ThinkMode, +) -> Result, RenderError> { + render_chat_choice_ex( + syntax, + msgs, + tool_schemas, + &[], + think_mode, + ToolChoice::Auto, + DsmlHead::V4, + ) +} + +pub fn render_chat_choice( + syntax: ModelSyntax, + msgs: &[ChatMsg], + tool_schemas: &str, + tool_orders: &[ToolSchemaOrder], + think_mode: ThinkMode, + tool_choice: ToolChoice, +) -> Result, RenderError> { + render_chat_choice_ex( + syntax, + msgs, + tool_schemas, + tool_orders, + think_mode, + tool_choice, + DsmlHead::V4, + ) +} + +pub fn render_chat_choice_ex( + syntax: ModelSyntax, + msgs: &[ChatMsg], + tool_schemas: &str, + tool_orders: &[ToolSchemaOrder], + think_mode: ThinkMode, + tool_choice: ToolChoice, + head: DsmlHead, +) -> Result, RenderError> { + if syntax != ModelSyntax::Inkling + && syntax != ModelSyntax::Mimo2 + && msgs + .iter() + .flat_map(|m| &m.parts) + .any(|p| matches!(p, ChatPart::Audio(_))) + { + return Err(RenderError("audio input requires Inkling or MiMo")); + } + match syntax { + ModelSyntax::Step37 => Err(RenderError( + "Step input requires its official Jinja template", + )), + ModelSyntax::Ling3Vl => Err(RenderError( + "Ling input requires its official Jinja template", + )), + ModelSyntax::Mimo2 => Err(RenderError( + "MiMo input requires its official Jinja template", + )), + ModelSyntax::NaiveN05 => Err(RenderError( + "Naive input requires its official Jinja template", + )), + ModelSyntax::IQuestQ1 => Err(RenderError( + "IQuest input requires its official Jinja template", + )), + ModelSyntax::BeamVq => Err(RenderError( + "Beam input requires its official Jinja template", + )), + ModelSyntax::Motif3 => render_motif3_chat_ex(msgs, tool_schemas, tool_orders, think_mode), + ModelSyntax::Exaone => render_exaone_chat(msgs, tool_schemas, think_mode), + ModelSyntax::Dots3 => render_dots3_chat(msgs, tool_schemas, think_mode), + ModelSyntax::SolarOpen2 => { + render_solar_chat_ex(msgs, tool_schemas, tool_orders, think_mode) + } + ModelSyntax::Qwen4Exp | ModelSyntax::Qwen35 => { + render_qwen_chat_ex(msgs, tool_schemas, tool_orders, think_mode) + } + ModelSyntax::Glm53 => render_glm_chat_ex(msgs, tool_schemas, tool_orders, think_mode), + ModelSyntax::K2Horizon => render_k2_chat(msgs, tool_schemas, think_mode), + ModelSyntax::Inkling => inkling::render(msgs, tool_schemas, think_mode), + ModelSyntax::DeepSeek => { + render_dsml_chat_choice_ex(msgs, tool_schemas, think_mode, tool_choice, head) + } + } +} + +/// Render the suffix appended to an exact live bank frontier for an +/// Anthropic/Responses tool-result-only continuation. +pub fn render_live_tool_tail( + syntax: ModelSyntax, + api: Api, + msgs: &[ChatMsg], + think_mode: ThinkMode, +) -> Result, RenderError> { + let start = match api { + Api::Responses => { + let mut start = msgs.len(); + while start > 0 && matches!(msgs[start - 1].role.as_str(), "tool" | "function") { + start -= 1; + } + (start < msgs.len()).then_some(start) + } + Api::Anthropic => { + let mut end = msgs.len(); + while end > 0 && role_is_system(&msgs[end - 1].role) { + end -= 1; + } + let mut start = end; + while start > 0 { + let msg = &msgs[start - 1]; + if msg.role != "user" + || (msg.tool_call_id.is_empty() && msg.tool_call_ids.is_empty()) + { + break; + } + start -= 1; + } + (start < end).then_some(start) + } + Api::Openai => None, + }; + let Some(start) = start else { + return Err(RenderError("live tool-result suffix is unavailable")); + }; + let tail = &msgs[start..]; + let mut out = Vec::new(); + match syntax { + ModelSyntax::Step37 => { + return Err(RenderError( + "Step tool results require retained history and Jinja", + )); + } + ModelSyntax::Ling3Vl => { + return Err(RenderError( + "Ling tool results require retained history and Jinja", + )); + } + ModelSyntax::Mimo2 => { + return Err(RenderError( + "MiMo tool results require retained history and Jinja", + )); + } + ModelSyntax::NaiveN05 => { + return Err(RenderError( + "Naive tool results require retained history and Jinja", + )); + } + ModelSyntax::IQuestQ1 => { + return Err(RenderError( + "IQuest tool results require retained history and Jinja", + )); + } + ModelSyntax::BeamVq => { + return Err(RenderError( + "Beam tool results require retained history and Jinja", + )); + } + ModelSyntax::Inkling => return inkling::live_tail(tail, msgs), + ModelSyntax::Glm53 => { + let think = think_mode_enabled(think_mode); + let mut pending_assistant = false; + let mut observation_open = false; + for m in tail { + if role_is_system(&m.role) { + observation_open = false; + } else if chat_msg_is_model_tool_result(m) { + if !observation_open { + put(&mut out, "<|observation|>"); + } + append_glm_tool_result_message(&mut out, m); + observation_open = true; + pending_assistant = true; + } else if m.role == "user" { + observation_open = false; + put(&mut out, "<|user|>"); + append_glm_message_content(&mut out, m)?; + pending_assistant = true; + } else if m.role == "assistant" { + observation_open = false; + put(&mut out, "<|assistant|>"); + append_glm_assistant_prefix(&mut out, m, think); + put_trimmed(&mut out, &m.content); + append_glm_tool_calls_text(&mut out, m, &[]); + pending_assistant = false; + } + } + if pending_assistant { + put(&mut out, "<|assistant|>"); + put(&mut out, if think { "" } else { "" }); + } + } + ModelSyntax::DeepSeek => { + // The engine tail is standalone (server_live_prep.c:21-69; the + // port own C: ds4_server.c:4700-4748): EOS + messages only - no + // effort prefix and no system token, whose head is already in KV. + let think = think_mode_enabled(think_mode); + let mut pending_assistant = false; + let mut pending_tool_result = false; + put(&mut out, DSML_EOS); + for m in tail { + if role_is_system(&m.role) { + continue; + } else if m.role == "user" { + put(&mut out, DSML_USER); + put(&mut out, &m.content); + pending_assistant = true; + pending_tool_result = false; + } else if m.role == "tool" || m.role == "function" { + if !pending_tool_result { + put(&mut out, DSML_USER); + } + put(&mut out, ""); + append_tool_result_text(&mut out, m.content.as_bytes()); + put(&mut out, ""); + pending_assistant = true; + pending_tool_result = true; + } else if m.role == "assistant" { + if pending_assistant { + put(&mut out, DSML_ASSISTANT); + if think { + put(&mut out, DSML_THINK_START); + put(&mut out, &m.reasoning); + put(&mut out, DSML_THINK_END); + } else { + put(&mut out, DSML_THINK_END); + } + } + put(&mut out, &m.content); + append_dsml_tool_calls_text(&mut out, m, false); + put(&mut out, DSML_EOS); + pending_assistant = false; + pending_tool_result = false; + } + } + if pending_assistant { + put(&mut out, DSML_ASSISTANT); + put( + &mut out, + if think { + DSML_THINK_START + } else { + DSML_THINK_END + }, + ); + } + } + ModelSyntax::SolarOpen2 => { + put(&mut out, SOLAR_IM_END); + out.push(b'\n'); + out.extend(render_solar_chat_ex(tail, "", &[], think_mode)?); + } + ModelSyntax::Motif3 => { + put(&mut out, "<|endofturn|>"); + let rendered = render_motif3_chat_ex(tail, "", &[], think_mode)?; + out.extend_from_slice( + rendered + .strip_prefix(b"<|beginoftext|>") + .unwrap_or(&rendered), + ); + } + ModelSyntax::Exaone => { + put(&mut out, "<|endofturn|>\n"); + out.extend(render_exaone_chat(tail, "", think_mode)?); + } + ModelSyntax::Dots3 => { + put(&mut out, "<|endofassistant|>"); + let mut rendered = render_dots3_chat(tail, "", think_mode)?; + let body = rendered + .windows(b"<|endofsystem|>".len()) + .position(|w| w == b"<|endofsystem|>") + .map(|at| rendered.split_off(at + b"<|endofsystem|>".len())) + .unwrap_or_default(); + out.extend(body); + } + ModelSyntax::Qwen4Exp | ModelSyntax::Qwen35 => { + put(&mut out, QWEN_IM_END); + out.push(b'\n'); + out.extend(render_qwen_chat_ex(tail, "", &[], think_mode)?); + } + ModelSyntax::K2Horizon => { + put(&mut out, K2_IM_END); + let rendered = render_k2_chat(tail, "", think_mode)?; + out.extend_from_slice( + rendered + .strip_prefix(K2_BOS.as_bytes()) + .unwrap_or(&rendered), + ); + } + } + Ok(out) +} + +fn append_k2_tools_system(out: &mut Vec, tool_schemas: &str, system: &str) { + put(out, K2_IM_START); + put(out, "system\n# Tools\nYou may call one or more tools to assist with the user query.\n\nAvailable tools are:\n\n\n"); + put(out, tool_schemas); + put(out, "\n\n\nWhen calling tools, you MUST follow the tool-call format below:\n\nWrap all tool calls in a single block. For each call, write the function name at the start of , followed by paired and tags for each argument:\n\n\n$FUNCTION_NAME\n$PARAMETER_NAME\n$PARAMETER_VALUE\n...\n\n\n\nString and scalar parameters should be written as plain text. Array and object parameters should be written as JSON literals."); + if !system.is_empty() { + put(out, "\n\n"); + put(out, system); + } + put(out, K2_IM_END); +} + +fn append_k2_arg(out: &mut Vec, arg: &crate::json::JsonArg) { + put(out, K2_ARG_KEY_START); + put(out, &arg.key); + put(out, K2_ARG_KEY_END); + out.push(b'\n'); + put(out, K2_ARG_VALUE_START); + put(out, &arg.value); + put(out, K2_ARG_VALUE_END); + out.push(b'\n'); +} + +fn append_k2_arguments_from_json(out: &mut Vec, json: &str) -> bool { + let Some(args) = json_args_parse(json) else { + return false; + }; + for arg in &args { + append_k2_arg(out, arg); + } + true +} + +fn append_k2_tool_calls(out: &mut Vec, m: &ChatMsg) { + if m.calls.is_empty() { + return; + } + if !m.raw_dsml.is_empty() { + put(out, &m.raw_dsml); + return; + } + put(out, K2_TOOL_CALLS_START); + for tc in &m.calls { + put(out, "\n"); + put(out, K2_TOOL_CALL_START); + put(out, &tc.name); + out.push(b'\n'); + if !append_k2_arguments_from_json(out, &tc.arguments) { + put(out, K2_ARG_KEY_START); + put(out, "arguments"); + put(out, K2_ARG_KEY_END); + out.push(b'\n'); + put(out, K2_ARG_VALUE_START); + put( + out, + if tc.arguments.is_empty() { + "{}" + } else { + &tc.arguments + }, + ); + put(out, K2_ARG_VALUE_END); + out.push(b'\n'); + } + put(out, K2_TOOL_CALL_END); + } + put(out, "\n"); + put(out, K2_TOOL_CALLS_END); +} + +/// Official IFM chat protocol using JSON tool presentation and default XML calls. +pub fn render_k2_chat( + msgs: &[ChatMsg], + tool_schemas: &str, + think_mode: ThinkMode, +) -> Result, RenderError> { + let mut out = Vec::new(); + put(&mut out, K2_BOS); + let first_system = msgs.first().filter(|m| role_is_system(&m.role)); + if !tool_schemas.is_empty() { + append_k2_tools_system( + &mut out, + tool_schemas, + first_system.map(|m| m.content.as_str()).unwrap_or(""), + ); + } + for (i, m) in msgs.iter().enumerate() { + if role_is_system(&m.role) { + if i == 0 && !tool_schemas.is_empty() { + continue; + } + put(&mut out, K2_IM_START); + put(&mut out, "system\n"); + put(&mut out, &m.content); + put(&mut out, K2_IM_END); + continue; + } + if m.role == "user" && !chat_msg_is_model_tool_result(m) { + put(&mut out, K2_IM_START); + put(&mut out, "user\n"); + put(&mut out, &m.content); + put(&mut out, K2_IM_END); + continue; + } + if chat_msg_is_model_tool_result(m) { + put(&mut out, K2_IM_START); + put(&mut out, "tool\n"); + put(&mut out, &m.content); + put(&mut out, K2_IM_END); + continue; + } + if m.role == "assistant" { + put(&mut out, K2_IM_START); + put(&mut out, "assistant\n"); + put(&mut out, K2_THINK_START); + put(&mut out, "\n"); + put(&mut out, &m.reasoning); + put(&mut out, K2_THINK_END); + put(&mut out, &m.content); + append_k2_tool_calls(&mut out, m); + put(&mut out, K2_IM_END); + } + } + put(&mut out, K2_IM_START); + put(&mut out, "assistant\n"); + put(&mut out, K2_THINK_START); + put(&mut out, "\n"); + if !think_mode_enabled(think_mode) { + put(&mut out, K2_THINK_END); + put(&mut out, "\n"); + } + Ok(out) +} diff --git a/crates/ds4-server/src/render/inkling.rs b/crates/ds4-server/src/render/inkling.rs new file mode 100644 index 0000000000000000000000000000000000000000..1f3ac065e631cfbc975f5e5fa7d63bf978b2ed5c --- /dev/null +++ b/crates/ds4-server/src/render/inkling.rs @@ -0,0 +1,316 @@ +//! Pinned Inkling message framing and sorted source-template JSON. + +use serde_json::{json, Value}; + +use super::{ + collect_tool_result_message, role_is_system, ChatMsg, ChatPart, RenderError, ThinkMode, +}; + +pub(crate) const MODEL: &str = "<|message_model|>"; +pub(crate) const TEXT: &str = "<|content_text|>"; +pub(crate) const THINK: &str = "<|content_thinking|>"; +pub(crate) const END: &str = "<|end_message|>"; +pub(crate) const EOS: &str = "<|content_model_end_sampling|>"; +pub(crate) const INVOKE: &str = "<|content_invoke_tool_json|>"; +const SYSTEM: &str = "<|message_system|>"; +const USER: &str = "<|message_user|>"; +const TOOL: &str = "<|message_tool|>"; +const IMAGE: &str = "<|content_image|><|unused_200054|>"; +const AUDIO: &str = "<|content_audio_input|><|unused_200053|><|audio_end|>"; + +fn source_json(value: &Value) -> String { + match value { + Value::Array(values) => format!( + "[{}]", + values.iter().map(source_json).collect::>().join(",") + ), + Value::Object(values) => { + let mut keys: Vec<_> = values.keys().collect(); + keys.sort(); + let members: Vec<_> = keys + .into_iter() + .map(|key| format!("{}:{}", json!(key), source_json(&values[key]))) + .collect(); + format!("{{{}}}", members.join(",")) + } + Value::Number(n) if n.is_f64() => { + let value = n.as_f64().unwrap(); + // Python's JSON encoder uses repr(float), including signed, + // two-digit exponents and .0 for integral finite floats. + if value != 0.0 && (value.abs() < 1e-4 || value.abs() >= 1e16) { + let scientific = format!("{value:e}"); + let (mantissa, exponent) = scientific.split_once('e').unwrap(); + let exponent: i32 = exponent.parse().unwrap(); + format!("{mantissa}e{exponent:+03}") + } else { + let text = value.to_string(); + if text.contains('.') { + text + } else { + format!("{text}.0") + } + } + } + _ => value.to_string(), + } +} + +fn message(out: &mut String, role: &str, channel: &str, text: &str) { + out.push_str(role); + out.push_str(channel); + out.push_str(text); + out.push_str(END); +} + +fn effort(out: &mut String, think: ThinkMode) { + let level = match think { + ThinkMode::None => "0", + ThinkMode::Low => "0.2", + ThinkMode::High => "0.9", + ThinkMode::Max => "0.99", + }; + message( + out, + SYSTEM, + TEXT, + &format!("Thinking effort level: {level}"), + ); +} + +fn declare(out: &mut String, schemas: &str) -> Result<(), RenderError> { + if schemas.is_empty() { + return Ok(()); + } + // The shared request parser stores consecutive schema objects, including + // multiline JSON. A line split would corrupt nested parameter schemas. + let tools: Vec = serde_json::Deserializer::from_str(schemas) + .into_iter() + .collect::>() + .map_err(|_| RenderError("invalid Inkling tool schemas"))?; + if tools.is_empty() { + return Ok(()); + } + let mut specs = Vec::with_capacity(tools.len()); + for tool in tools { + let function = tool.get("function").unwrap_or(&tool); + let name = function["name"] + .as_str() + .ok_or(RenderError("invalid Inkling tool name"))?; + specs.push(json!({ + "name": name, + "description": function["description"].as_str().unwrap_or(""), + "parameters": function.get("parameters").or_else(|| function.get("input_schema")).filter(|p| !p.is_null()).cloned().unwrap_or(json!({})), + "type": tool["type"].as_str().filter(|s| !s.is_empty()).unwrap_or("function"), + })); + } + message( + out, + SYSTEM, + "tool_declare<|content_xml|>", + &source_json(&json!(specs)), + ); + Ok(()) +} + +fn append_message(out: &mut String, msg: &ChatMsg, history: &[ChatMsg]) -> Result<(), RenderError> { + if super::chat_msg_is_model_tool_result(msg) { + let mut views = Vec::new(); + collect_tool_result_message(msg, &mut views); + for view in views { + let name = if !msg.name.is_empty() { + msg.name.as_str() + } else { + history + .iter() + .flat_map(|m| &m.calls) + .find(|call| !view.id.is_empty() && call.id == view.id) + .map(|call| call.name.as_str()) + .unwrap_or("") + }; + message( + out, + TOOL, + &format!("{name}{TEXT}"), + &String::from_utf8_lossy(view.text), + ); + } + return Ok(()); + } + let role = match msg.role.as_str() { + "system" | "developer" => SYSTEM, + "user" => USER, + "assistant" => MODEL, + _ => return Err(RenderError("invalid Inkling message role")), + }; + if msg.role == "assistant" && !msg.reasoning.is_empty() { + message(out, MODEL, THINK, &msg.reasoning); + } + if !msg.parts.is_empty() { + for part in &msg.parts { + match part { + ChatPart::Text(text) => message(out, role, TEXT, text), + ChatPart::Image(_) => message(out, role, IMAGE, ""), + ChatPart::Audio(_) => message(out, role, AUDIO, ""), + ChatPart::Video(_) => return Err(RenderError("video input requires MiMo")), + ChatPart::ToolResult { .. } => {} + } + } + } else if !msg.content.is_empty() || (msg.calls.is_empty() && msg.reasoning.is_empty()) { + message(out, role, TEXT, &msg.content); + } + if msg.role == "assistant" { + for call in &msg.calls { + let args: Value = serde_json::from_str(if call.arguments.is_empty() { + "{}" + } else { + &call.arguments + }) + .map_err(|_| RenderError("invalid Inkling tool arguments"))?; + if !args.is_object() { + return Err(RenderError("Inkling tool arguments must be an object")); + } + // The outer envelope has a source-defined order; args sort recursively. + let body = format!( + "{{\"name\":{},\"args\":{}}}", + json!(call.name), + source_json(&args) + ); + message(out, MODEL, &format!("{}{INVOKE}", call.name), &body); + } + out.push_str(EOS); + } + Ok(()) +} + +pub(crate) fn render( + msgs: &[ChatMsg], + schemas: &str, + think: ThinkMode, +) -> Result, RenderError> { + let mut out = String::new(); + declare(&mut out, schemas)?; + let mut emitted = false; + for msg in msgs { + if !emitted && !role_is_system(&msg.role) { + effort(&mut out, think); + emitted = true; + } + append_message(&mut out, msg, msgs)?; + } + if !emitted { + effort(&mut out, think); + } + out.push_str(MODEL); + Ok(out.into_bytes()) +} + +pub(crate) fn live_tail(tail: &[ChatMsg], history: &[ChatMsg]) -> Result, RenderError> { + let mut out = EOS.to_string(); + for msg in tail { + append_message(&mut out, msg, history)?; + } + out.push_str(MODEL); + Ok(out.into_bytes()) +} + +pub(crate) fn recovery(raw: &[u8], detail: &str) -> Vec { + let mut out = String::new(); + // EOS is sampled without being evaluated. Close an unfinished channel + // before the source-template tool result and next model turn. + if !raw.ends_with(EOS.as_bytes()) { + if !raw.ends_with(END.as_bytes()) { + out.push_str(END); + } + out.push_str(EOS); + } + let error = format!( + "Tool error: invalid Inkling tool call: {detail}\n\ +The previous assistant output was not executed because its tool syntax was malformed. \ +Emit a new valid native Inkling tool call, or answer normally if no tool is needed." + ); + message(&mut out, TOOL, TEXT, &error); + out.push_str(MODEL); + out.into_bytes() +} + +pub(crate) fn system_region(prompt: &[u8]) -> Vec { + let Ok(mut rest) = std::str::from_utf8(prompt) else { + return Vec::new(); + }; + while let Some(body) = rest.strip_prefix(SYSTEM) { + let Some((body, tail)) = body.split_once(END) else { + break; + }; + if let Some(text) = body.strip_prefix(TEXT) { + return text.as_bytes().to_vec(); + } + rest = tail; + } + Vec::new() +} + +pub(crate) fn parse(raw: &[u8]) -> Option { + let mut out = crate::tools::ParsedGenerated { + ok: true, + ..Default::default() + }; + let mut pos = 0; + while pos < raw.len() { + if raw[pos..].starts_with(MODEL.as_bytes()) { + pos += MODEL.len(); + } + if raw[pos..].starts_with(EOS.as_bytes()) { + pos += EOS.len(); + continue; + } + let Some((start, marker)) = [TEXT, THINK, INVOKE] + .into_iter() + .filter_map(|marker| { + raw[pos..] + .windows(marker.len()) + .position(|s| s == marker.as_bytes()) + .map(|offset| (pos + offset, marker)) + }) + .min_by_key(|(at, _)| *at) + else { + // A generation budget can end inside the next channel header. + // Never publish a pending tool name or partial control marker. + break; + }; + let name = std::str::from_utf8(&raw[pos..start]).ok()?; + if marker != INVOKE && !name.is_empty() { + return None; + } + let body_start = start + marker.len(); + let end = raw[body_start..] + .windows(END.len()) + .position(|s| s == END.as_bytes()) + .map(|offset| body_start + offset); + let body_end = end.unwrap_or(raw.len()); + let body = &raw[body_start..body_end]; + match marker { + TEXT => out.content.extend_from_slice(body), + THINK => out.reasoning.extend_from_slice(body), + INVOKE if end.is_some() => { + let value: Value = serde_json::from_slice(body).ok()?; + let function = value["name"].as_str().filter(|s| !s.is_empty())?; + let args = value.get("args").filter(|v| v.is_object())?; + if !name.is_empty() && name != function { + return None; + } + out.calls.push(crate::parse::ToolCall { + id: String::new(), + name: function.into(), + arguments: source_json(args), + }); + out.raw_tool_text.push_str(MODEL); + out.raw_tool_text + .push_str(std::str::from_utf8(&raw[pos..body_end + END.len()]).ok()?); + } + INVOKE => return None, + _ => {} + } + pos = end.map(|end| end + END.len()).unwrap_or(raw.len()); + } + Some(out) +} diff --git a/crates/ds4-server/src/retry.rs b/crates/ds4-server/src/retry.rs new file mode 100644 index 0000000000000000000000000000000000000000..c5e207991b8adc4c3340464c1fb2b06ab7705312 --- /dev/null +++ b/crates/ds4-server/src/retry.rs @@ -0,0 +1,825 @@ +//! Corrective tool-call retry from `ds4_server.c` at v0.6.5-dfm: +//! tag-completion repair, model-visible tool-error suffix, and the +//! `decode_again` decision. Session rewind/sync stays on `DecodeIo`. + +use crate::parse::ToolSchemaOrder; +use crate::render::{ + append_qwen_tool_response_text, append_solar_tool_response_text, append_tool_result_text, + solar_role_open, ModelSyntax, QWEN_IM_END, QWEN_IM_START, QWEN_TOOL_RESPONSE_END, + QWEN_TOOL_RESPONSE_START, SOLAR_IM_CONTENT, SOLAR_IM_END, SOLAR_IM_START, SOLAR_THINK_END, + SOLAR_THINK_START, SOLAR_TOOL_ARG_END, SOLAR_TOOL_ARG_START, SOLAR_TOOL_ARG_VALUE, + SOLAR_TOOL_CALLS, SOLAR_TOOL_CALL_END, SOLAR_TOOL_RESPONSE_END, SOLAR_TOOL_RESPONSE_START, + THINK_HIGH_PREFIX, THINK_MAX_PREFIX, +}; +use crate::route::{think_mode_enabled, ThinkMode}; +use crate::stream::{think_end, ChatFormat}; +use crate::tools::{ + parse_generated_message, DSML_INVOKE_END, DSML_INVOKE_END_SHORT, DSML_INVOKE_END_SPACED, + DSML_INVOKE_START, DSML_INVOKE_START_SHORT, DSML_INVOKE_START_SPACED, DSML_PARAM_END, + DSML_PARAM_END_SHORT, DSML_PARAM_END_SPACED, DSML_PARAM_START, DSML_PARAM_START_SHORT, + DSML_PARAM_START_SPACED, DSML_TOOL_CALLS_END, DSML_TOOL_CALLS_END_SHORT, + DSML_TOOL_CALLS_END_SPACED, DSML_TOOL_CALLS_START, DSML_TOOL_CALLS_START_SHORT, + DSML_TOOL_CALLS_START_SPACED, +}; + +fn is_c_space(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c) +} + +fn find_substr(hay: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + hay.windows(needle.len()).position(|w| w == needle) +} + +fn find_last_substr(hay: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + hay.windows(needle.len()) + .enumerate() + .rev() + .find(|(_, w)| *w == needle) + .map(|(i, _)| i) +} + +fn count_marker(s: &[u8], marker: &[u8]) -> usize { + if marker.is_empty() { + return 0; + } + let mut n = 0; + let mut i = 0; + while i + marker.len() <= s.len() { + if s[i..].starts_with(marker) { + n += 1; + i += marker.len(); + } else { + i += 1; + } + } + n +} + +pub fn syntax_skips_recovery(syntax: ModelSyntax) -> bool { + matches!( + syntax, + ModelSyntax::Motif3 | ModelSyntax::Exaone | ModelSyntax::Dots3 + ) +} + +/// C `rendered_dsml_system_region`. +pub fn rendered_dsml_system_region(prompt: &[u8]) -> Vec { + let bos = "<|begin▁of▁sentence|>".as_bytes(); + let mut p = if prompt.starts_with(bos) { + &prompt[bos.len()..] + } else { + prompt + }; + for prefix in [THINK_HIGH_PREFIX.as_bytes(), THINK_MAX_PREFIX.as_bytes()] { + if !prefix.is_empty() && p.starts_with(prefix) { + p = &p[prefix.len()..]; + break; + } + } + while !p.is_empty() && is_c_space(p[0]) { + p = &p[1..]; + } + let user = find_substr(p, "<|User|>".as_bytes()); + let assistant = find_substr(p, "<|Assistant|>".as_bytes()); + let mut end = match (user, assistant) { + (Some(u), Some(a)) => u.min(a), + (Some(u), None) => u, + (None, Some(a)) => a, + (None, None) => p.len(), + }; + while end > 0 && is_c_space(p[end - 1]) { + end -= 1; + } + p[..end].to_vec() +} + +/// C `rendered_solar_system_region`. +pub fn rendered_solar_system_region(prompt: &[u8]) -> Vec { + let mut prefix = Vec::new(); + prefix.extend_from_slice(SOLAR_IM_START.as_bytes()); + prefix.extend_from_slice(b"system"); + prefix.extend_from_slice(SOLAR_IM_CONTENT.as_bytes()); + if !prompt.starts_with(&prefix) { + return Vec::new(); + } + let p = &prompt[prefix.len()..]; + let mut end = find_substr(p, SOLAR_IM_END.as_bytes()).unwrap_or(p.len()); + while end > 0 && is_c_space(p[end - 1]) { + end -= 1; + } + p[..end].to_vec() +} + +pub fn rendered_chat_system_region(format: ChatFormat, prompt: &[u8]) -> Vec { + if format == ChatFormat::Inkling { + crate::render::inkling::system_region(prompt) + } else if format == ChatFormat::SolarOpen2 { + rendered_solar_system_region(prompt) + } else if format == ChatFormat::Qwen4Exp { + let prefix = b"<|im_start|>system\n"; + let Some(mut body) = prompt.strip_prefix(prefix) else { + return Vec::new(); + }; + if let Some(end) = find_substr(body, QWEN_IM_END.as_bytes()) { + body = &body[..end]; + } + body.trim_ascii().to_vec() + } else { + rendered_dsml_system_region(prompt) + } +} + +/// C `try_repair_dsml` (ds4_server.c:6707-6776). Only the enclosing wrapper +/// may be missing: an unfinished invocation can still be missing arguments and +/// an unfinished value can be a truncated command, so closing either "would +/// invent an executable action" (:6697-6700). The style is the one of the +/// earliest tool start in the scan (:6730-6745 find_any_tool_start), and the +/// think scope skips markers inside parameter bodies (:6714). +pub fn try_repair_dsml(s: &[u8]) -> Option> { + if s.is_empty() { + return None; + } + let think = b""; + let scan_start = match find_last_structural(s, think) { + Some(i) => i + think.len(), + None => 0, + }; + let scan = &s[scan_start..]; + let (ts, te, is, ie, ps, pe) = repair_tags(scan)?; + let mut tos = 0usize; + let mut toe = 0usize; + let mut ios = 0usize; + let mut ioe = 0usize; + let mut pos = 0usize; + let mut poe = 0usize; + let mut i = 0; + while i < scan.len() { + if scan[i..].starts_with(ts) { + tos += 1; + i += ts.len(); + } else if scan[i..].starts_with(te) { + toe += 1; + i += te.len(); + } else if scan[i..].starts_with(is) { + ios += 1; + i += is.len(); + } else if scan[i..].starts_with(ie) { + ioe += 1; + i += ie.len(); + } else if scan[i..].starts_with(ps) { + pos += 1; + i += ps.len(); + } else if scan[i..].starts_with(pe) { + poe += 1; + i += pe.len(); + } else { + i += 1; + } + } + // Every invocation and every parameter must already be closed; an extra + // closing tag is not a truncation pattern. Only then is the missing + // wrapper supplied, and only it (:6768-6776). + if ios != ioe || pos != poe { + return None; + } + if tos == toe || toe > tos { + return None; + } + let mut out = s.to_vec(); + for _ in 0..(tos - toe) { + out.extend_from_slice(te); + } + Some(out) +} + +/// The tag set of the earliest tool start in the scan (C `find_any_tool_start` +/// plus the style dispatch, ds4_server.c:6730-6745). `` starts a +/// block for the search but has no repair style, so it refuses like the +/// reference's trailing `else`. +fn repair_tags( + scan: &[u8], +) -> Option<( + &'static [u8], + &'static [u8], + &'static [u8], + &'static [u8], + &'static [u8], + &'static [u8], +)> { + let first = find_any_tool_start(scan)?; + let s = &scan[first..]; + if s.starts_with(DSML_TOOL_CALLS_START_SPACED.as_bytes()) { + Some(( + DSML_TOOL_CALLS_START_SPACED.as_bytes(), + DSML_TOOL_CALLS_END_SPACED.as_bytes(), + DSML_INVOKE_START_SPACED.as_bytes(), + DSML_INVOKE_END_SPACED.as_bytes(), + DSML_PARAM_START_SPACED.as_bytes(), + DSML_PARAM_END_SPACED.as_bytes(), + )) + } else if s.starts_with(DSML_TOOL_CALLS_START.as_bytes()) { + Some(( + DSML_TOOL_CALLS_START.as_bytes(), + DSML_TOOL_CALLS_END.as_bytes(), + DSML_INVOKE_START.as_bytes(), + DSML_INVOKE_END.as_bytes(), + DSML_PARAM_START.as_bytes(), + DSML_PARAM_END.as_bytes(), + )) + } else if s.starts_with(DSML_TOOL_CALLS_START_SHORT.as_bytes()) { + Some(( + DSML_TOOL_CALLS_START_SHORT.as_bytes(), + DSML_TOOL_CALLS_END_SHORT.as_bytes(), + DSML_INVOKE_START_SHORT.as_bytes(), + DSML_INVOKE_END_SHORT.as_bytes(), + DSML_PARAM_START_SHORT.as_bytes(), + DSML_PARAM_END_SHORT.as_bytes(), + )) + } else if s.starts_with(b"") { + Some(( + b"".as_slice(), + b"".as_slice(), + b"".as_slice(), + b"".as_slice(), + )) + } else { + None + } +} + +/// The earliest of the spellings a tool block can open with (C +/// `find_any_tool_start`, ds4_server.c:5842-5856), index into `s`. +fn find_any_tool_start(s: &[u8]) -> Option { + [ + DSML_TOOL_CALLS_START_SPACED.as_bytes(), + DSML_TOOL_CALLS_START.as_bytes(), + DSML_TOOL_CALLS_START_SHORT.as_bytes(), + b"".as_slice(), + b"".as_slice(), + ] + .iter() + .filter_map(|p| find_substr(s, p)) + .min() +} + +/// The last `needle` outside a parameter body (C `find_tool_structural_text`, +/// ds4_server.c:5890-5920): tool arguments are literal data, so a marker inside +/// them is not structure. A wrapper whose close is missing ends the scan at +/// what it had found, exactly as the reference's early return does. +fn find_last_structural(s: &[u8], needle: &[u8]) -> Option { + let wrappers: [(&[u8], &[u8]); 6] = [ + ( + DSML_PARAM_START_SPACED.as_bytes(), + DSML_PARAM_END_SPACED.as_bytes(), + ), + (DSML_PARAM_START.as_bytes(), DSML_PARAM_END.as_bytes()), + ( + DSML_PARAM_START_SHORT.as_bytes(), + DSML_PARAM_END_SHORT.as_bytes(), + ), + (b""), + (b"", b""), + (b"", b""), + ]; + let mut found = None; + let mut i = 0; + while i < s.len() { + if s[i..].starts_with(needle) { + found = Some(i); + } + let mut skipped = false; + for (open, close) in wrappers { + if !s[i..].starts_with(open) { + continue; + } + let Some(gt) = find_substr(&s[i..], b">") else { + return found; + }; + let body = &s[i + gt + 1..]; + let Some(rel) = find_substr(body, close) else { + return found; + }; + i += gt + 1 + rel + close.len(); + skipped = true; + break; + } + if !skipped { + i += 1; + } + } + found +} + +/// C `try_repair_solar_tool_call`. +pub fn try_repair_solar(s: &[u8]) -> Option> { + if s.is_empty() { + return None; + } + let scan = if let Some(i) = find_last_substr(s, SOLAR_THINK_END.as_bytes()) { + &s[i + SOLAR_THINK_END.len()..] + } else { + s + }; + let call_open = count_marker(scan, SOLAR_TOOL_CALLS.as_bytes()); + let call_close = count_marker(scan, SOLAR_TOOL_CALL_END.as_bytes()); + let arg_open = count_marker(scan, SOLAR_TOOL_ARG_START.as_bytes()); + let arg_value = count_marker(scan, SOLAR_TOOL_ARG_VALUE.as_bytes()); + let arg_close = count_marker(scan, SOLAR_TOOL_ARG_END.as_bytes()); + if call_open == 0 || call_close > call_open || arg_close > arg_open || arg_value > arg_open { + return None; + } + if call_open == call_close && arg_open == arg_close { + return None; + } + if call_open != call_close + 1 || arg_open > arg_close + 1 { + return None; + } + if arg_open != arg_value { + return None; + } + let mut out = s.to_vec(); + if arg_open == arg_close + 1 { + out.extend_from_slice(SOLAR_TOOL_ARG_END.as_bytes()); + } + out.extend_from_slice(SOLAR_TOOL_CALL_END.as_bytes()); + Some(out) +} + +pub fn try_repair_tool_call_format(format: ChatFormat, s: &[u8]) -> Option> { + match format { + ChatFormat::SolarOpen2 => try_repair_solar(s), + ChatFormat::Exaone + | ChatFormat::Qwen4Exp + | ChatFormat::K2Horizon + | ChatFormat::Inkling + | ChatFormat::IQuestQ1 + | ChatFormat::Beam => None, + ChatFormat::DeepSeek => try_repair_dsml(s), + } +} + +pub fn tool_call_format_has_repairable_truncation(format: ChatFormat, s: &[u8]) -> bool { + try_repair_tool_call_format(format, s).is_some() +} + +/// C `build_invalid_tool_error_suffix`. +pub fn build_invalid_tool_error_suffix( + format: ChatFormat, + think_mode: ThinkMode, + thinking_inside: bool, + prompt: &[u8], + detail: &str, +) -> Vec { + if format == ChatFormat::Inkling { + return crate::render::inkling::recovery(&[], detail); + } + let solar = format == ChatFormat::SolarOpen2; + let qwen = format == ChatFormat::Qwen4Exp; + let system = rendered_chat_system_region(format, prompt); + let mut tool_error = Vec::new(); + if solar { + tool_error.extend_from_slice(b"Tool error: invalid Solar tool call"); + } else if qwen { + tool_error.extend_from_slice(b"Tool error: invalid Qwen tool call"); + } else { + tool_error.extend_from_slice(b"Tool error: invalid DSML tool call"); + } + if !detail.is_empty() { + tool_error.extend_from_slice(b": "); + tool_error.extend_from_slice(detail.as_bytes()); + } + if solar { + tool_error.extend_from_slice( + b"\nThe previous assistant output was not executed because the Solar tool syntax was malformed. \ +Emit a new valid native Solar tool call, or answer normally if no tool is needed.", + ); + } else if qwen { + tool_error.extend_from_slice( + b"\nThe previous assistant output was not executed because the Qwen tool syntax was malformed. \ +Emit a new valid native Qwen tool call, or answer normally if no tool is needed.", + ); + } else { + tool_error.extend_from_slice( + b"\nThe previous assistant output was not executed because the DSML syntax was malformed. \ +Emit a new valid DSML tool call, or answer normally if no tool is needed.", + ); + } + if !system.is_empty() { + tool_error.extend_from_slice(b"\n\nSystem prompt reminder:\n"); + tool_error.extend_from_slice(&system); + } + + let mut suffix = Vec::new(); + if think_mode_enabled(think_mode) && thinking_inside { + suffix.extend_from_slice(think_end(format).as_bytes()); + } + if solar { + suffix.extend_from_slice(SOLAR_IM_END.as_bytes()); + suffix.push(b'\n'); + solar_role_open(&mut suffix, "tool"); + suffix.extend_from_slice(SOLAR_TOOL_RESPONSE_START.as_bytes()); + append_solar_tool_response_text(&mut suffix, &tool_error); + suffix.extend_from_slice(SOLAR_TOOL_RESPONSE_END.as_bytes()); + suffix.push(b'\n'); + suffix.extend_from_slice(SOLAR_IM_END.as_bytes()); + suffix.push(b'\n'); + solar_role_open(&mut suffix, "assistant"); + suffix.extend_from_slice(SOLAR_THINK_START.as_bytes()); + if !think_mode_enabled(think_mode) { + suffix.extend_from_slice(SOLAR_THINK_END.as_bytes()); + } + } else if qwen { + suffix.extend_from_slice(QWEN_IM_END.as_bytes()); + suffix.extend_from_slice(b"\n"); + suffix.extend_from_slice(QWEN_IM_START.as_bytes()); + suffix.extend_from_slice(b"user\n"); + suffix.extend_from_slice(QWEN_TOOL_RESPONSE_START.as_bytes()); + suffix.extend_from_slice(b"\n"); + append_qwen_tool_response_text(&mut suffix, &tool_error); + suffix.extend_from_slice(b"\n"); + suffix.extend_from_slice(QWEN_TOOL_RESPONSE_END.as_bytes()); + suffix.extend_from_slice(QWEN_IM_END.as_bytes()); + suffix.extend_from_slice(b"\n"); + suffix.extend_from_slice(QWEN_IM_START.as_bytes()); + suffix.extend_from_slice(b"assistant\n\n"); + if !think_mode_enabled(think_mode) { + suffix.extend_from_slice(b"\n\n\n"); + } + } else { + suffix.extend_from_slice("<|end▁of▁sentence|><|User|>".as_bytes()); + append_tool_result_text(&mut suffix, &tool_error); + suffix.extend_from_slice("<|Assistant|>".as_bytes()); + if think_mode_enabled(think_mode) { + suffix.extend_from_slice(b""); + } else { + suffix.extend_from_slice(b""); + } + } + suffix +} + +fn build_glm_tool_error_suffix( + think_mode: ThinkMode, + thinking_inside: bool, + detail: &str, +) -> Vec { + let mut error = b"Tool error: invalid GLM tool call".to_vec(); + if !detail.is_empty() { + error.extend_from_slice(b": "); + error.extend_from_slice(detail.as_bytes()); + } + error.extend_from_slice(b"\nThe previous assistant output was not executed because the syntax was malformed. Emit a new valid , or answer normally if no tool is needed."); + + let mut suffix = Vec::new(); + if think_mode_enabled(think_mode) && thinking_inside { + suffix.extend_from_slice(b""); + } + suffix.extend_from_slice(b"<|observation|>"); + for index in 0..error.len() { + if error[index..].starts_with(b"") { + suffix.extend_from_slice(b"<"); + } else { + suffix.push(error[index]); + } + } + suffix.extend_from_slice(b"<|assistant|>"); + if think_mode_enabled(think_mode) { + suffix.extend_from_slice(b""); + } else { + suffix.extend_from_slice(b""); + } + suffix +} + +pub(crate) fn build_recovery_suffix( + syntax: ModelSyntax, + format: ChatFormat, + think_mode: ThinkMode, + prompt: &[u8], + acc: &crate::tools::SemAccum, + detail: &str, +) -> Vec { + if syntax == ModelSyntax::Glm53 { + return build_glm_tool_error_suffix(think_mode, acc.thinking_inside(), detail); + } + if format == ChatFormat::Inkling { + return crate::render::inkling::recovery(&acc.text, detail); + } + build_invalid_tool_error_suffix(format, think_mode, acc.thinking_inside(), prompt, detail) +} + +#[cfg(test)] +mod glm_recovery_tests { + use super::*; + use crate::tools::SemAccum; + + #[test] + fn glm_retry_uses_native_observation() { + assert!(!syntax_skips_recovery(ModelSyntax::Glm53)); + let prompt = b"<|assistant|>"; + let acc = SemAccum::init(true, true, true, ChatFormat::DeepSeek, prompt); + let suffix = build_recovery_suffix( + ModelSyntax::Glm53, + ChatFormat::DeepSeek, + ThinkMode::High, + prompt, + &acc, + "missing arg_value", + ); + let suffix = String::from_utf8(suffix).unwrap(); + assert!(suffix.starts_with("<|observation|>")); + assert!(suffix.contains("Tool error: invalid GLM tool call: missing arg_value")); + assert!(suffix.contains(" syntax was malformed")); + assert!(suffix.ends_with("<|assistant|>")); + assert!(!suffix.contains("DSML")); + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TruncationOutcome { + None, + Repair(Vec), + RetryUnterminated, + ErrorUnterminated, +} + +/// C serial-loop truncation / repair / unterminated-recovery gate. +pub fn truncation_outcome( + syntax: ModelSyntax, + format: ChatFormat, + kind_chat: bool, + has_tools: bool, + saw_tool_start: bool, + saw_tool_end: bool, + finish: &str, + stream: bool, + attempted: bool, + text: &[u8], + orders: &[ToolSchemaOrder], +) -> TruncationOutcome { + if !kind_chat + || !has_tools + || syntax_skips_recovery(syntax) + || !saw_tool_start + || finish == "error" + { + return TruncationOutcome::None; + } + if saw_tool_end && !tool_call_format_has_repairable_truncation(format, text) { + return TruncationOutcome::None; + } + if let Some(repaired) = try_repair_tool_call_format(format, text) { + let parsed = parse_generated_message(syntax, &repaired, false, format, orders); + if parsed.ok && !parsed.calls.is_empty() { + return TruncationOutcome::Repair(repaired); + } + } + if finish == "length" { + return TruncationOutcome::None; + } + if !stream && !attempted { + TruncationOutcome::RetryUnterminated + } else { + TruncationOutcome::ErrorUnterminated + } +} + +/// C parse-failure `decode_again` gate after `parse_generated_message_for_response`. +pub fn parse_failure_should_retry( + syntax: ModelSyntax, + stream: bool, + attempted: bool, + finish: &str, + recovered: bool, + has_tools: bool, + saw_tool_start: bool, +) -> bool { + recovered + && has_tools + && saw_tool_start + && !syntax_skips_recovery(syntax) + && !stream + && !attempted + && finish != "length" +} + +/// Termination cause only. An open think block does not rewrite it. +pub fn terminal_finish(finish: &str) -> &'static str { + match finish { + "tool_calls" => "tool_calls", + "length" => "length", + "error" => "error", + _ => "stop", + } +} + +fn truncation_name(o: TruncationOutcome) -> &'static str { + match o { + TruncationOutcome::None => "none", + TruncationOutcome::Repair(_) => "repair", + TruncationOutcome::RetryUnterminated => "retry-unterminated", + TruncationOutcome::ErrorUnterminated => "error-unterminated", + } +} + +fn utf(s: &[u8]) -> String { + String::from_utf8_lossy(s).into_owned() +} + +fn dsml_think_prompt() -> Vec { + format!( + "<|begin▁of▁sentence|>## Tools\nschema\n\nSystem rule\n\n<|User|>Hi<|Assistant|>" + ) + .into_bytes() +} + +fn solar_think_prompt() -> Vec { + let mut p = Vec::new(); + p.extend_from_slice(SOLAR_IM_START.as_bytes()); + p.extend_from_slice(b"system"); + p.extend_from_slice(SOLAR_IM_CONTENT.as_bytes()); + p.extend_from_slice(b"## System Prompt\n\nStay precise."); + p.extend_from_slice(SOLAR_IM_END.as_bytes()); + p.push(b'\n'); + p.extend_from_slice(SOLAR_IM_START.as_bytes()); + p.extend_from_slice(b"user"); + p.extend_from_slice(SOLAR_IM_CONTENT.as_bytes()); + p.extend_from_slice(b"Look it up"); + p.extend_from_slice(SOLAR_IM_END.as_bytes()); + p.push(b'\n'); + p.extend_from_slice(SOLAR_IM_START.as_bytes()); + p.extend_from_slice(b"assistant"); + p.extend_from_slice(SOLAR_IM_CONTENT.as_bytes()); + p.extend_from_slice(SOLAR_THINK_START.as_bytes()); + p +} + +/// The wrapper is the only tag missing: the repair's one legal case. +fn repair_dsml_missing_wrapper() -> String { + format!( + "{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">ls{}\n{}\n", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START, DSML_PARAM_END, DSML_INVOKE_END + ) +} + +/// The same in the V4.1 spaced spelling. +fn repair_dsml_spaced() -> String { + format!( + "{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">ls{}\n{}\n", + DSML_TOOL_CALLS_START_SPACED, + DSML_INVOKE_START_SPACED, + DSML_PARAM_START_SPACED, + DSML_PARAM_END_SPACED, + DSML_INVOKE_END_SPACED + ) +} + +/// An unfinished invocation: its arguments may still be missing, so the +/// repair must refuse. +fn repair_dsml_missing_invoke() -> String { + format!( + "{}\n{} name=\"edit\">\n{} name=\"path\" string=\"true\">/tmp/x{}\n", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START, DSML_PARAM_END + ) +} + +/// An unfinished value: a truncated command, so the repair must refuse. +fn repair_dsml_param_open() -> String { + format!( + "{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">ls", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ) +} + +fn repair_solar_trunc() -> String { + format!( + "Need a lookup.{SOLAR_THINK_END}{SOLAR_TOOL_CALLS}lookup\n{SOLAR_TOOL_ARG_START}query{SOLAR_TOOL_ARG_VALUE}solar open2" + ) +} + +pub fn dump_script(name: &str) -> String { + match name { + "dsml-think" => utf(&build_invalid_tool_error_suffix( + ChatFormat::DeepSeek, + ThinkMode::Low, + true, + &dsml_think_prompt(), + "missing invoke name", + )), + "dsml-nothink" => utf(&build_invalid_tool_error_suffix( + ChatFormat::DeepSeek, + ThinkMode::None, + false, + &dsml_think_prompt(), + "invalid tool call", + )), + "solar-think" => utf(&build_invalid_tool_error_suffix( + ChatFormat::SolarOpen2, + ThinkMode::Low, + true, + &solar_think_prompt(), + "missing argument terminator", + )), + "system-dsml" => utf(&rendered_dsml_system_region(&dsml_think_prompt())), + "system-solar" => utf(&rendered_solar_system_region(&solar_think_prompt())), + "repair-dsml" => match try_repair_dsml(repair_dsml_missing_wrapper().as_bytes()) { + Some(v) => utf(&v), + None => "NONE".into(), + }, + "repair-dsml-param-open" => match try_repair_dsml(repair_dsml_param_open().as_bytes()) { + Some(v) => utf(&v), + None => "NONE".into(), + }, + "repair-dsml-invoke-open" => match try_repair_dsml(repair_dsml_missing_invoke().as_bytes()) + { + Some(v) => utf(&v), + None => "NONE".into(), + }, + "repair-dsml-spaced" => match try_repair_dsml(repair_dsml_spaced().as_bytes()) { + Some(v) => utf(&v), + None => "NONE".into(), + }, + "repair-solar" => match try_repair_solar(repair_solar_trunc().as_bytes()) { + Some(v) => utf(&v), + None => "NONE".into(), + }, + "repair-dsml-none" => match try_repair_dsml( + format!( + "{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">ls{}\n{}\n{}", + DSML_TOOL_CALLS_START, + DSML_INVOKE_START, + DSML_PARAM_START, + DSML_PARAM_END, + DSML_INVOKE_END, + DSML_TOOL_CALLS_END + ) + .as_bytes(), + ) { + Some(v) => utf(&v), + None => "NONE".into(), + }, + "decide-unterminated-stop" => { + let t = format!("{}\n{}>", DSML_TOOL_CALLS_START, DSML_INVOKE_START); + format!( + "{}\n", + truncation_name(truncation_outcome( + ModelSyntax::DeepSeek, + ChatFormat::DeepSeek, + true, + true, + true, + false, + "stop", + false, + false, + t.as_bytes(), + &[], + )) + ) + } + "decide-unterminated-length" => { + let t = format!("{}\n{}>", DSML_TOOL_CALLS_START, DSML_INVOKE_START); + format!( + "{}\n", + truncation_name(truncation_outcome( + ModelSyntax::DeepSeek, + ChatFormat::DeepSeek, + true, + true, + true, + false, + "length", + false, + false, + t.as_bytes(), + &[], + )) + ) + } + "decide-parse-retry" => format!( + "{}\n", + parse_failure_should_retry( + ModelSyntax::DeepSeek, + false, + false, + "stop", + true, + true, + true + ) + ), + "decide-parse-motif" => format!( + "{}\n", + parse_failure_should_retry(ModelSyntax::Motif3, false, false, "stop", true, true, true) + ), + _ => "ERROR unknown-script\n".into(), + } +} diff --git a/crates/ds4-server/src/route/env.rs b/crates/ds4-server/src/route/env.rs new file mode 100644 index 0000000000000000000000000000000000000000..f0118b2f7e4f4631d6244cefb8faa5dbd87f92f1 --- /dev/null +++ b/crates/ds4-server/src/route/env.rs @@ -0,0 +1,38 @@ +/// C `DS4_SERVER_CONT_TOOLS_{ANTHROPIC,RESPONSES}` at v0.6.3-dfm. +/// Default ON; only the exact C string `"0"` forces off. +pub const fn cont_tools_from_env( + anthropic: Option<&[u8]>, + responses: Option<&[u8]>, +) -> (bool, bool) { + ( + !matches!(anthropic, Some(b"0")), + !matches!(responses, Some(b"0")), + ) +} + +#[cfg(test)] +mod cont_tools_env_tests { + use super::cont_tools_from_env; + + #[test] + fn cont_tools_default_equals_c_when_env_unset() { + // Given: C getenv() returns NULL for both kill switches + // When: Rust applies the v0.6.3-dfm contract + // Then: both surfaces stay ON + assert_eq!(cont_tools_from_env(None, None), (true, true)); + } + + #[test] + fn cont_tools_forced_off_when_env_is_zero() { + // Given: exact C string "0" on each surface + // When: parse + // Then: that surface is off + assert_eq!(cont_tools_from_env(Some(b"0"), Some(b"0")), (false, false)); + } + + #[test] + fn cont_tools_env_zero_is_per_surface() { + assert_eq!(cont_tools_from_env(Some(b"0"), None), (false, true)); + assert_eq!(cont_tools_from_env(None, Some(b"0")), (true, false)); + } +} diff --git a/crates/ds4-server/src/route/mod.rs b/crates/ds4-server/src/route/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..c5a0d74490e1b6fd06fb4c7814ebb25b71814c0b --- /dev/null +++ b/crates/ds4-server/src/route/mod.rs @@ -0,0 +1,398 @@ +//! `route_decide` + `request_compute_needs` from `ds4_server.c` at v0.6.5-dfm. +//! Pure functions. Do not improve the table. +// allow: SIZE_OK — frozen C route_decide / compute_needs table; do not split. + +mod env; +pub use env::cont_tools_from_env; + +pub const NEED_STREAMING: u32 = 1 << 0; +pub const NEED_PER_ROW_SAMPLING: u32 = 1 << 1; +pub const NEED_THINKING: u32 = 1 << 2; +pub const NEED_STOP_SCAN: u32 = 1 << 3; +pub const NEED_TOOL_SCAN: u32 = 1 << 4; +pub const NEED_TOKEN_IDS: u32 = 1 << 5; +pub const NEED_LIVE_FRONTIER: u32 = 1 << 6; +pub const NEED_CONTINUATION_PUBLISH: u32 = 1 << 7; +pub const NEED_CORRECTIVE_RECOVERY: u32 = 1 << 8; +pub const NEED_DURABLE_RESPONSE: u32 = 1 << 9; +pub const NEED_PREFILL_ONLY: u32 = 1 << 10; +pub const NEED_BANK_FRONTIER: u32 = 1 << 11; +pub const NEED_IMAGE: u32 = 1 << 12; +// Audio and MiMo video stay on the serial session. This bit is outside +// ROUTE_CONT_MASK, so those requests cannot land on the continuous lane. +pub const NEED_AUDIO: u32 = 1 << 13; +pub const NEED_EOS_POLICY: u32 = 1 << 14; + +pub const ROUTE_CONT_MASK: u32 = NEED_STREAMING + | NEED_PER_ROW_SAMPLING + | NEED_THINKING + | NEED_STOP_SCAN + | NEED_TOOL_SCAN + | NEED_TOKEN_IDS + | NEED_IMAGE + | NEED_EOS_POLICY; +pub const ROUTE_STATIC_MASK: u32 = 0; + +pub const LANE_SERIAL: u8 = 0; +pub const LANE_CONTINUOUS: u8 = 1; +pub const LANE_STATIC: u8 = 2; +pub const LANE_NONE: u8 = 0xFF; + +pub const REASON_CONT: u8 = 0; +pub const REASON_STATIC_NO_CONT: u8 = 1; +pub const REASON_STATIC_PROMPT_BOUNDS: u8 = 2; +pub const REASON_COALESCE_OFF: u8 = 3; +pub const REASON_SURFACE: u8 = 4; +pub const REASON_NEED_LIVE_FRONTIER: u8 = 5; +pub const REASON_NEED_CONTINUATION_PUBLISH: u8 = 6; +pub const REASON_NEED_CORRECTIVE_RECOVERY: u8 = 7; +pub const REASON_NEED_DURABLE: u8 = 8; +pub const REASON_NEED_PREFILL_ONLY: u8 = 9; +pub const REASON_TOKEN_IDS_PROJECTION: u8 = 10; +pub const REASON_TOOLS_COMPLETION: u8 = 11; +pub const REASON_CONT_UNAVAILABLE: u8 = 12; +pub const REASON_CONT_BANK: u8 = 13; + +pub const REASON_NAMES: [&str; 14] = [ + "continuous", + "static_no_cont", + "static_prompt_bounds", + "coalesce_off", + "surface", + "need_live_frontier", + "need_continuation_publish", + "need_corrective_recovery", + "need_durable_response", + "need_prefill_only", + "token_ids_projection", + "tools_completion_kind", + "cont_unavailable", + "continuous_bank_continuation", +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WireSurface { + OpenaiChat = 0, + OpenaiCompletion = 1, + Anthropic = 2, + Responses = 3, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Api { + Openai = 0, + Anthropic = 1, + Responses = 2, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReqKind { + Chat = 0, + Completion = 1, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RouteEnv { + pub coalesce: bool, + pub have_cont: bool, + pub cont_anthropic: bool, + pub cont_responses: bool, + pub cont_tools_anthropic: bool, + pub cont_tools_responses: bool, + pub seq_cap: i32, + pub prompt_len: i32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RouteDecision { + pub lane: u8, + pub reason: u8, +} + +#[derive(Debug, Clone, Copy)] +pub struct NeedInput { + pub api: Api, + pub kind: ReqKind, + pub stream: bool, + pub temperature: f32, + pub think: bool, + pub stop_count: u32, + pub has_tools: bool, + pub return_token_ids: bool, + pub responses_requires_live_tool_state: bool, + pub responses_requires_live_reasoning: bool, + pub anthropic_requires_live_tool_state: bool, + pub live_state_bank_owned: bool, + pub max_tokens_set: bool, + pub max_tokens: i32, + pub has_images: bool, +} + +pub fn wire_surface_for(api: Api, kind: ReqKind) -> WireSurface { + match api { + Api::Anthropic => WireSurface::Anthropic, + Api::Responses => WireSurface::Responses, + Api::Openai => match kind { + ReqKind::Completion => WireSurface::OpenaiCompletion, + ReqKind::Chat => WireSurface::OpenaiChat, + }, + } +} + +pub fn compute_needs(r: &NeedInput) -> u32 { + let mut n = 0u32; + if r.stream { + n |= NEED_STREAMING; + } + if r.temperature > 0.0 { + n |= NEED_PER_ROW_SAMPLING; + } + if r.think { + n |= NEED_THINKING; + } + if r.stop_count > 0 { + n |= NEED_STOP_SCAN; + } + if r.has_tools { + n |= NEED_TOOL_SCAN; + } + if r.return_token_ids { + n |= NEED_TOKEN_IDS; + } + if r.has_images { + n |= NEED_IMAGE; + } + if r.responses_requires_live_tool_state + || r.responses_requires_live_reasoning + || r.anthropic_requires_live_tool_state + { + let bank = r.live_state_bank_owned && !r.responses_requires_live_reasoning; + n |= if bank { + NEED_BANK_FRONTIER + } else { + NEED_LIVE_FRONTIER + }; + } + if r.has_tools && r.api != Api::Openai { + n |= NEED_CONTINUATION_PUBLISH; + if !r.stream { + n |= NEED_CORRECTIVE_RECOVERY; + } + } + if r.api == Api::Anthropic && r.max_tokens_set && r.max_tokens <= 0 { + n |= NEED_PREFILL_ONLY; + } + n +} + +pub fn route_decide(needs: u32, surf: WireSurface, env: &RouteEnv) -> RouteDecision { + let mut d = RouteDecision { + lane: LANE_SERIAL, + reason: 0, + }; + if needs & NEED_DURABLE_RESPONSE != 0 { + d.lane = LANE_NONE; + d.reason = REASON_NEED_DURABLE; + return d; + } + let tools_promoted = + (env.cont_anthropic && env.cont_tools_anthropic && surf == WireSurface::Anthropic) + || (env.cont_responses && env.cont_tools_responses && surf == WireSurface::Responses); + if needs & NEED_LIVE_FRONTIER != 0 { + d.reason = REASON_NEED_LIVE_FRONTIER; + return d; + } + if (needs & NEED_BANK_FRONTIER) != 0 && !tools_promoted { + d.reason = REASON_NEED_LIVE_FRONTIER; + return d; + } + if needs & NEED_CONTINUATION_PUBLISH != 0 && !((needs & NEED_STREAMING) != 0 && tools_promoted) + { + d.reason = REASON_NEED_CONTINUATION_PUBLISH; + return d; + } + if needs & NEED_CORRECTIVE_RECOVERY != 0 { + d.reason = REASON_NEED_CORRECTIVE_RECOVERY; + return d; + } + if needs & NEED_PREFILL_ONLY != 0 { + d.reason = REASON_NEED_PREFILL_ONLY; + return d; + } + if (needs & NEED_TOKEN_IDS) != 0 + && !((needs & NEED_STREAMING) != 0 && surf == WireSurface::OpenaiChat) + { + d.reason = REASON_TOKEN_IDS_PROJECTION; + return d; + } + if (needs & NEED_TOOL_SCAN) != 0 && surf == WireSurface::OpenaiCompletion { + d.reason = REASON_TOOLS_COMPLETION; + return d; + } + let mut cont_mask = ROUTE_CONT_MASK; + if tools_promoted { + if needs & NEED_STREAMING != 0 { + cont_mask |= NEED_CONTINUATION_PUBLISH; + } + cont_mask |= NEED_BANK_FRONTIER; + } + if needs & !cont_mask != 0 { + d.reason = REASON_CONT_UNAVAILABLE; + return d; + } + let cont_promoted = (env.cont_anthropic && surf == WireSurface::Anthropic) + || (env.cont_responses && surf == WireSurface::Responses); + if surf != WireSurface::OpenaiChat && surf != WireSurface::OpenaiCompletion && !cont_promoted { + d.reason = REASON_SURFACE; + return d; + } + if !env.coalesce { + d.reason = REASON_COALESCE_OFF; + return d; + } + if env.have_cont && env.prompt_len > 0 && env.prompt_len <= env.seq_cap { + d.lane = LANE_CONTINUOUS; + d.reason = if needs & NEED_BANK_FRONTIER != 0 { + REASON_CONT_BANK + } else { + REASON_CONT + }; + return d; + } + if needs & NEED_IMAGE != 0 { + d.lane = LANE_NONE; + d.reason = REASON_CONT_UNAVAILABLE; + return d; + } + if (needs & !ROUTE_STATIC_MASK) == 0 + && (surf == WireSurface::OpenaiChat + || surf == WireSurface::OpenaiCompletion + || cont_promoted) + { + d.lane = LANE_STATIC; + d.reason = if env.have_cont { + REASON_STATIC_PROMPT_BOUNDS + } else { + REASON_STATIC_NO_CONT + }; + return d; + } + d.reason = REASON_CONT_UNAVAILABLE; + d +} + +/// Inc 0b three-state budget. Parsers preload the server default into +/// `max_tokens` when the client omitted the field. +pub fn decode_budget(max_tokens_set: bool, max_tokens: i32, server_default: i32) -> i32 { + if max_tokens_set && max_tokens <= 0 { + return 0; + } + if max_tokens > 0 { + max_tokens + } else { + server_default + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ThinkMode { + None = 0, + Low = 1, + High = 2, + Max = 3, +} + +pub fn think_mode_enabled(mode: ThinkMode) -> bool { + mode != ThinkMode::None +} + +pub fn think_mode_from_enabled(enabled: bool, effort: ThinkMode) -> ThinkMode { + if !enabled || effort == ThinkMode::None { + ThinkMode::None + } else { + effort + } +} + +#[cfg(test)] +mod eos_policy_tests { + use super::*; + + #[test] + fn eos_policy_never_uses_static_lane() { + let mut env = RouteEnv { + coalesce: true, + have_cont: true, + cont_anthropic: false, + cont_responses: false, + cont_tools_anthropic: false, + cont_tools_responses: false, + seq_cap: 1024, + prompt_len: 10, + }; + let needs = NEED_EOS_POLICY; + + assert_eq!( + route_decide(needs, WireSurface::OpenaiChat, &env).lane, + LANE_CONTINUOUS + ); + env.have_cont = false; + assert_eq!( + route_decide(needs, WireSurface::OpenaiChat, &env).lane, + LANE_SERIAL + ); + } +} + +/// The shared wire vocabulary (low/medium/minimal are their own level here: +/// other families' templates consume Low). The engine's own collapse -- +/// everything non-zero below max is HIGH (server_msgs.c:194-215) -- is applied +/// for the engine's model set by `engine_think_mode` in parse.rs, so a ds41 +/// request nets exactly the engine's mapping without changing what the other +/// families may send. "off" is a port extension (the engine rejects unknown +/// names, "off" included); recorded divergence, not reachable for ds41's net +/// mode except as none-vs-400. +pub fn parse_reasoning_effort_name(s: &str) -> Option { + match s { + "max" => Some(ThinkMode::Max), + "high" | "xhigh" => Some(ThinkMode::High), + "low" | "medium" | "minimal" => Some(ThinkMode::Low), + "none" | "off" => Some(ThinkMode::None), + _ => None, + } +} + +/// The numeric budget a mode writes in the V4.1 head: the oracle's +/// REASONING_EFFORT_MAPPINGS (encoding.py:448-453), low 50, high 75, max 100; +/// a mode with no thinking writes no line at all (encoding.py:472-473). +pub fn mode_budget(mode: ThinkMode) -> Option { + match mode { + ThinkMode::Low => Some(50), + ThinkMode::High => Some(75), + ThinkMode::Max => Some(100), + ThinkMode::None => None, + } +} + +/// The reasoning effort one request asked for: the engine-facing mode, plus the +/// explicit budget when the client gave the oracle's numeric form, which the +/// head renders verbatim (encoding.py:456-473). Kept apart from `think_mode` +/// on purpose: the budget moves the prompt bytes only, never the engine's own +/// collapsed mode, so the reasoning cap and the sampling switches stay put. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RequestedEffort { + pub mode: ThinkMode, + /// `Some` only for a numeric request; a name leaves the number to the mode. + pub budget: Option, +} + +impl RequestedEffort { + pub fn named(mode: ThinkMode) -> Self { + Self { mode, budget: None } + } + + /// The number the V4.1 head writes for this request. + pub fn head_budget(self) -> Option { + self.budget.or(mode_budget(self.mode)) + } +} diff --git a/crates/ds4-server/src/serve.rs b/crates/ds4-server/src/serve.rs new file mode 100644 index 0000000000000000000000000000000000000000..a1bfbb45cfe9f9388ae65a4b7acd644a6508bef4 --- /dev/null +++ b/crates/ds4-server/src/serve.rs @@ -0,0 +1,4883 @@ +//! Blocking accept loop: HTTP door + parsers + admission + projectors + +//! serial decode (including finalize tool_calls) when a `DecodeIo` is supplied. + +use std::ffi::OsStr; +use std::io::{Error, ErrorKind, Write}; +use std::net::{TcpListener, TcpStream}; +#[cfg(unix)] +use std::os::unix::ffi::OsStrExt; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::sync::{Arc, Condvar, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use ds4_sys::{libc_atof, libc_atoi, libc_strtoull10}; + +use crate::admit::{ + enqueue, enqueue_release, enqueue_shed_error, next_job_id, preparse_shed, queue_unlink_head, + AdmitState, EnqVerdict, SERVER_SHUTTING_DOWN, SHED_CLIENTS, SHED_CONT_HOLD, SHED_QUEUE_AGE, + SHED_SLOW_READER, +}; +use crate::cont::{monotonic_now, place_bank_continuation, ContOwner, ContPin, ContRegistry}; +use crate::error::{http_response_bytes, wire_http_error_bytes, wire_stream_error_bytes}; +use crate::generate::{ + generate_terminal_prepared, prepare_serial_prompt, DecodeIo, GenerateError, GenerateOutcome, + NativeGraphFit, +}; +use crate::http::{ + chunked_enabled, parse_surface_for_path, read_http_request, shed_surface_for_path, +}; +#[cfg(feature = "native")] +use crate::metrics::MemCell; +use crate::metrics::{ + gov_modes_from_env, render_metrics, render_stats_json_plan, RouteMetrics, RuntimeMetrics, +}; +use crate::models::{model_id_known, model_one_json, models_list_json}; +use crate::parse::{parse_request, EosPolicy, ParseEnv}; +use crate::render::{syntax_for_model_id, ModelSyntax}; +use crate::route::{ + decode_budget, route_decide, Api, RouteDecision, RouteEnv, ThinkMode, WireSurface, + LANE_CONTINUOUS, LANE_SERIAL, LANE_STATIC, NEED_BANK_FRONTIER, NEED_IMAGE, + REASON_CONT_UNAVAILABLE, +}; +use crate::serve_cont::{cont_prompt_tokens, ContExec, ContPreparedPrompt}; +use crate::serve_cont_roll::RejectReason; +use crate::serve_serial_fit::{ + serial_fit_bounds, serial_live_preserve_msg, serial_session_fit_plan, SerialFitPlan, + SerialFrameKind, +}; +use crate::serve_serial_reclaim::{ + resolve_serial_fit, serial_capacity_refuse_msg, serial_fit_from_native, serial_reclaim_gate, + serial_reclaim_want, AvailBytes, MemFloor, SerialFitQuote, SerialReclaimOutcome, +}; +use crate::serve_static::{ + run_static_routed, write_static_completion, DetachedStatic, StaticFinish, StaticJob, StaticRow, + StaticSettle, STATIC_WIDTH_ERR, +}; + +#[path = "serve_owner_cont.rs"] +mod owner_cont; +#[path = "serve_owner_static.rs"] +mod owner_static; +use crate::stream::{unix_now, STREAM_HEARTBEAT_INTERVAL}; + +#[derive(Debug, Clone)] +pub struct ServerConfig { + pub listen_host: String, + pub listen_port: u16, + pub model_id: String, + pub model_name: String, + /// The engine's request-omitted parse defaults for its own model set + /// (unit H): thinking off unless asked, effort HIGH, min_p 0.0, and the + /// effort-name collapse -- see ParseEnv::engine_defaults. The bin sets + /// this only when the model is V4.1 (the v41 route branch); every other + /// family keeps the port's wire vocabulary and defaults. + pub engine_defaults: bool, + /// The served model's template writes max for an absent level (GLM-5.3 + /// chat_template.jinja:2, Beam's reasoning_effort 512): a request that + /// names no level gets it. The bin sets it from the opened model. + pub max_effort_default: bool, + pub ctx: i32, + pub default_tokens: i32, + pub eos_policy: EosPolicy, + pub cors: bool, + pub codex_models_json: Option, + pub max_queue: i32, + pub max_queue_bytes: u64, + pub max_clients: i32, + pub max_queue_age_s: f64, + pub out_agg_cap_bytes: u64, + pub out_agg_evict_min_bytes: u64, + pub client_sndbuf: i32, + pub disconnect_abort: bool, + pub continuous: bool, + pub have_engine: bool, + pub stop_requested: Option bool>, + pub mem_floor_gb: u64, + /// C `DS4_SERVER_SERIAL_RIGHTSIZE` (v0.5.2 inc1): false only for the + /// exact value "0", which restores the fail-at-full-`-c` behavior. + pub serial_rightsize: bool, + pub serving_plan: Option, + pub(crate) serial_fit: Option, +} + +pub(crate) fn env_i32_bound(name: &str, default: i32) -> i32 { + let value = std::env::var_os(name); + parse_i32_bound(value.as_deref(), default) +} + +fn env_u64_bound(name: &str, default: u64) -> u64 { + let value = std::env::var_os(name); + parse_u64_bound(value.as_deref(), default) +} + +fn env_f64_bound(name: &str, default: f64) -> f64 { + let value = std::env::var_os(name); + parse_f64_bound(value.as_deref(), default) +} + +fn os_str_bytes(value: &OsStr) -> &[u8] { + #[cfg(unix)] + { + value.as_bytes() + } + #[cfg(not(unix))] + { + value.as_encoded_bytes() + } +} + +fn parse_i32_bound(value: Option<&OsStr>, default: i32) -> i32 { + match value { + Some(value) => libc_atoi(os_str_bytes(value)).max(0), + None => default, + } +} + +fn parse_u64_bound(value: Option<&OsStr>, default: u64) -> u64 { + match value { + Some(value) => libc_strtoull10(os_str_bytes(value)), + None => default, + } +} + +fn parse_f64_bound(value: Option<&OsStr>, default: f64) -> f64 { + match value { + Some(value) => { + let value = libc_atof(os_str_bytes(value)); + if value < 0.0 { + 0.0 + } else { + value + } + } + None => default, + } +} + +fn parse_default_on(value: Option<&OsStr>) -> bool { + value.map(os_str_bytes) != Some(b"0") +} + +fn process_cont_tools() -> (bool, bool) { + crate::route::cont_tools_from_env( + std::env::var_os("DS4_SERVER_CONT_TOOLS_ANTHROPIC") + .as_deref() + .map(os_str_bytes), + std::env::var_os("DS4_SERVER_CONT_TOOLS_RESPONSES") + .as_deref() + .map(os_str_bytes), + ) +} + +impl Default for ServerConfig { + fn default() -> Self { + Self { + listen_host: "127.0.0.1".into(), + listen_port: 8000, + model_id: "ds4".into(), + model_name: "ds4".into(), + engine_defaults: false, + max_effort_default: false, + ctx: 8192, + default_tokens: 393216, + eos_policy: EosPolicy::Default, + cors: false, + codex_models_json: None, + max_queue: env_i32_bound("DS4_SERVER_MAX_QUEUE", 256), + max_queue_bytes: env_u64_bound("DS4_SERVER_MAX_QUEUE_BYTES", 256 * 1024 * 1024), + max_clients: env_i32_bound("DS4_SERVER_MAX_CLIENTS", 256), + max_queue_age_s: env_f64_bound("DS4_SERVER_MAX_QUEUE_AGE_S", 600.0), + out_agg_cap_bytes: env_u64_bound("DS4_SERVER_OUT_AGG_CAP", JOB_SINK_AGG_CAP_BYTES), + out_agg_evict_min_bytes: env_u64_bound( + "DS4_SERVER_OUT_AGG_EVICT_MIN", + JOB_SINK_AGG_EVICT_MIN_BYTES, + ), + client_sndbuf: env_i32_bound("DS4_SERVER_CLIENT_SNDBUF", 0), + disconnect_abort: parse_default_on( + std::env::var_os("DS4_SERVER_DISCONNECT_ABORT").as_deref(), + ), + continuous: parse_default_on(std::env::var_os("DS4_SERVER_CONTINUOUS").as_deref()), + have_engine: false, + stop_requested: None, + mem_floor_gb: mem_floor_gb_from_env(), + serial_rightsize: parse_default_on( + std::env::var_os("DS4_SERVER_SERIAL_RIGHTSIZE").as_deref(), + ), + serving_plan: None, + serial_fit: None, + } + } +} + +fn mem_floor_gb_from_env() -> u64 { + let env = std::env::var_os("DS4_MEM_FLOOR_GB"); + MemFloor::from_env_gb(env.as_deref().map(os_str_bytes)).gb() +} + +impl ServerConfig { + /// Take the resolved plan into the captured config. The config reads its + /// own `DS4_SERVER_*` environment before the plan exists, so publishing + /// env alone leaves a legacy `DS4_SERVER_CONTINUOUS=0` routing every + /// request away from a lane the plan asked for and allocated. Serial-only + /// speculation must also retract an inherited continuous routing flag. + pub fn adopt_plan(&mut self, plan: &ds4_core::ResolvedPlan) { + self.mem_floor_gb = plan.effective.mem_floor_gb; + if plan.uses_serial_mtp() { + self.continuous = false; + } else if plan.wants_bank_lane() { + self.continuous = true; + } + self.serving_plan = Some(plan.clone()); + } + + /// Integration-test fixture. Built inside the crate so `serial_fit` stays + /// `pub(crate)` and external tests never need functional-update syntax. + pub fn test_cfg() -> Self { + let mut cfg = Self::default(); + cfg.have_engine = true; + cfg + } + + pub fn apply_mem_floor_gb(&mut self, raw: &str) { + self.mem_floor_gb = MemFloor::from_cli_or_env(Some(raw.as_bytes()), None).gb(); + } +} + +#[derive(Debug, Default)] +pub struct ServerInner { + pub admit: AdmitState, + pub metrics: RouteMetrics, + pub runtime: RuntimeMetrics, + pub creg: ContRegistry, + pub boot_stamp: u64, + pub have_engine: bool, + pub serving_plan: Option, + pub last_request: Option, + disconnect_abort: bool, + out_agg_cap_bytes: u64, + out_agg_evict_min_bytes: u64, +} + +impl ServerInner { + pub fn from_cfg(cfg: &ServerConfig) -> Self { + let mut s = Self::default(); + s.admit.max_queue = cfg.max_queue; + s.admit.max_queue_bytes = cfg.max_queue_bytes; + s.admit.max_clients = cfg.max_clients; + s.runtime.memgov.gov_modes = gov_modes_from_env(); + s.boot_stamp = unix_now() as u64; + s.have_engine = cfg.have_engine; + s.serving_plan = cfg.serving_plan.clone(); + s.disconnect_abort = cfg.disconnect_abort; + s.out_agg_cap_bytes = cfg.out_agg_cap_bytes; + s.out_agg_evict_min_bytes = cfg.out_agg_evict_min_bytes; + s + } + + pub fn render_runtime(&self, now: u64) -> RuntimeMetrics { + let mut rt = self.runtime.clone(); + rt.uptime_seconds = if self.boot_stamp != 0 && now >= self.boot_stamp { + now - self.boot_stamp + } else { + 0 + }; + rt.creg_records_live = self.creg.n_live() as u64; + #[cfg(feature = "native")] + if self.have_engine { + overlay_live_census(&mut rt.memgov); + } + #[cfg(any(feature = "native", test))] + if self.have_engine { + let spec = ds4_core::snapshot_spec(); + rt.spec_drafts = spec.drafts; + rt.spec_hits = spec.hits; + rt.spec_quench = spec.quench; + } + rt + } + + fn record_generation(&mut self, outcome: &GenerateOutcome) { + let t = outcome.timings; + self.record_tokens( + t.prefill_tokens, + t.prefill_cached, + t.decode_tokens, + t.decode_steps, + ); + let lane = outcome.lane.unwrap_or(if outcome.bank.is_some() { + "continuous" + } else { + "serial" + }); + self.last_request = Some(ds4_core::RequestTrace { + effective_lane: lane, + reuse_kind: outcome.reuse, + reuse_miss: outcome.reuse_miss, + speculation_active: outcome.speculation_active, + fallback_reason: outcome.fallback_reason.clone(), + }); + } + + fn record_tokens(&mut self, computed: i32, cached: i32, decoded: i32, steps: i32) { + self.runtime.tokens_prefilled_computed = self + .runtime + .tokens_prefilled_computed + .saturating_add(u64::try_from(computed).unwrap_or(0)); + self.runtime.tokens_prefilled_cached = self + .runtime + .tokens_prefilled_cached + .saturating_add(u64::try_from(cached).unwrap_or(0)); + self.runtime.tokens_decoded = self + .runtime + .tokens_decoded + .saturating_add(u64::try_from(decoded).unwrap_or(0)); + self.runtime.decode_steps = self + .runtime + .decode_steps + .saturating_add(u64::try_from(steps).unwrap_or(0)); + } +} + +#[cfg(feature = "native")] +fn overlay_live_census(g: &mut crate::metrics::MemgovSnap) { + let snap = ds4_core::snapshot_mem(); + g.census_supported = snap.census.supported; + g.census_faults = snap.census.faults; + g.census_epoch = snap.census.epoch; + g.torn_fallbacks = snap.census.torn_fallbacks; + for c in 0..17 { + for d in 0..2 { + let src = snap.census.cells[c][d]; + g.cells[c][d] = MemCell { + requested: src.requested, + committed: src.committed, + freed_requested: src.freed_requested, + freed_committed: src.freed_committed, + }; + } + } + g.obs_status = snap.observe.status.clamp(0, 2) as u8; + g.obs_source = snap.observe.source.clamp(0, 2) as u8; + g.obs_free = snap.observe.free_bytes; + g.obs_total = snap.observe.total_bytes; + g.obs_cuda_free = snap.observe.cuda_free_bytes; + g.obs_meminfo = snap.observe.meminfo_avail_bytes; + g.substrate_outstanding = snap.substrate_outstanding; + g.emit_substrate = true; +} + +fn write_all(stream: &mut TcpStream, bytes: &[u8]) { + let _ = stream.write_all(bytes); +} + +const JOB_SINK_CAP_BYTES: u64 = 16 * 1024 * 1024; +const JOB_SINK_AGG_CAP_BYTES: u64 = 64 * 1024 * 1024; +const JOB_SINK_AGG_EVICT_MIN_BYTES: u64 = 256 * 1024; +const CLIENT_POLL: Duration = Duration::from_millis(50); +const CLIENT_IO_TIMEOUT: Duration = Duration::from_secs(10); +const CLIENT_SEND_STALL_TIMEOUT: Duration = Duration::from_secs(2); + +#[derive(Default)] +struct SinkBuffer { + bytes: Vec, + gone: bool, + slow: bool, + closed: bool, + started: bool, +} + +struct SinkState { + inner: Arc>, + probe: Option, + buffer: Mutex, + ready: Condvar, +} + +impl SinkState { + fn lock(&self) -> std::sync::MutexGuard<'_, SinkBuffer> { + self.buffer.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn gone(&self) -> bool { + self.lock().gone + } + + fn slow(&self) -> bool { + self.lock().slow + } + + fn socket_disconnected(&self) -> bool { + self.probe.as_ref().is_some_and(client_disconnected) + } + + fn observe_disconnect(&self) -> bool { + if !self.socket_disconnected() { + return false; + } + self.cancel(false); + true + } + + #[cfg(test)] + fn backlog_bytes(&self) -> u64 { + self.lock().bytes.len() as u64 + } + + fn take(&self) -> Vec { + let mut buffer = self.lock(); + let bytes = std::mem::take(&mut buffer.bytes); + let mut g = lock_inner(&self.inner); + g.runtime.out_backlog_bytes = g + .runtime + .out_backlog_bytes + .saturating_sub(bytes.len() as u64); + bytes + } + + fn cancel(&self, slow: bool) { + let mut buffer = self.lock(); + buffer.gone = true; + buffer.slow |= slow; + let bytes = std::mem::take(&mut buffer.bytes); + let mut g = lock_inner(&self.inner); + g.runtime.out_backlog_bytes = g + .runtime + .out_backlog_bytes + .saturating_sub(bytes.len() as u64); + drop(g); + drop(buffer); + drop(bytes); + self.ready.notify_all(); + } +} + +struct JobSink { + state: Arc, + close_on_drop: bool, +} + +impl Clone for JobSink { + fn clone(&self) -> Self { + Self { + state: Arc::clone(&self.state), + close_on_drop: false, + } + } +} + +impl Write for JobSink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + let mut buffer = self.state.lock(); + if buffer.gone || buffer.closed { + return Err(Error::new(ErrorKind::BrokenPipe, "client disconnected")); + } + if bytes.is_empty() { + return Ok(0); + } + buffer.started = true; + let mut g = lock_inner(&self.state.inner); + let job_next = buffer.bytes.len().saturating_add(bytes.len()) as u64; + let aggregate_next = g + .runtime + .out_backlog_bytes + .saturating_add(bytes.len() as u64); + if job_next > JOB_SINK_CAP_BYTES + || (g.out_agg_cap_bytes > 0 + && aggregate_next > g.out_agg_cap_bytes + && job_next >= g.out_agg_evict_min_bytes) + { + let pending = std::mem::take(&mut buffer.bytes); + g.runtime.out_backlog_bytes = g + .runtime + .out_backlog_bytes + .saturating_sub(pending.len() as u64); + buffer.slow = true; + buffer.gone = true; + drop(g); + drop(buffer); + drop(pending); + self.state.ready.notify_all(); + return Err(Error::new(ErrorKind::WouldBlock, "client output sink full")); + } + buffer.bytes.extend_from_slice(bytes); + g.runtime.out_backlog_bytes = aggregate_next; + drop(g); + drop(buffer); + self.state.ready.notify_one(); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + let abort = lock_inner(&self.state.inner).disconnect_abort; + if self.state.gone() || (abort && self.state.observe_disconnect()) { + Err(Error::new(ErrorKind::BrokenPipe, "client disconnected")) + } else { + Ok(()) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TerminalCommit { + Committed, + GoneBeforeStart, + FailedAfterStart, + Full, +} + +trait TerminalSink: Write { + fn response_started(&self) -> bool; + + fn commit_tool_terminal( + &mut self, + inner: &Mutex, + api: Api, + generated: &GenerateOutcome, + terminal: Vec, + ) -> TerminalCommit; +} + +fn publish_tool_turn(inner: &Mutex, api: Api, generated: &GenerateOutcome) { + lock_inner(inner).creg.publish_serial( + api, + &generated.tool_ids, + generated.generation, + generated.frontier, + monotonic_now(), + ); +} + +fn reserve_terminal(inner: &Mutex, bytes: u64) -> bool { + let mut g = lock_inner(inner); + if bytes > JOB_SINK_CAP_BYTES + || (g.out_agg_cap_bytes > 0 + && g.runtime.out_backlog_bytes.saturating_add(bytes) > g.out_agg_cap_bytes) + { + return false; + } + g.runtime.out_backlog_bytes = g.runtime.out_backlog_bytes.saturating_add(bytes); + true +} + +fn release_terminal(inner: &Mutex, bytes: u64) { + let mut g = lock_inner(inner); + g.runtime.out_backlog_bytes = g.runtime.out_backlog_bytes.saturating_sub(bytes); +} + +enum DirectTerminalIo<'a> { + Disconnected, + Send(&'a [u8]), +} + +struct DirectSink<'a> { + stream: &'a mut TcpStream, + started: bool, +} + +impl Write for DirectSink<'_> { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if !bytes.is_empty() { + self.started = true; + } + send_all_nonblocking(self.stream, bytes)?; + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.stream.flush() + } +} + +fn direct_terminal_commit( + inner: &Mutex, + api: Api, + generated: &GenerateOutcome, + terminal: &[u8], + mut io: impl FnMut(DirectTerminalIo<'_>) -> std::io::Result, +) -> TerminalCommit { + let bytes = terminal.len() as u64; + if !reserve_terminal(inner, bytes) { + return TerminalCommit::Full; + } + publish_tool_turn(inner, api, generated); + if !matches!(io(DirectTerminalIo::Disconnected), Ok(false)) { + lock_inner(inner).creg.demote_serial(); + release_terminal(inner, bytes); + return TerminalCommit::GoneBeforeStart; + } + let sent = io(DirectTerminalIo::Send(terminal)).is_ok(); + release_terminal(inner, bytes); + if sent { + TerminalCommit::Committed + } else { + TerminalCommit::FailedAfterStart + } +} + +impl TerminalSink for DirectSink<'_> { + fn response_started(&self) -> bool { + self.started + } + + fn commit_tool_terminal( + &mut self, + inner: &Mutex, + api: Api, + generated: &GenerateOutcome, + terminal: Vec, + ) -> TerminalCommit { + direct_terminal_commit(inner, api, generated, &terminal, |action| match action { + DirectTerminalIo::Disconnected => Ok(client_disconnected(self.stream)), + DirectTerminalIo::Send(bytes) => { + self.started = true; + send_all_nonblocking(self.stream, bytes).map(|_| false) + } + }) + } +} + +impl TerminalSink for JobSink { + fn response_started(&self) -> bool { + self.state.lock().started + } + + fn commit_tool_terminal( + &mut self, + inner: &Mutex, + api: Api, + generated: &GenerateOutcome, + terminal: Vec, + ) -> TerminalCommit { + let n = terminal.len() as u64; + let mut buffer = self.state.lock(); + let mut g = lock_inner(inner); + let job_next = buffer.bytes.len().saturating_add(terminal.len()) as u64; + let aggregate_next = g.runtime.out_backlog_bytes.saturating_add(n); + if job_next > JOB_SINK_CAP_BYTES + || (g.out_agg_cap_bytes > 0 && aggregate_next > g.out_agg_cap_bytes) + || buffer.bytes.try_reserve(terminal.len()).is_err() + { + let pending = std::mem::take(&mut buffer.bytes); + g.runtime.out_backlog_bytes = g + .runtime + .out_backlog_bytes + .saturating_sub(pending.len() as u64); + buffer.slow = true; + buffer.gone = true; + drop(g); + drop(buffer); + drop(pending); + self.state.ready.notify_all(); + return TerminalCommit::Full; + } + + g.runtime.out_backlog_bytes = aggregate_next; + g.creg.publish_serial( + api, + &generated.tool_ids, + generated.generation, + generated.frontier, + monotonic_now(), + ); + if buffer.gone || self.state.socket_disconnected() { + g.creg.demote_serial(); + let pending = std::mem::take(&mut buffer.bytes); + g.runtime.out_backlog_bytes = g + .runtime + .out_backlog_bytes + .saturating_sub(n.saturating_add(pending.len() as u64)); + buffer.gone = true; + drop(g); + drop(buffer); + drop(pending); + self.state.ready.notify_all(); + return TerminalCommit::GoneBeforeStart; + } + buffer.started = true; + buffer.bytes.extend_from_slice(&terminal); + drop(g); + drop(buffer); + self.state.ready.notify_one(); + TerminalCommit::Committed + } +} + +impl Drop for JobSink { + fn drop(&mut self) { + if self.close_on_drop { + self.state.lock().closed = true; + self.state.ready.notify_all(); + } + } +} + +#[cfg(test)] +fn job_sink(inner: Arc>) -> (JobSink, Arc) { + job_sink_with_probe(inner, None) +} + +fn job_sink_with_probe( + inner: Arc>, + probe: Option, +) -> (JobSink, Arc) { + let state = Arc::new(SinkState { + inner, + probe, + buffer: Mutex::new(SinkBuffer::default()), + ready: Condvar::new(), + }); + ( + JobSink { + state: Arc::clone(&state), + close_on_drop: true, + }, + state, + ) +} + +fn api_for_surface(surf: WireSurface) -> Api { + match surf { + WireSurface::Anthropic => Api::Anthropic, + WireSurface::Responses => Api::Responses, + _ => Api::Openai, + } +} + +fn continuation_conflict_msg(api: Api) -> &'static str { + match api { + Api::Responses => { + "Responses continuation state is not available; retry by replaying the full input history" + } + _ => { + "Anthropic continuation state is not available; retry by replaying the full messages history" + } + } +} + +fn refuse_bank_continuation( + cfg: &ServerConfig, + inner: &Mutex, + job: &mut PreparedJob, + exec: &dyn ContExec, + out: &mut W, +) -> Option { + if resolve_bank_continuation(inner, job, exec) { + return None; + } + Some(write_continuation_conflict(cfg, job, out)) +} + +fn resolve_bank_continuation( + inner: &Mutex, + job: &mut PreparedJob, + exec: &dyn ContExec, +) -> bool { + resolve_bank_continuation_with(inner, job, |bank| exec.bank_live(bank)) +} + +fn resolve_bank_continuation_with( + inner: &Mutex, + job: &mut PreparedJob, + bank_live: impl FnOnce(i32) -> Option<(u64, i32)>, +) -> bool { + if job.parsed.needs & NEED_BANK_FRONTIER == 0 { + return true; + } + let now = monotonic_now(); + let claim = lock_inner(inner) + .creg + .bank_claim(job.parsed.api, &job.parsed.live_call_ids, now); + let live = claim.and_then(|(bank, _, _)| bank_live(bank)); + match place_bank_continuation(claim, live) { + Ok(bank) => { + job.parsed.directed_bank = Some(bank); + true + } + Err(_) => false, + } +} + +fn write_continuation_conflict( + cfg: &ServerConfig, + job: &PreparedJob, + out: &mut W, +) -> Settlement { + let ok = out + .write_all(&wire_http_error_bytes( + job.surface, + 409, + continuation_conflict_msg(job.parsed.api), + cfg.cors, + None, + )) + .is_ok(); + if ok { + Settlement::COMPLETED + } else { + Settlement::CANCELED + } +} + +fn publish_continuous_tool_turn(inner: &Mutex, api: Api, generated: &GenerateOutcome) { + if !matches!(api, Api::Anthropic | Api::Responses) { + return; + } + if generated.tool_ids.is_empty() || generated.finish == "error" || generated.finish == "length" + { + return; + } + let Some(bank) = generated.bank.filter(|bank| *bank >= 0) else { + return; + }; + lock_inner(inner).creg.publish_bank( + api, + &generated.tool_ids, + bank, + generated.generation, + generated.frontier, + monotonic_now(), + ); +} + +fn settle_bank_continuation( + cfg: &ServerConfig, + job: &PreparedJob, + result: Result, + out: &mut W, +) -> Settlement { + match result { + Err(GenerateError::Unsupported(_)) => write_continuation_conflict(cfg, job, out), + other => settle_generation_result(cfg, job, other, out), + } +} + +struct PreparedJob { + parsed: crate::parse::ParsedRequest, + cont_prompt: Option, + surface: WireSurface, + body_bytes: u64, + arrived_at: Instant, +} + +fn prepare_cont_prompt<'a>( + job: &'a mut PreparedJob, + exec: &dyn ContExec, +) -> Option<&'a ContPreparedPrompt> { + if job.cont_prompt.is_none() { + let (prompt, tokens) = cont_prompt_tokens(exec, &mut job.parsed).ok()?; + job.cont_prompt = Some(ContPreparedPrompt { prompt, tokens }); + } + job.cont_prompt.as_ref() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Settle { + Completed, + Failed, + Canceled, + Shed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Settlement { + outcome: Settle, + shed: Option, +} + +impl Settlement { + const COMPLETED: Self = Self { + outcome: Settle::Completed, + shed: None, + }; + const FAILED: Self = Self { + outcome: Settle::Failed, + shed: None, + }; + const CANCELED: Self = Self { + outcome: Settle::Canceled, + shed: None, + }; + + fn shed(reason: u8) -> Self { + Self { + outcome: Settle::Shed, + shed: Some(reason), + } + } + + fn transport_gone(mut self) -> Self { + if self.outcome == Settle::Completed { + self.outcome = Settle::Canceled; + } + self + } + + fn slow_reader(mut self) -> Self { + if self.shed.is_none() { + self.shed = Some(SHED_SLOW_READER); + } + if !matches!(self.outcome, Settle::Failed | Settle::Shed) { + self.outcome = Settle::Canceled; + } + self + } +} + +fn acquire_continuation_pin( + inner: &mut ServerInner, + parsed: &mut crate::parse::ParsedRequest, + now: f64, +) -> Option { + let pin = inner + .creg + .pin_live(parsed.api, parsed.live_call_ids.first()?, now)?; + if inner.creg.pin_owner(pin) == Some(ContOwner::BatchBank) { + parsed.live_state_bank_owned = true; + parsed.finish_needs(); + } + Some(pin) +} + +struct BorrowedPin<'a> { + inner: &'a Mutex, + pin: Option, +} + +impl Drop for BorrowedPin<'_> { + fn drop(&mut self) { + if let Some(pin) = self.pin.take() { + lock_inner(self.inner).creg.unpin(pin); + } + } +} + +struct JobLease { + inner: Arc>, + body_bytes: u64, + queued: bool, + settlement: Settlement, + pin: Option, +} + +impl JobLease { + fn new(inner: Arc>, body_bytes: u64, pin: Option) -> Self { + Self { + inner, + body_bytes, + queued: true, + settlement: Settlement::CANCELED, + pin, + } + } + + fn start(&mut self) { + if !self.queued { + return; + } + queue_unlink_head(&mut lock_inner(&self.inner).admit); + self.queued = false; + } +} + +impl Drop for JobLease { + fn drop(&mut self) { + let mut g = lock_inner(&self.inner); + if self.queued { + queue_unlink_head(&mut g.admit); + } + if let Some(pin) = self.pin.take() { + g.creg.unpin(pin); + } + g.admit.inflight_body_bytes = g.admit.inflight_body_bytes.saturating_sub(self.body_bytes); + g.runtime.requests_inflight = g.runtime.requests_inflight.saturating_sub(1); + if let Some(reason) = self.settlement.shed { + g.metrics.record_shed(reason); + } + match self.settlement.outcome { + Settle::Completed => g.runtime.requests_completed += 1, + Settle::Failed => g.runtime.requests_failed += 1, + Settle::Canceled => g.runtime.requests_canceled += 1, + Settle::Shed => {} + } + } +} + +struct ClientLease { + inner: Arc>, +} + +impl ClientLease { + fn try_new(inner: Arc>) -> Option { + let mut g = lock_inner(&inner); + if g.admit.max_clients > 0 && g.admit.clients >= g.admit.max_clients { + g.metrics.record_shed(SHED_CLIENTS); + return None; + } + g.admit.clients += 1; + drop(g); + Some(Self { inner }) + } +} + +impl Drop for ClientLease { + fn drop(&mut self) { + let mut g = lock_inner(&self.inner); + g.admit.clients = g.admit.clients.saturating_sub(1); + } +} + +struct OwnerJob { + prepared: PreparedJob, + sink: JobSink, + done: Sender, + lease: JobLease, +} + +struct JobDrain { + done: Receiver, + state: Arc, + streaming: bool, +} + +impl Drop for JobDrain { + fn drop(&mut self) { + self.state.cancel(false); + } +} + +#[cfg(test)] +fn owner_job(prepared: PreparedJob, lease: JobLease) -> (OwnerJob, JobDrain) { + owner_job_with_probe(prepared, lease, None) +} + +fn owner_job_with_probe( + prepared: PreparedJob, + lease: JobLease, + probe: Option, +) -> (OwnerJob, JobDrain) { + let streaming = prepared.parsed.stream; + let (sink, state) = job_sink_with_probe(Arc::clone(&lease.inner), probe); + let (done_tx, done_rx) = mpsc::channel(); + ( + OwnerJob { + prepared, + sink, + done: done_tx, + lease, + }, + JobDrain { + done: done_rx, + state, + streaming, + }, + ) +} + +pub fn handle_client(cfg: &ServerConfig, stream: &mut TcpStream) { + let inner = Mutex::new(ServerInner::from_cfg(cfg)); + handle_client_inner(cfg, &inner, stream, None, None); +} + +fn lock_inner(inner: &Mutex) -> std::sync::MutexGuard<'_, ServerInner> { + inner.lock().unwrap_or_else(|e| e.into_inner()) +} + +pub fn handle_client_inner( + cfg: &ServerConfig, + inner: &Mutex, + stream: &mut TcpStream, + engine: Option<&mut dyn DecodeIo>, + cont: Option<&mut dyn ContExec>, +) { + let Some(mut job) = prepare_client(cfg, inner, stream) else { + return; + }; + let _ = stream.set_nonblocking(true); + let mut out = DirectSink { + stream, + started: false, + }; + let body_bytes = job.body_bytes; + let have_engine = engine.is_some(); + let (verdict, pin) = { + let mut g = lock_inner(inner); + let verdict = enqueue(&mut g.admit, body_bytes); + let pin = if verdict == EnqVerdict::Ok && have_engine { + acquire_continuation_pin(&mut g, &mut job.parsed, monotonic_now()) + } else { + None + }; + (verdict, pin) + }; + if let Some((reason, code, retry, msg)) = enqueue_shed_error(verdict) { + lock_inner(inner).metrics.record_shed(reason); + let _ = out.write_all(&wire_http_error_bytes( + job.surface, + code, + msg, + cfg.cors, + retry, + )); + return; + } + let pin = BorrowedPin { inner, pin }; + let settlement = run_prepared(cfg, inner, &mut job, engine, cont, &mut out, None); + if let Some(reason) = settlement.shed { + lock_inner(inner).metrics.record_shed(reason); + } + drop(pin); + enqueue_release(&mut lock_inner(inner).admit, body_bytes); +} + +/// The level of a request that names none: the template's own default for +/// GLM-5.3 and Beam (max), the port's low for every other family. +fn default_effort(cfg: &ServerConfig) -> ThinkMode { + if cfg.max_effort_default { + ThinkMode::Max + } else { + ThinkMode::Low + } +} + +fn prepare_client( + cfg: &ServerConfig, + inner: &Mutex, + stream: &mut TcpStream, +) -> Option { + let req = match read_http_request(stream, chunked_enabled()) { + Some(r) => r, + None => { + write_all( + stream, + &wire_http_error_bytes( + WireSurface::OpenaiChat, + 400, + "bad HTTP request", + cfg.cors, + None, + ), + ); + return None; + } + }; + let arrived_at = Instant::now(); + if req.method == "OPTIONS" { + write_all(stream, &http_response_bytes(204, None, None, cfg.cors, "")); + return None; + } + if req.method == "GET" && req.path == "/v1/models" { + let body = models_list_json( + &cfg.model_id, + &cfg.model_name, + cfg.ctx, + cfg.default_tokens, + cfg.codex_models_json.as_deref(), + ); + write_all( + stream, + &http_response_bytes(200, Some("application/json"), None, cfg.cors, &body), + ); + return None; + } + if req.method == "GET" && req.path.starts_with("/v1/models/") { + let id = &req.path["/v1/models/".len()..]; + if model_id_known(&cfg.model_id, id) { + let body = model_one_json(id, &cfg.model_name, cfg.ctx, cfg.default_tokens); + write_all( + stream, + &http_response_bytes(200, Some("application/json"), None, cfg.cors, &body), + ); + return None; + } + } + if req.method == "GET" && req.path == "/metrics" { + let body = { + let g = lock_inner(inner); + let rt = g.render_runtime(unix_now() as u64); + render_metrics(&g.metrics, &g.admit, &rt) + }; + write_all( + stream, + &http_response_bytes( + 200, + Some("text/plain; version=0.0.4"), + None, + cfg.cors, + &body, + ), + ); + return None; + } + if req.method == "GET" && req.path == "/v1/stats" { + let body = { + let g = lock_inner(inner); + let rt = g.render_runtime(unix_now() as u64); + let serving = g + .serving_plan + .as_ref() + .map(|plan| plan.to_json().to_string()); + let last = g + .last_request + .as_ref() + .map(|trace| trace.to_json().to_string()); + render_stats_json_plan( + &g.metrics, + &g.admit, + &rt, + serving.as_deref(), + last.as_deref(), + ) + }; + write_all( + stream, + &http_response_bytes(200, Some("application/json"), None, cfg.cors, &body), + ); + return None; + } + if req.method == "POST" { + let inference = shed_surface_for_path(&req.path).is_some(); + let generation = inference || req.path == "/v1/batch"; + if let Some((reason, code, retry, msg)) = { + let g = lock_inner(inner); + preparse_shed(&g.admit, inference, generation, req.body.len() as u64) + } { + lock_inner(inner).metrics.record_shed(reason); + let surf = shed_surface_for_path(&req.path).unwrap_or(WireSurface::OpenaiChat); + write_all( + stream, + &wire_http_error_bytes(surf, code, msg, cfg.cors, Some(retry)), + ); + return None; + } + if let Some(surf) = parse_surface_for_path(&req.path) { + let now = monotonic_now(); + let live_ids = { + let mut g = lock_inner(inner); + g.creg.expire(now); + g.creg.live_ids(api_for_surface(surf)) + }; + let env = ParseEnv { + default_model: cfg.model_id.clone(), + default_tokens: cfg.default_tokens, + default_effort: default_effort(cfg), + default_temp: crate::parse::default_temperature(), + live_ids, + engine_defaults: cfg.engine_defaults, + }; + let body = std::str::from_utf8(&req.body).unwrap_or(""); + match parse_request(surf, &env, body) { + Err(e) => { + let msg = if e.is_empty() { + "invalid JSON request" + } else { + &e + }; + write_all( + stream, + &wire_http_error_bytes(surf, 400, msg, cfg.cors, None), + ); + return None; + } + Ok(mut parsed) => { + parsed.eos_policy = cfg.eos_policy; + parsed.finish_needs(); + let body_bytes = req.body.len() as u64; + return Some(PreparedJob { + parsed, + cont_prompt: None, + surface: surf, + body_bytes, + arrived_at, + }); + } + } + } + } + write_all( + stream, + &wire_http_error_bytes( + WireSurface::OpenaiChat, + 404, + "unknown endpoint", + cfg.cors, + None, + ), + ); + None +} + +fn route_for_model( + model_id: i32, + needs: u32, + surface: WireSurface, + env: &RouteEnv, +) -> RouteDecision { + // MiMo feature spans live in the serial session, including under a banked boot. + if needs & NEED_IMAGE != 0 && syntax_for_model_id(model_id) == ModelSyntax::Mimo2 { + return RouteDecision { + lane: LANE_SERIAL, + reason: REASON_CONT_UNAVAILABLE, + }; + } + route_decide(needs, surface, env) +} + +fn run_prepared( + cfg: &ServerConfig, + inner: &Mutex, + job: &mut PreparedJob, + engine: Option<&mut dyn DecodeIo>, + cont: Option<&mut dyn ContExec>, + out: &mut W, + arrived_at: Option, +) -> Settlement { + let model_id = engine.as_ref().map_or(0, |engine| engine.model_id()); + let mimo_image = + !job.parsed.images.is_empty() && syntax_for_model_id(model_id) == ModelSyntax::Mimo2; + let cont_gate = if mimo_image { + None + } else { + match (cont.as_deref(), engine.is_some()) { + (Some(exec), true) => prepare_cont_prompt(job, exec) + .map(|prepared| (prepared.tokens.len() as i32, exec.seq_cap())), + _ => None, + } + }; + let (cont_tools_anthropic, cont_tools_responses) = process_cont_tools(); + let route_env = RouteEnv { + coalesce: cont_gate.is_some(), + have_cont: cont_gate.is_some() && cfg.continuous, + cont_anthropic: parse_default_on(std::env::var_os("DS4_SERVER_CONT_ANTHROPIC").as_deref()), + cont_responses: parse_default_on(std::env::var_os("DS4_SERVER_CONT_RESPONSES").as_deref()), + cont_tools_anthropic, + cont_tools_responses, + seq_cap: cont_gate.map_or(cfg.ctx, |(_, cap)| cap), + prompt_len: cont_gate.map_or(0, |(len, _)| len), + }; + let dec = route_for_model(model_id, job.parsed.needs, job.surface, &route_env); + let id = next_job_id(&mut lock_inner(inner).admit, job.parsed.kind); + let arrived_at = arrived_at.unwrap_or_else(Instant::now); + let (actual_lane, settlement) = match engine { + Some(engine) => run_engine(cfg, inner, job, &id, dec, engine, cont, out, arrived_at), + None => { + let ok = out + .write_all(&wire_http_error_bytes( + job.surface, + 503, + SERVER_SHUTTING_DOWN, + cfg.cors, + None, + )) + .is_ok(); + ( + dec.lane, + if ok { + Settlement::COMPLETED + } else { + Settlement::CANCELED + }, + ) + } + }; + lock_inner(inner).metrics.record_route( + job.surface, + actual_lane, + dec.reason, + job.parsed.think_mode, + ); + settlement +} + +fn run_engine( + cfg: &ServerConfig, + inner: &Mutex, + job: &mut PreparedJob, + id: &str, + dec: crate::route::RouteDecision, + engine: &mut dyn DecodeIo, + mut cont: Option<&mut dyn ContExec>, + out: &mut W, + arrived_at: Instant, +) -> (u8, Settlement) { + let mut serial_fallback = None; + if dec.lane == LANE_CONTINUOUS { + if let Some(exec) = cont.as_mut() { + exec.set_stop_requested(cfg.stop_requested); + if let Some(settlement) = refuse_bank_continuation(cfg, inner, job, *exec, out) { + return (LANE_CONTINUOUS, settlement); + } + let store = engine.kv_store_mut(); + let mut bank_hold_retry = |bank, live| { + lock_inner(inner) + .creg + .bank_hold_retry(bank, live, monotonic_now()) + }; + let result = exec.generate_prepared( + &job.parsed, + job.cont_prompt.as_ref(), + id, + unix_now(), + cfg.cors, + cfg.default_tokens, + arrived_at, + &mut bank_hold_retry, + store, + out, + ); + if let Ok(outcome) = &result { + lock_inner(inner).record_generation(outcome); + publish_continuous_tool_turn(inner, job.parsed.api, outcome); + } + let bank_follow = job.parsed.needs & NEED_BANK_FRONTIER != 0; + if bank_follow || !matches!(result, Err(GenerateError::Unsupported(_))) { + let settlement = if bank_follow { + settle_bank_continuation(cfg, job, result, out) + } else { + settle_generation_result(cfg, job, result, out) + }; + return (LANE_CONTINUOUS, settlement); + } + if let Err(GenerateError::Unsupported(msg)) = result { + serial_fallback = Some(msg.to_string()); + } + // C lane-entry counters (ds4_server.c route_metrics_record): + // cont_admit success ticks continuous BEFORE the engine's + // funding verdict, and the stranded fallback's generate_job + // entry then ticks serial too. Mirror the two-entry shape on + // the Unsupported fold — this return path ends in run_serial, + // whose caller records the serial entry. + lock_inner(inner) + .metrics + .record_lane_entry(job.surface, LANE_CONTINUOUS); + } + } + if dec.lane == LANE_STATIC { + let tokens = match cont.as_deref() { + Some(exec) => prepare_cont_prompt(job, exec) + .map(|prepared| prepared.tokens.clone()) + .unwrap_or_default(), + None => Vec::new(), + }; + let current = StaticJob { + tokens: &tokens, + max_new_tokens: job.parsed.max_tokens, + eos: -1, + }; + let mut detached = DetachedStatic; + let result = match cont.as_mut() { + Some(exec) => match exec.as_static() { + Some(owner) => run_static_routed(owner, current), + None => run_static_routed(&mut detached, current), + }, + None => run_static_routed(&mut detached, current), + }; + // C worker_main: after coalesce_gather, n==1 collapses to + // run_job_single. generate_static is never called with n<2 on + // the HTTP path. The decision stays STATIC (already recorded); + // the entry lane is SERIAL — decisions and entries are different + // metric families. n>=2 generate_static failures stay here and + // never fall through to serial. + if matches!(&result, Err(GenerateError::Engine(msg)) if msg == STATIC_WIDTH_ERR) { + return ( + crate::route::LANE_SERIAL, + run_serial(cfg, inner, job, id, engine, cont, out, arrived_at, None), + ); + } + return ( + LANE_STATIC, + settle_static_lane( + cfg, + inner, + job, + id, + engine, + i32::try_from(tokens.len()).unwrap_or(i32::MAX), + result, + out, + ), + ); + } + ( + crate::route::LANE_SERIAL, + run_serial( + cfg, + inner, + job, + id, + engine, + cont, + out, + arrived_at, + serial_fallback, + ), + ) +} + +fn settle_static_lane( + cfg: &ServerConfig, + inner: &Mutex, + job: &PreparedJob, + id: &str, + engine: &dyn DecodeIo, + prompt_n: i32, + result: Result, GenerateError>, + out: &mut W, +) -> Settlement { + match result { + Ok(rows) => { + let empty = StaticRow { + tokens: Vec::new(), + finish: StaticFinish::Length, + }; + let row = rows.last().unwrap_or(&empty); + let decoded = i32::try_from(row.tokens.len()).unwrap_or(i32::MAX); + lock_inner(inner).record_generation(&GenerateOutcome { + timings: crate::stream::ReqTimings { + prefill_tokens: prompt_n, + decode_tokens: decoded, + ..crate::stream::ReqTimings::default() + }, + lane: Some("static"), + ..GenerateOutcome::default() + }); + let bytes = write_static_completion( + StaticSettle { + parsed: &job.parsed, + job_id: id, + created: unix_now(), + cors: cfg.cors, + default_tokens: cfg.default_tokens, + model_id: engine.model_id(), + prompt_n, + row, + }, + |token| engine.token_is_stop(token), + |token| engine.token_text(token).unwrap_or_default(), + ); + let ok = out.write_all(&bytes).is_ok(); + if ok { + Settlement::COMPLETED + } else { + Settlement::CANCELED + } + } + Err(GenerateError::Engine(msg)) if msg == STATIC_WIDTH_ERR => { + refuse_static_width(cfg, job, &msg, out) + } + Err(error) => settle_generation_result(cfg, job, Err(error), out), + } +} + +fn refuse_static_width( + cfg: &ServerConfig, + job: &PreparedJob, + msg: &str, + out: &mut W, +) -> Settlement { + let ok = out + .write_all(&wire_http_error_bytes( + job.surface, + 400, + msg, + cfg.cors, + None, + )) + .is_ok(); + if ok { + Settlement::COMPLETED + } else { + Settlement::CANCELED + } +} + +fn run_serial( + cfg: &ServerConfig, + inner: &Mutex, + job: &PreparedJob, + id: &str, + engine: &mut dyn DecodeIo, + mut cont: Option<&mut dyn ContExec>, + out: &mut W, + arrived_at: Instant, + fallback: Option, +) -> Settlement { + let parsed = &job.parsed; + let now = monotonic_now(); + let retry = { + let mut g = lock_inner(inner); + g.creg.serial_hold(parsed.api, &parsed.live_call_ids, now) + }; + if let Some(retry) = retry { + let _ = out.write_all(&wire_http_error_bytes( + job.surface, + 503, + "serial capacity is reserved for a live tool continuation; retry shortly", + cfg.cors, + Some(retry), + )); + return Settlement::shed(SHED_CONT_HOLD); + } + + let generation = engine.generation(); + let pos = engine.pos(); + let resolved = { + let mut g = lock_inner(inner); + g.creg + .resolve_serial(parsed.api, &parsed.live_call_ids, generation, pos, now) + }; + let requires_live = + parsed.anthropic_requires_live_tool_state || parsed.responses_requires_live_tool_state; + if requires_live && !resolved { + let ok = out + .write_all(&wire_http_error_bytes( + job.surface, + 409, + continuation_conflict_msg(parsed.api), + cfg.cors, + None, + )) + .is_ok(); + return if ok { + Settlement::COMPLETED + } else { + Settlement::CANCELED + }; + } + + // Host prompt preparation does not allocate the inference graph. Resize + // first so admission quotes this request, rather than the unused boot ctx. + let prep = match prepare_serial_prompt(engine, parsed) { + Ok(prep) => prep, + Err(error) => return settle_generation_result(cfg, job, Err(error), out), + }; + // RequiredMiss frames were answered with the protocol-native 409 above; + // a still-live requires_live frame is C's SERIAL_FRAME_RESOLVED_LIVE. + let frame = if requires_live { + SerialFrameKind::ResolvedLive + } else { + SerialFrameKind::Canonical + }; + let mut prompt_len = i64::try_from(prep.tokens.len()).unwrap_or(i64::MAX); + if frame == SerialFrameKind::ResolvedLive { + // C preflight sizes the effective live frame (live tokens + suffix), + // not the client frame. The retained frontier is a hard lower bound. + prompt_len = prompt_len.max(i64::from(engine.pos())); + } + let budget = i64::from(decode_budget( + prep.parsed.max_tokens_set, + prep.parsed.max_tokens, + cfg.default_tokens, + )); + let serial_cont = cont.as_mut().map(|exec| &mut **exec as &mut dyn ContExec); + if let Some(refusal) = ensure_serial_session_fit( + cfg, + inner, + job, + engine, + serial_cont, + frame, + prompt_len, + budget, + out, + ) { + return refusal; + } + + let probe = engine.serial_session_probe(); + let fit_ctx = probe.map_or(cfg.ctx, |p| p.cur_ctx); + let live = engine.native_graph_fit(fit_ctx).and_then(|mut quote| { + // A retained graph is already charged to live memory. + if probe.is_some_and(|p| !p.graph_pending) { + quote.need_bytes = 0; + } + serial_native_fit(quote) + }); + let mut quote = resolve_serial_fit(cfg.serial_fit, live); + let floor = MemFloor::from_gb(cfg.mem_floor_gb); + let mut ask = quote.ask(floor); + let want = serial_reclaim_want(ask); + if want > 0 { + if let Some(exec) = cont.as_deref_mut() { + let released = exec.trim_idle_banks(want); + quote.avail = AvailBytes::from_raw(quote.avail.raw().saturating_add(released)); + ask = quote.ask(floor); + } + } + match serial_reclaim_gate(ask) { + SerialReclaimOutcome::Admit { .. } => {} + SerialReclaimOutcome::Refuse { .. } => { + return refuse_serial_capacity(cfg, inner, job, prompt_len, out); + } + } + + lock_inner(inner).runtime.requests_serial += 1; + let result = generate_terminal_prepared( + engine, + prep, + id, + unix_now(), + cfg.cors, + cfg.default_tokens, + arrived_at, + cfg.stop_requested, + out, + ); + let (generated, terminal) = match result { + Ok(result) => result, + Err(error) => return settle_generation_result(cfg, job, Err(error), out), + }; + let mut generated = generated; + generated.fallback_reason = fallback; + lock_inner(inner).record_generation(&generated); + let publish = matches!(parsed.api, Api::Anthropic | Api::Responses) + && !generated.tool_ids.is_empty() + && generated.finish != "error" + && generated.finish != "length"; + if publish { + return match out.commit_tool_terminal(inner, parsed.api, &generated, terminal) { + TerminalCommit::Committed => Settlement::COMPLETED, + TerminalCommit::GoneBeforeStart | TerminalCommit::FailedAfterStart => { + Settlement::CANCELED + } + TerminalCommit::Full => Settlement::shed(SHED_SLOW_READER), + }; + } + lock_inner(inner).creg.demote_serial(); + if out.write_all(&terminal).is_ok() { + if generated.finish == "error" { + Settlement::FAILED + } else { + Settlement::COMPLETED + } + } else { + Settlement::CANCELED + } +} + +/// C `serial_session_ensure_fit` driver. The serial session was created for +/// boot `-c` and its S6 lazy graph is sized by that ctx, not the request — +/// on a bank-holding boot the full graph can never fit and every serial +/// request dies at the alloc. A serial request needs only prompt+budget +/// positions: when the full graph cannot fit, re-create the (still +/// graphless) session at the largest ctx the fit probe passes, bounded by +/// what the request could use. When even a minimal output window cannot +/// fit, refuse 503 (retryable capacity), never the doomed alloc. +/// Returns `Some(settlement)` after writing a refusal. +#[allow(clippy::too_many_arguments)] +fn ensure_serial_session_fit( + cfg: &ServerConfig, + inner: &Mutex, + job: &PreparedJob, + engine: &mut dyn DecodeIo, + cont: Option<&mut dyn ContExec>, + frame: SerialFrameKind, + prompt_len: i64, + budget: i64, + out: &mut W, +) -> Option { + // Stub/test engines have no native serial session: pass native and + // never invent a rightsize or a refuse. + let probe = engine.serial_session_probe()?; + let mut bounds = serial_fit_bounds(prompt_len, budget, i64::from(cfg.ctx)); + let ling = ds4_core::shape_for_variant(ds4_core::Variant::Ling30FlashVl); + let ling_orig_ctx = ling.rope_orig_ctx as i64; + if engine.model_id() == ling.model_id() && i64::from(cfg.ctx) > ling_orig_ctx { + // Serial/image requests must retain the boot YaRN factor even when + // their graph shrinks beside the banks. Refuse if this floor cannot fit. + bounds.need_min = bounds.need_min.max(ling_orig_ctx + 1); + bounds.request_cap = bounds.request_cap.max(bounds.need_min); + } + let cur_ctx = i64::from(probe.cur_ctx); + // C: a mandatory live continuation is never resized — the retained + // session holds the only copy of the frontier. Preserve and refuse. + if frame == SerialFrameKind::ResolvedLive && cur_ctx < bounds.live_need_min { + return Some(refuse_serial_live_preserve( + cfg, + inner, + job, + prompt_len, + bounds.live_need_min, + cur_ctx, + out, + )); + } + // C scope gates: DS4_SERVER_SERIAL_RIGHTSIZE=0 and serial-only boots + // (no batch ctx) keep the v0.5.1 contract — alloc at -c, loud native + // fit refusal when genuinely doomed. + if !cfg.serial_rightsize { + return None; + } + let exec = cont?; + let current_fits = if probe.graph_pending && cur_ctx <= bounds.request_cap { + serial_ctx_fits(cfg, engine, probe.cur_ctx) + } else { + true + }; + match serial_session_fit_plan( + cur_ctx, + probe.graph_pending, + bounds.need_min, + bounds.request_cap, + current_fits, + frame, + ) { + SerialFitPlan::Reuse | SerialFitPlan::PassNative => None, + SerialFitPlan::RefusePreserve => Some(refuse_serial_live_preserve( + cfg, + inner, + job, + prompt_len, + bounds.need_min, + cur_ctx, + out, + )), + SerialFitPlan::Resize => { + // Free BEFORE probing so a committed right-sized graph's own + // GiBs count as available for its replacement (the C regrow + // case). The live continuation records described that session's + // content — demote so continuations get the honest native 409. + engine.serial_session_reset(); + lock_inner(inner).creg.demote_serial(); + let need_min_ctx = i32::try_from(bounds.need_min).unwrap_or(i32::MAX); + let mut min_fits = fit_settle_probe(cfg, engine, need_min_ctx); + if !min_fits { + // C deepmem D4-2: collect from the commons before refusing. + // The quote's exact deficit bounds the collect (deficit + + // headroom slack); a numberless quote keeps the + // whole-commons shape. One C trim, then the same settle + // window; a still-unfittable request refuses as before. + let want = match engine.native_graph_fit(need_min_ctx) { + Some(quote) if !quote.fail_open => { + let live = serial_native_fit(quote); + let fit = resolve_serial_fit(cfg.serial_fit, live); + let floor = MemFloor::from_gb(cfg.mem_floor_gb); + serial_reclaim_want(fit.ask(floor)) + .max(quote.deficit_bytes.saturating_add(quote.headroom_bytes)) + } + _ => u64::MAX, + }; + if exec.trim_idle_banks(want) > 0 { + min_fits = fit_settle_probe(cfg, engine, need_min_ctx); + } + } + let mut target: i64 = 0; + if min_fits { + // Search bound: what the request could use, plus + // continuation headroom so a serial conversation grows for + // many turns inside one session. Invariant: lo fits. + let mut lo = bounds.need_min; + let mut hi = bounds.request_cap; + while hi - lo > 1024 { + let mid = lo + (hi - lo) / 2; + let fits = serial_ctx_fits(cfg, engine, i32::try_from(mid).unwrap_or(i32::MAX)); + if fits { + lo = mid; + } else { + hi = mid; + } + } + target = lo; + } + // Check before creating: lazy-graph=0 allocates at creation. + if target > 0 && serial_ctx_fits(cfg, engine, target as i32) { + if let Ok(()) = + engine.serial_session_rightsize(i32::try_from(target).unwrap_or(i32::MAX)) + { + eprintln!( + "ds4-server-rs: serial session right-sized ctx={cur_ctx} -> {target} \ + (boot -c {}; request-bounded beside the batch banks, prompt={prompt_len} \ + budget={budget}) (DS4_SERVER_SERIAL_RIGHTSIZE=0 restores full--c alloc)", + cfg.ctx + ); + return None; + } + } + // Refusal: leave the lane in boot shape (the Rust host restores + // the session lazily) so later requests re-probe fresh. + engine.serial_session_reset(); + eprintln!( + "ds4-server-rs: serial right-size: no graph fits (prompt={prompt_len} \ + need_min={} boot -c {}); refusing 503", + bounds.need_min, cfg.ctx + ); + Some(refuse_serial_capacity(cfg, inner, job, prompt_len, out)) + } + } +} + +/// C ensure-fit settle window: freed graph bytes reach MemAvailable with +/// driver-reclaim lag (the WAIT_MEM class); give the fit probe a bounded +/// settle window. An engine without a native probe fails open. +fn fit_settle_probe(cfg: &ServerConfig, engine: &dyn DecodeIo, ctx: i32) -> bool { + for _ in 0..20 { + if serial_ctx_fits(cfg, engine, ctx) { + return true; + } + thread::sleep(Duration::from_millis(100)); + } + false +} + +fn serial_ctx_fits(cfg: &ServerConfig, engine: &dyn DecodeIo, ctx: i32) -> bool { + let Some(quote) = engine.native_graph_fit(ctx) else { + return true; + }; + if !quote.fits { + return false; + } + let fit = resolve_serial_fit(cfg.serial_fit, serial_native_fit(quote)); + serial_reclaim_gate(fit.ask(MemFloor::from_gb(cfg.mem_floor_gb))).admitted() +} + +fn serial_native_fit(quote: NativeGraphFit) -> Option { + // Motif/Dots/Solar/EXAONE report a verdict without a byte budget. + let numberless = quote.fits + && quote.need_bytes == 0 + && quote.avail_bytes == 0 + && quote.headroom_bytes == 0 + && quote.deficit_bytes == 0; + serial_fit_from_native( + quote.need_bytes, + quote.avail_bytes, + quote.headroom_bytes, + 0, + quote.fail_open || numberless, + ) +} + +/// C `serial_session_ensure_fit` final refusal: typed 503, retryable +/// capacity. Counts the C `JOB_OUT_REFUSED_DEEP_SERIAL` outcome and the +/// typed serial live-headroom rejection. +fn refuse_serial_capacity( + cfg: &ServerConfig, + inner: &Mutex, + job: &PreparedJob, + prompt_len: i64, + out: &mut W, +) -> Settlement { + { + let mut g = lock_inner(inner); + g.runtime.requests_refused_deep_serial += 1; + g.runtime.rejected[1][RejectReason::LiveHeadroom as usize] += 1; + } + let prompt_n = i32::try_from(prompt_len).unwrap_or(i32::MAX); + let ok = out + .write_all(&wire_http_error_bytes( + job.surface, + 503, + &serial_capacity_refuse_msg(prompt_n), + cfg.cors, + None, + )) + .is_ok(); + if ok { + Settlement::COMPLETED + } else { + Settlement::CANCELED + } +} + +/// C `serial_live_capacity_refuse`: the retained live continuation cannot +/// extend inside its serial context; preserve the frontier and refuse. +fn refuse_serial_live_preserve( + cfg: &ServerConfig, + inner: &Mutex, + job: &PreparedJob, + effective_len: i64, + need_min: i64, + cur_ctx: i64, + out: &mut W, +) -> Settlement { + eprintln!( + "ds4-server-rs: live serial continuation needs {effective_len} tokens \ + (minimum={need_min}) but retained session ctx={cur_ctx}; preserving frontier \ + and refusing 503" + ); + lock_inner(inner).runtime.requests_refused_deep_serial += 1; + let ok = out + .write_all(&wire_http_error_bytes( + job.surface, + 503, + &serial_live_preserve_msg(effective_len, cur_ctx), + cfg.cors, + None, + )) + .is_ok(); + if ok { + Settlement::COMPLETED + } else { + Settlement::CANCELED + } +} + +fn write_generation_error( + cfg: &ServerConfig, + job: &PreparedJob, + out: &mut W, + code: i32, + msg: &str, + retry_after: Option, +) { + let bytes = if job.parsed.stream && out.response_started() { + wire_stream_error_bytes(job.surface, msg, 0) + } else { + wire_http_error_bytes(job.surface, code, msg, cfg.cors, retry_after) + }; + let _ = out.write_all(&bytes); +} + +fn settle_generation_result( + cfg: &ServerConfig, + job: &PreparedJob, + result: Result, + out: &mut W, +) -> Settlement { + match result { + Ok(outcome) if outcome.finish == "error" => Settlement::FAILED, + Ok(_) => Settlement::COMPLETED, + Err(GenerateError::Io) => Settlement::CANCELED, + Err(GenerateError::Streamed(_)) => Settlement::FAILED, + Err(GenerateError::BadRequest(msg)) => { + write_generation_error(cfg, job, out, 400, msg, None); + Settlement::FAILED + } + Err(GenerateError::Unsupported(msg)) => { + write_generation_error(cfg, job, out, 503, msg, None); + Settlement::FAILED + } + Err(GenerateError::ContinuationHold { retry_after }) => { + write_generation_error( + cfg, + job, + out, + 503, + "batch capacity is reserved for live tool continuations; retry shortly", + Some(retry_after.max(1)), + ); + Settlement::shed(SHED_CONT_HOLD) + } + Err(GenerateError::Engine(msg)) => { + write_generation_error(cfg, job, out, 500, &msg, None); + Settlement::FAILED + } + } +} + +pub fn listen(cfg: &ServerConfig) -> std::io::Result { + TcpListener::bind((cfg.listen_host.as_str(), cfg.listen_port)) +} + +/// Bind only when the plan may listen. An impossible mix returns `Ok(None)` +/// and never opens a socket. +pub fn listen_if_allowed( + cfg: &ServerConfig, + plan: &ds4_core::ResolvedPlan, +) -> std::io::Result> { + if !plan.may_listen() { + return Ok(None); + } + listen(cfg).map(Some) +} + +fn accept_error_retryable(_kind: ErrorKind) -> bool { + true +} + +fn retry_accept(kind: ErrorKind, polling_stop: bool) { + if kind != ErrorKind::Interrupted { + thread::sleep(Duration::from_millis(if polling_stop { 10 } else { 1 })); + } +} + +fn stop_requested(cfg: &ServerConfig) -> bool { + cfg.stop_requested.is_some_and(|stop| stop()) +} + +fn refuse_client_capacity(stream: &mut TcpStream, cfg: &ServerConfig) { + configure_client_socket(stream, cfg.client_sndbuf); + write_all( + stream, + &wire_http_error_bytes( + WireSurface::OpenaiChat, + 503, + "server connection capacity reached; retry later", + cfg.cors, + Some(10), + ), + ); +} + +pub fn accept_loop(listener: TcpListener, cfg: ServerConfig) { + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let polling_stop = cfg.stop_requested.is_some(); + if polling_stop { + if let Err(error) = listener.set_nonblocking(true) { + eprintln!("ds4-server-rs: stop polling unavailable: {error}"); + lock_inner(&inner).admit.stopping = true; + return; + } + } + loop { + if stop_requested(&cfg) { + break; + } + let mut stream = match listener.accept() { + Ok((stream, _)) => { + if stop_requested(&cfg) { + break; + } + stream + } + Err(e) if accept_error_retryable(e.kind()) => { + retry_accept(e.kind(), polling_stop); + continue; + } + Err(_) => break, + }; + let Some(client) = ClientLease::try_new(Arc::clone(&inner)) else { + refuse_client_capacity(&mut stream, &cfg); + continue; + }; + let cfg = cfg.clone(); + let inner = Arc::clone(&inner); + let _ = thread::Builder::new().spawn(move || { + let _client = client; + let _ = stream.set_nodelay(true); + configure_client_socket(&stream, cfg.client_sndbuf); + handle_client_inner(&cfg, &inner, &mut stream, None, None); + }); + } + lock_inner(&inner).admit.stopping = true; +} + +fn configure_client_socket(stream: &TcpStream, client_sndbuf: i32) { + let _ = stream.set_read_timeout(Some(CLIENT_IO_TIMEOUT)); + let _ = stream.set_write_timeout(Some(CLIENT_IO_TIMEOUT)); + let _ = ds4_sys::set_socket_send_buffer(stream, client_sndbuf); +} + +fn send_all_nonblocking(stream: &mut TcpStream, mut bytes: &[u8]) -> std::io::Result<()> { + let mut deadline = Instant::now() + CLIENT_SEND_STALL_TIMEOUT; + while !bytes.is_empty() { + match stream.write(bytes) { + Ok(0) => { + return Err(Error::new( + ErrorKind::WriteZero, + "socket write returned zero", + )) + } + Ok(n) => { + bytes = &bytes[n..]; + deadline = Instant::now() + CLIENT_SEND_STALL_TIMEOUT; + } + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) if e.kind() == ErrorKind::WouldBlock => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(Error::new(ErrorKind::TimedOut, "socket send stalled")); + } + thread::sleep(CLIENT_POLL.min(remaining)); + } + Err(e) => return Err(e), + } + } + Ok(()) +} + +fn client_disconnected(stream: &TcpStream) -> bool { + if stream.set_nonblocking(true).is_err() { + return false; + } + let mut byte = [0u8; 1]; + let result = stream.peek(&mut byte); + match result { + Ok(0) => true, + Err(e) => matches!( + e.kind(), + ErrorKind::BrokenPipe | ErrorKind::ConnectionReset | ErrorKind::NotConnected + ), + Ok(_) => false, + } +} + +fn drain_job(mut stream: TcpStream, drain: JobDrain) { + drain_job_with_interval(&mut stream, &drain, STREAM_HEARTBEAT_INTERVAL); + let Ok(mut lease) = drain.done.recv() else { + return; + }; + if drain.state.gone() { + lease.settlement = lease.settlement.transport_gone(); + } +} + +fn drain_job_with_interval(stream: &mut TcpStream, drain: &JobDrain, interval: Duration) { + configure_client_socket(stream, 0); + loop { + let mut buffer = drain.state.lock(); + let mut heartbeat = false; + if buffer.bytes.is_empty() && !buffer.closed && !buffer.gone { + let waited = drain + .state + .ready + .wait_timeout(buffer, interval) + .unwrap_or_else(|e| e.into_inner()); + buffer = waited.0; + heartbeat = waited.1.timed_out() + && drain.streaming + && buffer.started + && buffer.bytes.is_empty() + && !buffer.closed + && !buffer.gone; + } + let have_bytes = !buffer.bytes.is_empty(); + let finished = buffer.closed || buffer.gone; + drop(buffer); + if have_bytes { + if send_all_nonblocking(stream, &drain.state.take()).is_err() { + drain.state.cancel(false); + break; + } + } else if finished { + break; + } else if drain.state.observe_disconnect() { + break; + } else if heartbeat && send_all_nonblocking(stream, b": keep-alive\n\n").is_err() { + drain.state.cancel(false); + break; + } + } +} + +fn queue_client( + cfg: ServerConfig, + inner: Arc>, + jobs: mpsc::Sender, + mut stream: TcpStream, + _client: ClientLease, +) { + let _ = stream.set_nodelay(true); + configure_client_socket(&stream, cfg.client_sndbuf); + let Some(mut prepared) = prepare_client(&cfg, &inner, &mut stream) else { + return; + }; + let surface = prepared.surface; + let _ = stream.set_nonblocking(true); + let probe = match stream.try_clone() { + Ok(probe) => Some(probe), + Err(_) => { + let _ = send_all_nonblocking( + &mut stream, + &wire_http_error_bytes(surface, 503, SERVER_SHUTTING_DOWN, cfg.cors, None), + ); + return; + } + }; + let body_bytes = prepared.body_bytes; + let mut g = lock_inner(&inner); + let verdict = enqueue(&mut g.admit, body_bytes); + if let Some((reason, code, retry, msg)) = enqueue_shed_error(verdict) { + g.metrics.record_shed(reason); + drop(g); + let _ = send_all_nonblocking( + &mut stream, + &wire_http_error_bytes(surface, code, msg, cfg.cors, retry), + ); + return; + } + let pin = acquire_continuation_pin(&mut g, &mut prepared.parsed, monotonic_now()); + g.runtime.requests_started += 1; + g.runtime.requests_inflight += 1; + let lease = JobLease::new(Arc::clone(&inner), body_bytes, pin); + let (job, drain) = owner_job_with_probe(prepared, lease, probe); + let sent = jobs.send(job); + drop(g); + if let Err(err) = sent { + let mut job = err.0; + let wrote = send_all_nonblocking( + &mut stream, + &wire_http_error_bytes(surface, 503, SERVER_SHUTTING_DOWN, cfg.cors, None), + ) + .is_ok(); + job.lease.settlement = if wrote { + Settlement::FAILED + } else { + Settlement::CANCELED + }; + return; + } + drain_job(stream, drain); +} + +fn run_owner_job( + cfg: &ServerConfig, + inner: &Arc>, + engine: &mut dyn DecodeIo, + cont: Option<&mut dyn ContExec>, + job: OwnerJob, +) { + let OwnerJob { + mut prepared, + mut sink, + done, + mut lease, + } = job; + lease.start(); + let queue_age = prepared.arrived_at.elapsed().as_secs_f64(); + let mut settlement = if sink.state.gone() || sink.state.observe_disconnect() { + Settlement::CANCELED + } else if cfg.max_queue_age_s > 0.0 && queue_age > cfg.max_queue_age_s { + let msg = format!( + "request waited {queue_age:.1}s in queue, over the {:.0}s limit; server overloaded, retry later", + cfg.max_queue_age_s + ); + let _ = sink.write_all(&wire_http_error_bytes( + prepared.surface, + 503, + &msg, + cfg.cors, + Some(30), + )); + Settlement::shed(SHED_QUEUE_AGE) + } else { + let arrived_at = prepared.arrived_at; + run_prepared( + cfg, + inner, + &mut prepared, + Some(engine), + cont, + &mut sink, + Some(arrived_at), + ) + }; + if sink.state.slow() { + settlement = settlement.slow_reader(); + } + drop(sink); + lease.settlement = settlement; + if let Err(err) = done.send(lease) { + let mut lease = err.0; + lease.settlement = lease.settlement.transport_gone(); + } +} + +fn owner_loop( + listener: TcpListener, + cfg: ServerConfig, + engine: &mut dyn DecodeIo, + mut cont: Option<&mut dyn ContExec>, +) { + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + if let Some(exec) = cont.as_deref_mut() { + exec.set_stop_requested(cfg.stop_requested); + } + let polling_stop = cfg.stop_requested.is_some(); + if polling_stop { + if let Err(error) = listener.set_nonblocking(true) { + eprintln!("ds4-server-rs: stop polling unavailable: {error}"); + shutdown_owner(engine, cont); + return; + } + } + let (jobs_tx, jobs_rx) = mpsc::channel(); + let accept_cfg = cfg.clone(); + let accept_inner = Arc::clone(&inner); + let accept = thread::Builder::new().spawn(move || { + loop { + if stop_requested(&accept_cfg) { + break; + } + let mut stream = match listener.accept() { + Ok((stream, _)) => { + if stop_requested(&accept_cfg) { + break; + } + stream + } + Err(e) if accept_error_retryable(e.kind()) => { + retry_accept(e.kind(), polling_stop); + continue; + } + Err(_) => break, + }; + let Some(client) = ClientLease::try_new(Arc::clone(&accept_inner)) else { + refuse_client_capacity(&mut stream, &accept_cfg); + continue; + }; + let cfg = accept_cfg.clone(); + let inner = Arc::clone(&accept_inner); + let jobs = jobs_tx.clone(); + let _ = thread::Builder::new() + .spawn(move || queue_client(cfg, inner, jobs, stream, client)); + } + lock_inner(&accept_inner).admit.stopping = true; + }); + if accept.is_err() { + lock_inner(&inner).admit.stopping = true; + shutdown_owner(engine, cont); + return; + } + + let mut lookahead = None; + loop { + let job = match lookahead.take() { + Some(job) => job, + None => match jobs_rx.recv() { + Ok(job) => job, + Err(_) => break, + }, + }; + lookahead = match cont.as_mut() { + Some(exec) => owner_static::run_owner_maybe_coalesce( + &cfg, + &inner, + engine, + &mut **exec, + job, + &jobs_rx, + ), + None => { + run_owner_job(&cfg, &inner, engine, None, job); + None + } + }; + } + shutdown_owner(engine, cont); +} + +fn shutdown_owner(engine: &mut dyn DecodeIo, mut cont: Option<&mut dyn ContExec>) { + if let Err(error) = engine.shutdown() { + eprintln!("ds4-server-rs: serial shutdown checkpoint failed: {error}"); + } + if let Some(exec) = cont.as_deref_mut() { + exec.shutdown(engine.kv_store_mut()); + } +} + +/// Client threads read and parse; this caller remains the sole non-`Send` +/// inference owner. Continuous jobs roll: a second queued request can admit +/// while the first is generating. Static jobs may coalesce. Serial stays FIFO. +pub fn accept_loop_with_engine( + listener: TcpListener, + cfg: ServerConfig, + engine: &mut dyn DecodeIo, +) { + owner_loop(listener, cfg, engine, None); +} + +/// The same owner selects continuous or serial execution after FIFO dequeue. +pub fn accept_loop_with_engine_cont( + listener: TcpListener, + cfg: ServerConfig, + engine: &mut dyn DecodeIo, + cont: &mut dyn ContExec, +) { + owner_loop(listener, cfg, engine, Some(cont)); +} + +/// Test helper: unused EnqVerdict keep-alive for the admit oracle dump. +pub fn enq_verdict_name(v: EnqVerdict) -> &'static str { + match v { + EnqVerdict::Ok => "ok", + EnqVerdict::Stopping => "stopping", + EnqVerdict::ShedQueueDepth => "shed_queue_depth", + EnqVerdict::ShedQueueBytes => "shed_queue_bytes", + } +} + +#[cfg(test)] +mod owner_tests { + use super::*; + use crate::generate::{ + generate_and_write_at, generate_terminal_at, GenerateOutcome, ScriptedDecode, + }; + use crate::parse::{ParseEnv, ParsedRequest}; + use crate::route::WireSurface; + use std::io::{Read, Write}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + static TEST_STOP: AtomicBool = AtomicBool::new(false); + static TEST_STOP_POLLS: AtomicUsize = AtomicUsize::new(0); + static DECODE_STOP: AtomicBool = AtomicBool::new(false); + + // A GLM-5.3 request that names no level takes its template's max; every + // other family keeps the port's low. + #[test] + fn glm53_requests_default_to_max_effort() { + let mut cfg = ServerConfig::default(); + assert_eq!(default_effort(&cfg), ThinkMode::Low); + cfg.max_effort_default = true; + assert_eq!(default_effort(&cfg), ThinkMode::Max); + } + + #[test] + fn mimo_image_keeps_serial_lane() { + let env = RouteEnv { + coalesce: false, + have_cont: false, + cont_anthropic: false, + cont_responses: false, + cont_tools_anthropic: false, + cont_tools_responses: false, + seq_cap: 262144, + prompt_len: 0, + }; + let dec = route_for_model( + ModelSyntax::Mimo2 as i32, + crate::route::NEED_IMAGE, + WireSurface::OpenaiChat, + &env, + ); + assert_eq!(dec.lane, crate::route::LANE_SERIAL); + } + + fn test_stop_requested() -> bool { + TEST_STOP_POLLS.fetch_add(1, Ordering::Relaxed); + TEST_STOP.load(Ordering::Relaxed) + } + + fn decode_stop_requested() -> bool { + DECODE_STOP.load(Ordering::Relaxed) + } + + enum TestCont { + Accept, + Reject, + Fail, + Hold(i32), + Shutdown(Arc>>), + } + + struct SlowPrepDecode { + sampled: bool, + pos: i32, + shutdown_order: Option>>>, + } + + struct DisconnectDecode { + started: Option>, + resume: Receiver<()>, + samples: usize, + evals: usize, + pos: i32, + } + + impl DecodeIo for DisconnectDecode { + fn model_id(&self) -> i32 { + 0 + } + fn tokenize_text(&self, _text: &str) -> Result, GenerateError> { + Ok(vec![1]) + } + fn tokenize_rendered_chat(&self, _text: &[u8]) -> Result, GenerateError> { + Ok(vec![1]) + } + fn tokenizes_control_literals(&self) -> bool { + false + } + fn token_text(&self, _token: i32) -> Result, GenerateError> { + Ok(b"x".to_vec()) + } + fn token_is_stop(&self, _token: i32) -> bool { + false + } + fn sync(&mut self, tokens: &[i32]) -> Result<(), GenerateError> { + self.pos = tokens.len() as i32; + Ok(()) + } + fn eval(&mut self, _token: i32) -> Result<(), GenerateError> { + self.evals += 1; + self.pos += 1; + Ok(()) + } + fn sample( + &mut self, + _temperature: f32, + _top_k: i32, + _top_p: f32, + _min_p: f32, + _rng: &mut u64, + ) -> i32 { + if let Some(started) = self.started.take() { + started.send(()).unwrap(); + self.resume.recv_timeout(Duration::from_secs(1)).unwrap(); + } + if self.samples >= 5 { + return -1; + } + self.samples += 1; + 1 + } + fn pos(&self) -> i32 { + self.pos + } + fn ctx(&self) -> i32 { + 8192 + } + fn generation(&self) -> u64 { + 1 + } + fn invalidate(&mut self) { + self.pos = 0; + } + } + + impl DecodeIo for SlowPrepDecode { + fn model_id(&self) -> i32 { + 0 + } + fn tokenize_text(&self, _text: &str) -> Result, GenerateError> { + thread::sleep(Duration::from_millis(100)); + Ok(vec![1]) + } + fn tokenize_rendered_chat(&self, _text: &[u8]) -> Result, GenerateError> { + self.tokenize_text("") + } + fn tokenizes_control_literals(&self) -> bool { + false + } + fn token_text(&self, _token: i32) -> Result, GenerateError> { + Ok(b"x".to_vec()) + } + fn token_is_stop(&self, _token: i32) -> bool { + false + } + fn sync(&mut self, tokens: &[i32]) -> Result<(), GenerateError> { + thread::sleep(Duration::from_millis(40)); + self.pos = tokens.len() as i32; + Ok(()) + } + fn eval(&mut self, _token: i32) -> Result<(), GenerateError> { + self.pos += 1; + Ok(()) + } + fn sample( + &mut self, + _temperature: f32, + _top_k: i32, + _top_p: f32, + _min_p: f32, + _rng: &mut u64, + ) -> i32 { + if self.sampled { + -1 + } else { + self.sampled = true; + 1 + } + } + fn pos(&self) -> i32 { + self.pos + } + fn ctx(&self) -> i32 { + 8192 + } + fn generation(&self) -> u64 { + 1 + } + fn invalidate(&mut self) { + self.pos = 0; + } + fn shutdown(&mut self) -> Result<(), GenerateError> { + if let Some(order) = &self.shutdown_order { + order.lock().unwrap().push("serial"); + } + Ok(()) + } + } + + impl ContExec for TestCont { + fn model_id(&self) -> i32 { + 0 + } + + fn seq_cap(&self) -> i32 { + 8192 + } + + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + vec![1] + } + + fn encode_text(&self, _text: &str) -> Vec { + vec![1] + } + + fn generate( + &mut self, + _parsed: &ParsedRequest, + _job_id: &str, + _created: i64, + cors: bool, + _default_tokens: i32, + _t_arrive: Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut ds4_kv::Store>, + out: &mut dyn Write, + ) -> Result { + match self { + Self::Accept => { + out.write_all(&http_response_bytes( + 200, + Some("application/json"), + None, + cors, + "{}", + )) + .map_err(|_| GenerateError::Io)?; + Ok(GenerateOutcome { + generation: 1, + frontier: 1, + finish: "stop".into(), + ..GenerateOutcome::default() + }) + } + Self::Reject => Err(GenerateError::Unsupported("serial fallback")), + Self::Fail => Err(GenerateError::Engine("native failure".into())), + Self::Hold(retry_after) => Err(GenerateError::ContinuationHold { + retry_after: *retry_after, + }), + Self::Shutdown(_) => unreachable!("shutdown-only continuous lane generated"), + } + } + + fn shutdown(&mut self, _store: Option<&mut ds4_kv::Store>) { + if let Self::Shutdown(order) = self { + order.lock().unwrap().push("bank"); + } + } + } + + fn test_cfg() -> ServerConfig { + let mut cfg = ServerConfig::test_cfg(); + cfg.default_tokens = 8; + cfg + } + + #[test] + fn test_cfg_enables_engine_and_keeps_serial_fit_unset() { + let cfg = ServerConfig::test_cfg(); + assert!(cfg.have_engine); + assert!(cfg.serial_fit.is_none()); + } + + #[test] + fn c_admission_defaults_are_preserved() { + let cfg = ServerConfig::default(); + assert_eq!(cfg.max_clients, 256); + assert_eq!(cfg.max_queue, 256); + assert_eq!(cfg.max_queue_bytes, 256 * 1024 * 1024); + assert_eq!(cfg.max_queue_age_s, 600.0); + assert_eq!(cfg.out_agg_cap_bytes, 64 * 1024 * 1024); + assert_eq!(cfg.out_agg_evict_min_bytes, 256 * 1024); + assert_eq!( + cfg.mem_floor_gb, + crate::serve_serial_reclaim::DEFAULT_MEM_FLOOR_GB + ); + assert!(cfg.serial_fit.is_none()); + } + + #[test] + fn generation_outcome_updates_common_token_counters() { + let mut inner = ServerInner::default(); + inner.record_generation(&GenerateOutcome { + timings: crate::stream::ReqTimings { + prefill_tokens: 18, + prefill_cached: 260, + decode_tokens: 4, + decode_steps: 4, + ..crate::stream::ReqTimings::default() + }, + speculation_active: true, + reuse: ds4_core::ReuseTaken::Exact, + ..GenerateOutcome::default() + }); + + assert_eq!(inner.runtime.tokens_prefilled_computed, 18); + assert_eq!(inner.runtime.tokens_prefilled_cached, 260); + assert_eq!(inner.runtime.tokens_decoded, 4); + assert_eq!(inner.runtime.decode_steps, 4); + let last = inner.last_request.as_ref().unwrap(); + assert_eq!(last.effective_lane, "serial"); + assert_eq!(last.reuse_kind, ds4_core::ReuseTaken::Exact); + assert!(last.speculation_active); + assert!(last.fallback_reason.is_none()); + } + + #[test] + fn adopting_a_bank_plan_overrides_a_legacy_serial_config() { + let req = ds4_core::ServingRequest { + max_seqs: ds4_core::MaxSeqs::Fixed(2), + mem_floor_gb: 12, + ..ds4_core::ServingRequest::default() + }; + let plan = ds4_core::resolve_plan( + &req, + Some(ds4_core::serving_caps( + ds4_core::ModelFamily::Qwen4Exp, + ds4_core::Variant::Qwen38FlashNext, + )), + &ds4_core::EngineFacts::default(), + ); + // `DS4_SERVER_CONTINUOUS=0` was captured before the plan existed. + let mut cfg = ServerConfig::default(); + cfg.continuous = false; + cfg.adopt_plan(&plan); + assert!(cfg.continuous); + assert_eq!(cfg.mem_floor_gb, 12); + assert!(cfg.serving_plan.is_some()); + } + + #[test] + fn adopting_a_serial_plan_leaves_the_lane_flag_alone() { + let req = ds4_core::ServingRequest { + max_seqs: ds4_core::MaxSeqs::Off, + ..ds4_core::ServingRequest::default() + }; + let plan = ds4_core::resolve_plan( + &req, + Some(ds4_core::serving_caps( + ds4_core::ModelFamily::Qwen4Exp, + ds4_core::Variant::Qwen38FlashNext, + )), + &ds4_core::EngineFacts::default(), + ); + let mut cfg = ServerConfig::default(); + cfg.continuous = false; + cfg.adopt_plan(&plan); + assert!(!cfg.continuous); + } + + #[test] + fn dots3_mtp_retracts_bank_route() { + for max_seqs in [ds4_core::MaxSeqs::Auto, ds4_core::MaxSeqs::Fixed(1)] { + let req = ds4_core::ServingRequest { + max_seqs, + mtp_mode: ds4_core::MtpMode::On, + ..ds4_core::ServingRequest::default() + }; + let plan = ds4_core::resolve_plan( + &req, + Some(ds4_core::serving_caps( + ds4_core::ModelFamily::Dots3Note, + ds4_core::Variant::Dots3NotePrev, + )), + &ds4_core::EngineFacts::default(), + ); + assert!(plan.may_listen()); + let mut cfg = ServerConfig::default(); + cfg.continuous = true; + cfg.adopt_plan(&plan); + assert!( + !cfg.continuous, + "serial MTP must retract routing: {max_seqs:?}" + ); + assert_eq!( + cfg.serving_plan.unwrap().effective.mtp_mode, + ds4_core::MtpMode::On + ); + } + } + + #[test] + fn static_settlement_records_static_lane() { + let mut inner = ServerInner::default(); + inner.record_generation(&GenerateOutcome { + lane: Some("static"), + timings: crate::stream::ReqTimings { + prefill_tokens: 8, + decode_tokens: 3, + ..crate::stream::ReqTimings::default() + }, + ..GenerateOutcome::default() + }); + let last = inner.last_request.as_ref().unwrap(); + assert_eq!(last.effective_lane, "static"); + assert_eq!(last.reuse_kind, ds4_core::ReuseTaken::Cold); + assert!(!last.speculation_active); + assert_eq!(last.reuse_miss, ds4_core::ReuseMiss::None); + } + + #[test] + fn stats_exposes_serving_and_last_request() { + let req = ds4_core::ServingRequest { + max_seqs: ds4_core::MaxSeqs::Fixed(2), + mem_floor_gb: 12, + ..ds4_core::ServingRequest::default() + }; + let plan = ds4_core::resolve_plan( + &req, + Some(ds4_core::serving_caps( + ds4_core::ModelFamily::Qwen4Exp, + ds4_core::Variant::Qwen38FlashNext, + )), + &ds4_core::EngineFacts::default(), + ); + let mut cfg = ServerConfig::default(); + cfg.serving_plan = Some(plan); + + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + lock_inner(&inner).record_generation(&GenerateOutcome { + bank: Some(0), + timings: crate::stream::ReqTimings { + prefill_tokens: 18, + prefill_cached: 260, + decode_tokens: 7, + decode_steps: 4, + ..crate::stream::ReqTimings::default() + }, + speculation_active: true, + // A request that took nothing is the one that carries a reason. + reuse: ds4_core::ReuseTaken::Cold, + reuse_miss: ds4_core::ReuseMiss::RenderedPrefix, + ..GenerateOutcome::default() + }); + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + write!(client, "GET /v1/stats HTTP/1.1\r\n\r\n").unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + handle_client_inner(&cfg, &inner, &mut server, None, None); + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + let text = String::from_utf8(response).unwrap(); + assert!(text.starts_with("HTTP/1.1 200 OK"), "{text}"); + let json_start = text.find("{\"routes\"").expect(&text); + let body: serde_json::Value = + serde_json::from_str(text[json_start..].trim()).expect(&text[json_start..]); + assert!(body["serving"]["requested"].is_object(), "{body}"); + assert!(body["serving"]["effective"].is_object(), "{body}"); + assert!(body["serving"]["qualified"].is_object(), "{body}"); + assert_eq!(body["last_request"]["effective_lane"], "continuous"); + assert_eq!(body["last_request"]["reuse_kind"], "cold"); + assert_eq!( + body["last_request"]["reuse_miss"], + "rendered prefix changed" + ); + assert_eq!(body["last_request"]["speculation_active"], true); + assert!(body["last_request"].get("fallback_reason").is_some()); + } + + #[cfg(not(feature = "native"))] + #[test] + fn live_speculation_replaces_cached_counters() { + let mut inner = ServerInner::default(); + inner.have_engine = true; + inner.runtime.spec_drafts = 99; + inner.runtime.spec_hits = 88; + inner.runtime.spec_quench = 77; + // The model-free bridge fixture supplies fresh process totals. + SPEC_COUNTS.with(|counts| counts.set([9, 6, 1])); + for _ in 0..2 { + let rt = inner.render_runtime(0); + assert_eq!((rt.spec_drafts, rt.spec_hits, rt.spec_quench), (9, 6, 1)); + } + SPEC_COUNTS.with(|counts| counts.set([12, 8, 2])); + let rt = inner.render_runtime(0); + assert_eq!((rt.spec_drafts, rt.spec_hits, rt.spec_quench), (12, 8, 2)); + inner.have_engine = false; + let rt = inner.render_runtime(0); + assert_eq!((rt.spec_drafts, rt.spec_hits, rt.spec_quench), (99, 88, 77)); + SPEC_COUNTS.with(|counts| counts.set([0; 3])); + } + + #[cfg(not(feature = "native"))] + thread_local! { + static SPEC_COUNTS: std::cell::Cell<[u64; 3]> = const { std::cell::Cell::new([0; 3]) }; + } + + #[cfg(not(feature = "native"))] + #[no_mangle] + extern "C" fn ds4_bridge_spec_snapshot(out: *mut ds4_sys::ds4_bridge_spec_metrics) { + let [drafts, hits, quench] = SPEC_COUNTS.with(std::cell::Cell::get); + // SAFETY: snapshot_spec lends a live output slot for this fixture call. + unsafe { + *out = ds4_sys::ds4_bridge_spec_metrics { + drafts, + hits, + quench, + } + }; + } + + #[test] + fn client_capacity_is_reserved_before_request_body_read() { + let mut cfg = test_cfg(); + cfg.max_clients = 1; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + + let first = ClientLease::try_new(Arc::clone(&inner)).expect("first client"); + assert!(ClientLease::try_new(Arc::clone(&inner)).is_none()); + assert_eq!(lock_inner(&inner).admit.clients, 1); + + drop(first); + assert_eq!(lock_inner(&inner).admit.clients, 0); + } + + #[test] + fn c_admission_environment_overrides_parse_without_unsafe_env_mutation() { + let os = OsStr::new; + assert_eq!(parse_i32_bound(Some(os("7")), 256), 7); + assert_eq!(parse_i32_bound(Some(os(" 7jobs")), 256), 7); + assert_eq!(parse_i32_bound(Some(os("\u{2003}7")), 256), 0); + assert_eq!(parse_i32_bound(Some(os("-1")), 256), 0); + assert_eq!(parse_i32_bound(Some(os("bad")), 256), 0); + assert_eq!(parse_u64_bound(Some(os("0")), 64), 0); + assert_eq!(parse_u64_bound(Some(os("1048576")), 64), 1_048_576); + assert_eq!(parse_u64_bound(Some(os("1048576bytes")), 64), 1_048_576); + assert_eq!(parse_u64_bound(Some(os("-1")), 64), u64::MAX); + assert_eq!( + parse_u64_bound(Some(os("18446744073709551616")), 64), + u64::MAX + ); + assert_eq!(parse_u64_bound(Some(os("bad")), 64), 0); + assert_eq!(parse_f64_bound(Some(os("2.5")), 600.0), 2.5); + assert_eq!(parse_f64_bound(Some(os(" 2.5seconds")), 600.0), 2.5); + assert_eq!(parse_f64_bound(Some(os("0x1p2seconds")), 600.0), 4.0); + assert!(parse_f64_bound(Some(os("nan")), 600.0).is_nan()); + assert_eq!(parse_f64_bound(Some(os("-1")), 600.0), 0.0); + assert_eq!(parse_f64_bound(Some(os("bad")), 600.0), 0.0); + } + + #[cfg(unix)] + #[test] + fn c_admission_environment_overrides_preserve_non_utf8_bytes() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + assert_eq!(parse_i32_bound(Some(OsStr::from_bytes(b"7\xff")), 256), 7); + assert_eq!( + parse_u64_bound(Some(OsStr::from_bytes(b"1048576\xff")), 64), + 1_048_576 + ); + assert_eq!( + parse_f64_bound(Some(OsStr::from_bytes(b"2.5\xff")), 600.0), + 2.5 + ); + assert!(!parse_default_on(Some(OsStr::from_bytes(b"0")))); + assert!(parse_default_on(Some(OsStr::from_bytes(b"0\xff")))); + } + + #[test] + fn c_disconnect_abort_environment_contract_is_preserved() { + assert!(parse_default_on(None)); + assert!(!parse_default_on(Some(OsStr::new("0")))); + assert!(parse_default_on(Some(OsStr::new("00")))); + assert!(parse_default_on(Some(OsStr::new("1")))); + assert!(parse_default_on(Some(OsStr::new("bad")))); + } + + #[test] + fn default_queue_bound_limits_owner_channel_metadata() { + let mut cfg = test_cfg(); + cfg.max_clients = 0; + assert_eq!(cfg.max_queue, 256); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (jobs_tx, jobs_rx) = mpsc::channel(); + let body = r#"{"prompt":"queued","max_tokens":0}"#; + let request = format!( + "POST /v1/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let mut clients = Vec::with_capacity(256); + let mut threads = Vec::with_capacity(256); + + for expected in 1..=256 { + let mut client = TcpStream::connect(addr).unwrap(); + let (server, _) = listener.accept().unwrap(); + client.write_all(request.as_bytes()).unwrap(); + let client_cfg = cfg.clone(); + let client_inner = Arc::clone(&inner); + let client_jobs = jobs_tx.clone(); + let lease = ClientLease::try_new(Arc::clone(&inner)).unwrap(); + threads.push(thread::spawn(move || { + queue_client(client_cfg, client_inner, client_jobs, server, lease) + })); + for _ in 0..1000 { + if lock_inner(&inner).admit.queued == expected { + break; + } + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(lock_inner(&inner).admit.queued, expected); + clients.push(client); + } + + let mut refused = TcpStream::connect(addr).unwrap(); + refused + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let (server, _) = listener.accept().unwrap(); + refused.write_all(request.as_bytes()).unwrap(); + let refuse_cfg = cfg.clone(); + let refuse_inner = Arc::clone(&inner); + let refuse_jobs = jobs_tx.clone(); + let refuse_lease = ClientLease::try_new(Arc::clone(&inner)).unwrap(); + let refuse_thread = thread::spawn(move || { + queue_client(refuse_cfg, refuse_inner, refuse_jobs, server, refuse_lease) + }); + let mut response = Vec::new(); + refused.read_to_end(&mut response).unwrap(); + refuse_thread.join().unwrap(); + assert!( + String::from_utf8_lossy(&response).starts_with("HTTP/1.1 429 Too Many Requests"), + "{}", + String::from_utf8_lossy(&response) + ); + assert_eq!(lock_inner(&inner).admit.queued, 256); + + let jobs: Vec<_> = jobs_rx.try_iter().collect(); + assert_eq!( + jobs.len(), + 256, + "unbounded channel must be admission-bounded" + ); + drop(jobs); + drop(clients); + for thread in threads { + thread.join().unwrap(); + } + assert_eq!(lock_inner(&inner).admit.queued, 0); + } + + #[test] + // C lane-entry contract (ds4_server.c ~10794 + live 2026-08-26 + // surfaces evidence): cont_admit success ticks continuous before the + // engine funding verdict; the stranded fallback's generate_job entry + // ticks serial too — C /v1/stats shows openai_chat.serial:1 AND + // openai_chat.continuous:1 for one stranded request. + fn fallback_records_actual_serial_lane() { + let cfg = test_cfg(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let body = + r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#; + write!( + client, + "POST /v1/chat/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[b"ok"]); + let mut cont = TestCont::Reject; + + handle_client_inner( + &cfg, + &inner, + &mut server, + Some(&mut engine), + Some(&mut cont), + ); + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + + assert!(String::from_utf8_lossy(&response).starts_with("HTTP/1.1 200 OK")); + let g = inner.lock().unwrap(); + assert_eq!(g.metrics.route_requests[0][0], 1); + assert_eq!(g.metrics.route_requests[0][1], 1); + let last = g.last_request.as_ref().unwrap(); + assert_eq!(last.effective_lane, "serial"); + assert_eq!(last.fallback_reason.as_deref(), Some("serial fallback")); + } + + #[test] + fn continuous_lane_does_not_take_serial_hold() { + let cfg = test_cfg(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + lock_inner(&inner).creg.publish_serial( + Api::Responses, + &["live-call".into()], + 1, + 1, + monotonic_now(), + ); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let body = + r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#; + write!( + client, + "POST /v1/chat/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[]); + let mut cont = TestCont::Accept; + + handle_client_inner( + &cfg, + &inner, + &mut server, + Some(&mut engine), + Some(&mut cont), + ); + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + + assert!(String::from_utf8_lossy(&response).starts_with("HTTP/1.1 200 OK")); + let g = lock_inner(&inner); + assert_eq!(g.metrics.route_requests[0][1], 1); + assert_eq!(g.metrics.shed[SHED_CONT_HOLD as usize], 0); + assert_eq!(g.runtime.requests_serial, 0); + } + + #[test] + fn protected_bank_saturation_sheds_with_retry_without_serial_fallback() { + let cfg = test_cfg(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let body = + r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#; + write!( + client, + "POST /v1/chat/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[b"serial fallback"]); + let mut cont = TestCont::Hold(7); + + handle_client_inner( + &cfg, + &inner, + &mut server, + Some(&mut engine), + Some(&mut cont), + ); + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + let response = String::from_utf8(response).unwrap(); + + assert!( + response.starts_with("HTTP/1.1 503 Service Unavailable"), + "{response}" + ); + assert!(response.contains("Retry-After: 7\r\n"), "{response}"); + let (_, body) = response.split_once("\r\n\r\n").unwrap(); + assert_eq!( + body, + "{\"error\":{\"message\":\"batch capacity is reserved for live tool continuations; retry shortly\",\"type\":\"server_error\"}}\n" + ); + let g = lock_inner(&inner); + assert_eq!(g.metrics.shed[SHED_CONT_HOLD as usize], 1); + assert_eq!(g.runtime.requests_serial, 0); + } + + #[test] + fn stateless_anthropic_uses_continuous_lane() { + let cfg = test_cfg(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let body = r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":8}"#; + write!( + client, + "POST /v1/messages HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[]); + let mut cont = TestCont::Accept; + + handle_client_inner( + &cfg, + &inner, + &mut server, + Some(&mut engine), + Some(&mut cont), + ); + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + + assert!(String::from_utf8_lossy(&response).starts_with("HTTP/1.1 200 OK")); + let g = lock_inner(&inner); + assert_eq!( + g.metrics.route_requests[WireSurface::Anthropic as usize][1], + 1 + ); + assert_eq!(g.runtime.requests_serial, 0); + } + + #[test] + fn stateless_responses_uses_continuous_lane() { + let cfg = test_cfg(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let body = r#"{"input":"hi","max_output_tokens":8}"#; + write!( + client, + "POST /v1/responses HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[]); + let mut cont = TestCont::Accept; + + handle_client_inner( + &cfg, + &inner, + &mut server, + Some(&mut engine), + Some(&mut cont), + ); + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + + assert!(String::from_utf8_lossy(&response).starts_with("HTTP/1.1 200 OK")); + let g = lock_inner(&inner); + assert_eq!( + g.metrics.route_requests[WireSurface::Responses as usize][1], + 1 + ); + assert_eq!(g.runtime.requests_serial, 0); + } + + #[test] + fn job_sink_is_bounded_and_disconnect_aware() { + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&ServerConfig::default()))); + let (mut slow, slow_state) = job_sink(Arc::clone(&inner)); + let err = slow + .write_all(&vec![0; JOB_SINK_CAP_BYTES as usize + 1]) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::WouldBlock); + assert!(slow_state.slow()); + assert_eq!(lock_inner(&inner).runtime.out_backlog_bytes, 0); + + lock_inner(&inner).runtime.out_backlog_bytes = JOB_SINK_AGG_CAP_BYTES; + let (mut aggregate, aggregate_state) = job_sink(Arc::clone(&inner)); + aggregate.write_all(b"x").unwrap(); + assert_eq!(aggregate_state.backlog_bytes(), 1); + assert_eq!( + aggregate + .write_all(&vec![0; 256 * 1024 - 1]) + .unwrap_err() + .kind(), + std::io::ErrorKind::WouldBlock + ); + assert!(aggregate_state.slow()); + lock_inner(&inner).runtime.out_backlog_bytes = 0; + + let (mut gone, gone_state) = job_sink(Arc::clone(&inner)); + gone_state.cancel(false); + let err = gone.write_all(b"x").unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::BrokenPipe); + assert!(gone_state.gone()); + assert!(!gone_state.slow()); + + let (mut sink, state) = job_sink(Arc::clone(&inner)); + let expected: Vec = (0..100_000).map(|i| (i % 251) as u8).collect(); + for (i, byte) in expected.iter().enumerate() { + sink.write_all(std::slice::from_ref(byte)).unwrap(); + if i == 256 { + assert_eq!(state.backlog_bytes(), 257); + } + } + assert_eq!(state.backlog_bytes(), expected.len() as u64); + assert!(!state.slow()); + assert_eq!( + lock_inner(&inner).runtime.out_backlog_bytes, + expected.len() as u64 + ); + assert_eq!(state.take(), expected); + assert_eq!(state.backlog_bytes(), 0); + assert_eq!(lock_inner(&inner).runtime.out_backlog_bytes, 0); + } + + #[test] + fn rolling_sink_clone_does_not_close_the_owner_sink() { + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&ServerConfig::default()))); + let (mut owner, state) = job_sink(inner); + let mut rolling = owner.clone(); + rolling.write_all(b"head").unwrap(); + drop(rolling); + + assert!(!state.lock().closed); + owner.write_all(b"tail").unwrap(); + drop(owner); + assert!(state.lock().closed); + assert_eq!(state.take(), b"headtail"); + } + + fn tool_outcome(id: &str, generation: u64, frontier: i32) -> GenerateOutcome { + GenerateOutcome { + tool_ids: vec![id.into()], + bank: None, + generation, + frontier, + finish: "tool_calls".into(), + ..GenerateOutcome::default() + } + } + + #[test] + fn terminal_commit_ladder_matches_c_ordering() { + let cfg = test_cfg(); + + let clean_inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (mut clean, clean_state) = job_sink(Arc::clone(&clean_inner)); + let clean_watch = Arc::clone(&clean_state); + let clean_registry = Arc::clone(&clean_inner); + let wake = thread::spawn(move || { + let mut buffer = clean_watch.lock(); + while buffer.bytes.is_empty() { + buffer = clean_watch + .ready + .wait(buffer) + .unwrap_or_else(|e| e.into_inner()); + } + drop(buffer); + assert!( + lock_inner(&clean_registry).creg.live_has_id( + Api::Responses, + "call-clean", + monotonic_now() + ), + "terminal became drainable before publication" + ); + }); + assert_eq!( + clean.commit_tool_terminal( + &clean_inner, + Api::Responses, + &tool_outcome("call-clean", 2, 20), + b"terminal".to_vec(), + ), + TerminalCommit::Committed + ); + wake.join().unwrap(); + assert_eq!(clean_state.take(), b"terminal"); + assert_eq!(lock_inner(&clean_inner).runtime.out_backlog_bytes, 0); + + let gone_inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (mut gone, gone_state) = job_sink(Arc::clone(&gone_inner)); + gone_state.cancel(false); + assert_eq!( + gone.commit_tool_terminal( + &gone_inner, + Api::Responses, + &tool_outcome("call-gone", 3, 30), + b"terminal".to_vec(), + ), + TerminalCommit::GoneBeforeStart + ); + { + let mut g = lock_inner(&gone_inner); + assert!(g.creg.id_known("call-gone")); + assert!(!g + .creg + .live_has_id(Api::Responses, "call-gone", monotonic_now())); + assert_eq!(g.runtime.out_backlog_bytes, 0); + } + + let full_inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + { + let mut g = lock_inner(&full_inner); + g.creg + .publish_serial(Api::Responses, &["call-old".into()], 1, 10, monotonic_now()); + g.runtime.out_backlog_bytes = g.out_agg_cap_bytes; + } + let (mut full, full_state) = job_sink(Arc::clone(&full_inner)); + assert_eq!( + full.commit_tool_terminal( + &full_inner, + Api::Responses, + &tool_outcome("call-full", 4, 40), + b"terminal".to_vec(), + ), + TerminalCommit::Full + ); + let mut g = lock_inner(&full_inner); + assert!(!g.creg.id_known("call-full")); + assert!(g + .creg + .live_has_id(Api::Responses, "call-old", monotonic_now())); + assert!(full_state.slow()); + } + + #[test] + fn production_terminal_probe_demotes_a_closed_tcp_client() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + server.set_nonblocking(true).unwrap(); + client.shutdown(std::net::Shutdown::Both).unwrap(); + drop(client); + let (mut sink, state) = job_sink_with_probe(Arc::clone(&inner), Some(server)); + + assert_eq!( + sink.commit_tool_terminal( + &inner, + Api::Responses, + &tool_outcome("call-tcp-gone", 5, 50), + b"terminal".to_vec(), + ), + TerminalCommit::GoneBeforeStart + ); + assert!(state.gone()); + let mut g = lock_inner(&inner); + assert!(g.creg.id_known("call-tcp-gone")); + assert!(!g + .creg + .live_has_id(Api::Responses, "call-tcp-gone", monotonic_now())); + assert_eq!(g.runtime.out_backlog_bytes, 0); + } + + #[test] + fn disconnect_abort_zero_disables_only_mid_generation_probe() { + for (disconnect_abort, expect_error) in [(false, false), (true, true)] { + let mut cfg = test_cfg(); + cfg.disconnect_abort = disconnect_abort; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + server.set_nonblocking(true).unwrap(); + client.shutdown(std::net::Shutdown::Both).unwrap(); + drop(client); + let (mut sink, state) = job_sink_with_probe(Arc::clone(&inner), Some(server)); + + assert_eq!(sink.flush().is_err(), expect_error); + assert_eq!(state.gone(), expect_error); + } + } + + #[test] + fn terminal_capture_preserves_anthropic_and_responses_stream_prefix() { + let env = ParseEnv { + default_model: "ds4".into(), + default_tokens: 16, + default_effort: ThinkMode::None, + default_temp: 0.0, + live_ids: Vec::new(), + engine_defaults: false, + }; + let mut parsed = parse_request( + WireSurface::Anthropic, + &env, + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":16,"stream":true,"thinking":{"type":"disabled"},"tools":[{"name":"bash","input_schema":{"type":"object"}}]}"#, + ) + .unwrap(); + parsed.think_mode = ThinkMode::None; + parsed.temperature = 0.0; + let block = concat!( + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"bash\">\n", + "<|DSML|parameter name=\"command\" string=\"true\">ls", + "\n", + "\n", + "" + ); + + for (api, terminal_marker) in [ + (Api::Anthropic, "event: message_stop"), + (Api::Responses, "\"type\":\"response.completed\""), + ] { + parsed.api = api; + let arrive = Instant::now(); + let mut captured_engine = ScriptedDecode::from_pieces(&[block.as_bytes()]); + let mut prefix = Vec::new(); + let (outcome, terminal) = generate_terminal_at( + &mut captured_engine, + &parsed, + "tool-stream", + 1, + false, + 16, + arrive, + &mut prefix, + ) + .unwrap(); + assert!(!outcome.tool_ids.is_empty()); + let prefix_text = String::from_utf8_lossy(&prefix); + let terminal_text = String::from_utf8_lossy(&terminal); + assert!(!prefix_text.contains(terminal_marker), "{prefix_text}"); + assert!(terminal_text.contains(terminal_marker), "{terminal_text}"); + + let mut full_engine = ScriptedDecode::from_pieces(&[block.as_bytes()]); + let mut full = Vec::new(); + generate_and_write_at( + &mut full_engine, + &parsed, + "tool-stream", + 1, + false, + 16, + arrive, + &mut full, + ) + .unwrap(); + let stable = [b"call_".as_slice(), b"toolu_".as_slice()] + .iter() + .filter_map(|needle| { + prefix + .windows(needle.len()) + .position(|window| window == *needle) + }) + .min() + .unwrap_or(prefix.len()); + assert!(full.starts_with(&prefix[..stable])); + assert!(String::from_utf8_lossy(&full).contains(terminal_marker)); + } + } + + #[test] + fn direct_send_failure_after_start_keeps_tool_turn_live() { + let cfg = test_cfg(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let outcome = tool_outcome("call-partial", 5, 50); + let result = + direct_terminal_commit(&inner, Api::Responses, &outcome, b"terminal", |action| { + match action { + DirectTerminalIo::Disconnected => Ok(false), + DirectTerminalIo::Send(_) => { + Err(Error::new(ErrorKind::BrokenPipe, "late failure")) + } + } + }); + + assert_eq!(result, TerminalCommit::FailedAfterStart); + let mut g = lock_inner(&inner); + assert!(g + .creg + .live_has_id(Api::Responses, "call-partial", monotonic_now())); + assert_eq!(g.runtime.out_backlog_bytes, 0); + } + + #[test] + fn terminal_full_sheds_once_without_publishing_new_turn() { + let mut cfg = test_cfg(); + cfg.out_agg_cap_bytes = 1; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + lock_inner(&inner).runtime.out_backlog_bytes = 1; + let env = ParseEnv { + default_model: "ds4".into(), + default_tokens: 16, + default_effort: ThinkMode::None, + default_temp: 0.0, + live_ids: Vec::new(), + engine_defaults: false, + }; + let parsed = parse_request( + WireSurface::Anthropic, + &env, + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":16,"thinking":{"type":"disabled"},"tools":[{"name":"bash","input_schema":{"type":"object"}}]}"#, + ) + .unwrap(); + let mut prepared = PreparedJob { + parsed, + cont_prompt: None, + surface: WireSurface::Anthropic, + body_bytes: 11, + arrived_at: Instant::now(), + }; + let lease = admit_test_job(&inner, &mut prepared); + let (job, drain) = owner_job(prepared, lease); + let block = concat!( + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"bash\">\n", + "<|DSML|parameter name=\"command\" string=\"true\">ls", + "\n", + "\n", + "" + ); + let mut engine = ScriptedDecode::from_pieces(&[block.as_bytes()]); + + run_owner_job(&cfg, &inner, &mut engine, None, job); + assert!(settle_drain(drain).is_empty()); + + let g = lock_inner(&inner); + assert_eq!(g.metrics.shed[SHED_SLOW_READER as usize], 1); + assert_eq!(g.runtime.requests_completed, 0); + assert_eq!(g.runtime.requests_canceled, 0); + assert_eq!(g.runtime.requests_failed, 0); + assert_eq!(g.creg.n_live(), 0); + } + + fn admit_test_job(inner: &Arc>, prepared: &mut PreparedJob) -> JobLease { + let mut g = lock_inner(inner); + assert_eq!(enqueue(&mut g.admit, prepared.body_bytes), EnqVerdict::Ok); + let pin = acquire_continuation_pin(&mut g, &mut prepared.parsed, monotonic_now()); + g.runtime.requests_started += 1; + g.runtime.requests_inflight += 1; + drop(g); + JobLease::new(Arc::clone(inner), prepared.body_bytes, pin) + } + + fn queued_completion(prompt: &str, body_bytes: u64) -> PreparedJob { + let env = ParseEnv { + default_model: "ds4".into(), + default_tokens: 8, + default_effort: ThinkMode::None, + default_temp: 0.0, + live_ids: Vec::new(), + engine_defaults: false, + }; + let body = format!(r#"{{"prompt":"{prompt}","max_tokens":0}}"#); + let parsed = parse_request(WireSurface::OpenaiCompletion, &env, &body).unwrap(); + PreparedJob { + parsed, + cont_prompt: None, + surface: WireSurface::OpenaiCompletion, + body_bytes, + arrived_at: Instant::now(), + } + } + + #[test] + fn started_stream_failure_never_appends_a_second_http_response() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (mut sink, state) = job_sink(Arc::clone(&inner)); + let mut job = queued_completion("hello", 1); + job.parsed.stream = true; + sink.write_all(&crate::stream::sse_headers(false)).unwrap(); + + let settlement = settle_generation_result( + &cfg, + &job, + Err(GenerateError::Engine("boom".into())), + &mut sink, + ); + + assert_eq!(settlement.outcome, Settle::Failed); + let wire = String::from_utf8(state.take()).unwrap(); + assert_eq!(wire.matches("HTTP/1.1").count(), 1, "{wire}"); + assert!(wire.contains("event: error\ndata:"), "{wire}"); + assert!(!wire.contains("HTTP/1.1 500"), "{wire}"); + } + + fn response_number(response: &str, key: &str) -> f64 { + let start = response.find(key).unwrap() + key.len(); + let end = response[start..] + .find(|c: char| !c.is_ascii_digit() && c != '.') + .map(|n| start + n) + .unwrap(); + response[start..end].parse().unwrap() + } + + fn settle_drain(drain: JobDrain) -> Vec { + let bytes = drain.state.take(); + let mut lease = drain.done.recv().unwrap(); + if drain.state.gone() { + lease.settlement = lease.settlement.transport_gone(); + } + drop(lease); + bytes + } + + #[test] + fn owner_fifo_settles_body_once() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut first = queued_completion("first", 10); + let mut second = queued_completion("second", 20); + let first_lease = admit_test_job(&inner, &mut first); + let second_lease = admit_test_job(&inner, &mut second); + let (first, first_drain) = owner_job(first, first_lease); + let (second, second_drain) = owner_job(second, second_lease); + let (tx, rx) = mpsc::channel(); + tx.send(first).unwrap(); + tx.send(second).unwrap(); + drop(tx); + let mut engine = ScriptedDecode::from_pieces(&[]); + + let first = rx.recv().unwrap(); + assert_eq!(first.prepared.parsed.prompt_text.as_deref(), Some("first")); + run_owner_job(&cfg, &inner, &mut engine, None, first); + assert!(first_drain.state.backlog_bytes() > 0); + { + let g = lock_inner(&inner); + assert_eq!(g.admit.queued, 1); + assert_eq!(g.admit.inflight_body_bytes, 30); + assert_eq!(g.runtime.requests_completed, 0); + assert_eq!(g.runtime.requests_inflight, 2); + } + + let second = rx.recv().unwrap(); + assert_eq!( + second.prepared.parsed.prompt_text.as_deref(), + Some("second") + ); + run_owner_job(&cfg, &inner, &mut engine, None, second); + let second_bytes = settle_drain(second_drain); + assert!(String::from_utf8_lossy(&second_bytes).contains("\"id\":\"cmpl-2\"")); + { + let g = lock_inner(&inner); + assert_eq!(g.admit.queued, 0); + assert_eq!(g.admit.inflight_body_bytes, 10); + assert_eq!(g.runtime.requests_completed, 1); + assert_eq!(g.runtime.requests_inflight, 1); + } + let first_bytes = settle_drain(first_drain); + assert!(String::from_utf8_lossy(&first_bytes).contains("\"id\":\"cmpl-1\"")); + let g = lock_inner(&inner); + assert_eq!(g.admit.queued, 0); + assert_eq!(g.admit.inflight_body_bytes, 0); + assert_eq!(g.runtime.requests_started, 2); + assert_eq!(g.runtime.requests_completed, 2); + assert_eq!(g.runtime.requests_inflight, 0); + assert_eq!(g.runtime.requests_serial, 2); + } + + #[test] + fn queued_pin_releases_and_ttl_uses_execution_time() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let now = monotonic_now(); + { + let mut g = lock_inner(&inner); + g.creg.ttl_s = 0.01; + g.creg + .publish_serial(Api::Responses, &["call-1".into()], 1, 1, now); + } + let mut prepared = queued_completion("tool output", 19); + prepared.surface = WireSurface::Responses; + prepared.parsed.api = Api::Responses; + prepared.parsed.live_call_ids = vec!["call-1".into()]; + prepared.parsed.responses_requires_live_tool_state = true; + let lease = admit_test_job(&inner, &mut prepared); + assert_eq!(lock_inner(&inner).creg.serial_live_hard_refs(), 1); + thread::sleep(Duration::from_millis(20)); + let (job, drain) = owner_job(prepared, lease); + let mut engine = ScriptedDecode::from_pieces(&[]); + engine.pos = 1; + + run_owner_job(&cfg, &inner, &mut engine, None, job); + let response = String::from_utf8(settle_drain(drain)).unwrap(); + + assert!(response.starts_with("HTTP/1.1 409 Conflict"), "{response}"); + let g = lock_inner(&inner); + assert_eq!(g.creg.serial_live_hard_refs(), 0); + assert_eq!(g.admit.inflight_body_bytes, 0); + assert_eq!(g.runtime.requests_completed, 1); + assert_eq!(g.runtime.requests_inflight, 0); + } + + #[test] + fn queued_bank_pin_releases_after_failed_job() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let now = monotonic_now(); + { + let mut g = lock_inner(&inner); + g.creg.grace_s = 0.0; + g.creg + .publish_bank(Api::Responses, &["call-bank".into()], 2, 7, 100, now); + } + let mut prepared = queued_completion("tool output", 19); + prepared.surface = WireSurface::Responses; + prepared.parsed.api = Api::Responses; + prepared.parsed.live_call_ids = vec!["call-bank".into()]; + prepared.parsed.responses_requires_live_tool_state = true; + let lease = admit_test_job(&inner, &mut prepared); + assert!(lock_inner(&inner) + .creg + .bank_protected(2, Some((7, 100)), monotonic_now())); + let (job, drain) = owner_job(prepared, lease); + let mut engine = ScriptedDecode::from_pieces(&[]); + engine.pos = 100; + + run_owner_job(&cfg, &inner, &mut engine, None, job); + let response = String::from_utf8(settle_drain(drain)).unwrap(); + + assert!(response.starts_with("HTTP/1.1 409 Conflict"), "{response}"); + assert!(!lock_inner(&inner) + .creg + .bank_protected(2, Some((7, 100)), monotonic_now())); + } + + #[test] + fn continuation_hold_is_shed_only_once() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + lock_inner(&inner).creg.publish_serial( + Api::Responses, + &["live-call".into()], + 1, + 1, + monotonic_now(), + ); + let mut prepared = queued_completion("unrelated", 17); + let lease = admit_test_job(&inner, &mut prepared); + let (job, drain) = owner_job(prepared, lease); + let mut engine = ScriptedDecode::from_pieces(&[]); + + run_owner_job(&cfg, &inner, &mut engine, None, job); + let response = String::from_utf8(settle_drain(drain)).unwrap(); + + assert!( + response.starts_with("HTTP/1.1 503 Service Unavailable"), + "{response}" + ); + let g = lock_inner(&inner); + assert_eq!(g.metrics.shed[SHED_CONT_HOLD as usize], 1); + assert_eq!(g.runtime.requests_completed, 0); + assert_eq!(g.runtime.requests_failed, 0); + assert_eq!(g.runtime.requests_canceled, 0); + assert_eq!(g.runtime.requests_inflight, 0); + assert_eq!(g.runtime.requests_serial, 0); + } + + #[test] + fn stale_queue_head_is_reaped_before_inference() { + let mut cfg = test_cfg(); + cfg.max_queue_age_s = 0.01; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = queued_completion("stale", 17); + prepared.arrived_at = Instant::now() - Duration::from_millis(20); + let lease = admit_test_job(&inner, &mut prepared); + let (job, drain) = owner_job(prepared, lease); + let mut engine = ScriptedDecode::from_pieces(&[]); + + run_owner_job(&cfg, &inner, &mut engine, None, job); + let response = String::from_utf8(settle_drain(drain)).unwrap(); + + assert!( + response.starts_with("HTTP/1.1 503 Service Unavailable"), + "{response}" + ); + assert!(response.contains("Retry-After: 30"), "{response}"); + assert_eq!(engine.pos, 0); + let g = lock_inner(&inner); + assert_eq!(g.metrics.shed[SHED_QUEUE_AGE as usize], 1); + assert_eq!(g.runtime.requests_completed, 0); + assert_eq!(g.runtime.requests_canceled, 0); + } + + #[test] + fn engine_failure_precedes_transport_cancel() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = queued_completion("fail", 13); + prepared.surface = WireSurface::OpenaiChat; + prepared.parsed.kind = crate::route::ReqKind::Chat; + prepared.parsed.prompt_text = None; + prepared.parsed.needs = 0; + let lease = admit_test_job(&inner, &mut prepared); + let (job, drain) = owner_job(prepared, lease); + let mut engine = ScriptedDecode::from_pieces(&[]); + let mut cont = TestCont::Fail; + + run_owner_job(&cfg, &inner, &mut engine, Some(&mut cont), job); + drain.state.cancel(false); + let mut lease = drain.done.recv().unwrap(); + lease.settlement = lease.settlement.transport_gone(); + assert_eq!(lease.settlement.outcome, Settle::Failed); + drop(lease); + + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_failed, 1); + assert_eq!(g.runtime.requests_canceled, 0); + assert_eq!(g.runtime.requests_inflight, 0); + assert_eq!(g.runtime.out_backlog_bytes, 0); + } + + #[test] + fn queued_wait_is_included_in_serial_ttft() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = queued_completion("timed", 15); + prepared.parsed.max_tokens = 1; + prepared.parsed.max_tokens_set = true; + prepared.arrived_at = Instant::now() - Duration::from_millis(100); + let lease = admit_test_job(&inner, &mut prepared); + let (job, drain) = owner_job(prepared, lease); + let mut engine = ScriptedDecode::from_pieces(&[b"x"]); + + run_owner_job(&cfg, &inner, &mut engine, None, job); + let response = String::from_utf8(settle_drain(drain)).unwrap(); + let ttft_ms = response_number(&response, "\"ttft_ms\":"); + assert!(ttft_ms >= 80.0, "ttft={ttft_ms} response={response}"); + } + + #[test] + fn active_serial_decode_finishes_as_failed_when_shutdown_is_requested() { + let mut cfg = test_cfg(); + cfg.stop_requested = Some(decode_stop_requested); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = queued_completion("shutdown", 15); + prepared.parsed.stream = true; + prepared.parsed.max_tokens = 8; + prepared.parsed.max_tokens_set = true; + let lease = admit_test_job(&inner, &mut prepared); + let (job, drain) = owner_job(prepared, lease); + let mut engine = ScriptedDecode::from_pieces(&[b"must-not-run"]); + DECODE_STOP.store(true, Ordering::Relaxed); + + run_owner_job(&cfg, &inner, &mut engine, None, job); + DECODE_STOP.store(false, Ordering::Relaxed); + let response = String::from_utf8(settle_drain(drain)).unwrap(); + + assert_eq!(engine.idx, 0, "decode sampled after shutdown: {response}"); + assert!( + response.contains("\"finish_reason\":\"error\""), + "{response}" + ); + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_failed, 1); + assert_eq!(g.runtime.requests_completed, 0); + } + + #[test] + fn prefill_timer_starts_immediately_before_sync() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = queued_completion("timed", 15); + prepared.parsed.max_tokens = 1; + prepared.parsed.max_tokens_set = true; + let lease = admit_test_job(&inner, &mut prepared); + let (job, drain) = owner_job(prepared, lease); + let mut engine = SlowPrepDecode { + sampled: false, + pos: 0, + shutdown_order: None, + }; + + run_owner_job(&cfg, &inner, &mut engine, None, job); + let response = String::from_utf8(settle_drain(drain)).unwrap(); + let prefill_tok_s = response_number(&response, "\"prefill_tok_s\":"); + assert!( + prefill_tok_s > 15.0, + "tokenization leaked into prefill timing={prefill_tok_s}: {response}" + ); + } + + #[test] + fn request_arrival_is_stamped_after_body_read() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + let (jobs_tx, jobs_rx) = mpsc::channel(); + let client_lease = ClientLease::try_new(Arc::clone(&inner)).unwrap(); + let client_cfg = cfg.clone(); + let client_inner = Arc::clone(&inner); + let h = thread::spawn(move || { + queue_client(client_cfg, client_inner, jobs_tx, server, client_lease) + }); + let body = r#"{"prompt":"delayed","max_tokens":1}"#; + write!( + client, + "POST /v1/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .unwrap(); + thread::sleep(Duration::from_millis(120)); + client.write_all(body.as_bytes()).unwrap(); + let job = jobs_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[b"x"]); + + run_owner_job(&cfg, &inner, &mut engine, None, job); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + h.join().unwrap(); + + let response = String::from_utf8(response).unwrap(); + let ttft_ms = response_number(&response, "\"ttft_ms\":"); + assert!( + ttft_ms < 80.0, + "body read leaked into ttft={ttft_ms}: {response}" + ); + } + + #[test] + fn c_accept_loop_retries_errors_until_shutdown() { + assert!(accept_error_retryable(ErrorKind::Interrupted)); + assert!(accept_error_retryable(ErrorKind::WouldBlock)); + assert!(accept_error_retryable(ErrorKind::ConnectionAborted)); + assert!(accept_error_retryable(ErrorKind::TimedOut)); + assert!(accept_error_retryable(ErrorKind::InvalidInput)); + assert!(accept_error_retryable(ErrorKind::PermissionDenied)); + } + + #[test] + fn stop_drains_then_saves_serial_before_continuous_banks() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let mut cfg = test_cfg(); + cfg.stop_requested = Some(test_stop_requested); + TEST_STOP.store(false, Ordering::Relaxed); + TEST_STOP_POLLS.store(0, Ordering::Relaxed); + let order = Arc::new(Mutex::new(Vec::new())); + let thread_order = Arc::clone(&order); + let (done_tx, done_rx) = mpsc::channel(); + let owner = thread::spawn(move || { + let mut engine = SlowPrepDecode { + sampled: false, + pos: 0, + shutdown_order: Some(Arc::clone(&thread_order)), + }; + let mut cont = TestCont::Shutdown(thread_order); + owner_loop(listener, cfg, &mut engine, Some(&mut cont)); + done_tx.send(()).unwrap(); + }); + + let deadline = Instant::now() + Duration::from_secs(1); + while TEST_STOP_POLLS.load(Ordering::Relaxed) == 0 && Instant::now() < deadline { + thread::yield_now(); + } + assert!(TEST_STOP_POLLS.load(Ordering::Relaxed) > 0); + TEST_STOP.store(true, Ordering::Relaxed); + if done_rx.recv_timeout(Duration::from_secs(1)).is_err() { + let _ = TcpStream::connect(addr); + panic!("owner loop did not stop after the stop predicate changed"); + } + owner.join().unwrap(); + + assert_eq!(*order.lock().unwrap(), ["serial", "bank"]); + } + + #[test] + fn disconnect_probe_keeps_post_parse_socket_nonblocking() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + server.set_nonblocking(true).unwrap(); + + assert!(!client_disconnected(&server)); + let writer = thread::spawn(move || { + thread::sleep(Duration::from_millis(100)); + client.write_all(b"x").unwrap(); + }); + let mut byte = [0u8; 1]; + let err = server.read(&mut byte).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::WouldBlock); + writer.join().unwrap(); + } + + #[test] + fn direct_nonblocking_sink_retries_partial_writes() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + server.set_nonblocking(true).unwrap(); + let payload = vec![b'x'; 16 * 1024 * 1024]; + let expected = payload.len(); + let reader = thread::spawn(move || { + thread::sleep(Duration::from_millis(100)); + let mut received = Vec::new(); + client.read_to_end(&mut received).unwrap(); + received.len() + }); + + DirectSink { + stream: &mut server, + started: false, + } + .write_all(&payload) + .unwrap(); + server.shutdown(std::net::Shutdown::Write).unwrap(); + + assert_eq!(reader.join().unwrap(), expected); + } + + #[test] + fn tcp_client_runs_through_owner_sink() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + let (jobs_tx, jobs_rx) = mpsc::channel(); + let client_lease = ClientLease::try_new(Arc::clone(&inner)).unwrap(); + let client_cfg = cfg.clone(); + let client_inner = Arc::clone(&inner); + let h = thread::spawn(move || { + queue_client(client_cfg, client_inner, jobs_tx, server, client_lease) + }); + let body = r#"{"prompt":"hello","max_tokens":0}"#; + write!( + client, + "POST /v1/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + let job = jobs_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[]); + + run_owner_job(&cfg, &inner, &mut engine, None, job); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + h.join().unwrap(); + + let response = String::from_utf8_lossy(&response); + assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}"); + assert!(response.contains("\"id\":\"cmpl-1\""), "{response}"); + let g = lock_inner(&inner); + assert_eq!(g.admit.clients, 0); + assert_eq!(g.admit.queued, 0); + assert_eq!(g.admit.inflight_body_bytes, 0); + assert_eq!(g.runtime.requests_completed, 1); + } + + #[test] + fn owner_drain_keeps_a_started_stream_alive_while_engine_is_silent() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (mut sink, state) = job_sink(inner); + let (_done_tx, done_rx) = mpsc::channel(); + let drain = JobDrain { + done: done_rx, + state, + streaming: true, + }; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + client + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let (mut server, _) = listener.accept().unwrap(); + sink.write_all(&crate::stream::sse_headers(false)).unwrap(); + let drain_thread = thread::spawn(move || { + drain_job_with_interval(&mut server, &drain, Duration::from_millis(20)); + }); + + let mut wire = Vec::new(); + let mut chunk = [0; 512]; + while !wire.ends_with(b": keep-alive\n\n") { + let n = client.read(&mut chunk).unwrap(); + wire.extend_from_slice(&chunk[..n]); + } + + drop(sink); + drain_thread.join().unwrap(); + let wire = String::from_utf8(wire).unwrap(); + assert_eq!(wire.matches("HTTP/1.1").count(), 1, "{wire}"); + assert!(wire.ends_with(": keep-alive\n\n"), "{wire}"); + } + + #[test] + fn slow_sink_cancels_once() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = queued_completion("slow", 31); + prepared.parsed.stream = true; + let lease = admit_test_job(&inner, &mut prepared); + let (mut sink, state) = job_sink(Arc::clone(&inner)); + sink.write_all(&vec![0; JOB_SINK_CAP_BYTES as usize]) + .unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + let job = OwnerJob { + lease, + prepared, + sink, + done: done_tx, + }; + let mut engine = ScriptedDecode::from_pieces(&[]); + + run_owner_job(&cfg, &inner, &mut engine, None, job); + assert!(state.slow()); + let lease = done_rx.recv().unwrap(); + assert_eq!(lease.settlement.outcome, Settle::Canceled); + drop(lease); + + let g = lock_inner(&inner); + assert_eq!(g.admit.inflight_body_bytes, 0); + assert_eq!(g.runtime.requests_canceled, 1); + assert_eq!(g.runtime.requests_inflight, 0); + assert_eq!(g.metrics.shed[SHED_SLOW_READER as usize], 1); + } + + #[test] + fn disconnected_drain_cancels_and_releases_body() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = queued_completion("gone", 23); + let lease = admit_test_job(&inner, &mut prepared); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + server.set_nonblocking(true).unwrap(); + client.shutdown(std::net::Shutdown::Both).unwrap(); + drop(client); + let (job, drain) = owner_job_with_probe(prepared, lease, Some(server)); + let mut engine = ScriptedDecode::from_pieces(&[]); + + run_owner_job(&cfg, &inner, &mut engine, None, job); + assert!(settle_drain(drain).is_empty()); + assert_eq!(engine.pos, 0, "disconnected queue head must not prefill"); + + let g = lock_inner(&inner); + assert_eq!(g.admit.queued, 0); + assert_eq!(g.admit.inflight_body_bytes, 0); + assert_eq!(g.runtime.requests_canceled, 1); + assert_eq!(g.runtime.requests_inflight, 0); + assert_eq!(g.metrics.shed[SHED_SLOW_READER as usize], 0); + } + + #[test] + fn queued_drain_observes_disconnect_before_output_starts() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + let probe = server.try_clone().unwrap(); + let (sink, state) = job_sink_with_probe(inner, Some(probe)); + let (_done_tx, done_rx) = mpsc::channel(); + let drain = JobDrain { + done: done_rx, + state: Arc::clone(&state), + streaming: false, + }; + client.shutdown(std::net::Shutdown::Both).unwrap(); + drop(client); + let thread = thread::spawn(move || { + drain_job_with_interval(&mut server, &drain, Duration::from_millis(10)); + }); + + thread::sleep(Duration::from_millis(30)); + let observed_before_close = state.gone(); + drop(sink); + thread.join().unwrap(); + + assert!(observed_before_close); + } + + #[test] + fn serial_owner_aborts_after_started_client_disconnect() { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = queued_completion("gone-after-start", 23); + prepared.parsed.max_tokens = 5; + prepared.parsed.max_tokens_set = true; + let lease = admit_test_job(&inner, &mut prepared); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + server.set_nonblocking(true).unwrap(); + let probe = server.try_clone().unwrap(); + let (job, drain) = owner_job_with_probe(prepared, lease, Some(probe)); + let state = Arc::clone(&drain.state); + let drain_thread = thread::spawn(move || drain_job(server, drain)); + let (started_tx, started_rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let disconnect = thread::spawn(move || { + started_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + client.shutdown(std::net::Shutdown::Both).unwrap(); + drop(client); + resume_tx.send(()).unwrap(); + }); + let mut engine = DisconnectDecode { + started: Some(started_tx), + resume: resume_rx, + samples: 0, + evals: 0, + pos: 0, + }; + + run_owner_job(&cfg, &inner, &mut engine, None, job); + disconnect.join().unwrap(); + drain_thread.join().unwrap(); + + assert!(state.gone(), "decode probe must observe the TCP disconnect"); + assert!( + engine.evals <= 1, + "decode continued for {} tokens", + engine.evals + ); + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_canceled, 1); + assert_eq!(g.runtime.requests_inflight, 0); + } + + mod stream_disconnect; + + // C `serial_session_ensure_fit` driver behavior (v0.5.2 inc1). + use crate::generate::SerialSessionProbe; + use ds4_kv::Store as KvStore; + + struct FitProbeDecode { + model_id: i32, + probe: SerialSessionProbe, + fits_at_or_below: i32, + rightsized: Option, + resets: u32, + memory_budget: Option, + bytes_per_ctx: Option, + } + + impl FitProbeDecode { + fn new(cur_ctx: i32, graph_pending: bool, fits_at_or_below: i32) -> Self { + Self { + model_id: 0, + probe: SerialSessionProbe { + cur_ctx, + graph_pending, + }, + fits_at_or_below, + rightsized: None, + resets: 0, + memory_budget: None, + bytes_per_ctx: None, + } + } + } + + impl DecodeIo for FitProbeDecode { + fn model_id(&self) -> i32 { + self.model_id + } + fn tokenize_text(&self, _text: &str) -> Result, GenerateError> { + Ok(vec![1]) + } + fn tokenize_rendered_chat(&self, _text: &[u8]) -> Result, GenerateError> { + Ok(vec![1]) + } + fn token_text(&self, _token: i32) -> Result, GenerateError> { + Ok(b"x".to_vec()) + } + fn token_is_stop(&self, _token: i32) -> bool { + true + } + fn sync(&mut self, _tokens: &[i32]) -> Result<(), GenerateError> { + Ok(()) + } + fn eval(&mut self, _token: i32) -> Result<(), GenerateError> { + Ok(()) + } + fn sample(&mut self, _t: f32, _k: i32, _p: f32, _m: f32, _rng: &mut u64) -> i32 { + 0 + } + fn native_graph_fit(&self, ctx: i32) -> Option { + let (need_bytes, avail_bytes) = self.memory_budget.map_or((0, 0), |budget| { + let need = self.bytes_per_ctx.map_or_else( + || { + if ctx <= self.fits_at_or_below { + budget / 2 + } else { + budget + 1 + } + }, + |bytes| ctx as u64 * bytes, + ); + (need, budget) + }); + Some(NativeGraphFit { + fits: ctx <= self.fits_at_or_below + && self.memory_budget.is_none_or(|avail| need_bytes <= avail), + need_bytes, + avail_bytes, + headroom_bytes: 0, + deficit_bytes: 0, + fail_open: false, + }) + } + fn serial_session_probe(&self) -> Option { + Some(self.probe) + } + fn serial_session_rightsize(&mut self, target: i32) -> Result<(), GenerateError> { + self.rightsized = Some(target); + self.probe = SerialSessionProbe { + cur_ctx: target, + graph_pending: true, + }; + Ok(()) + } + fn serial_session_reset(&mut self) { + self.resets += 1; + } + fn pos(&self) -> i32 { + 0 + } + fn ctx(&self) -> i32 { + self.probe.cur_ctx + } + fn generation(&self) -> u64 { + 0 + } + fn invalidate(&mut self) {} + } + + struct NoTrimCont; + + impl ContExec for NoTrimCont { + fn model_id(&self) -> i32 { + 0 + } + fn seq_cap(&self) -> i32 { + 0 + } + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + Vec::new() + } + fn encode_text(&self, _text: &str) -> Vec { + Vec::new() + } + #[allow(clippy::too_many_arguments)] + fn generate( + &mut self, + _parsed: &ParsedRequest, + _job_id: &str, + _created: i64, + _cors: bool, + _default_tokens: i32, + _t_arrive: Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut KvStore>, + _out: &mut dyn Write, + ) -> Result { + Err(GenerateError::Unsupported("test cont")) + } + } + + fn serial_fit_cfg(boot_ctx: i32) -> ServerConfig { + let mut cfg = ServerConfig::test_cfg(); + cfg.ctx = boot_ctx; + cfg.serial_rightsize = true; + cfg + } + + #[test] + fn serial_admit_uses_sized_ctx() { + // Verdict-only family probes have no byte quote; both forms serve. + for budget in [Some(8 << 30), None] { + let mut cfg = serial_fit_cfg(262_144); + cfg.mem_floor_gb = 2; + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let mut job = queued_completion("hello", 32); + job.parsed.max_tokens = 128; + let mut engine = FitProbeDecode::new(cfg.ctx, true, 40_000); + engine.memory_budget = budget; + let mut cont = NoTrimCont; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + + // The request-sized graph fits; the unused boot graph does not. + run_serial( + &cfg, + &inner, + &job, + "sized-fit", + &mut engine, + Some(&mut cont), + &mut DirectSink { + stream: &mut server, + started: false, + }, + Instant::now(), + None, + ); + server.shutdown(std::net::Shutdown::Write).unwrap(); + let mut response = String::new(); + client.read_to_string(&mut response).unwrap(); + + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!(engine.rightsized.is_some_and(|ctx| ctx <= 40_000)); + } + } + + #[test] + fn serial_admit_keeps_graph() { + for (avail, status) in [(3 << 30, "HTTP/1.1 200"), (1 << 30, "HTTP/1.1 503")] { + let mut cfg = serial_fit_cfg(262_144); + cfg.mem_floor_gb = 2; + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let job = queued_completion("hello", 32); + let mut engine = FitProbeDecode::new(33_000, false, 40_000); + engine.memory_budget = Some(avail); + let mut cont = NoTrimCont; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + + // Retained KV needs no new graph allocation, but keeps the floor. + run_serial( + &cfg, + &inner, + &job, + "retained-fit", + &mut engine, + Some(&mut cont), + &mut DirectSink { + stream: &mut server, + started: false, + }, + Instant::now(), + None, + ); + server.shutdown(std::net::Shutdown::Write).unwrap(); + let mut response = String::new(); + client.read_to_string(&mut response).unwrap(); + + assert!(response.starts_with(status), "{response}"); + assert_eq!(engine.resets, 0); + assert_eq!(engine.rightsized, None); + } + } + + #[test] + fn serial_fit_checks_floor() { + let mut cfg = serial_fit_cfg(262_144); + cfg.mem_floor_gb = 8; + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let job = queued_completion("hello", 32); + let mut engine = FitProbeDecode::new(cfg.ctx, true, cfg.ctx); + engine.memory_budget = Some(12 << 30); + engine.bytes_per_ctx = Some(128 << 10); + let mut cont = NoTrimCont; + let mut out = Vec::new(); + + // A small graph fits above the floor; a native-only target does not. + assert!(ensure_serial_session_fit( + &cfg, + &inner, + &job, + &mut engine, + Some(&mut cont), + SerialFrameKind::Canonical, + 1, + 65_536, + &mut out, + ) + .is_none()); + let target = engine.rightsized.expect("a smaller graph fits"); + let quote = engine.native_graph_fit(target).unwrap(); + let floor = MemFloor::from_gb(cfg.mem_floor_gb).bytes(); + assert!(quote.avail_bytes >= quote.need_bytes + floor); + } + + #[test] + fn serial_fit_resizes_pending_full_ctx_session() { + let cfg = serial_fit_cfg(250_000); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let job = queued_completion("hello", 32); + let mut engine = FitProbeDecode::new(250_000, true, 40_000); + let mut cont = NoTrimCont; + let mut out = Vec::new(); + + let refusal = ensure_serial_session_fit( + &cfg, + &inner, + &job, + &mut engine, + Some(&mut cont), + SerialFrameKind::Canonical, + 26, + 384, + &mut out, + ); + assert!(refusal.is_none()); + assert_eq!(engine.resets, 1, "free before probing (the regrow case)"); + // request_cap = prompt + budget + 32768; the search stops within + // 1024 of the cap and the target must fit. + let target = engine.rightsized.expect("session re-created"); + let cap = 26 + 384 + 32768; + assert!(target >= cap - 1024 && target <= cap, "target={target}"); + assert!(out.is_empty(), "no refusal bytes on the resize path"); + } + + #[test] + fn serial_fit_keeps_ling_yarn() { + for boot_ctx in [131_072, 131_073, 262_144] { + let cfg = serial_fit_cfg(boot_ctx); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let job = queued_completion("hello", 32); + let mut engine = FitProbeDecode::new(boot_ctx, true, boot_ctx); + engine.model_id = ds4_core::Variant::Ling30FlashVl as i32; + let mut cont = NoTrimCont; + let mut out = Vec::new(); + + assert!(ensure_serial_session_fit( + &cfg, + &inner, + &job, + &mut engine, + Some(&mut cont), + SerialFrameKind::Canonical, + 26, + 384, + &mut out, + ) + .is_none()); + let target = engine.probe.cur_ctx; + assert_eq!(target > 131_072, boot_ctx > 131_072, "ctx={target}"); + assert!(target <= boot_ctx); + } + } + + #[test] + fn serial_fit_refuses_yarn_switch() { + let cfg = serial_fit_cfg(262_144); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let job = queued_completion("hello", 32); + let mut engine = FitProbeDecode::new(cfg.ctx, true, 131_072); + engine.model_id = ds4_core::Variant::Ling30FlashVl as i32; + let mut cont = NoTrimCont; + let mut out = Vec::new(); + + assert!(ensure_serial_session_fit( + &cfg, + &inner, + &job, + &mut engine, + Some(&mut cont), + SerialFrameKind::Canonical, + 26, + 384, + &mut out, + ) + .is_some()); + assert!(engine.rightsized.is_none()); + assert!(String::from_utf8_lossy(&out).starts_with("HTTP/1.1 503")); + } + + #[test] + fn serial_fit_refuses_when_no_graph_fits() { + let cfg = serial_fit_cfg(250_000); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let job = queued_completion("hello", 32); + let mut engine = FitProbeDecode::new(250_000, true, 0); + let mut cont = NoTrimCont; + let mut out = Vec::new(); + + let refusal = ensure_serial_session_fit( + &cfg, + &inner, + &job, + &mut engine, + Some(&mut cont), + SerialFrameKind::Canonical, + 26, + 384, + &mut out, + ); + assert!(refusal.is_some()); + assert!(engine.rightsized.is_none()); + assert_eq!(engine.resets, 2, "free before probing + boot-shape restore"); + let response = String::from_utf8_lossy(&out); + assert!(response.starts_with("HTTP/1.1 503"), "{response}"); + assert!( + response.contains("no session graph fits beside the batch banks"), + "{response}" + ); + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_refused_deep_serial, 1); + assert_eq!( + g.runtime.rejected[1][RejectReason::LiveHeadroom as usize], + 1 + ); + } + + #[test] + fn serial_fit_preserves_live_continuation_frontier() { + let cfg = serial_fit_cfg(250_000); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let job = queued_completion("hello", 32); + // Committed 2048-ctx session retaining the only continuation copy. + let mut engine = FitProbeDecode::new(2048, false, 40_000); + let mut cont = NoTrimCont; + let mut out = Vec::new(); + + let refusal = ensure_serial_session_fit( + &cfg, + &inner, + &job, + &mut engine, + Some(&mut cont), + SerialFrameKind::ResolvedLive, + 5000, + 1024, + &mut out, + ); + assert!(refusal.is_some()); + assert_eq!(engine.resets, 0, "the frontier must be preserved"); + assert!(engine.rightsized.is_none()); + let response = String::from_utf8_lossy(&out); + assert!(response.starts_with("HTTP/1.1 503"), "{response}"); + assert!(response.contains("replay the full history"), "{response}"); + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_refused_deep_serial, 1); + assert_eq!( + g.runtime.rejected[1][RejectReason::LiveHeadroom as usize], + 0 + ); + } + + #[test] + fn serial_fit_reuses_committed_session_and_skips_scope_gates() { + let cfg = serial_fit_cfg(8192); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let job = queued_completion("hello", 32); + + // Committed and big enough: reuse. + let mut engine = FitProbeDecode::new(8192, false, 0); + let mut cont = NoTrimCont; + let mut out = Vec::new(); + assert!(ensure_serial_session_fit( + &cfg, + &inner, + &job, + &mut engine, + Some(&mut cont), + SerialFrameKind::Canonical, + 100, + 384, + &mut out, + ) + .is_none()); + assert_eq!(engine.resets, 0); + + // DS4_SERVER_SERIAL_RIGHTSIZE=0 keeps the v0.5.1 contract. + let mut off = serial_fit_cfg(8192); + off.serial_rightsize = false; + let mut engine = FitProbeDecode::new(8192, true, 0); + assert!(ensure_serial_session_fit( + &off, + &inner, + &job, + &mut engine, + Some(&mut cont), + SerialFrameKind::Canonical, + 9000, + 384, + &mut out, + ) + .is_none()); + assert_eq!(engine.resets, 0); + + // Serial-only boot (no batch ctx): full--c session IS the serving + // configuration. + let mut engine = FitProbeDecode::new(8192, true, 0); + assert!(ensure_serial_session_fit( + &cfg, + &inner, + &job, + &mut engine, + None, + SerialFrameKind::Canonical, + 9000, + 384, + &mut out, + ) + .is_none()); + assert_eq!(engine.resets, 0); + } +} diff --git a/crates/ds4-server/src/serve/owner_tests/stream_disconnect.rs b/crates/ds4-server/src/serve/owner_tests/stream_disconnect.rs new file mode 100644 index 0000000000000000000000000000000000000000..d65aff90cc2c8e305f08619db5fb2160695cc933 --- /dev/null +++ b/crates/ds4-server/src/serve/owner_tests/stream_disconnect.rs @@ -0,0 +1,252 @@ +//! Mid-stream disconnect / backpressure fixtures for Anthropic and Responses. +//! Event names stay the C automata; client drop must not panic. + +use super::{ + admit_test_job, drain_job, job_sink, lock_inner, owner_job_with_probe, run_owner_job, test_cfg, + DisconnectDecode, OwnerJob, PreparedJob, Settle, JOB_SINK_CAP_BYTES, +}; +use crate::admit::SHED_SLOW_READER; +use crate::generate::{generate_and_write, GenerateError, ScriptedDecode}; +use crate::parse::{parse_request, ParseEnv, ParsedRequest}; +use crate::route::{ThinkMode, WireSurface}; +use crate::stream::TAPE_PLAIN; +use crate::ServerInner; +use std::io::{self, Write}; +use std::net::{TcpListener, TcpStream}; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::{mpsc, Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +const ANTHROPIC_STREAM: &str = r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":16,"stream":true,"thinking":{"type":"disabled"}}"#; +const RESPONSES_STREAM: &str = r#"{"input":"hi","max_output_tokens":8,"stream":true}"#; + +struct FailOnWrite { + fail_at: usize, + writes: usize, + kind: io::ErrorKind, + captured: Vec, +} + +impl Write for FailOnWrite { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.writes += 1; + if self.writes >= self.fail_at { + return Err(io::Error::new(self.kind, "client gone")); + } + self.captured.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + if self.writes >= self.fail_at { + Err(io::Error::new(self.kind, "client gone")) + } else { + Ok(()) + } + } +} + +fn parse_env() -> ParseEnv { + ParseEnv { + default_model: "ds4".into(), + default_tokens: 16, + default_effort: ThinkMode::None, + default_temp: 0.0, + live_ids: Vec::new(), + engine_defaults: false, + } +} + +fn streamed(surface: WireSurface, body: &str) -> ParsedRequest { + let mut parsed = parse_request(surface, &parse_env(), body).unwrap(); + parsed.think_mode = ThinkMode::None; + parsed.temperature = 0.0; + parsed +} + +fn tape_engine() -> ScriptedDecode { + ScriptedDecode::from_pieces(&TAPE_PLAIN.iter().map(|s| s.as_bytes()).collect::>()) +} + +fn generate_mid_stream_io( + surface: WireSurface, + body: &str, + job_id: &str, + kind: io::ErrorKind, + start_marker: &str, + stop_marker: &str, +) { + let parsed = streamed(surface, body); + let mut engine = tape_engine(); + let mut out = FailOnWrite { + // Headers are sent before prefill; protocol start is the second write. + fail_at: 3, + writes: 0, + kind, + captured: Vec::new(), + }; + let result = catch_unwind(AssertUnwindSafe(|| { + generate_and_write(&mut engine, &parsed, job_id, 1, false, 16, &mut out) + })); + let err = result + .expect("client drop must not panic") + .expect_err("mid-stream write fail must abort"); + assert!(matches!(err, GenerateError::Io), "{err}"); + let prefix = String::from_utf8_lossy(&out.captured); + assert!(prefix.contains(start_marker), "{prefix}"); + assert!(!prefix.contains(stop_marker), "{prefix}"); +} + +fn streamed_job(surface: WireSurface, body: &str, body_bytes: u64) -> PreparedJob { + PreparedJob { + parsed: streamed(surface, body), + cont_prompt: None, + surface, + body_bytes, + arrived_at: Instant::now(), + } +} + +fn assert_slow_sink_cancels(surface: WireSurface, body: &str) { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = streamed_job(surface, body, 31); + let lease = admit_test_job(&inner, &mut prepared); + let (mut sink, state) = job_sink(Arc::clone(&inner)); + sink.write_all(&vec![0; JOB_SINK_CAP_BYTES as usize]) + .unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + let job = OwnerJob { + lease, + prepared, + sink, + done: done_tx, + }; + let mut engine = tape_engine(); + run_owner_job(&cfg, &inner, &mut engine, None, job); + assert!(state.slow()); + let lease = done_rx.recv().unwrap(); + assert_eq!(lease.settlement.outcome, Settle::Canceled); + drop(lease); + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_canceled, 1); + assert_eq!(g.runtime.requests_failed, 0); + assert_eq!(g.metrics.shed[SHED_SLOW_READER as usize], 1); +} + +fn assert_client_drop_cancels(surface: WireSurface, body: &str) { + let cfg = test_cfg(); + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let mut prepared = streamed_job(surface, body, 23); + prepared.parsed.max_tokens = 5; + prepared.parsed.max_tokens_set = true; + let lease = admit_test_job(&inner, &mut prepared); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + server.set_nonblocking(true).unwrap(); + let probe = server.try_clone().unwrap(); + let (job, drain) = owner_job_with_probe(prepared, lease, Some(probe)); + let state = Arc::clone(&drain.state); + let drain_thread = thread::spawn(move || drain_job(server, drain)); + let (started_tx, started_rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let disconnect = thread::spawn(move || { + started_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + client.shutdown(std::net::Shutdown::Both).unwrap(); + drop(client); + resume_tx.send(()).unwrap(); + }); + let mut engine = DisconnectDecode { + started: Some(started_tx), + resume: resume_rx, + samples: 0, + evals: 0, + pos: 0, + }; + let result = catch_unwind(AssertUnwindSafe(|| { + run_owner_job(&cfg, &inner, &mut engine, None, job); + })); + assert!(result.is_ok(), "client drop must not panic"); + disconnect.join().unwrap(); + drain_thread.join().unwrap(); + assert!(state.gone(), "decode probe must observe the TCP disconnect"); + assert!( + engine.evals <= 1, + "decode continued for {} tokens", + engine.evals + ); + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_canceled, 1); + assert_eq!(g.runtime.requests_failed, 0); + assert_eq!(g.runtime.requests_inflight, 0); +} + +#[test] +fn anthropic_stream_broken_pipe_mid_delta_is_io_without_panic() { + generate_mid_stream_io( + WireSurface::Anthropic, + ANTHROPIC_STREAM, + "msg-drop", + io::ErrorKind::BrokenPipe, + "event: message_start", + "event: message_stop", + ); +} + +#[test] +fn responses_stream_broken_pipe_mid_delta_is_io_without_panic() { + generate_mid_stream_io( + WireSurface::Responses, + RESPONSES_STREAM, + "resp-drop", + io::ErrorKind::BrokenPipe, + "\"type\":\"response.created\"", + "\"type\":\"response.completed\"", + ); +} + +#[test] +fn anthropic_stream_would_block_mid_delta_is_io_without_panic() { + generate_mid_stream_io( + WireSurface::Anthropic, + ANTHROPIC_STREAM, + "msg-slow", + io::ErrorKind::WouldBlock, + "event: message_start", + "event: message_stop", + ); +} + +#[test] +fn responses_stream_would_block_mid_delta_is_io_without_panic() { + generate_mid_stream_io( + WireSurface::Responses, + RESPONSES_STREAM, + "resp-slow", + io::ErrorKind::WouldBlock, + "\"type\":\"response.created\"", + "\"type\":\"response.completed\"", + ); +} + +#[test] +fn anthropic_stream_slow_sink_cancels_once() { + assert_slow_sink_cancels(WireSurface::Anthropic, ANTHROPIC_STREAM); +} + +#[test] +fn responses_stream_slow_sink_cancels_once() { + assert_slow_sink_cancels(WireSurface::Responses, RESPONSES_STREAM); +} + +#[test] +fn anthropic_stream_client_drop_mid_decode_cancels_without_panic() { + assert_client_drop_cancels(WireSurface::Anthropic, ANTHROPIC_STREAM); +} + +#[test] +fn responses_stream_client_drop_mid_decode_cancels_without_panic() { + assert_client_drop_cancels(WireSurface::Responses, RESPONSES_STREAM); +} diff --git a/crates/ds4-server/src/serve_503_body_test.rs b/crates/ds4-server/src/serve_503_body_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..328b75447a3372a546567133d738d44c89f6b184 --- /dev/null +++ b/crates/ds4-server/src/serve_503_body_test.rs @@ -0,0 +1,129 @@ +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::Mutex; + +use crate::admit::enqueue_shed_error; +use crate::error::wire_http_error_bytes; +use crate::generate::ScriptedDecode; +use crate::route::WireSurface; +use crate::serve::{handle_client_inner, ServerConfig, ServerInner}; +use crate::serve_serial_reclaim::serial_capacity_refuse_msg; + +const C_SHUTTING_DOWN: &str = "server shutting down"; + +fn drive_chat(cfg: &ServerConfig, inner: &Mutex, engine: bool) -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let body = r#"{"messages":[{"role":"user","content":"hi"}]}"#; + write!( + client, + "POST /v1/chat/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + if engine { + let mut decode = ScriptedDecode::from_pieces(&[]); + handle_client_inner(cfg, inner, &mut server, Some(&mut decode), None); + } else { + handle_client_inner(cfg, inner, &mut server, None, None); + } + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + String::from_utf8(response).unwrap() +} + +fn assert_c_503_message(response: &str, msg: &str) { + assert!( + response.starts_with("HTTP/1.1 503 Service Unavailable"), + "{response}" + ); + let expected = String::from_utf8(wire_http_error_bytes( + WireSurface::OpenaiChat, + 503, + msg, + false, + None, + )) + .unwrap(); + assert_eq!(response, expected); + assert!(!response.contains("Retry-After:"), "{response}"); + assert!( + !response.contains("generation remains on C ds4-server"), + "{response}" + ); + assert!(!response.contains("model not loaded"), "{response}"); + assert!( + !response.contains("server could not track the client connection"), + "{response}" + ); + assert!(!response.contains("server is shutting down"), "{response}"); +} + +#[test] +fn missing_engine_503_reuses_c_shutting_down() { + // Given: no DecodeIo (C never serves HTTP without a loaded engine) + let cfg = ServerConfig::default(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + + // When: a chat request is admitted + let response = drive_chat(&cfg, &inner, false); + + // Then: public 503 is the C Stopping body, not an invented twin + assert_c_503_message(&response, C_SHUTTING_DOWN); +} + +#[test] +fn have_engine_without_decode_io_503_reuses_c_shutting_down() { + // Given: have_engine advertised but no DecodeIo was supplied + let cfg = ServerConfig::test_cfg(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + + // When: a chat request is admitted + let response = drive_chat(&cfg, &inner, false); + + // Then: no "generation remains on C ds4-server" + assert_c_503_message(&response, C_SHUTTING_DOWN); +} + +#[test] +fn stopping_enqueue_503_matches_c_client_main() { + // Given: admit.stopping like C enqueue Stopping + let cfg = ServerConfig::test_cfg(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + inner.lock().unwrap().admit.stopping = true; + + // When: a chat request is admitted + let response = drive_chat(&cfg, &inner, true); + + // Then: C "server shutting down" + let (_, _, retry, msg) = enqueue_shed_error(crate::admit::EnqVerdict::Stopping).unwrap(); + assert_eq!(msg, C_SHUTTING_DOWN); + assert_eq!(retry, None); + let expected = String::from_utf8(wire_http_error_bytes( + WireSurface::OpenaiChat, + 503, + C_SHUTTING_DOWN, + false, + retry, + )) + .unwrap(); + assert_eq!(response, expected); + assert!(!response.contains("Retry-After:"), "{response}"); +} + +#[test] +fn serial_capacity_refuse_stays_c_bytes() { + // Given: C serial_session_ensure_fit + // When: format the refuse + let msg = serial_capacity_refuse_msg(8); + + // Then: existing C production string + assert_eq!( + msg, + "Server is temporarily at capacity for a 8-token serial request \ + (no session graph fits beside the batch banks); retry shortly" + ); +} diff --git a/crates/ds4-server/src/serve_cont.rs b/crates/ds4-server/src/serve_cont.rs new file mode 100644 index 0000000000000000000000000000000000000000..b01ca44aab20bf5098171f4b3c97ced20fafba4e --- /dev/null +++ b/crates/ds4-server/src/serve_cont.rs @@ -0,0 +1,5579 @@ +//! Continuous-lane execution: a push-based per-token stepper (pure, tape +//! testable) plus the native `ContLane` that drives the engine's rolling +//! scheduler through `ds4-core::BatchCtx`. The accept loop admits another +//! request while one is generating; the persistent batch context owns every +//! configured bank. The engine-side bank/admit machinery is the same one C's +//! continuous lane uses. Corrective retry (`decode_again`) and continuation +//! publish route serial by the needs word, so this path never re-decodes. + +use std::io::Write; +#[cfg(any(feature = "native", test))] +use std::path::Path; +use std::time::Instant; + +use ds4_kv::Store as KvStore; +#[cfg(feature = "native")] +use ds4_kv::{bank_checkpoint_due_from_host, HostKvView}; +#[cfg(any(feature = "native", test))] +use ds4_kv::{bank_persist_ext_flags, PrefixAnswer, Reason as KvReason, EXT_IMAGE_PIXELS_V2}; + +use crate::dsml::{SampleOverride, SamplePolicy}; +#[cfg(any(feature = "native", test))] +use crate::generate::thinking_visible_key; +use crate::generate::{ + prepare_required_prefixes, render_prompt_ex, responses_ids, stream_req_from_parsed, + GenerateError, GenerateOutcome, +}; +use crate::parse::{fixed_thinking_sampling, DEFAULT_MIN_P, DEFAULT_TEMPERATURE, DEFAULT_TOP_P}; +use crate::parse::{EosPolicy, ParsedRequest, ToolCall, ToolChoice}; +#[cfg(feature = "native")] +use crate::render::render_live_tool_tail; +use crate::render::syntax_for_model_id; +#[cfg(any(feature = "native", test))] +use crate::render::ModelSyntax; +use crate::retry::{terminal_finish, truncation_outcome, TruncationOutcome}; +use crate::route::{decode_budget, think_mode_enabled, Api, ReqKind, NEED_BANK_FRONTIER}; +#[cfg(feature = "native")] +use crate::stream::stream_error; +#[cfg(any(feature = "native", test))] +use crate::stream::stream_heartbeat_if_due; +use crate::stream::{ + anthropic_final_response, anthropic_sse_finish_live, anthropic_sse_start_live, + anthropic_sse_stream_update, final_response, openai_sse_finish_live, openai_sse_stream_update, + openai_stream_start, responses_final_response, responses_sse_created, + responses_sse_finish_live, responses_sse_stream_update, responses_stream_init, sse_chunk, + sse_done, sse_headers, AnthropicStream, OpenaiStream, ReqTimings, ResponsesStream, StreamReq, + ThinkBlock, Writer, +}; +#[cfg(any(feature = "native", test))] +use crate::stream::{think_end, ChatFormat}; +use crate::tools::{assign_tool_ids, parse_generated_for_response, SemAccum, SemFeed}; + +#[cfg(any(feature = "native", test))] +/// One definition with the serving plan, so `/v1/stats` cannot advertise a +/// threshold retirement does not use. +const DEFAULT_BANK_PERSIST_MIN_TOKENS: i32 = ds4_core::DEFAULT_BANK_PERSIST; + +#[cfg(any(feature = "native", test))] +fn bank_persist_eligible(committed: i32, persist_min: i32) -> bool { + persist_min > 0 && committed >= persist_min +} + +/// Pure continuous-request stepper. The caller feeds decoded pieces and +/// receives wire bytes; the engine (or a tape) owns token production. +/// The four frozen wire surfaces share this stepper; route eligibility still +/// decides which request shapes may enter the continuous lane. +pub struct ContStepper { + pub req: StreamReq, + pub job_id: String, + pub model_id: i32, + pub prompt: Vec, + #[cfg(feature = "native")] + chat_request: Option, + #[cfg(feature = "native")] + chat_history: Option, + #[cfg(feature = "native")] + cache_prompt: Option>, + #[cfg(feature = "native")] + image_cache_spans: Vec, + pub prompt_n: i32, + pub max_tokens: i32, + acc: SemAccum, + eos_policy: EosPolicy, + w: Writer, + oa: Option, + anth: Option, + resp: Option, + finish: &'static str, + stop_token: bool, + stops: Vec, + think_mode: crate::route::ThinkMode, + tool_choice: ToolChoice, + has_tool_results: bool, + effective_usage_frame: bool, + #[cfg(any(feature = "native", test))] + cors: bool, + #[cfg(feature = "native")] + prompt_preserves_reasoning: bool, + parsed_max_tokens: i32, + required_tool_prefix: Vec, + required_think_end_prefix: Vec, + tool_replay: Option<(Vec, String)>, + started: bool, + #[cfg(any(feature = "native", test))] + last_heartbeat: Instant, +} + +pub struct ContStep { + pub bytes: Vec, + /// Set when the host wants the sequence aborted (stop hit / tool block + /// closed). The engine's own EOS/budget finish arrives via `finalize`. + pub done: bool, +} + +impl ContStepper { + pub fn new( + parsed: &ParsedRequest, + model_id: i32, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + prompt: Vec, + prompt_n: i32, + seq_room: i32, + ) -> (Self, Vec) { + let req = stream_req_from_parsed(parsed, model_id); + let mut max_tokens = + decode_budget(parsed.max_tokens_set, parsed.max_tokens, default_tokens); + let room = seq_room - prompt_n; + if room >= 0 && max_tokens > room { + max_tokens = room; + } + let mut acc = SemAccum::init( + parsed.kind == ReqKind::Chat, + parsed.has_tools, + think_mode_enabled(parsed.think_mode), + req.chat_format, + &prompt, + ); + // The request's tools, for the start scan that recognises the + // artifact's quoted root (find_tool_start). + acc.set_tool_orders(&parsed.tool_orders); + acc.set_model_syntax(req.syntax); + let mut w = Writer::new(created); + let mut oa = None; + let mut anth = None; + let mut resp = None; + if req.stream { + w.out.extend_from_slice(&sse_headers(cors)); + match req.api { + Api::Openai if req.kind == ReqKind::Chat => { + let mut st = openai_stream_start(&req); + st.tool.use_random_ids(); + oa = Some(st); + sse_chunk(&mut w, &req, job_id, None, None); + } + Api::Anthropic => { + let mut st = anthropic_sse_start_live(&mut w, &req, job_id, prompt_n); + st.tool.use_random_ids(); + anth = Some(st); + } + Api::Responses => { + let (response_id, reasoning_id, message_id) = responses_ids(job_id); + let mut st = + responses_stream_init(&req, &response_id, &reasoning_id, &message_id); + responses_sse_created(&mut w, &req, &mut st, created); + resp = Some(st); + } + Api::Openai => {} + } + } + let head = std::mem::take(&mut w.out); + ( + Self { + req, + job_id: job_id.to_string(), + model_id, + prompt, + #[cfg(feature = "native")] + chat_request: None, + #[cfg(feature = "native")] + chat_history: None, + #[cfg(feature = "native")] + cache_prompt: None, + #[cfg(feature = "native")] + image_cache_spans: Vec::new(), + prompt_n, + max_tokens, + acc, + eos_policy: parsed.eos_policy, + w, + oa, + anth, + resp, + finish: "length", + stop_token: false, + stops: parsed.stops.clone(), + think_mode: parsed.think_mode, + tool_choice: parsed.tool_choice, + has_tool_results: parsed.has_tool_results, + effective_usage_frame: parsed.needs & NEED_BANK_FRONTIER != 0, + #[cfg(any(feature = "native", test))] + cors, + #[cfg(feature = "native")] + prompt_preserves_reasoning: prompt_preserves_reasoning(parsed), + parsed_max_tokens: parsed.max_tokens, + required_tool_prefix: parsed.required_tool_prefix.clone(), + required_think_end_prefix: parsed.required_think_end_prefix.clone(), + tool_replay: None, + started: true, + #[cfg(any(feature = "native", test))] + last_heartbeat: Instant::now(), + }, + head, + ) + } + + fn apply_engine_usage(&mut self, n_cached: i32, n_computed: i32) { + let cached = n_cached.max(0); + let computed = n_computed.max(0); + if self.effective_usage_frame { + self.prompt_n = cached.saturating_add(computed); + self.req.cache_read_tokens = cached; + self.req.cache_write_tokens = computed; + return; + } + let prompt = self.prompt_n.max(0); + let write = computed.min(prompt); + self.req.cache_write_tokens = write; + self.req.cache_read_tokens = prompt - write; + } + + #[cfg(any(feature = "native", test))] + fn admitted_head(&mut self, head: Vec, n_cached: i32, n_computed: i32) -> Vec { + self.apply_engine_usage(n_cached, n_computed); + if !self.req.stream || self.req.api != Api::Anthropic { + return head; + } + let mut w = Writer::new(self.w.created); + w.out.extend_from_slice(&sse_headers(self.cors)); + let mut stream = anthropic_sse_start_live(&mut w, &self.req, &self.job_id, self.prompt_n); + stream.tool.use_random_ids(); + self.anth = Some(stream); + w.out + } + + /// Per-token sampling override, same policy the serial `decode_pass` + /// consults (required prefixes + DSML structural greedy). + pub fn sample_override(&mut self) -> SampleOverride { + let policy = SamplePolicy { + tool_choice: self.tool_choice, + has_tool_results: self.has_tool_results, + think_mode: self.think_mode, + max_tokens: self.parsed_max_tokens, + required_tool_prefix: &self.required_tool_prefix, + required_think_end_prefix: &self.required_think_end_prefix, + }; + self.acc.sampling_override(&policy) + } + + pub fn sample_exclude(&self, eos: i32) -> i32 { + self.eos_policy + .excluded(eos, self.acc.thinking_inside()) + .unwrap_or(-1) + } + + /// Match serial sampling: IQuest and GLM-5.3 keep the requested sampler + /// in thinking; other families retain their legacy thinking defaults. + pub fn sampling(&self, parsed: &ParsedRequest) -> (f32, i32, f32, f32) { + if think_mode_enabled(parsed.think_mode) + && fixed_thinking_sampling(syntax_for_model_id(self.model_id)) + { + (DEFAULT_TEMPERATURE, 0, DEFAULT_TOP_P, DEFAULT_MIN_P) + } else { + (parsed.temperature, parsed.top_k, parsed.top_p, parsed.min_p) + } + } + + pub fn feed(&mut self, piece: &[u8]) -> ContStep { + let feed = self.acc.feed(piece, &self.stops); + self.step(piece, feed) + } + + /// A stop token's text: streamed, not counted (`SemAccum::feed_stop`). + pub fn feed_stop(&mut self, piece: &[u8]) -> ContStep { + let feed = self.acc.feed_stop(piece, &self.stops); + self.step(piece, feed) + } + + fn step(&mut self, piece: &[u8], feed: SemFeed) -> ContStep { + if self.req.stream { + let view = &self.acc.text[..feed.emit_limit.min(self.acc.text.len())]; + match (self.req.api, self.req.kind) { + (Api::Openai, ReqKind::Completion) => { + if let Some(delta) = last_delta(&self.acc.text, feed.emit_limit, piece.len()) { + sse_chunk(&mut self.w, &self.req, &self.job_id, Some(delta), None); + } + } + (Api::Openai, ReqKind::Chat) => { + if let Some(st) = self.oa.as_mut() { + openai_sse_stream_update( + &mut self.w, + &self.req, + &self.job_id, + st, + view, + false, + ); + } + } + (Api::Anthropic, _) => { + if let Some(st) = self.anth.as_mut() { + anthropic_sse_stream_update( + &mut self.w, + &self.req, + &self.job_id, + st, + view, + false, + ); + } + } + (Api::Responses, _) => { + if let Some(st) = self.resp.as_mut() { + responses_sse_stream_update(&mut self.w, &self.req, st, view, false); + } + } + } + } + let mut done = false; + if feed.hit_stop { + self.finish = "stop"; + done = true; + } else if self.acc.track_tools + && self.acc.saw_tool_end + && self.req.chat_format == crate::stream::ChatFormat::DeepSeek + { + self.finish = "tool_calls"; + done = true; + } else if self.acc.completion >= self.max_tokens { + self.finish = "length"; + done = true; + } + ContStep { + bytes: std::mem::take(&mut self.w.out), + done, + } + } + + #[cfg(any(feature = "native", test))] + fn heartbeat(&mut self, now: Instant) -> Vec { + stream_heartbeat_if_due( + &mut self.w, + &self.req, + self.resp.as_mut(), + &mut self.last_heartbeat, + now, + ": keep-alive\n\n", + ); + std::mem::take(&mut self.w.out) + } + + #[cfg(feature = "native")] + fn fail(&mut self, message: &str) -> Vec { + if self.req.stream { + stream_error(&mut self.w, &self.req, self.resp.as_mut(), message); + } + std::mem::take(&mut self.w.out) + } + + /// Engine finished (EOS = 1, budget/abort = 0). Runs the serial tail: + /// tag-completion repair (no re-decode on this lane), generated-message + /// parse, tool ids, usage/timings, stream finish or buffered final. + pub fn finalize( + &mut self, + engine_eos: bool, + n_cached: i32, + n_computed: i32, + timings: ReqTimings, + cors: bool, + ) -> (Vec, GenerateOutcome) { + assert!(self.started); + if self.finish == "length" && engine_eos { + self.finish = "stop"; + } + let syntax = syntax_for_model_id(self.model_id); + self.finish = terminal_finish(self.finish); + if let TruncationOutcome::Repair(text) = truncation_outcome( + syntax, + self.req.chat_format, + self.req.kind == ReqKind::Chat, + self.acc.track_tools, + self.acc.saw_tool_start, + self.acc.saw_tool_end, + self.finish, + self.req.stream, + true, /* recovery_attempted: decode_again routes serial */ + &self.acc.text, + &self.req.tool_orders, + ) { + self.acc.text = text; + self.acc.saw_tool_end = true; + } + let mut parsed_gen = if self.req.kind == ReqKind::Chat { + let (pg, recovered_finish) = parse_generated_for_response( + syntax, + &self.acc.text, + self.acc.track_tools, + self.acc.saw_tool_start, + crate::route::think_mode_enabled(self.think_mode), + self.req.chat_format, + &self.req.tool_orders, + self.finish, + ); + self.finish = recovered_finish; + pg + } else { + crate::tools::ParsedGenerated { + content: self.acc.text.clone(), + ok: true, + ..Default::default() + } + }; + let completion = self.acc.completion; + if !parsed_gen.calls.is_empty() { + if let Some(st) = self.oa.as_ref() { + st.tool.apply_ids(&mut parsed_gen.calls); + } + if let Some(st) = self.anth.as_ref() { + st.tool.apply_ids(&mut parsed_gen.calls); + } + assign_tool_ids( + &mut parsed_gen.calls, + if self.req.api == Api::Anthropic { + "toolu_" + } else { + "call_" + }, + ); + let exact = if parsed_gen.raw_dsml.is_empty() { + &parsed_gen.raw_tool_text + } else { + &parsed_gen.raw_dsml + }; + if !exact.is_empty() { + self.tool_replay = Some((parsed_gen.calls.clone(), exact.clone())); + } + self.finish = "tool_calls"; + } + self.apply_engine_usage(n_cached, n_computed); + self.req.timings = timings; + if self.req.stream { + match (self.req.api, self.req.kind) { + (Api::Openai, ReqKind::Completion) => { + sse_chunk( + &mut self.w, + &self.req, + &self.job_id, + None, + Some(self.finish), + ); + sse_done( + &mut self.w, + &self.req, + &self.job_id, + self.prompt_n, + completion, + ); + } + (Api::Openai, ReqKind::Chat) => { + if let Some(st) = self.oa.as_mut() { + openai_sse_finish_live( + &mut self.w, + &self.req, + &self.job_id, + st, + &self.acc.text, + self.finish, + self.prompt_n, + completion, + &parsed_gen.calls, + ); + } + } + (Api::Anthropic, _) => { + if let Some(st) = self.anth.as_mut() { + anthropic_sse_finish_live( + &mut self.w, + &self.req, + &self.job_id, + st, + &self.acc.text, + self.finish, + self.acc.matched_stop.as_deref(), + completion, + &parsed_gen.calls, + ); + } + } + (Api::Responses, _) => { + if let Some(st) = self.resp.as_mut() { + let created = self.w.created; + responses_sse_finish_live( + &mut self.w, + &self.req, + st, + &self.acc.text, + self.finish, + self.prompt_n, + completion, + self.acc.reasoning_tokens, + created, + &parsed_gen.calls, + ); + } + } + } + } else { + let bytes = match self.req.api { + Api::Anthropic => anthropic_final_response( + &self.req, + &self.job_id, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + self.finish, + self.acc.matched_stop.as_deref(), + self.prompt_n, + completion, + cors, + &parsed_gen.calls, + ), + Api::Responses => { + let (response_id, reasoning_id, message_id) = responses_ids(&self.job_id); + let think = if self.acc.thinking_inside() { + ThinkBlock::Open + } else { + ThinkBlock::Closed + }; + responses_final_response( + &self.req, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + self.finish, + think, + self.prompt_n, + completion, + self.acc.reasoning_tokens, + self.w.created, + cors, + &response_id, + &reasoning_id, + &message_id, + &parsed_gen.calls, + ) + } + Api::Openai => final_response( + &self.req, + &self.job_id, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + self.finish, + self.prompt_n, + completion, + self.w.created, + cors, + &parsed_gen.calls, + &[], + ), + }; + self.w.out.extend_from_slice(&bytes); + } + #[cfg(feature = "native")] + if self.finish != "error" && self.finish != "length" && !parsed_gen.calls.is_empty() { + self.chat_history = self + .chat_request + .take() + .map(|request| crate::chat_input::History::capture(request, &parsed_gen)); + } + let outcome = GenerateOutcome { + tool_ids: parsed_gen + .calls + .iter() + .map(|c| c.id.clone()) + .filter(|id| !id.is_empty()) + .collect(), + bank: None, + generation: 0, + frontier: self.prompt_n + completion, + finish: self.finish.to_string(), + timings: self.req.timings, + ..GenerateOutcome::default() + }; + (std::mem::take(&mut self.w.out), outcome) + } + + #[cfg(any(feature = "native", test))] + fn take_tool_replay(&mut self) -> Option<(Vec, String)> { + self.tool_replay.take() + } + + #[cfg(feature = "native")] + fn has_complete_tool_turn(&self) -> bool { + self.acc.saw_tool_end + } +} + +#[cfg(any(feature = "native", test))] +const QWEN_IMAGE_CACHE_PREFIX: &[u8] = b"\xffDS4IMG2:"; +#[cfg(any(feature = "native", test))] +const QWEN_IMAGE_CACHE_MARKER_LEN: usize = 53; +#[cfg(any(feature = "native", test))] +const QWEN_IMAGE_PAD: &[u8] = b"<|image_pad|>"; + +#[cfg(any(feature = "native", test))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ImageCacheSpan { + marker_start: usize, + marker_end: usize, + token_offset: u32, +} + +#[cfg(any(feature = "native", test))] +#[derive(Clone, Copy)] +struct ImageCacheIdentity { + pixel_hash: u64, + grid_h: u32, + grid_w: u32, + token_count: u32, + token_offset: u32, +} + +#[cfg(any(feature = "native", test))] +fn qwen_image_cache_text_build( + text: &[u8], + images: &[ImageCacheIdentity], +) -> Result<(Vec, Vec), &'static str> { + if images.is_empty() || images.len() > 4 { + return Err("image cache key requires an image"); + } + let capacity = text + .len() + .checked_add( + images + .len() + .checked_mul(QWEN_IMAGE_CACHE_MARKER_LEN) + .ok_or("image cache key overflow")?, + ) + .ok_or("image cache key overflow")?; + let mut key = Vec::with_capacity(capacity); + let mut spans = Vec::with_capacity(images.len()); + let mut cursor = 0; + for image in images { + let pad = text[cursor..] + .windows(QWEN_IMAGE_PAD.len()) + .position(|window| window == QWEN_IMAGE_PAD) + .map(|offset| cursor + offset) + .ok_or("image cache key is missing an image placeholder")?; + let through_pad = pad + QWEN_IMAGE_PAD.len(); + key.extend_from_slice(&text[cursor..through_pad]); + let marker_start = key.len(); + key.extend_from_slice(QWEN_IMAGE_CACHE_PREFIX); + key.extend_from_slice( + format!( + "{:016x}:{:08x}:{:08x}:{:08x}", + image.pixel_hash, image.grid_h, image.grid_w, image.token_count + ) + .as_bytes(), + ); + key.push(0xfe); + let marker_end = key.len(); + if marker_end - marker_start != QWEN_IMAGE_CACHE_MARKER_LEN { + return Err("image cache key overflow"); + } + spans.push(ImageCacheSpan { + marker_start, + marker_end, + token_offset: image.token_offset, + }); + cursor = through_pad; + } + key.extend_from_slice(&text[cursor..]); + Ok((key, spans)) +} + +#[cfg(any(feature = "native", test))] +fn qwen_image_cache_marker_at(bytes: &[u8]) -> bool { + bytes.len() >= QWEN_IMAGE_CACHE_MARKER_LEN + && bytes.starts_with(QWEN_IMAGE_CACHE_PREFIX) + && bytes[25] == b':' + && bytes[34] == b':' + && bytes[43] == b':' + && bytes[52] == 0xfe + && (QWEN_IMAGE_CACHE_PREFIX.len()..52) + .all(|i| matches!(i, 25 | 34 | 43) || bytes[i].is_ascii_hexdigit()) +} + +#[cfg(any(feature = "native", test))] +fn qwen_image_cache_text_strip(key: &[u8]) -> Option> { + let mut text = Vec::with_capacity(key.len()); + let mut markers = 0; + let mut i = 0; + while i < key.len() { + if qwen_image_cache_marker_at(&key[i..]) { + i += QWEN_IMAGE_CACHE_MARKER_LEN; + markers += 1; + } else { + text.push(key[i]); + i += 1; + } + } + (markers > 0).then_some(text) +} + +#[cfg(any(feature = "native", test))] +fn qwen_image_cache_token_cap(spans: &[ImageCacheSpan], cache_lcp: usize) -> usize { + spans + .iter() + .find(|span| cache_lcp >= span.marker_start && cache_lcp < span.marker_end) + .map_or(usize::MAX, |span| span.token_offset as usize) +} + +/// Matches `solar_batch_runtime_create`: max(4096, ceil(ctx/24)) then +/// aligned up to 4096. Token-only fallback must not admit a cut with no +/// native checkpoint; that would also hide a deeper disk restore. +#[cfg(any(feature = "native", test))] +const SOLAR_PARTIAL_ALIGN: usize = 4096; + +#[cfg(any(feature = "native", test))] +const SOLAR_PARTIAL_PERIOD: usize = 24; + +#[cfg(any(feature = "native", test))] +fn solar_partial_stride(ctx: i32) -> usize { + let ctx = usize::try_from(ctx.max(0)).unwrap_or(0); + let raw = ctx.div_ceil(SOLAR_PARTIAL_PERIOD).max(SOLAR_PARTIAL_ALIGN); + raw.div_ceil(SOLAR_PARTIAL_ALIGN) * SOLAR_PARTIAL_ALIGN +} + +#[cfg(any(feature = "native", test))] +fn warm_record_has_image(record: Option<&WarmRecord>) -> bool { + record.is_some_and(|record| { + record.cache_text.is_some() || (record.ext_flags & EXT_IMAGE_PIXELS_V2) != 0 + }) +} + +#[cfg(any(feature = "native", test))] +fn solar_stride_floor(cached: usize, ctx: i32) -> Option { + let stride = solar_partial_stride(ctx); + if stride == 0 || cached < stride { + return None; + } + Some((cached / stride) * stride) +} + +fn last_delta(raw: &[u8], emit_limit: usize, piece_len: usize) -> Option<&[u8]> { + if emit_limit == 0 { + return None; + } + let start = raw.len().saturating_sub(piece_len).min(emit_limit); + if start >= emit_limit { + return None; + } + Some(&raw[start..emit_limit]) +} + +#[cfg(any(feature = "native", test))] +fn bank_scope(parsed: &ParsedRequest) -> bool { + parsed.kind == ReqKind::Chat +} + +#[cfg(any(feature = "native", test))] +fn prompt_preserves_reasoning(parsed: &ParsedRequest) -> bool { + parsed.has_tools + || parsed.messages.iter().any(|message| { + (message.role == "assistant" && !message.calls.is_empty()) + || message.role == "tool" + || message.role == "function" + }) +} + +#[cfg(any(feature = "native", test))] +#[derive(Debug, Default)] +pub(crate) struct WarmBank { + pub(crate) record: Option, + /// Exact native committed frontier recorded at retire/restore. Banks are + /// idle between owner calls, so this is also the C victim-picker depth. + pub(crate) committed_tokens: i32, + pub(crate) stored_tokens: i32, + pub(crate) last_use: u64, +} + +#[cfg(any(feature = "native", test))] +#[derive(Debug)] +pub(crate) struct WarmRecord { + pub(crate) text: Vec, + pub(crate) cache_text: Option>, + pub(crate) exact_text: Option>, + pub(crate) exact_cache_text: Option>, + pub(crate) partial_only: bool, + pub(crate) generation: u64, + pub(crate) ext_flags: u8, + pub(crate) trailer: Vec, +} + +/// Keep Responses/title bits across retire so the next persist still +/// writes them. Trailer is rebuilt at persist time. +#[cfg(any(feature = "native", test))] +fn retired_record( + prev: Option<&WarmRecord>, + key: Vec, + cache_text: Option>, + generation: u64, +) -> Option { + let mut ext_flags = prev.map(|record| record.ext_flags).unwrap_or(0) & !EXT_IMAGE_PIXELS_V2; + if cache_text.is_some() { + ext_flags |= EXT_IMAGE_PIXELS_V2; + } + (!key.is_empty()).then(|| WarmRecord { + text: key, + cache_text, + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation, + ext_flags, + trailer: Vec::new(), + }) +} + +#[cfg(any(feature = "native", test))] +fn restored_record(key: Vec, generation: u64, ext_flags: u8) -> Option { + if ext_flags & EXT_IMAGE_PIXELS_V2 != 0 { + Some(WarmRecord { + text: qwen_image_cache_text_strip(&key)?, + cache_text: Some(key), + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation, + ext_flags, + trailer: Vec::new(), + }) + } else { + Some(WarmRecord { + text: key, + cache_text: None, + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation, + ext_flags, + trailer: Vec::new(), + }) + } +} + +#[cfg(any(feature = "native", test))] +fn extended_image_cache_key(prompt: &[u8], cache_prompt: &[u8], key: &[u8]) -> Option> { + let suffix = key.strip_prefix(prompt)?; + let mut cache_key = Vec::with_capacity(cache_prompt.len().checked_add(suffix.len())?); + cache_key.extend_from_slice(cache_prompt); + cache_key.extend_from_slice(suffix); + Some(cache_key) +} + +#[cfg(any(feature = "native", test))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct WarmMatch { + bank: usize, + exact: bool, +} + +#[cfg(any(feature = "native", test))] +fn warm_match_pick( + banks: &[WarmBank], + prompt: &[u8], + cache_prompt: Option<&[u8]>, +) -> Option { + let request_key = cache_prompt.unwrap_or(prompt); + let mut best = None; + let mut best_len = 0; + for (bank, state) in banks.iter().enumerate() { + let Some(record) = state.record.as_ref() else { + continue; + }; + if record.partial_only || record.cache_text.is_some() != cache_prompt.is_some() { + continue; + } + let primary = record.cache_text.as_deref().unwrap_or(&record.text); + let exact = if cache_prompt.is_some() { + record.exact_cache_text.as_deref() + } else { + record.exact_text.as_deref() + }; + for (key, is_exact) in [(Some(primary), false), (exact, true)] { + let Some(key) = key else { continue }; + if !key.is_empty() + && key.len() < request_key.len() + && key.len() > best_len + && request_key.starts_with(key) + { + best = Some(WarmMatch { + bank, + exact: is_exact, + }); + best_len = key.len(); + } + } + } + best +} + +#[cfg(any(feature = "native", test))] +fn warm_partial_match_pick( + banks: &[WarmBank], + prompt: &[u8], + cache_prompt: Option<&[u8]>, + min_prefix: usize, +) -> Option<(usize, usize)> { + let request_key = cache_prompt.unwrap_or(prompt); + if min_prefix == 0 || request_key.len() < min_prefix { + return None; + } + let mut best = None; + let mut best_len = 0; + for (bank, state) in banks.iter().enumerate() { + let Some(record) = state.record.as_ref() else { + continue; + }; + if record.cache_text.is_some() != cache_prompt.is_some() { + continue; + } + let key = record.cache_text.as_deref().unwrap_or(&record.text); + let prefix = request_key + .iter() + .zip(key) + .take_while(|(left, right)| left == right) + .count(); + if prefix >= min_prefix && prefix > best_len { + best = Some((bank, prefix)); + best_len = prefix; + } + } + best +} + +#[cfg(any(feature = "native", test))] +fn warm_partial_token_cut( + source: &[i32], + prompt: &[i32], + min_tokens: i32, + token_cap: usize, +) -> Option { + let cap = source + .len() + .min(prompt.len().saturating_sub(1)) + .min(token_cap); + let cut = source + .iter() + .zip(prompt) + .take(cap) + .take_while(|(left, right)| left == right) + .count(); + i32::try_from(cut) + .ok() + .filter(|cut| *cut >= min_tokens.max(1)) +} + +#[cfg(any(feature = "native", test))] +fn warm_victim_pick( + banks: &[WarmBank], + protected: &[bool], + exclude: Option, + pin_min: i32, + evict_lru: bool, +) -> Option { + debug_assert!(protected.is_empty() || protected.len() == banks.len()); + let mut superseded = None; + let mut superseded_use = u64::MAX; + let mut shallow = None; + let mut shallow_use = u64::MAX; + let mut deep = None; + let mut deep_use = u64::MAX; + + for (bank, state) in banks.iter().enumerate() { + if Some(bank) == exclude || protected.get(bank).copied().unwrap_or(false) { + continue; + } + if state.record.is_none() { + return Some(bank); + } + let is_deep = pin_min > 0 && state.committed_tokens >= pin_min; + if is_deep { + if state.last_use < deep_use { + deep = Some(bank); + deep_use = state.last_use; + } + } else if state.last_use < shallow_use { + shallow = Some(bank); + shallow_use = state.last_use; + } + if state.last_use < superseded_use && warm_record_superseded(banks, bank) { + superseded = Some(bank); + superseded_use = state.last_use; + } + } + superseded.or_else(|| evict_lru.then_some(shallow.or(deep)).flatten()) +} + +#[cfg(any(feature = "native", test))] +fn warm_record_superseded(banks: &[WarmBank], bank: usize) -> bool { + let Some(record) = banks.get(bank).and_then(|state| state.record.as_ref()) else { + return false; + }; + let key = record.cache_text.as_deref().unwrap_or(&record.text); + if key.is_empty() { + return false; + } + banks.iter().enumerate().any(|(other_bank, state)| { + other_bank != bank + && state.record.as_ref().is_some_and(|other| { + let other_key = other.cache_text.as_deref().unwrap_or(&other.text); + !other.partial_only + && other.cache_text.is_some() == record.cache_text.is_some() + && !other_key.is_empty() + && other_key.starts_with(key) + }) + }) +} + +#[cfg(any(feature = "native", test))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct WarmPlacement { + source: usize, + target: usize, + fork: bool, +} + +#[cfg(any(feature = "native", test))] +fn warm_placement( + banks: &[WarmBank], + protected: &[bool], + source: usize, + cached: i32, + pin_min: i32, + fork_enabled: bool, +) -> Option { + let target = (fork_enabled && (pin_min <= 0 || cached < pin_min)) + .then(|| warm_victim_pick(banks, protected, Some(source), pin_min, false)) + .flatten(); + if let Some(target) = target { + return Some(WarmPlacement { + source, + target, + fork: true, + }); + } + if protected.get(source).copied().unwrap_or(false) { + return None; + } + Some(WarmPlacement { + source, + target: source, + fork: false, + }) +} + +#[cfg(any(feature = "native", test))] +fn history_retire_prompt(prompt: &[u8], syntax: ModelSyntax) -> &[u8] { + // Motif and Step snapshots retain the generation-only think pair. + // Their separate history checkpoints were captured before that suffix; + // shortening a retired key would describe different native KV. + // + // Ling, MiMo, Naive and GLM replay the pair before assistant content. A + // stripped key cannot prefix the next rendered turn or restore disk KV. + if matches!( + syntax, + ModelSyntax::Motif3 + | ModelSyntax::Ling3Vl + | ModelSyntax::Mimo2 + | ModelSyntax::NaiveN05 + | ModelSyntax::Glm53 + ) { + return prompt; + } + prompt.strip_suffix(b"").unwrap_or(prompt) +} + +#[cfg(any(feature = "native", test))] +fn committed_key( + prompt: &[u8], + tokens: &[i32], + syntax: ModelSyntax, + mut token_text: impl FnMut(i32) -> Vec, +) -> Vec { + let mut key = history_retire_prompt(prompt, syntax).to_vec(); + for &token in tokens.iter().take(tokens.len().saturating_sub(1)) { + key.extend(token_text(token)); + } + key +} + +#[cfg(any(feature = "native", test))] +#[derive(Debug, PartialEq, Eq)] +struct BankRetireKey { + text: Vec, + exact_text: Option>, + partial_only: bool, + retained_existing: bool, +} + +#[cfg(any(feature = "native", test))] +fn text_ends_with_proper_prefix(text: &[u8], marker: &[u8]) -> bool { + (1..marker.len()) + .rev() + .any(|len| text.ends_with(&marker[..len])) +} + +#[cfg(any(feature = "native", test))] +fn thinking_bank_retire_key( + prompt: &[u8], + done_tokens: &[i32], + engine_finished: bool, + semantic_or_transport_cut: bool, + prompt_preserves_reasoning: bool, + has_tools: bool, + saw_tool_start: bool, + syntax: ModelSyntax, + format: ChatFormat, + mut token_text: impl FnMut(i32) -> Vec, +) -> Option { + if done_tokens.is_empty() { + return None; + } + let mut generation = Vec::new(); + for &token in done_tokens.iter().take(done_tokens.len().saturating_sub(1)) { + generation.extend(token_text(token)); + } + let mut exact = prompt.to_vec(); + exact.extend_from_slice(&generation); + + let close = think_end(format).as_bytes(); + let close_at = generation + .windows(close.len()) + .position(|window| window == close); + + // Official IQuest, Bailing V3 (Ling) and Beam history retains supplied + // reasoning even without tools. Keep that exact prefix; a visible-only + // rewrite cannot describe its KV. + if prompt_preserves_reasoning + || matches!( + syntax, + ModelSyntax::IQuestQ1 | ModelSyntax::Ling3Vl | ModelSyntax::BeamVq + ) + { + if engine_finished && has_tools && !saw_tool_start { + if let Some(at) = close_at { + if let Some(visible) = thinking_visible_key( + prompt, + &generation[at + close.len()..], + syntax, + format, + false, + ) { + return Some(BankRetireKey { + text: visible, + exact_text: Some(exact), + partial_only: false, + retained_existing: false, + }); + } + } + } + return Some(BankRetireKey { + text: exact, + exact_text: None, + partial_only: false, + retained_existing: false, + }); + } + + if engine_finished { + let at = close_at?; + return Some(BankRetireKey { + text: thinking_visible_key( + prompt, + &generation[at + close.len()..], + syntax, + format, + false, + )?, + exact_text: Some(exact), + partial_only: false, + retained_existing: false, + }); + } + + let partial_only = done_tokens.len() > 1 + && (semantic_or_transport_cut || text_ends_with_proper_prefix(&generation, close)); + if !partial_only { + if let Some(at) = close_at { + return Some(BankRetireKey { + text: thinking_visible_key( + prompt, + &generation[at + close.len()..], + syntax, + format, + false, + )?, + exact_text: None, + partial_only: false, + retained_existing: false, + }); + } + } + Some(BankRetireKey { + text: prompt.to_vec(), + exact_text: None, + partial_only, + retained_existing: false, + }) +} + +#[cfg(any(feature = "native", test))] +fn bank_retire_key( + prompt: &[u8], + snapshot_tokens: &[i32], + done_tokens: &[i32], + allow_generated_snapshot: bool, + syntax: ModelSyntax, + mut token_text: impl FnMut(i32) -> Vec, +) -> Option<(Vec, bool)> { + if !done_tokens.is_empty() { + return Some(( + committed_key(prompt, done_tokens, syntax, token_text), + false, + )); + } + let retained = committed_key(&[], snapshot_tokens, syntax, &mut token_text); + if retained.is_empty() { + return None; + } + if prompt.starts_with(&retained) { + return Some((retained, true)); + } + (allow_generated_snapshot && retained.starts_with(prompt)).then_some((retained, false)) +} + +#[cfg(any(feature = "native", test))] +fn warm_admit_tokens( + warm: &WarmBank, + prompt: &[u8], + cache_prompt: Option<&[u8]>, + snapshot_tokens: &[i32], + generation: u64, + seq_cap: i32, + exact: bool, + tokenize_suffix: impl FnOnce(&[u8]) -> Vec, +) -> Option<(Vec, i32)> { + let record = warm.record.as_ref()?; + if record.generation != generation || record.partial_only { + return None; + } + if record.cache_text.is_some() != cache_prompt.is_some() { + return None; + } + let text = if exact { + record.exact_text.as_deref()? + } else { + &record.text + }; + let key = if cache_prompt.is_some() { + if exact { + record.exact_cache_text.as_deref()? + } else { + record.cache_text.as_deref()? + } + } else { + text + }; + let request_key = cache_prompt.unwrap_or(prompt); + if key.is_empty() + || key.len() >= request_key.len() + || !request_key.starts_with(key) + || text.len() >= prompt.len() + || !prompt.starts_with(text) + { + return None; + } + let cached = i32::try_from(snapshot_tokens.len()) + .ok() + .filter(|n| *n > 0)?; + let mut tokens = snapshot_tokens.to_vec(); + tokens.extend(tokenize_suffix(&prompt[text.len()..])); + if tokens.len() <= snapshot_tokens.len() + || i32::try_from(tokens.len()) + .ok() + .filter(|n| *n <= seq_cap) + .is_none() + { + return None; + } + Some((tokens, cached)) +} + +#[cfg(any(feature = "native", test))] +fn live_continuation_tokens( + exact_prefix: &[i32], + suffix: &[u8], + seq_cap: i32, + tokenize_suffix: impl FnOnce(&[u8]) -> Vec, +) -> Option<(Vec, i32)> { + let cached = i32::try_from(exact_prefix.len()).ok().filter(|n| *n > 0)?; + let mut tokens = exact_prefix.to_vec(); + tokens.extend(tokenize_suffix(suffix)); + (tokens.len() > exact_prefix.len() && i32::try_from(tokens.len()).is_ok_and(|n| n <= seq_cap)) + .then_some((tokens, cached)) +} + +#[cfg(any(feature = "native", test))] +fn warm_partial_admit_tokens( + warm: &WarmBank, + prompt_tokens: &[i32], + snapshot_tokens: &[i32], + generation: u64, + min_tokens: i32, + seq_cap: i32, + token_cap: usize, +) -> Option<(Vec, i32)> { + let record = warm.record.as_ref()?; + if record.generation != generation + || i32::try_from(prompt_tokens.len()) + .ok() + .filter(|tokens| *tokens <= seq_cap) + .is_none() + { + return None; + } + let cached = warm_partial_token_cut(snapshot_tokens, prompt_tokens, min_tokens, token_cap)?; + Some((prompt_tokens.to_vec(), cached)) +} + +#[cfg(any(feature = "native", test))] +fn disk_restore_allowed( + warm: &WarmBank, + live: Option<(u64, usize)>, + disk_tokens: u32, + pin_min: i32, +) -> bool { + let Some(record) = warm.record.as_ref() else { + return true; + }; + let Some((generation, live_tokens)) = live else { + return true; + }; + if record.generation != generation { + return true; + } + let Ok(pin_min) = usize::try_from(pin_min) else { + return false; + }; + pin_min > 0 && disk_tokens as usize >= pin_min && live_tokens < pin_min +} + +#[cfg(any(feature = "native", test))] +fn disk_restore_target_allowed( + banks: &[WarmBank], + target: usize, + disk_tokens: u32, + pin_min: i32, +) -> bool { + let Some(state) = banks.get(target) else { + return false; + }; + if warm_record_superseded(banks, target) { + return true; + } + let live = state + .record + .as_ref() + .map(|record| (record.generation, state.committed_tokens.max(0) as usize)); + disk_restore_allowed(state, live, disk_tokens, pin_min) +} + +#[cfg(any(feature = "native", test))] +fn bank_retire_allowed(bank_enabled: bool, admitted: bool, done_called: bool) -> bool { + bank_enabled && admitted && done_called +} + +#[cfg(any(feature = "native", test))] +fn retired_bank( + bank_enabled: bool, + admitted: bool, + done_called: bool, + actual_bank: Option, + max_seq: i32, +) -> Option { + if !bank_retire_allowed(bank_enabled, admitted, done_called) { + return None; + } + actual_bank.filter(|bank| *bank >= 0 && *bank < max_seq) +} + +#[cfg(any(feature = "native", test))] +pub(crate) fn save_bank_record( + store: &mut KvStore, + warm: &mut WarmBank, + identity: (u8, u8, u32), + committed: i32, + generation: u64, + reason: KvReason, + save_payload: impl FnOnce(&Path) -> Result<(), GenerateError>, +) -> Result { + if committed <= 0 { + return Ok(false); + } + let Some(record) = warm.record.as_ref().filter(|record| { + record.generation == generation && !record.text.is_empty() && !record.partial_only + }) else { + return Ok(false); + }; + let text = record.cache_text.as_deref().unwrap_or(&record.text); + let tokens = u32::try_from(committed) + .map_err(|_| GenerateError::Engine("bank token count exceeds u32".into()))?; + let keep_ext = record.ext_flags; + let trailer = &record.trailer; + let (model_id, quant_bits, ctx) = identity; + let mut header = crate::generate::kv_header(model_id, quant_bits, ctx, tokens); + header.reason = reason; + header.ext_flags = bank_persist_ext_flags(keep_ext, false, !trailer.is_empty()); + let payload = store + .payload_temp() + .map_err(|error| GenerateError::Engine(error.to_string()))?; + save_payload(payload.path())?; + store + .write_payload_file(header, text, payload.path(), trailer) + .map_err(|error| GenerateError::Engine(error.to_string()))?; + warm.stored_tokens = committed; + Ok(true) +} + +/// One continuous work item for a rolling admit. +#[derive(Clone, Debug)] +pub struct ContPreparedPrompt { + pub prompt: Vec, + pub tokens: Vec, +} + +pub struct ContWork<'a> { + pub parsed: &'a ParsedRequest, + pub prepared: Option<&'a ContPreparedPrompt>, + pub job_id: &'a str, + pub created: i64, + pub cors: bool, + pub default_tokens: i32, + pub t_arrive: Instant, + pub out: &'a mut dyn Write, +} + +/// Owned request handed to the native rolling loop. The owner keeps the +/// original job for settlement; `out` is a clone of its shared sink. +pub struct ContOwnedWork { + pub key: usize, + pub parsed: ParsedRequest, + pub prepared: Option, + pub job_id: String, + pub created: i64, + pub cors: bool, + pub default_tokens: i32, + pub t_arrive: Instant, + pub out: Box, +} + +pub struct ContOwnedResult { + pub key: usize, + pub result: Result, +} + +/// Read-only engine view used by the owner while native generation is active. +pub trait ContProbe { + fn prompt_tokens( + &self, + parsed: &mut ParsedRequest, + ) -> Result<(Vec, Vec), GenerateError>; + fn seq_cap(&self) -> i32; + fn bank_live(&self, bank: i32) -> Option<(u64, i32)>; +} + +/// FIFO source polled again whenever the native scheduler exposes a free bank. +pub trait ContSource { + fn next(&mut self, probe: &dyn ContProbe) -> Option; + + /// Called after the bank record is retired and before terminal bytes are + /// appended to the shared client sink. + fn publish(&mut self, _key: usize, _outcome: &GenerateOutcome) {} + + /// Called after terminal bytes have been appended. The owner can release + /// completed clients without waiting for the whole rolling epoch to drain. + fn settled(&mut self, _key: usize, _result: &Result) {} +} + +/// Trait seam so `handle_client_inner` can drive a continuous lane without +/// the native feature (tests supply a scripted implementation). +pub trait ContExec { + fn model_id(&self) -> i32; + /// C `ds4_chat_system_token()` (core_engine_api.c:94-96): "" when the + /// loaded vocab has no <|System|> (the V4 fallback). + fn chat_system_token(&self) -> &'static str { + "" + } + fn render_request(&self, parsed: &ParsedRequest) -> Result, GenerateError> { + render_prompt_ex( + parsed, + self.model_id(), + crate::render::dsml_head(self.model_id(), self.chat_system_token()) + .with_effort(parsed.effort.head_budget()), + ) + } + fn restore_chat(&self, _parsed: &mut ParsedRequest) -> Result<(), GenerateError> { + Ok(()) + } + fn seq_cap(&self) -> i32; + fn set_stop_requested(&mut self, _stop_requested: Option bool>) {} + /// Number of persistent continuous banks available to this executor. + fn max_seq(&self) -> i32 { + 1 + } + /// One C trim (`ds4_batch_ctx_trim_free`). Host does not implement trim. + fn trim_idle_banks(&mut self, _want_bytes: u64) -> u64 { + 0 + } + fn encode_chat(&self, rendered: &[u8]) -> Vec; + fn encode_text(&self, text: &str) -> Vec; + fn prepare_tokens( + &self, + parsed: &ParsedRequest, + _prompt: &[u8], + tokens: Vec, + ) -> Result, GenerateError> { + if parsed.images.is_empty() { + Ok(tokens) + } else { + Err(GenerateError::Unsupported( + "image input requires native continuous runtime", + )) + } + } + /// `bank_hold_retry` keeps registry locking outside the native owner; a + /// missing live reference asks the registry to preserve C's fail-closed rule. + fn generate( + &mut self, + parsed: &ParsedRequest, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + t_arrive: Instant, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + store: Option<&mut KvStore>, + out: &mut dyn Write, + ) -> Result; + + fn generate_prepared( + &mut self, + parsed: &ParsedRequest, + _prepared: Option<&ContPreparedPrompt>, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + t_arrive: Instant, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + store: Option<&mut KvStore>, + out: &mut dyn Write, + ) -> Result { + self.generate( + parsed, + job_id, + created, + cors, + default_tokens, + t_arrive, + bank_hold_retry, + store, + out, + ) + } + + /// Drive up to `max_seq()` jobs in one rolling admit loop. The default + /// keeps non-native implementations correct by running them sequentially. + fn generate_batch( + &mut self, + jobs: Vec>, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + mut store: Option<&mut KvStore>, + ) -> Vec> { + jobs.into_iter() + .map(|job| { + self.generate_prepared( + job.parsed, + job.prepared, + job.job_id, + job.created, + job.cors, + job.default_tokens, + job.t_arrive, + bank_hold_retry, + store.as_deref_mut(), + job.out, + ) + }) + .collect() + } + + /// Native implementations return `Some` and keep polling `source` while + /// any bank remains active. Other executors retain the bounded batch path. + fn generate_rolling( + &mut self, + _source: &mut dyn ContSource, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut KvStore>, + ) -> Option> { + None + } + + fn shutdown(&mut self, _store: Option<&mut KvStore>) {} + + fn bank_live(&self, _bank: i32) -> Option<(u64, i32)> { + None + } + + /// Static owner when this lane also holds a `BatchCtx`. + fn as_static(&mut self) -> Option<&mut dyn crate::serve_static::StaticExec> { + None + } +} + +/// Render + tokenize a request for routing (`prompt_len` feeds +/// `route_decide` before any lane is entered), mirroring the C server's +/// job-prep order. +pub fn cont_prompt_tokens( + exec: &dyn ContExec, + parsed: &mut ParsedRequest, +) -> Result<(Vec, Vec), GenerateError> { + prepare_required_prefixes(parsed, syntax_for_model_id(exec.model_id()), |literal| { + Ok(exec.encode_chat(literal)) + })?; + exec.restore_chat(parsed)?; + let prompt = exec.render_request(parsed)?; + let tokens = match parsed.kind { + ReqKind::Completion => exec.encode_text(std::str::from_utf8(&prompt).unwrap_or("")), + ReqKind::Chat => exec.encode_chat(&prompt), + }; + let tokens = exec.prepare_tokens(parsed, &prompt, tokens)?; + Ok((prompt, tokens)) +} + +#[cfg(any(feature = "native", test))] +fn can_reuse_cont_prompt(attached_tool_blocks: usize, has_images: bool) -> bool { + attached_tool_blocks == 0 && !has_images +} + +#[cfg(feature = "native")] +pub use native::ContLane; + +#[cfg(feature = "native")] +mod native { + use super::*; + + use ds4_core::{ + qwen_image_pixel_hash, qwen_image_probe, BatchCtx, ContAdmit, ContDone, ContDriver, + QwenImageInput, ReuseKind, ReuseMiss, ReuseTaken, Vocab, CONT_SAMPLE_GREEDY, + CONT_SAMPLE_NONE, + }; + + use crate::serve_static::{BatchStatic, CoalesceLimits, StaticExec, StaticJob, StaticRow}; + use crate::tool_memory::ToolMemory; + + /// Native continuous lane: a rolling `ContDriver` over every bank + /// exposed by the persistent native batch context. + pub struct ContLane<'m> { + batch: BatchCtx<'m>, + host: ContHost<'m>, + } + + struct BankHistory { + generation: u64, + frontier: usize, + history: crate::chat_input::History, + } + + struct ContHost<'m> { + vocab: &'m Vocab, + template: Option<&'m ds4_core::chat_template::Template>, + histories: Vec>, + model_id: i32, + quant_bits: i32, + ctx: i32, + /// Family EOT for the per-seq stop, like the C server's job prep; + /// the engine's `-1` default is the base EOS, not the family EOT. + eos: i32, + warm: Vec, + warm_clock: u64, + /// Resolved `--prefix-reuse`. `None` skips every warm plan: fork is + /// not the only reuse, an in-place bank hit is one too. + warm_reuse: ReuseKind, + warm_fork: bool, + warm_fork_partial: bool, + warm_disk_partial: bool, + warm_partial_min: i32, + warm_pin_min: i32, + warm_persist_min: i32, + warm_checkpoint: bool, + tool_memory: ToolMemory, + memgov: Box, + stop_requested: Option bool>, + } + + struct WarmAdmitPlan { + source: usize, + tokens: Vec, + cached: i32, + partial: bool, + } + + struct PreparedSlot { + admit: ContAdmit, + head: Vec, + stepper: ContStepper, + capture_done: bool, + reuse: ReuseTaken, + miss: ReuseMiss, + t_arrive: Instant, + stop_requested: Option bool>, + history: Option, + } + + struct HistoryCheckpoint { + text: Vec, + tokens: Vec, + payload: ds4_kv::PayloadTemp, + captured: bool, + } + + impl HistoryCheckpoint { + fn capture(&mut self, batch: &BatchCtx<'_>, bank: i32, tokens: &[i32]) { + if tokens != self.tokens || self.captured { + return; + } + match batch.save_bank_payload(bank, self.payload.path()) { + Ok(()) => self.captured = true, + Err(error) => eprintln!("ds4-server-rs: history bank capture failed: {error}"), + } + } + + fn persist(self, store: &mut KvStore, identity: (u8, u8, u32)) { + if !self.captured { + return; + } + let (model_id, quant_bits, ctx) = identity; + let mut header = + crate::generate::kv_header(model_id, quant_bits, ctx, self.tokens.len() as u32); + header.reason = KvReason::BankCheckpoint; + header.ext_flags = bank_persist_ext_flags(0, false, false); + if let Err(error) = + store.write_payload_file(header, &self.text, self.payload.path(), &[]) + { + eprintln!("ds4-server-rs: history bank checkpoint failed: {error}"); + } + } + } + + struct PreparedImages { + tokens: Vec, + images: Vec, + cache_prompt: Option>, + cache_spans: Vec, + } + + fn prepare_qwen_images( + model_id: i32, + parsed: &ParsedRequest, + prompt: Option<&[u8]>, + tokens: Vec, + ) -> Result { + const IMAGE_PAD_TOKEN: i32 = 248056; + if parsed.images.is_empty() { + return Ok(PreparedImages { + tokens, + images: Vec::new(), + cache_prompt: None, + cache_spans: Vec::new(), + }); + } + if model_id != 6 { + return Err(GenerateError::Unsupported( + "image input is supported only by Qwen4Exp", + )); + } + let mut probed = Vec::with_capacity(parsed.images.len()); + let mut expanded_len = tokens.len(); + for image in &parsed.images { + let info = qwen_image_probe(&image.data) + .map_err(|error| GenerateError::Engine(error.to_string()))?; + if info.token_count == 0 { + return Err(GenerateError::Engine( + "Qwen image probe returned zero tokens".into(), + )); + } + expanded_len = expanded_len + .checked_add(info.token_count as usize - 1) + .ok_or_else(|| { + GenerateError::Engine("expanded image prompt is too large".into()) + })?; + probed.push(info); + } + if expanded_len > i32::MAX as usize { + return Err(GenerateError::Engine( + "expanded image prompt is too large".into(), + )); + } + let mut expanded = Vec::with_capacity(expanded_len); + let mut images = Vec::with_capacity(parsed.images.len()); + let mut image_index = 0; + for token in tokens { + if token != IMAGE_PAD_TOKEN { + expanded.push(token); + continue; + } + let Some((image, info)) = parsed.images.get(image_index).zip(probed.get(image_index)) + else { + return Err(GenerateError::Engine( + "ambiguous literal <|image_pad|> in prompt".into(), + )); + }; + let token_offset = u32::try_from(expanded.len()) + .map_err(|_| GenerateError::Engine("expanded image prompt is too large".into()))?; + expanded.extend(std::iter::repeat_n( + IMAGE_PAD_TOKEN, + info.token_count as usize, + )); + images.push(QwenImageInput { + data: image.data.clone(), + token_offset, + grid_h: info.grid_h, + grid_w: info.grid_w, + }); + image_index += 1; + } + if image_index != parsed.images.len() || expanded.len() != expanded_len { + return Err(GenerateError::Engine( + "Qwen image placeholder count does not match payloads".into(), + )); + } + let (cache_prompt, cache_spans) = if let Some(prompt) = prompt { + let identities = images + .iter() + .zip(&probed) + .map(|(image, info)| { + Ok(ImageCacheIdentity { + pixel_hash: qwen_image_pixel_hash(&image.data) + .map_err(|error| GenerateError::Engine(error.to_string()))?, + grid_h: info.grid_h, + grid_w: info.grid_w, + token_count: info.token_count, + token_offset: image.token_offset, + }) + }) + .collect::, GenerateError>>()?; + let (key, spans) = qwen_image_cache_text_build(prompt, &identities) + .map_err(|error| GenerateError::Engine(error.into()))?; + (Some(key), spans) + } else { + (None, Vec::new()) + }; + Ok(PreparedImages { + tokens: expanded, + images, + cache_prompt, + cache_spans, + }) + } + + impl PreparedSlot { + fn into_slot<'a>(self, out: Box) -> JobSlot<'a> { + JobSlot { + admit: Some(self.admit), + head: if self.head.is_empty() { + None + } else { + Some(self.head) + }, + stepper: self.stepper, + out, + admitted: false, + bank: None, + io_failed: false, + host_abort: false, + engine_eos: false, + capture_done: self.capture_done, + reuse: self.reuse, + miss: self.miss, + speculated: false, + stop_requested: self.stop_requested, + done_tokens: Vec::new(), + n_cached: 0, + n_computed: 0, + t_arrive: self.t_arrive, + t_admit: None, + t_first: None, + t_done: None, + decode_ms: 0.0, + decode_tokens: 0, + decode_steps: 0, + history: self.history, + } + } + } + + struct JobSlot<'a> { + admit: Option, + /* Client-visible transport is committed at on_admitted, the C + * cont_stream_start point: a rejected request never sees bytes and + * can fall back to the serial lane transport-clean. */ + head: Option>, + stepper: ContStepper, + out: Box, + admitted: bool, + bank: Option, + io_failed: bool, + host_abort: bool, + engine_eos: bool, + capture_done: bool, + /// Mechanism chosen at admission; only real once native reports cache. + reuse: ReuseTaken, + /// Why a candidate was refused, when one was. + miss: ReuseMiss, + /// Native ran draft rows for this sequence. + speculated: bool, + stop_requested: Option bool>, + done_tokens: Vec, + n_cached: i32, + n_computed: i32, + t_arrive: Instant, + t_admit: Option, + t_first: Option, + t_done: Option, + decode_ms: f64, + decode_tokens: i32, + decode_steps: i32, + history: Option, + } + + impl JobSlot<'_> { + fn transport_alive(&mut self) -> bool { + if self.stepper.stop_for_shutdown(self.stop_requested) { + self.host_abort = true; + return false; + } + if self.admitted { + let heartbeat = self.stepper.heartbeat(Instant::now()); + self.push(&heartbeat); + } + if !self.io_failed && self.out.flush().is_err() { + self.io_failed = true; + } + !self.io_failed + } + + fn push(&mut self, bytes: &[u8]) { + if bytes.is_empty() || self.io_failed { + return; + } + if self.out.write_all(bytes).is_err() || self.out.flush().is_err() { + self.io_failed = true; + } + } + + fn on_token(&mut self, vocab: &Vocab, token: i32) -> bool { + if !self.transport_alive() { + self.host_abort = true; + return false; + } + if self.t_first.is_none() { + self.t_first = Some(Instant::now()); + } + if vocab.is_stop(token) { + let format = self.stepper.req.chat_format; + if let Some(text) = crate::tools::beam::stop_text(format, token, vocab.eot_id) { + let step = self.stepper.feed_stop(text); + self.push(&step.bytes); + } + self.stepper.mark_stop(); + self.host_abort = true; + return false; + } + let step = self.stepper.feed(&vocab.token_text(token)); + self.push(&step.bytes); + if self.io_failed || step.done { + self.host_abort = true; + return false; + } + true + } + + fn sample_override(&mut self) -> i32 { + match self.stepper.sample_override() { + SampleOverride::None => CONT_SAMPLE_NONE, + SampleOverride::Greedy => CONT_SAMPLE_GREEDY, + SampleOverride::Token(token) => ds4_core::cont_sample_token(token), + } + } + + fn sample_exclude(&self, vocab: &Vocab) -> i32 { + self.stepper.sample_exclude(vocab.eos_id) + } + + fn admitted(&mut self, n_cached: i32, n_computed: i32, bank: i32) -> bool { + self.n_cached = n_cached; + self.n_computed = n_computed; + self.bank = Some(bank); + self.t_admit = Some(Instant::now()); + self.admitted = true; + let head = self.stepper.admitted_head( + self.head.take().unwrap_or_default(), + n_cached, + n_computed, + ); + self.push(&head); + self.transport_alive() + } + + fn done(&mut self, tokens: &[i32], finish: i32, stats: ContDone) { + self.engine_eos = finish == 1; + if self.capture_done { + self.done_tokens.extend_from_slice(tokens); + } + self.decode_ms = stats.decode_ms; + self.decode_tokens = stats.decode_tokens; + self.decode_steps = stats.decode_steps; + self.speculated = stats.spec_drafts > 0; + self.t_done = Some(Instant::now()); + } + } + + struct RollDriver<'a, 'm> { + roll: crate::serve_cont_roll::ContRoll, + slots: std::collections::HashMap>, + vocab: &'a Vocab, + next_user: usize, + batch: &'a BatchCtx<'m>, + } + + impl<'a, 'm> RollDriver<'a, 'm> { + fn new(vocab: &'a Vocab, batch: &'a BatchCtx<'m>) -> Self { + Self { + roll: crate::serve_cont_roll::ContRoll::new(), + slots: std::collections::HashMap::new(), + vocab, + next_user: 1, + batch, + } + } + + fn push(&mut self, mut slot: JobSlot<'a>) -> usize { + let user = self.next_user; + self.next_user += 1; + if let Some(admit) = slot.admit.as_mut() { + admit.user = user; + } + self.roll.enqueue(user); + self.slots.insert(user, slot); + user + } + } + + impl ContDriver for RollDriver<'_, '_> { + fn admit(&mut self) -> Option { + let user = self.roll.admit()?; + self.slots.get_mut(&user)?.admit.take() + } + + fn on_token(&mut self, user: usize, token: i32) -> bool { + let Some(slot) = self.slots.get_mut(&user) else { + return false; + }; + slot.on_token(self.vocab, token) + } + + fn on_done(&mut self, user: usize, tokens: &[i32], finish: i32, stats: ContDone) { + let Some(slot) = self.slots.get_mut(&user) else { + return; + }; + slot.done(tokens, finish, stats); + self.roll.complete(user); + } + + fn sample_override(&mut self, user: usize) -> i32 { + let Some(slot) = self.slots.get_mut(&user) else { + return CONT_SAMPLE_NONE; + }; + slot.sample_override() + } + + fn sample_exclude(&mut self, user: usize) -> i32 { + self.slots + .get(&user) + .map_or(-1, |slot| slot.sample_exclude(self.vocab)) + } + + fn alive(&mut self, user: usize) -> bool { + self.slots + .get_mut(&user) + .is_none_or(|slot| slot.transport_alive()) + } + + fn on_admitted(&mut self, user: usize, n_cached: i32, n_computed: i32, bank: i32) -> bool { + let Some(slot) = self.slots.get_mut(&user) else { + return false; + }; + slot.admitted(n_cached, n_computed, bank) + } + + fn on_checkpoint(&mut self, user: usize, bank: i32, tokens: &[i32]) { + if let Some(history) = self + .slots + .get_mut(&user) + .and_then(|slot| slot.history.as_mut()) + { + history.capture(self.batch, bank, tokens); + } + } + } + + struct LaneProbe<'a, 'm> { + batch: &'a BatchCtx<'m>, + host: &'a ContHost<'m>, + } + + impl ContProbe for LaneProbe<'_, '_> { + fn prompt_tokens( + &self, + parsed: &mut ParsedRequest, + ) -> Result<(Vec, Vec), GenerateError> { + prepare_required_prefixes( + parsed, + syntax_for_model_id(self.host.model_id), + |literal| Ok(self.host.vocab.encode_rendered_bytes(literal)), + )?; + self.host.restore_request(self.batch, parsed)?; + let prompt = self.host.render_request(parsed)?; + let tokens = match parsed.kind { + ReqKind::Completion => self + .host + .vocab + .encode_text(std::str::from_utf8(&prompt).unwrap_or("")), + ReqKind::Chat => self.host.vocab.encode_rendered_bytes(&prompt), + }; + let tokens = prepare_qwen_images(self.host.model_id, parsed, None, tokens)?.tokens; + Ok((prompt, tokens)) + } + + fn seq_cap(&self) -> i32 { + self.batch.seq_cap() + } + + fn bank_live(&self, bank: i32) -> Option<(u64, i32)> { + let snapshot = self.batch.bank_snapshot(bank).ok()?; + let frontier = i32::try_from(snapshot.tokens.len()).ok()?; + (frontier > 0).then_some((snapshot.generation, frontier)) + } + } + + struct RollingSlot { + key: usize, + cors: bool, + job: JobSlot<'static>, + } + + struct RollingDriver<'a, 'm> { + host: &'a mut ContHost<'m>, + batch: &'a BatchCtx<'m>, + source: &'a mut dyn ContSource, + bank_hold_retry: &'a mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + store: Option<&'a mut KvStore>, + slots: std::collections::HashMap, + pending: Option, + results: Vec, + next_user: usize, + } + + impl RollingDriver<'_, '_> { + fn reserve(&self) -> crate::serve_cont_roll::RollReserve { + let mut reserve = crate::serve_cont_roll::RollReserve::new(); + for slot in self.slots.values() { + if let Some(bank) = slot.job.bank { + reserve.note_place(bank.saturating_add(1)); + } + } + reserve + } + + fn finish_stranded(&mut self, error: String) { + if let Some(work) = self.pending.take() { + self.results.push(ContOwnedResult { + key: work.key, + result: Err(GenerateError::Engine(error.clone())), + }); + } + for (_, slot) in std::mem::take(&mut self.slots) { + let result = self.host.finish_driven( + self.batch, + slot.job, + Some(error.clone()), + self.store.as_deref_mut(), + slot.cors, + None, + ); + self.results.push(ContOwnedResult { + key: slot.key, + result, + }); + } + } + } + + impl ContDriver for RollingDriver<'_, '_> { + fn admit(&mut self) -> Option { + loop { + let work = match self.pending.take() { + Some(work) => work, + None => { + let probe = LaneProbe { + batch: self.batch, + host: self.host, + }; + self.source.next(&probe)? + } + }; + let reserve = self.reserve(); + if work + .parsed + .directed_bank + .is_some_and(|bank| reserve.contains(bank)) + { + self.pending = Some(work); + return None; + } + let key = work.key; + let cors = work.cors; + let mut work = work; + let prepared = { + let borrowed = ContWork { + parsed: &work.parsed, + prepared: work.prepared.as_ref(), + job_id: &work.job_id, + created: work.created, + cors: work.cors, + default_tokens: work.default_tokens, + t_arrive: work.t_arrive, + out: &mut *work.out, + }; + self.host.prepare_slot( + self.batch, + &borrowed, + self.bank_hold_retry, + self.store.as_deref_mut(), + &reserve, + ) + }; + let slot = match prepared { + Ok(slot) => slot, + Err(error) => { + self.results.push(ContOwnedResult { + key, + result: Err(error), + }); + continue; + } + }; + let user = self.next_user; + self.next_user += 1; + let mut job = slot.into_slot(work.out); + let admit = job.admit.as_mut()?; + admit.user = user; + let result = job.admit.take(); + self.slots.insert(user, RollingSlot { key, cors, job }); + return result; + } + } + + fn on_token(&mut self, user: usize, token: i32) -> bool { + self.slots + .get_mut(&user) + .is_some_and(|slot| slot.job.on_token(self.host.vocab, token)) + } + + fn on_done(&mut self, user: usize, tokens: &[i32], finish: i32, stats: ContDone) { + let Some(mut slot) = self.slots.remove(&user) else { + return; + }; + slot.job.done(tokens, finish, stats); + let key = slot.key; + let source = &mut self.source; + let mut publish = |outcome: &GenerateOutcome| source.publish(key, outcome); + let result = self.host.finish_driven( + self.batch, + slot.job, + None, + self.store.as_deref_mut(), + slot.cors, + Some(&mut publish), + ); + drop(publish); + self.source.settled(key, &result); + self.results.push(ContOwnedResult { key, result }); + } + + fn sample_override(&mut self, user: usize) -> i32 { + self.slots + .get_mut(&user) + .map_or(CONT_SAMPLE_NONE, |slot| slot.job.sample_override()) + } + + fn sample_exclude(&mut self, user: usize) -> i32 { + self.slots + .get(&user) + .map_or(-1, |slot| slot.job.sample_exclude(self.host.vocab)) + } + + fn alive(&mut self, user: usize) -> bool { + self.slots + .get_mut(&user) + .is_none_or(|slot| slot.job.transport_alive()) + } + + fn on_admitted(&mut self, user: usize, n_cached: i32, n_computed: i32, bank: i32) -> bool { + self.slots + .get_mut(&user) + .is_some_and(|slot| slot.job.admitted(n_cached, n_computed, bank)) + } + + fn on_checkpoint(&mut self, user: usize, bank: i32, tokens: &[i32]) { + if let Some(history) = self + .slots + .get_mut(&user) + .and_then(|slot| slot.job.history.as_mut()) + { + history.capture(self.batch, bank, tokens); + } + } + } + + impl<'m> ContLane<'m> { + pub fn with_template( + mut self, + template: Option<&'m ds4_core::chat_template::Template>, + ) -> Self { + self.host.template = template; + self + } + + /// The plan owns reuse. Without this the lane keeps whatever the + /// published `DS4_SERVER_FORK` said. + pub fn with_prefix_reuse(mut self, reuse: ReuseKind) -> Self { + self.host.warm_reuse = reuse; + self + } + + /// The boot exact-replay switches, the same map the serial engine gets + /// (`NativeDecode::with_tool_memory`). The reference has one server + /// struct, so a flag that reached only one lane would be a half port. + pub fn with_tool_memory(mut self, config: crate::tool_memory::ToolMemoryConfig) -> Self { + self.host.tool_memory = crate::tool_memory::ToolMemory::from_config(config); + self + } + + pub fn new( + batch: BatchCtx<'m>, + vocab: &'m Vocab, + model_id: i32, + quant_bits: i32, + ctx: i32, + eos: i32, + ) -> Self { + let max_seq = usize::try_from(batch.max_seq().max(0)).unwrap_or(0); + let warm_pin_min = crate::serve::env_i32_bound("DS4_SERVER_PIN_MIN_TOKENS", 65536); + let warm_persist_min = crate::serve::env_i32_bound( + "DS4_SERVER_PERSIST_MIN_TOKENS", + DEFAULT_BANK_PERSIST_MIN_TOKENS, + ); + let warm_fork = std::env::var_os("DS4_SERVER_FORK").is_none_or(|value| value != "0"); + let partial_requested = + std::env::var_os("DS4_SERVER_FORK_PARTIAL").is_none_or(|value| value != "0"); + let warm_fork_partial = partial_requested && batch.supports_partial_reuse(); + let warm_disk_partial = warm_fork_partial + && std::env::var_os("DS4_SERVER_DISK_PARTIAL").is_none_or(|value| value != "0"); + let warm_partial_min = + crate::serve::env_i32_bound("DS4_SERVER_FORK_PARTIAL_MIN", 192).max(136); + let warm_checkpoint = + std::env::var_os("DS4_SERVER_BANK_CHECKPOINT").is_none_or(|value| value != "0"); + if partial_requested && !warm_fork_partial { + eprintln!("ds4-server-rs: partial bank reuse disabled by model runtime"); + } + Self { + batch, + host: ContHost { + vocab, + template: None, + histories: (0..max_seq).map(|_| None).collect(), + model_id, + quant_bits, + ctx, + eos, + warm: (0..max_seq).map(|_| WarmBank::default()).collect(), + warm_clock: 0, + warm_reuse: if warm_fork { + ReuseKind::Partial + } else { + ReuseKind::None + }, + warm_fork, + warm_fork_partial, + warm_disk_partial, + warm_partial_min, + warm_pin_min, + warm_persist_min, + warm_checkpoint, + tool_memory: ToolMemory::default(), + memgov: Box::new(crate::serve_cont_roll::AdmitAlways), + stop_requested: None, + }, + } + } + } + + impl ContHost<'_> { + fn restore_request( + &self, + batch: &BatchCtx<'_>, + parsed: &mut ParsedRequest, + ) -> Result { + if self.template.is_none() || parsed.live_call_ids.is_empty() { + return Ok(false); + } + let saved = self.histories.iter().enumerate().find_map(|(bank, saved)| { + if parsed + .directed_bank + .is_some_and(|directed| directed != bank as i32) + { + return None; + } + saved + .as_ref() + .filter(|saved| saved.history.matches(parsed)) + .map(|saved| (bank, saved)) + }); + let Some((bank, saved)) = saved else { + if parsed.responses_requires_live_tool_state + || parsed.anthropic_requires_live_tool_state + || parsed.directed_bank.is_some() + { + return Err(GenerateError::Unsupported( + "retained chat history is unavailable", + )); + } + return Ok(false); + }; + let snapshot = batch + .bank_snapshot(bank as i32) + .map_err(|e| GenerateError::Engine(e.to_string()))?; + if snapshot.generation != saved.generation || snapshot.tokens.len() != saved.frontier { + return Err(GenerateError::Unsupported( + "retained chat frontier is no longer live", + )); + } + saved.history.restore(parsed) + } + + fn render_request(&self, parsed: &ParsedRequest) -> Result, GenerateError> { + crate::chat_input::render_model( + self.template, + self.model_id, + parsed, + self.vocab.chat_system_token(), + ) + } + fn identity(&self) -> Option<(u8, u8, u32)> { + crate::generate::kv_identity(self.model_id, self.quant_bits, self.ctx) + } + + fn note_use(&mut self, bank: usize) { + self.warm_clock = self.warm_clock.wrapping_add(1); + if let Some(state) = self.warm.get_mut(bank) { + state.last_use = self.warm_clock; + } + } + + fn warm_full_plan( + &mut self, + batch: &BatchCtx<'_>, + prompt: &[u8], + cache_prompt: Option<&[u8]>, + ) -> Option { + let matched = warm_match_pick(&self.warm, prompt, cache_prompt)?; + let source = matched.bank; + let snapshot = batch.bank_snapshot(i32::try_from(source).ok()?).ok()?; + let (tokens, cached) = warm_admit_tokens( + self.warm.get(source)?, + prompt, + cache_prompt, + &snapshot.tokens, + snapshot.generation, + batch.seq_cap(), + matched.exact, + |suffix| self.vocab.encode_rendered_bytes(suffix), + )?; + self.warm[source].committed_tokens = cached; + Some(WarmAdmitPlan { + source, + tokens, + cached, + partial: false, + }) + } + + fn warm_partial_plan( + &mut self, + batch: &BatchCtx<'_>, + prompt: &[u8], + cache_prompt: Option<&[u8]>, + cache_spans: &[ImageCacheSpan], + prompt_tokens: &[i32], + miss: &mut ReuseMiss, + ) -> Option { + if !self.warm_fork_partial { + return None; + } + let min_prefix = usize::try_from(self.warm_partial_min).ok()?; + if let Some((source, cache_lcp)) = + warm_partial_match_pick(&self.warm, prompt, cache_prompt, min_prefix) + { + let snapshot = batch.bank_snapshot(i32::try_from(source).ok()?).ok()?; + if let Some((tokens, cached)) = warm_partial_admit_tokens( + self.warm.get(source)?, + prompt_tokens, + &snapshot.tokens, + snapshot.generation, + self.warm_partial_min, + batch.seq_cap(), + qwen_image_cache_token_cap(cache_spans, cache_lcp), + ) { + self.warm[source].committed_tokens = + i32::try_from(snapshot.tokens.len()).ok()?; + return Some(WarmAdmitPlan { + source, + tokens, + cached, + partial: true, + }); + } + + // The text LCP passed; the tokens it covers can still be + // under the partial minimum. Only then is this the threshold + // — a stale bank record or a prompt past the sequence is not. + let fresh = self.warm.get(source).is_some_and(|warm| { + warm.record + .as_ref() + .is_some_and(|record| record.generation == snapshot.generation) + }); + let fits = i32::try_from(prompt_tokens.len()).is_ok_and(|n| n <= batch.seq_cap()); + if fresh && fits { + Self::note_miss(miss, ReuseMiss::BelowThreshold); + } + } + // Host text records can be missing after a one-bank Solar retire + // while the native committed tokens are still the reuse source. + // Qwen/image banks stay on the text/image-aware plan: token IDs + // can match across different pixels, including a text-only + // follow-up against a multimodal source. + if syntax_for_model_id(self.model_id) != ModelSyntax::SolarOpen2 + || !cache_spans.is_empty() + { + return None; + } + if i32::try_from(prompt_tokens.len()) + .ok() + .filter(|tokens| *tokens <= batch.seq_cap()) + .is_none() + { + return None; + } + let n_banks = usize::try_from(batch.max_seq().max(0)).ok()?; + let mut best = None; + for bank in 0..n_banks { + let Ok(bank_i32) = i32::try_from(bank) else { + continue; + }; + let Ok(snapshot) = batch.bank_snapshot(bank_i32) else { + continue; + }; + if warm_record_has_image( + self.warm.get(bank).and_then(|state| state.record.as_ref()), + ) { + continue; + } + let cut = warm_partial_token_cut( + &snapshot.tokens, + prompt_tokens, + self.warm_partial_min, + qwen_image_cache_token_cap(cache_spans, 0), + ); + let Some(cached) = cut else { + continue; + }; + let Ok(cached_n) = usize::try_from(cached) else { + continue; + }; + let Some(cached_n) = solar_stride_floor(cached_n, self.ctx) else { + continue; + }; + let Ok(cached) = i32::try_from(cached_n) else { + continue; + }; + if best.is_none_or(|(_, current)| cached > current) { + best = Some((bank, cached)); + } + } + let (source, cached) = best?; + Some(WarmAdmitPlan { + source, + tokens: prompt_tokens.to_vec(), + cached, + partial: true, + }) + } + + /// The first refusal wins: a later, broader one would mask the + /// decision that actually turned the candidate down. + fn note_miss(miss: &mut ReuseMiss, reason: ReuseMiss) { + if *miss == ReuseMiss::None { + *miss = reason; + } + } + + fn warm_plan( + &mut self, + batch: &BatchCtx<'_>, + prompt: &[u8], + cache_prompt: Option<&[u8]>, + cache_spans: &[ImageCacheSpan], + prompt_tokens: &[i32], + miss: &mut ReuseMiss, + ) -> Option { + // Jinja may rewrite earlier messages. Text prefix keys only pick + // candidates; the complete rendered token sequence validates reuse. + let template = self.template.is_none(); + let full = self + .warm_full_plan(batch, prompt, cache_prompt) + .filter(|plan| { + let kept = template || plan.tokens == prompt_tokens; + if !kept { + // A live bank held this conversation; the template + // re-rendered it. + Self::note_miss(miss, ReuseMiss::RenderedPrefix); + } + kept + }); + let partial = self.warm_partial_plan( + batch, + prompt, + cache_prompt, + cache_spans, + prompt_tokens, + miss, + ); + match (full, partial) { + (Some(full), Some(partial)) if partial.cached > full.cached => Some(partial), + (Some(full), _) => Some(full), + (None, partial) => partial, + } + } + + fn protected_banks( + &self, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + ) -> (Vec, Option) { + let mut protected = Vec::with_capacity(self.warm.len()); + let mut retry_min = None; + for (bank, state) in self.warm.iter().enumerate() { + let live = state + .record + .as_ref() + .map(|record| (record.generation, state.committed_tokens)); + let retry = bank_hold_retry(i32::try_from(bank).unwrap_or(-1), live); + protected.push(retry.is_some()); + if let Some(retry) = retry { + retry_min = Some(retry_min.map_or(retry, |current: i32| current.min(retry))); + } + } + (protected, retry_min) + } + + fn disk_victim(&self, protected: &[bool], disk_tokens: u32) -> Option { + if let Some(bank) = + warm_victim_pick(&self.warm, protected, None, self.warm_pin_min, false) + { + return Some(bank); + } + if self.warm_pin_min <= 0 || disk_tokens < self.warm_pin_min as u32 { + return None; + } + let bank = warm_victim_pick(&self.warm, protected, None, self.warm_pin_min, true)?; + (self.warm[bank].committed_tokens < self.warm_pin_min).then_some(bank) + } + + fn disk_plan( + &mut self, + batch: &BatchCtx<'_>, + store: &mut KvStore, + prompt: &[u8], + cache_prompt: Option<&[u8]>, + protected: &[bool], + miss: &mut ReuseMiss, + ) -> Option { + let identity = self.identity()?; + let request_key = cache_prompt.unwrap_or(prompt); + let identity_flags = cache_prompt + .is_some() + .then_some(EXT_IMAGE_PIXELS_V2) + .unwrap_or(0); + // An envelope that will not read is a refusal by the record, + // not an absence; `Ok(None)` is the absence. + let candidate = match store.bank_text_prefix_candidate_identity( + request_key, + identity.0, + identity.1, + identity.2, + identity_flags, + ) { + Ok(candidate) => candidate, + Err(_) => { + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + return None; + } + }; + let (path, envelope) = candidate?; + let target = self.disk_victim(protected, envelope.header.tokens)?; + if !disk_restore_target_allowed( + &self.warm, + target, + envelope.header.tokens, + self.warm_pin_min, + ) { + return None; + } + // An image record whose key carries no usable marker cannot be + // rebuilt: the record is malformed, not missing. + let payload = match store.open_payload(&path, &envelope) { + Ok(payload) => payload, + Err(_) => { + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + return None; + } + }; + let Some(mut record) = restored_record(envelope.text, 0, envelope.header.ext_flags) + else { + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + return None; + }; + let snapshot = match batch.load_bank_payload_range( + i32::try_from(target).ok()?, + payload.path(), + envelope.payload_offset, + envelope.header.payload_bytes, + ) { + Ok(snapshot) => snapshot, + Err(error) => { + eprintln!("ds4-server-rs: bank restore skipped: {error}"); + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + return None; + } + }; + if usize::try_from(envelope.header.tokens).ok() != Some(snapshot.tokens.len()) { + // The payload disagrees with its own envelope. The record is + // discarded, so the next request finds nothing — but this + // one was refused by a mismatch, not by an absence. + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + let _ = store.discard_bank(&path); + return None; + } + let committed = i32::try_from(snapshot.tokens.len()).ok()?; + record.generation = snapshot.generation; + self.warm[target].record = Some(record); + self.warm[target].committed_tokens = committed; + self.warm[target].stored_tokens = committed; + self.note_use(target); + let _ = store.touch_hit(&path); + let (tokens, cached) = warm_admit_tokens( + &self.warm[target], + prompt, + cache_prompt, + &snapshot.tokens, + snapshot.generation, + batch.seq_cap(), + false, + |suffix| self.vocab.encode_rendered_bytes(suffix), + )?; + Some(WarmAdmitPlan { + source: target, + tokens, + cached, + partial: false, + }) + } + + fn disk_partial_plan( + &mut self, + batch: &BatchCtx<'_>, + store: &mut KvStore, + prompt: &[u8], + cache_prompt: Option<&[u8]>, + cache_spans: &[ImageCacheSpan], + prompt_tokens: &[i32], + protected: &[bool], + miss: &mut ReuseMiss, + ) -> Option { + if !self.warm_disk_partial { + return None; + } + let identity = self.identity()?; + let min_prefix = usize::try_from(self.warm_partial_min).ok()?; + let request_key = cache_prompt.unwrap_or(prompt); + let identity_flags = cache_prompt + .is_some() + .then_some(EXT_IMAGE_PIXELS_V2) + .unwrap_or(0); + let candidate = match store.bank_text_lcp_candidate_identity( + request_key, + identity.0, + identity.1, + identity.2, + min_prefix, + identity_flags, + ) { + Ok(candidate) => candidate, + Err(_) => { + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + return None; + } + }; + let (path, envelope, cache_lcp) = candidate?; + let target = self.disk_victim(protected, envelope.header.tokens)?; + if !disk_restore_target_allowed( + &self.warm, + target, + envelope.header.tokens, + self.warm_pin_min, + ) { + return None; + } + let payload = match store.open_payload(&path, &envelope) { + Ok(payload) => payload, + Err(_) => { + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + return None; + } + }; + let Some(mut record) = restored_record(envelope.text, 0, envelope.header.ext_flags) + else { + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + return None; + }; + let snapshot = match batch.load_bank_payload_range( + i32::try_from(target).ok()?, + payload.path(), + envelope.payload_offset, + envelope.header.payload_bytes, + ) { + Ok(snapshot) => snapshot, + Err(error) => { + eprintln!("ds4-server-rs: partial bank restore skipped: {error}"); + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + return None; + } + }; + if usize::try_from(envelope.header.tokens).ok() != Some(snapshot.tokens.len()) { + Self::note_miss(miss, ReuseMiss::PayloadMismatch); + let _ = store.discard_bank(&path); + return None; + } + let committed = i32::try_from(snapshot.tokens.len()).ok()?; + record.generation = snapshot.generation; + self.warm[target].record = Some(record); + self.warm[target].committed_tokens = committed; + self.warm[target].stored_tokens = committed; + self.note_use(target); + let _ = store.touch_hit(&path); + // The store matched on bytes; the tokens those bytes cover can + // still fall short of the partial minimum. That is a refusal by + // the threshold, not the bank budget that stays quiet. + let fits = i32::try_from(prompt_tokens.len()).is_ok_and(|n| n <= batch.seq_cap()); + let Some((tokens, cached)) = warm_partial_admit_tokens( + &self.warm[target], + prompt_tokens, + &snapshot.tokens, + snapshot.generation, + self.warm_partial_min, + batch.seq_cap(), + qwen_image_cache_token_cap(cache_spans, cache_lcp), + ) else { + if fits { + Self::note_miss(miss, ReuseMiss::BelowThreshold); + } + return None; + }; + Some(WarmAdmitPlan { + source: target, + tokens, + cached, + partial: true, + }) + } + + fn retire( + &mut self, + batch: &BatchCtx<'_>, + bank: i32, + stepper: &ContStepper, + done_tokens: &[i32], + engine_finished: bool, + transport_failed: bool, + allow_generated_snapshot: bool, + ) { + let Ok(bank) = usize::try_from(bank) else { + return; + }; + if bank >= self.warm.len() { + return; + } + let Ok(snapshot) = batch.bank_snapshot(bank as i32) else { + self.warm[bank].record = None; + return; + }; + self.warm[bank].committed_tokens = snapshot.tokens.len() as i32; + self.note_use(bank); + if snapshot.tokens.is_empty() { + self.warm[bank].record = None; + return; + } + let plan = if think_mode_enabled(stepper.think_mode) && !done_tokens.is_empty() { + thinking_bank_retire_key( + &stepper.prompt, + done_tokens, + engine_finished || stepper.stopped_by_token(), + transport_failed || stepper.acc.verdict.is_some(), + stepper.prompt_preserves_reasoning, + stepper.acc.track_tools, + stepper.acc.saw_tool_start, + syntax_for_model_id(stepper.model_id), + stepper.req.chat_format, + |token| self.vocab.token_text(token), + ) + } else { + bank_retire_key( + &stepper.prompt, + &snapshot.tokens, + done_tokens, + allow_generated_snapshot, + syntax_for_model_id(stepper.model_id), + |token| self.vocab.token_text(token), + ) + .map(|(text, retained_existing)| BankRetireKey { + text, + exact_text: None, + partial_only: false, + retained_existing, + }) + }; + let Some(plan) = plan else { + self.warm[bank].record = None; + return; + }; + if plan.partial_only && !self.warm_fork_partial { + self.warm[bank].record = None; + return; + } + if plan.retained_existing { + self.warm[bank].stored_tokens = self.warm[bank] + .stored_tokens + .min(snapshot.tokens.len() as i32); + } + let cache_text = stepper.cache_prompt.as_deref().and_then(|cache_prompt| { + extended_image_cache_key(&stepper.prompt, cache_prompt, &plan.text) + }); + let exact_cache_text = stepper.cache_prompt.as_deref().and_then(|cache_prompt| { + plan.exact_text.as_deref().and_then(|exact| { + extended_image_cache_key(&stepper.prompt, cache_prompt, exact) + }) + }); + if stepper.cache_prompt.is_some() + && (cache_text.is_none() + || (plan.exact_text.is_some() && exact_cache_text.is_none())) + { + self.warm[bank].record = None; + return; + } + let mut record = retired_record( + self.warm[bank].record.as_ref(), + plan.text, + cache_text, + snapshot.generation, + ); + if let Some(record) = record.as_mut() { + record.exact_text = plan.exact_text; + record.exact_cache_text = exact_cache_text; + record.partial_only = plan.partial_only; + } + self.warm[bank].record = record; + } + + /// Live evict: same `save_bank_record` path as `persist_bank`, + /// then drop the warm record. Pinned banks are left untouched. + fn evict_bank( + &mut self, + batch: &BatchCtx<'_>, + store: Option<&mut KvStore>, + bank: usize, + pinned: bool, + ) -> bool { + if pinned { + return false; + } + if let Some(store) = store { + self.persist_bank( + batch, + store, + bank, + KvReason::BankEvict, + self.warm_persist_min, + false, + ); + } + if let Some(warm) = self.warm.get_mut(bank) { + warm.record = None; + } + true + } + + fn persist_bank( + &mut self, + batch: &BatchCtx<'_>, + store: &mut KvStore, + bank: usize, + reason: KvReason, + min_committed: i32, + due_only: bool, + ) { + if bank >= self.warm.len() { + return; + } + let Some(identity) = self.identity() else { + return; + }; + let Ok(bank_i32) = i32::try_from(bank) else { + return; + }; + let Ok(snapshot) = batch.bank_snapshot(bank_i32) else { + return; + }; + let Ok(committed) = i32::try_from(snapshot.tokens.len()) else { + return; + }; + if self.warm[bank].record.as_ref().is_none_or(|record| { + record.generation != snapshot.generation || record.partial_only + }) || !bank_persist_eligible(committed, min_committed) + || (due_only + && !bank_checkpoint_due_from_host( + &store.opt, + HostKvView { + live_tokens: committed, + stored_tokens: self.warm[bank].stored_tokens, + }, + )) + { + return; + } + let Some(trailer) = self.warm[bank] + .record + .as_ref() + .and_then(|record| self.tool_memory.checkpoint(&record.text)) + else { + eprintln!( + "ds4-server-rs: bank checkpoint skipped bank={bank}: tool-map exceeds bound" + ); + return; + }; + if let Some(record) = self.warm[bank].record.as_mut() { + record.trailer = trailer; + } + let state = &mut self.warm[bank]; + if let Err(error) = save_bank_record( + store, + state, + identity, + committed, + snapshot.generation, + reason, + |path| { + batch + .save_bank_payload(bank_i32, path) + .map_err(|error| GenerateError::Engine(error.to_string())) + }, + ) { + eprintln!( + "ds4-server-rs: bank checkpoint failed bank={bank} reason={reason:?}: {error}" + ); + } + } + + fn place_warm( + &mut self, + batch: &BatchCtx<'_>, + plan: &WarmAdmitPlan, + protected: &[bool], + mut store: Option<&mut KvStore>, + ) -> Option { + let placement = warm_placement( + &self.warm, + protected, + plan.source, + plan.cached, + self.warm_pin_min, + self.warm_fork, + )?; + if placement.fork && protected.get(placement.target).copied().unwrap_or(false) { + return None; + } + self.note_use(placement.source); + if placement.fork { + let _ = self.evict_bank(batch, store.as_deref_mut(), placement.target, false); + let stored = if plan.partial { + self.warm[placement.source].stored_tokens.min(plan.cached) + } else { + self.warm[placement.source].stored_tokens + }; + self.warm[placement.target].stored_tokens = stored; + } else if plan.partial { + // In-place truncate must keep native hist; persisting and + // dropping the bank here made the engine cold-admit. + self.warm[placement.source].stored_tokens = + self.warm[placement.source].stored_tokens.min(plan.cached); + } else { + self.warm[placement.source].record = None; + } + Some(placement) + } + + fn place_cold( + &mut self, + batch: &BatchCtx<'_>, + protected: &[bool], + store: Option<&mut KvStore>, + ) -> Option { + let target = warm_victim_pick(&self.warm, protected, None, self.warm_pin_min, true)?; + if !self.evict_bank( + batch, + store, + target, + protected.get(target).copied().unwrap_or(false), + ) { + return None; + } + self.warm[target].committed_tokens = 0; + self.warm[target].stored_tokens = 0; + Some(target) + } + + fn shutdown_banks(&mut self, batch: &BatchCtx<'_>, store: &mut KvStore) { + for bank in 0..self.warm.len() { + self.persist_bank(batch, store, bank, KvReason::BankShutdown, 1, false); + } + } + + fn prepare_slot( + &mut self, + batch: &BatchCtx<'_>, + work: &ContWork<'_>, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + mut store: Option<&mut KvStore>, + reserve: &crate::serve_cont_roll::RollReserve, + ) -> Result { + let mut parsed = work.parsed.clone(); + let restored_chat = self.restore_request(batch, &mut parsed)?; + let attached_tool_blocks = if parsed.kind == ReqKind::Chat && self.template.is_none() { + if let Ok(model_id) = u8::try_from(self.model_id) { + if let Some(store) = store.as_deref() { + self.tool_memory + .restore_store(store, model_id, &parsed.messages); + } + } + self.tool_memory.attach(&mut parsed.messages) + } else { + 0 + }; + let parsed = &parsed; + let prepared = work.prepared.filter(|_| { + !restored_chat + && can_reuse_cont_prompt(attached_tool_blocks, !parsed.images.is_empty()) + }); + let (prompt, tokens) = match prepared { + Some(prepared) => (prepared.prompt.clone(), prepared.tokens.clone()), + None => { + let prompt = self.render_request(parsed)?; + let tokens = match parsed.kind { + ReqKind::Completion => self + .vocab + .encode_text(std::str::from_utf8(&prompt).unwrap_or("")), + ReqKind::Chat => self.vocab.encode_rendered_bytes(&prompt), + }; + (prompt, tokens) + } + }; + let mut miss = ReuseMiss::None; + let directed = parsed.directed_bank.filter(|bank| *bank >= 0); + let (tokens, images, cache_prompt, cache_spans, directed_cached) = if let Some(bank) = + directed + { + if reserve.contains(bank) { + return Err(GenerateError::Unsupported( + "continuation bank is already in flight", + )); + } + let snapshot = batch.bank_snapshot(bank).map_err(|_| { + GenerateError::Unsupported("continuation bank is no longer live") + })?; + if self.template.is_some() { + let prepared = + prepare_qwen_images(self.model_id, parsed, Some(&prompt), tokens)?; + if prepared.tokens.len() > batch.seq_cap() as usize { + return Err(GenerateError::Unsupported( + "rendered continuation exceeds sequence capacity", + )); + } + let media_match = prepared.images.is_empty() + || self + .warm + .get(bank as usize) + .and_then(|warm| warm.record.as_ref()) + .is_some_and(|record| { + prepared.cache_prompt.as_deref().is_some_and(|key| { + [ + record.cache_text.as_deref(), + record.exact_cache_text.as_deref(), + ] + .into_iter() + .flatten() + .any(|prefix| key.starts_with(prefix)) + }) + }); + let cached = if media_match + && prepared.tokens.len() > snapshot.tokens.len() + && prepared.tokens.starts_with(&snapshot.tokens) + { + snapshot.tokens.len() as i32 + } else { + // A directed continuation never reaches `warm_plan`, + // so this is the only place that can say why its own + // bank was refused: the media it was keyed by, or a + // template that re-rendered the prefix. + if self.warm_reuse != ReuseKind::None { + Self::note_miss( + &mut miss, + if media_match { + ReuseMiss::RenderedPrefix + } else { + ReuseMiss::PayloadMismatch + }, + ); + } + 0 + }; + ( + prepared.tokens, + prepared.images, + prepared.cache_prompt, + prepared.cache_spans, + Some(cached), + ) + } else { + let suffix = render_live_tool_tail( + syntax_for_model_id(self.model_id), + parsed.api, + &parsed.messages, + parsed.think_mode, + )?; + let (tokens, cached) = live_continuation_tokens( + &snapshot.tokens, + &suffix, + batch.seq_cap(), + |suffix| self.vocab.encode_rendered_bytes(suffix), + ) + .ok_or(GenerateError::Unsupported( + "continuation suffix is empty or exceeds the sequence capacity", + ))?; + (tokens, Vec::new(), None, Vec::new(), Some(cached)) + } + } else { + let prepared = prepare_qwen_images(self.model_id, parsed, Some(&prompt), tokens)?; + ( + prepared.tokens, + prepared.images, + prepared.cache_prompt, + prepared.cache_spans, + None, + ) + }; + let prompt_n = tokens.len() as i32; + let (mut stepper, head) = ContStepper::new( + parsed, + self.model_id, + work.job_id, + work.created, + work.cors, + work.default_tokens, + prompt, + prompt_n, + batch.seq_cap(), + ); + stepper.cache_prompt = cache_prompt; + stepper.chat_request = + (self.template.is_some() && parsed.kind == ReqKind::Chat).then(|| parsed.clone()); + stepper.image_cache_spans = cache_spans; + let (temperature, top_k, top_p, min_p) = stepper.sampling(parsed); + let capture_done = bank_scope(parsed); + crate::serve_cont_roll::charge_roll_admit( + &mut *self.memgov, + prompt_n, + stepper.max_tokens, + ) + .map_err(|_| GenerateError::Unsupported(crate::serve_cont_roll::CONT_ADMIT_REFUSED))?; + // The trace records the mechanism, not the counters: an appended + // frontier turn also prefills tokens, and a fork looks like any + // other prefix hit. + let mut reuse = ReuseTaken::Exact; + let mut admit = if let Some(bank) = directed { + let mut admit = ContAdmit::cold(1, tokens, stepper.max_tokens.max(1)); + admit.place_bank = bank.saturating_add(1); + // A continuation reuses live bank KV like any other hit, so + // reuse-off has to refuse it too. + admit.n_cached = match self.warm_reuse { + ReuseKind::None => { + reuse = ReuseTaken::Cold; + 0 + } + _ => directed_cached.unwrap_or(0), + }; + admit + } else { + let (hold, hold_retry) = self.protected_banks(bank_hold_retry); + let protected = reserve.protect(&hold); + let warm = if capture_done && self.warm_reuse != ReuseKind::None { + self.warm_plan( + batch, + &stepper.prompt, + stepper.cache_prompt.as_deref(), + &stepper.image_cache_spans, + &tokens, + &mut miss, + ) + .or_else(|| { + store.as_deref_mut().and_then(|store| { + self.disk_plan( + batch, + store, + &stepper.prompt, + stepper.cache_prompt.as_deref(), + &protected, + &mut miss, + ) + .filter(|plan| { + // An official template re-rendered this + // conversation, so the stored tokens are no + // longer this prompt's prefix. + let kept = self.template.is_none() || plan.tokens == tokens; + if !kept { + Self::note_miss(&mut miss, ReuseMiss::RenderedPrefix); + } + kept + }) + }) + }) + .or_else(|| { + store.as_deref_mut().and_then(|store| { + self.disk_partial_plan( + batch, + store, + &stepper.prompt, + stepper.cache_prompt.as_deref(), + &stepper.image_cache_spans, + &tokens, + &protected, + &mut miss, + ) + }) + }) + } else { + None + }; + let placement = warm.as_ref().and_then(|plan| { + self.place_warm(batch, plan, &protected, store.as_deref_mut()) + }); + // A plan the lane found but could not place is a bank-budget + // refusal, not a cache answer. + let planned = warm.is_some(); + if let (Some(plan), Some(placement)) = (warm, placement) { + let mut admit = ContAdmit::cold(1, plan.tokens, stepper.max_tokens.max(1)); + admit.place_bank = i32::try_from(placement.target + 1).unwrap_or(0); + admit.n_cached = plan.cached; + reuse = match (plan.partial, placement.fork) { + (true, _) => ReuseTaken::Partial, + (false, true) => ReuseTaken::Fork, + (false, false) => ReuseTaken::Exact, + }; + if placement.fork || plan.partial { + admit.fork_bank = i32::try_from(placement.source + 1).unwrap_or(0); + } + admit + } else { + let target = self + .place_cold(batch, &protected, store.as_deref_mut()) + .ok_or(GenerateError::ContinuationHold { + retry_after: hold_retry.unwrap_or(1), + })?; + let mut admit = ContAdmit::cold(1, tokens, stepper.max_tokens.max(1)); + admit.place_bank = i32::try_from(target + 1).unwrap_or(0); + reuse = ReuseTaken::Cold; + // Only a lookup that ran can report an absence; with + // reuse off nothing was examined. A stored record ruled + // out by identity is a mismatch, not an absence. + if miss == ReuseMiss::None + && !planned + && capture_done + && self.warm_reuse != ReuseKind::None + { + // Ask with the key `disk_plan` searched under: an + // image request is stored by its media-marked cache + // text, and the rendered prompt would match nothing. + let request_key = stepper + .cache_prompt + .as_deref() + .unwrap_or(stepper.prompt.as_slice()); + let identity_flags = stepper + .cache_prompt + .is_some() + .then_some(EXT_IMAGE_PIXELS_V2) + .unwrap_or(0); + // `warm_persist_min` is not a floor on what exists: + // `shutdown_banks` persists every live bank at one + // token. The store's own record minimum is, because + // the searches skip anything below it, so only a + // conversation under that could never be found. + let write_min = store.as_deref().map(|store| store.opt.min_tokens); + let min_lcp = usize::try_from(self.warm_partial_min) + .ok() + .filter(|_| self.warm_disk_partial) + .unwrap_or(0); + // Both searches in one pass each: the partial one + // keys on the longest common prefix, so an edited + // prompt needs the question the exact one cannot ask. + let (exact, partial) = self + .identity() + .zip(store.as_deref_mut()) + .map(|((model_id, quant_bits, ctx), store)| { + let exact = store.bank_prefix_answer( + request_key, + model_id, + quant_bits, + ctx, + identity_flags, + ); + let partial = store.bank_lcp_answer( + request_key, + model_id, + quant_bits, + ctx, + min_lcp, + identity_flags, + ); + (exact, partial) + }) + .unwrap_or((PrefixAnswer::None, PrefixAnswer::None)); + let answered = |answer| exact == answer || partial == answer; + // The record this request would have used decides the + // answer. Only when none exists does an incompatible + // one speak, then a conversation too short to have + // been stored, then plain absence. + miss = if answered(PrefixAnswer::Usable) { + // A record this request could have used, refused + // by the bank budget rather than by the cache. + // The contract has no reason for that, and an + // absence would be the wrong one. + ReuseMiss::None + } else if answered(PrefixAnswer::Mismatch) { + ReuseMiss::PayloadMismatch + } else if write_min.is_some_and(|min| prompt_n < min) + || answered(PrefixAnswer::Shallow) + { + ReuseMiss::BelowThreshold + } else { + ReuseMiss::NoCheckpoint + }; + } + admit + } + }; + admit.eos = self.eos; + admit.exclude_eos = parsed.eos_policy != EosPolicy::Default; + admit.temperature = temperature; + admit.top_k = top_k; + admit.top_p = top_p; + admit.min_p = min_p; + admit.seed = parsed.seed; + admit.images = images; + let frontier = (self.template.is_some() + && self.warm_reuse != ReuseKind::None + && parsed.kind == ReqKind::Chat + && parsed.images.is_empty() + && parsed.audios.is_empty() + && parsed.videos.is_empty() + && crate::generate::ordinary_disk_cache_eligible(parsed)) + .then(|| { + crate::generate::history_frontier( + self.model_id, + &stepper.prompt, + &admit.tokens, + |text| Ok(self.vocab.encode_rendered_bytes(text)), + ) + }) + .flatten(); + // Capture the native prefix while it is live. Disk persistence + // follows at retirement; the advancing bank keeps its own state. + let history = frontier.and_then(|(text, tokens)| { + let checkpoint_at = tokens.len() as i32; + let store = store.as_deref()?; + if !self.warm_checkpoint + || !bank_persist_eligible(checkpoint_at, self.warm_persist_min) + || checkpoint_at < store.opt.min_tokens + { + return None; + } + let payload = store.payload_temp().ok()?; + admit.checkpoint_at = checkpoint_at; + Some(HistoryCheckpoint { + text, + tokens, + payload, + captured: false, + }) + }); + Ok(PreparedSlot { + admit, + head, + stepper, + capture_done, + reuse, + miss, + t_arrive: work.t_arrive, + stop_requested: self.stop_requested, + history, + }) + } + + fn finish_driven( + &mut self, + batch: &BatchCtx<'_>, + mut job: JobSlot<'_>, + native_err: Option, + mut store: Option<&mut KvStore>, + cors: bool, + publish: Option<&mut dyn FnMut(&GenerateOutcome)>, + ) -> Result { + if let (Some(history), Some(store), Some(identity)) = + (job.history.take(), store.as_deref_mut(), self.identity()) + { + history.persist(store, identity); + } + let timings = { + let completion = job.stepper.completion(); + let mut timings = ReqTimings { + prefill_tokens: job.n_computed, + prefill_cached: job.n_cached, + decode_tokens: job.decode_tokens, + decode_steps: job.decode_steps, + ..ReqTimings::default() + }; + if let Some(first) = job.t_first.filter(|_| completion > 0) { + timings = ReqTimings { + valid: true, + ttft_ms: first.duration_since(job.t_arrive).as_secs_f64() * 1e3, + prefill_ms: first + .duration_since(job.t_admit.unwrap_or(job.t_arrive)) + .as_secs_f64() + * 1e3, + decode_ms: job.decode_ms, + prefill_tokens: job.n_computed, + prefill_cached: job.n_cached, + decode_tokens: job.decode_tokens, + decode_steps: job.decode_steps, + }; + } + timings + }; + let engine_eos = job.engine_eos && !job.host_abort; + let n_cached = job.n_cached; + let n_computed = job.n_computed; + let io_ok = !job.io_failed; + let done_called = job.t_done.is_some(); + let done_tokens = std::mem::take(&mut job.done_tokens); + let admitted = job.admitted; + let actual_bank = job.bank; + let capture_done = job.capture_done; + let allow_generated_snapshot = + native_err.is_none() && io_ok && job.stepper.has_complete_tool_turn(); + let retired = retired_bank( + capture_done, + admitted, + done_called, + actual_bank, + batch.max_seq(), + ); + if let Some(bank) = retired { + self.retire( + batch, + bank, + &job.stepper, + &done_tokens, + job.engine_eos, + !io_ok, + allow_generated_snapshot, + ); + if self.warm_checkpoint { + if let Some(store) = store.as_deref_mut() { + self.persist_bank( + batch, + store, + bank as usize, + KvReason::BankCheckpoint, + self.warm_persist_min, + true, + ); + } + } + } + if let Some(error) = native_err { + if admitted && io_ok && job.stepper.req.stream { + let failure = job.stepper.fail(&error); + job.push(&failure); + return if job.io_failed { + Err(GenerateError::Io) + } else { + Err(GenerateError::Streamed(error)) + }; + } + return Err(GenerateError::Engine(error)); + } + if !admitted { + return Err(GenerateError::Unsupported( + "continuous admission rejected; serial fallback", + )); + } + if !io_ok { + return Err(GenerateError::Io); + } + let (tail, mut outcome) = job + .stepper + .finalize(engine_eos, n_cached, n_computed, timings, cors); + outcome.speculation_active = job.speculated; + // Native confirms the plan: no committed prefix means the warm + // placement did not land, whatever admission intended. + outcome.reuse = if n_cached > 0 { + job.reuse + } else { + ReuseTaken::Cold + }; + // One probe can refuse while the next admits — a partial bank + // too shallow to fork, then an exact hit. The reason belongs to + // a request that took nothing. + outcome.reuse_miss = if outcome.reuse == ReuseTaken::Cold { + job.miss + } else { + ReuseMiss::None + }; + if let Some(bank) = actual_bank { + if let Ok(snapshot) = batch.bank_snapshot(bank) { + outcome.bank = Some(bank); + outcome.generation = snapshot.generation; + outcome.frontier = i32::try_from(snapshot.tokens.len()).unwrap_or(0); + self.histories[bank as usize] = + job.stepper.chat_history.take().map(|history| BankHistory { + generation: snapshot.generation, + frontier: snapshot.tokens.len(), + history, + }); + } + } + let mut tool_remembered = false; + if let Some((calls, exact)) = job.stepper.take_tool_replay() { + tool_remembered = self.tool_memory.remember(&calls, &exact) > 0; + } + if tool_remembered && self.warm_checkpoint { + if let (Some(store), Some(bank)) = ( + store.as_deref_mut(), + retired.and_then(|bank| usize::try_from(bank).ok()), + ) { + self.persist_bank( + batch, + store, + bank, + KvReason::BankCheckpoint, + self.warm_persist_min, + false, + ); + } + } + if let Some(publish) = publish { + publish(&outcome); + } + if !tail.is_empty() { + job.out.write_all(&tail).map_err(|_| GenerateError::Io)?; + let _ = job.out.flush(); + } + Ok(outcome) + } + } + + impl StaticExec for ContLane<'_> { + fn generate_static( + &mut self, + jobs: &[StaticJob<'_>], + ) -> Result, GenerateError> { + BatchStatic::new(&mut self.batch).generate_static(jobs) + } + + fn ctx_max_seq(&self) -> i32 { + self.batch.max_seq() + } + + fn coalesce_limits(&self) -> CoalesceLimits { + CoalesceLimits { + cap: self.batch.max_seq().max(1) as usize, + max_tok_total: 0, + } + } + } + + impl ContExec for ContLane<'_> { + fn model_id(&self) -> i32 { + self.host.model_id + } + + fn render_request(&self, parsed: &ParsedRequest) -> Result, GenerateError> { + self.host.render_request(parsed) + } + + fn restore_chat(&self, parsed: &mut ParsedRequest) -> Result<(), GenerateError> { + self.host.restore_request(&self.batch, parsed).map(|_| ()) + } + + fn set_stop_requested(&mut self, stop_requested: Option bool>) { + self.host.stop_requested = stop_requested; + } + + fn as_static(&mut self) -> Option<&mut dyn StaticExec> { + Some(self) + } + + fn seq_cap(&self) -> i32 { + self.batch.seq_cap() + } + + fn max_seq(&self) -> i32 { + self.batch.max_seq() + } + + fn trim_idle_banks(&mut self, want_bytes: u64) -> u64 { + // One-bank Solar workers sit just above the memory floor. Serial + // reclaim would trim the only hist-valid bank and the next Chat + // request could not partial-fork. Step 3.7 and multi-bank Solar + // still trim idle banks / unreferenced checkpoints. + if self.host.warm_fork_partial + && self.batch.max_seq() == 1 + && syntax_for_model_id(self.host.model_id) == ModelSyntax::SolarOpen2 + { + return 0; + } + self.batch.trim_free(want_bytes) + } + + fn encode_chat(&self, rendered: &[u8]) -> Vec { + self.host.vocab.encode_rendered_bytes(rendered) + } + + fn encode_text(&self, text: &str) -> Vec { + self.host.vocab.encode_text(text) + } + + fn prepare_tokens( + &self, + parsed: &ParsedRequest, + _prompt: &[u8], + tokens: Vec, + ) -> Result, GenerateError> { + prepare_qwen_images(self.host.model_id, parsed, None, tokens) + .map(|prepared| prepared.tokens) + } + + fn bank_live(&self, bank: i32) -> Option<(u64, i32)> { + let snapshot = self.batch.bank_snapshot(bank).ok()?; + let frontier = i32::try_from(snapshot.tokens.len()).ok()?; + if frontier <= 0 { + return None; + } + Some((snapshot.generation, frontier)) + } + + fn generate( + &mut self, + parsed: &ParsedRequest, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + t_arrive: Instant, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + store: Option<&mut KvStore>, + out: &mut dyn Write, + ) -> Result { + self.generate_prepared( + parsed, + None, + job_id, + created, + cors, + default_tokens, + t_arrive, + bank_hold_retry, + store, + out, + ) + } + + fn generate_prepared( + &mut self, + parsed: &ParsedRequest, + prepared: Option<&ContPreparedPrompt>, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + t_arrive: Instant, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + mut store: Option<&mut KvStore>, + out: &mut dyn Write, + ) -> Result { + let work = ContWork { + parsed, + prepared, + job_id, + created, + cors, + default_tokens, + t_arrive, + out, + }; + let prepared = self.host.prepare_slot( + &self.batch, + &work, + bank_hold_retry, + store.as_deref_mut(), + &crate::serve_cont_roll::RollReserve::new(), + )?; + let mut driver = RollDriver::new(self.host.vocab, &self.batch); + let user = driver.push(prepared.into_slot(Box::new(work.out))); + let native_err = self + .batch + .continuous_generate(&mut driver) + .err() + .map(|error| error.to_string()); + let job = driver.slots.remove(&user).ok_or_else(|| { + GenerateError::Engine("continuous driver lost the admitted job".into()) + })?; + self.host + .finish_driven(&self.batch, job, native_err, store, cors, None) + } + + fn generate_batch( + &mut self, + jobs: Vec>, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + mut store: Option<&mut KvStore>, + ) -> Vec> { + let count = jobs.len(); + let mut results: Vec>> = + (0..count).map(|_| None).collect(); + let mut reserve = crate::serve_cont_roll::RollReserve::new(); + let mut prepared = Vec::with_capacity(count); + for (index, work) in jobs.into_iter().enumerate() { + let cors = work.cors; + match self.host.prepare_slot( + &self.batch, + &work, + bank_hold_retry, + store.as_deref_mut(), + &reserve, + ) { + Ok(slot) => { + reserve.note_place(slot.admit.place_bank); + prepared.push((index, cors, slot, work.out)); + } + Err(error) => results[index] = Some(Err(error)), + } + } + let mut driver = RollDriver::new(self.host.vocab, &self.batch); + let order: Vec<_> = prepared + .into_iter() + .map(|(index, cors, slot, out)| { + let user = driver.push(slot.into_slot(Box::new(out))); + (index, user, cors) + }) + .collect(); + let native_err = (!order.is_empty()).then(|| { + self.batch + .continuous_generate(&mut driver) + .err() + .map(|error| error.to_string()) + }); + let native_err = native_err.flatten(); + for (index, user, cors) in order { + results[index] = Some(match driver.slots.remove(&user) { + Some(job) => self.host.finish_driven( + &self.batch, + job, + native_err.clone(), + store.as_deref_mut(), + cors, + None, + ), + None => Err(GenerateError::Engine( + "continuous driver lost an admitted job".into(), + )), + }); + } + results + .into_iter() + .map(|result| { + result.unwrap_or_else(|| { + Err(GenerateError::Engine( + "continuous batch lost a job result".into(), + )) + }) + }) + .collect() + } + + fn generate_rolling( + &mut self, + source: &mut dyn ContSource, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + store: Option<&mut KvStore>, + ) -> Option> { + let mut driver = RollingDriver { + host: &mut self.host, + batch: &self.batch, + source, + bank_hold_retry, + store, + slots: std::collections::HashMap::new(), + pending: None, + results: Vec::new(), + next_user: 1, + }; + loop { + match self.batch.continuous_generate(&mut driver) { + Ok(()) if driver.pending.is_some() => continue, + Ok(()) => break, + Err(error) => { + driver.finish_stranded(error.to_string()); + break; + } + } + } + Some(driver.results) + } + + fn shutdown(&mut self, store: Option<&mut KvStore>) { + if let Some(store) = store { + self.host.shutdown_banks(&self.batch, store); + } + } + } +} + +impl ContStepper { + pub fn completion(&self) -> i32 { + self.acc.completion + } + + /// Host stop-token verdict (family EOT / eos / role starts), the same + /// pre-eval check the serial `decode_pass` runs; the stop token's text + /// is never fed. + pub fn mark_stop(&mut self) { + self.finish = "stop"; + self.stop_token = true; + } + + /// The model closed its turn with a family stop token (GLM `<|user|>`) + /// instead of the engine EOS; the reply before it is still complete. + #[cfg(any(feature = "native", test))] + fn stopped_by_token(&self) -> bool { + self.stop_token + } + + #[cfg(any(feature = "native", test))] + fn stop_for_shutdown(&mut self, stop_requested: Option bool>) -> bool { + if !stop_requested.is_some_and(|stop| stop()) { + return false; + } + self.finish = "error"; + true + } +} + +#[cfg(test)] +mod bank_tests { + use super::*; + use crate::parse::{ + parse_anthropic_request, parse_chat_request, parse_responses_request, ChatMsg, ParseEnv, + }; + use crate::route::{Api, ReqKind, ThinkMode}; + use ds4_kv::{Options, Reason, Store, EXT_BANK_REPLAY_V1, EXT_TOOL_MAP}; + use std::fs; + + fn shutdown_requested() -> bool { + true + } + + #[test] + fn iquest_thinking_sampling() { + let glm = ds4_core::Variant::Glm53FlashVq as i32; + for model_id in [ds4_core::Variant::IQuestQ1 as i32, glm, 2] { + for temperature in [0.0, 0.4] { + let parsed = parse_chat_request( + &ParseEnv::default(), + &format!(r#"{{"messages":[{{"role":"user","content":"2 + 2"}}],"reasoning_effort":"high","temperature":{temperature},"top_k":9,"top_p":0.9,"min_p":0.03}}"#), + ).unwrap(); + let (stepper, _) = ContStepper::new( + &parsed, + model_id, + "thinking-sample", + 0, + false, + 16, + b"".to_vec(), + 1, + 8192, + ); + let expected = if model_id == ds4_core::Variant::IQuestQ1 as i32 || model_id == glm + { + (temperature, 9, 0.9, 0.03) + } else { + (DEFAULT_TEMPERATURE, 0, DEFAULT_TOP_P, DEFAULT_MIN_P) + }; + assert_eq!(stepper.sampling(&parsed), expected); + } + } + } + + #[test] + fn ling_thinking_retire() { + // Bailing V3 sets preserved_thinking: history replays reasoning, so a + // finished turn without tools retires its exact prompt + generation. + let prompt = b"SYSTEMdetailed thinking on<|role_end|>HUMAN2 + 2<|role_end|>ASSISTANT\n".to_vec(); + let pieces = [b"Compute.".as_slice(), b"", b"4", b"<|role_end|>"]; + let key = thinking_bank_retire_key( + &prompt, + &[0, 1, 2, 3], + true, + false, + false, + false, + false, + ModelSyntax::Ling3Vl, + ChatFormat::DeepSeek, + |token| pieces[token as usize].to_vec(), + ) + .expect("Ling reasoning history must retain a reusable record"); + let mut expected = prompt.clone(); + expected.extend_from_slice(b"Compute.4"); + assert_eq!(key.text, expected); + assert!(!key.partial_only); + // The official render of the next turn replays that reasoning. + let mut follow = expected.clone(); + follow.extend_from_slice( + b"<|role_end|>HUMAN4 + 1<|role_end|>ASSISTANT\n", + ); + assert!(follow.starts_with(&key.text)); + } + + #[test] + fn iquest_thinking_retire() { + use ds4_core::chat_template::{RenderClock, Template}; + let model_id = ds4_core::Variant::IQuestQ1 as i32; + let template = Template::compile( + include_str!("../../../tests/fixtures/chat-template/models/iquest/chat_template.jinja"), + RenderClock::Fixed(0), + ) + .unwrap(); + let parsed = parse_chat_request( + &ParseEnv::default(), + r#"{"messages":[{"role":"user","content":"2 + 2"}],"reasoning_effort":"high"}"#, + ) + .unwrap(); + let prompt = crate::chat_input::render(&template, model_id, &parsed).unwrap(); + let pieces = [b"Compute.".as_slice(), b"", b"4", b"<|iquest_end|>"]; + let key = thinking_bank_retire_key( + &prompt, + &[0, 1, 2, 3], + true, + false, + prompt_preserves_reasoning(&parsed), + false, + false, + ModelSyntax::IQuestQ1, + ChatFormat::IQuestQ1, + |token| pieces[token as usize].to_vec(), + ) + .expect("IQuest reasoning history must retain a reusable record"); + let mut expected = prompt.clone(); + expected.extend_from_slice(b"Compute.4"); + assert_eq!(key.text, expected); + assert!(!key.partial_only); + + let follow = parse_chat_request(&ParseEnv::default(), + r#"{"messages":[{"role":"user","content":"2 + 2"},{"role":"assistant","content":"4","reasoning_content":"Compute."},{"role":"user","content":"4 + 1"}],"reasoning_effort":"high"}"#, + ).unwrap(); + let rendered = crate::chat_input::render(&template, model_id, &follow).unwrap(); + assert!(rendered.starts_with(&key.text)); + assert!(rendered[key.text.len()..].starts_with(b"<|iquest_end|><|iquest_user|>")); + + // Exercise the real persistence gate with a mock native payload. The + // record must retain the same reasoning-bearing key across disk IO. + let (dir, mut store) = temp_store("iquest-thinking"); + let mut bank = warm("", 0); + bank.record.as_mut().unwrap().text = key.text.clone(); + assert!(save_bank_record( + &mut store, + &mut bank, + (model_id as u8, 2, 8192), + 4, + 1, + Reason::BankShutdown, + |path| fs::write(path, b"native-payload") + .map_err(|e| GenerateError::Engine(e.to_string())), + ) + .unwrap()); + let record = store.read(&store.entries()[0].path).unwrap(); + assert_eq!(record.text, key.text); + assert_eq!(record.payload, b"native-payload"); + fs::remove_dir_all(dir).unwrap(); + } + + /// Renders one turn (`2 + 2`) or the next (`4` replied, `4 + 1` asked) + /// with GLM's official template. + fn glm_turn(effort: &str, next: bool) -> Vec { + use ds4_core::chat_template::{RenderClock, Template}; + let template = Template::compile( + include_str!("../../../tests/fixtures/chat-template/models/glm/chat_template.jinja"), + RenderClock::Fixed(0), + ) + .unwrap(); + let history = if next { + r#",{"role":"assistant","content":"4"},{"role":"user","content":"4 + 1"}"# + } else { + "" + }; + let body = format!( + r#"{{"messages":[{{"role":"user","content":"2 + 2"}}{history}],"reasoning_effort":"{effort}"}}"# + ); + let parsed = parse_chat_request(&ParseEnv::default(), &body).unwrap(); + let model_id = ds4_core::Variant::Glm53FlashVq as i32; + crate::chat_input::render(&template, model_id, &parsed).unwrap() + } + + #[test] + fn glm_disabled_thinking_key() { + // GLM replays the empty pair before history content; the key keeps it. + let pieces = [b"4".as_slice(), b"<|user|>"]; + let (key, _) = bank_retire_key( + &glm_turn("none", false), + &[], + &[0, 1], + false, + ModelSyntax::Glm53, + |token| pieces[token as usize].to_vec(), + ) + .unwrap(); + assert!(glm_turn("none", true).starts_with(&key)); + } + + #[test] + fn glm_visible_thinking_key() { + let prompt = glm_turn("high", false); + let parsed = parse_chat_request( + &ParseEnv::default(), + r#"{"messages":[{"role":"user","content":"2 + 2"}],"reasoning_effort":"high"}"#, + ) + .unwrap(); + let model_id = ds4_core::Variant::Glm53FlashVq as i32; + let (mut stepper, _) = ContStepper::new( + &parsed, + model_id, + "glm", + 0, + false, + 16, + prompt.clone(), + 1, + 8192, + ); + // `<|user|>` ends the reply as a host stop token, never as engine EOS. + assert!(!stepper.stopped_by_token()); + stepper.mark_stop(); + let pieces = [b"Compute.".as_slice(), b"", b"4", b"<|user|>"]; + let key = thinking_bank_retire_key( + &prompt, + &[0, 1, 2, 3], + stepper.stopped_by_token(), + false, + false, + false, + false, + ModelSyntax::Glm53, + ChatFormat::DeepSeek, + |token| pieces[token as usize].to_vec(), + ) + .unwrap(); + // History without reasoning renders `4`. + assert!(glm_turn("high", true).starts_with(&key.text)); + let mut exact = prompt; + exact.extend_from_slice(b"Compute.4"); + assert_eq!(key.exact_text, Some(exact)); + } + + #[test] + fn continuous_eos_policy_follows_reasoning_state() { + let mut parsed = parse_chat_request( + &ParseEnv::default(), + r#"{"messages":[{"role":"user","content":"Explain"}],"stream":true}"#, + ) + .unwrap(); + parsed.eos_policy = EosPolicy::Reasoning; + let (mut stepper, _) = ContStepper::new( + &parsed, + ModelSyntax::Qwen4Exp as i32, + "eos-policy", + 7, + false, + 16, + b"".to_vec(), + 1, + 32, + ); + + assert_eq!(stepper.sample_exclude(99), 99); + stepper.feed(b"reasoninganswer"); + assert_eq!(stepper.sample_exclude(99), -1); + } + + struct RequiredPrefixExec; + + impl ContExec for RequiredPrefixExec { + fn model_id(&self) -> i32 { + ModelSyntax::K2Horizon as i32 + } + + fn seq_cap(&self) -> i32 { + 8192 + } + + fn encode_chat(&self, rendered: &[u8]) -> Vec { + if rendered == crate::render::K2_TOOL_CALLS_START.as_bytes() { + vec![101] + } else if rendered == crate::render::K2_THINK_END.as_bytes() { + vec![202] + } else { + vec![1] + } + } + + fn encode_text(&self, _text: &str) -> Vec { + vec![1] + } + + fn generate( + &mut self, + _parsed: &ParsedRequest, + _job_id: &str, + _created: i64, + _cors: bool, + _default_tokens: i32, + _t_arrive: Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut KvStore>, + _out: &mut dyn Write, + ) -> Result { + unreachable!() + } + } + + #[test] + fn continuous_k2_required_tool_choice_prepares_sampling_prefixes() { + let mut parsed = parse_chat_request( + &ParseEnv::default(), + r#"{"messages":[{"role":"user","content":"weather"}],"tools":[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}],"tool_choice":"required"}"#, + ) + .unwrap(); + + cont_prompt_tokens(&RequiredPrefixExec, &mut parsed).unwrap(); + + assert_eq!(parsed.required_tool_prefix, [101]); + assert_eq!(parsed.required_think_end_prefix, [202]); + } + + #[test] + fn prepared_prompt_is_reused_only_when_native_inputs_are_unchanged() { + assert!(can_reuse_cont_prompt(0, false)); + assert!(!can_reuse_cont_prompt(1, false)); + assert!(!can_reuse_cont_prompt(0, true)); + } + + #[test] + fn shallow_agent_sessions_persist_without_becoming_resident_pinned() { + let session_tokens = 58_000; + assert!(bank_persist_eligible( + session_tokens, + DEFAULT_BANK_PERSIST_MIN_TOKENS + )); + assert!(session_tokens < 65_536); + assert!(!bank_persist_eligible(session_tokens, 65_536)); + } + + #[test] + fn anthropic_stream_starts_with_the_admitted_cache_split() { + let parsed = parse_anthropic_request( + &ParseEnv::default(), + r#"{"messages":[{"role":"user","content":"hello"}],"max_tokens":8,"stream":true}"#, + ) + .unwrap(); + let (mut stepper, head) = ContStepper::new( + &parsed, + 0, + "msg-cache-usage", + 7, + false, + 16, + b"prompt".to_vec(), + 278, + 512, + ); + + let head = String::from_utf8(stepper.admitted_head(head, 260, 18)).unwrap(); + assert!(head.contains("\"input_tokens\":0"), "{head}"); + assert!(head.contains("\"cache_read_input_tokens\":260"), "{head}"); + assert!( + head.contains("\"cache_creation_input_tokens\":18"), + "{head}" + ); + } + + #[test] + fn output_only_usage_reports_the_full_effective_input() { + let mut parsed = + parse_responses_request(&ParseEnv::default(), r#"{"input":"result"}"#).unwrap(); + parsed.needs |= NEED_BANK_FRONTIER; + let (mut stepper, _) = ContStepper::new( + &parsed, + 0, + "resp-cache-usage", + 7, + false, + 16, + b"suffix".to_vec(), + 16, + 512, + ); + + stepper.feed(b"answer"); + let (body, _) = stepper.finalize(true, 260, 18, ReqTimings::default(), false); + let body = String::from_utf8(body).unwrap(); + assert!(body.contains("\"input_tokens\":278"), "{body}"); + assert!(body.contains("\"cached_tokens\":260"), "{body}"); + assert!(body.contains("\"cache_write_tokens\":18"), "{body}"); + } + + #[test] + fn continuous_responses_heartbeat_advances_the_live_sequence() { + let parsed = + parse_responses_request(&ParseEnv::default(), r#"{"input":"hello","stream":true}"#) + .unwrap(); + let (mut stepper, head) = ContStepper::new( + &parsed, + 0, + "heartbeat", + 7, + false, + 16, + b"prompt".to_vec(), + 1, + 32, + ); + assert!(String::from_utf8_lossy(&head).contains("\"sequence_number\":0")); + + let due = stepper.last_heartbeat + crate::stream::STREAM_HEARTBEAT_INTERVAL; + let heartbeat = stepper.heartbeat(due); + + let heartbeat = String::from_utf8(heartbeat).unwrap(); + assert!(heartbeat.contains("\"type\":\"response.in_progress\"")); + assert!(heartbeat.contains("\"sequence_number\":1")); + } + + #[test] + fn continuous_shutdown_marks_a_protocol_failure_before_row_abort() { + let parsed = + parse_responses_request(&ParseEnv::default(), r#"{"input":"hello","stream":true}"#) + .unwrap(); + let (mut stepper, _) = ContStepper::new( + &parsed, + 0, + "shutdown", + 7, + false, + 16, + b"prompt".to_vec(), + 1, + 32, + ); + + assert!(stepper.stop_for_shutdown(Some(shutdown_requested))); + assert_eq!(stepper.finish, "error"); + } + + #[test] + fn bank_continuation_keeps_exact_prefix_and_tokenizes_only_the_suffix() { + let (tokens, cached) = live_continuation_tokens(&[10, 11], b" suffix", 4, |suffix| { + assert_eq!(suffix, b" suffix"); + vec![20, 21] + }) + .unwrap(); + assert_eq!(tokens, [10, 11, 20, 21]); + assert_eq!(cached, 2); + assert!(live_continuation_tokens(&[10, 11], b" suffix", 3, |_| vec![20, 21]).is_none()); + } + + fn temp_store(tag: &str) -> (std::path::PathBuf, Store) { + let dir = + std::env::temp_dir().join(format!("ds4-server-bank-{tag}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let store = Store::open(&dir, 16, false, Options::default()).unwrap(); + (dir, store) + } + + fn warm(text: &str, last_use: u64) -> WarmBank { + WarmBank { + record: Some(WarmRecord { + text: text.as_bytes().to_vec(), + cache_text: None, + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation: 1, + ext_flags: 0, + trailer: Vec::new(), + }), + committed_tokens: 10, + stored_tokens: 0, + last_use, + } + } + + #[test] + fn bank_scope_accepts_chat_turns_including_thinking() { + let mut env = ParseEnv::default(); + env.default_effort = ThinkMode::None; + let mut parsed = parse_chat_request( + &env, + r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#, + ) + .unwrap(); + assert!(bank_scope(&parsed)); + + parsed.api = Api::Anthropic; + assert!(bank_scope(&parsed)); + parsed.api = Api::Responses; + assert!(bank_scope(&parsed)); + parsed.api = Api::Openai; + parsed.kind = ReqKind::Completion; + assert!(!bank_scope(&parsed)); + parsed.kind = ReqKind::Chat; + parsed.think_mode = ThinkMode::Low; + assert!(bank_scope(&parsed)); + parsed.think_mode = ThinkMode::None; + parsed.has_tools = true; + assert!(bank_scope(&parsed)); + parsed.has_tools = false; + parsed.has_tool_results = true; + assert!(bank_scope(&parsed)); + parsed.has_tool_results = false; + parsed.live_call_ids.push("call_1".into()); + assert!(bank_scope(&parsed)); + } + + #[test] + fn completed_tool_history_preserves_reasoning_without_active_tools() { + let parsed = parse_chat_request( + &ParseEnv::default(), + r#"{"messages":[{"role":"user","content":"call f"},{"role":"assistant","content":"","tool_calls":[{"id":"call_1","type":"function","function":{"name":"f","arguments":"{}"}}]},{"role":"tool","tool_call_id":"call_1","content":"ok"},{"role":"assistant","content":"done"},{"role":"user","content":"next"}],"reasoning_effort":"high"}"#, + ) + .unwrap(); + assert!(!parsed.has_tools); + assert!(prompt_preserves_reasoning(&parsed)); + } + + #[test] + fn qwen_thinking_bank_key_matches_the_next_visible_history() { + let user = ChatMsg { + role: "user".into(), + content: "hello".into(), + ..Default::default() + }; + let prompt = crate::render::render_chat( + ModelSyntax::Qwen4Exp, + std::slice::from_ref(&user), + "", + ThinkMode::High, + ) + .unwrap(); + let key = thinking_visible_key( + &prompt, + b" visible answer ", + ModelSyntax::Qwen4Exp, + ChatFormat::Qwen4Exp, + false, + ) + .unwrap(); + let next = crate::render::render_chat( + ModelSyntax::Qwen4Exp, + &[ + user, + ChatMsg { + role: "assistant".into(), + content: "visible answer".into(), + ..Default::default() + }, + ChatMsg { + role: "user".into(), + content: "next".into(), + ..Default::default() + }, + ], + "", + ThinkMode::High, + ) + .unwrap(); + assert!(next.starts_with(&key)); + assert!(!key.ends_with(b"<|im_end|>\n")); + } + + #[test] + fn mimo_disk_key_keeps_think() { + let prompt = b"<|im_start|>assistant\n"; + let key = bank_retire_key(prompt, &[], &[0, 1], false, ModelSyntax::Mimo2, |token| { + [b"4".as_slice(), b"<|im_end|>".as_slice()][token as usize].to_vec() + }) + .unwrap(); + + // MiMo re-emits the empty pair before replayed assistant content. + assert_eq!(key.0, b"<|im_start|>assistant\n4"); + } + + #[test] + fn qwen_thinking_retire_keeps_visible_key_and_tools_alias() { + let prompt = b"<|im_start|>assistant\n\n"; + let pieces = [ + b"hidden".as_slice(), + b"".as_slice(), + b"\n\nanswer".as_slice(), + b"<|im_end|>".as_slice(), + ]; + let retired = thinking_bank_retire_key( + prompt, + &[0, 1, 2, 3], + true, + false, + true, + true, + false, + crate::render::ModelSyntax::Qwen4Exp, + crate::stream::ChatFormat::Qwen4Exp, + |token| pieces[token as usize].to_vec(), + ) + .unwrap(); + assert_eq!( + retired.text, + b"<|im_start|>assistant\n\n\n\n\nanswer" + ); + assert_eq!( + retired.exact_text.as_deref(), + Some(b"<|im_start|>assistant\n\nhidden\n\nanswer".as_slice()) + ); + assert!(!retired.partial_only); + } + + #[test] + fn qwen_thinking_retire_keeps_reasoning_alias_without_tools() { + let prompt = b"<|im_start|>assistant\n\n"; + let pieces = [ + b"hidden".as_slice(), + b"".as_slice(), + b"\n\nanswer".as_slice(), + b"<|im_end|>".as_slice(), + ]; + let retired = thinking_bank_retire_key( + prompt, + &[0, 1, 2, 3], + true, + false, + false, + false, + false, + ModelSyntax::Qwen4Exp, + ChatFormat::Qwen4Exp, + |token| pieces[token as usize].to_vec(), + ) + .unwrap(); + assert_eq!( + retired.text, + b"<|im_start|>assistant\n\n\n\n\nanswer" + ); + assert_eq!( + retired.exact_text.as_deref(), + Some(b"<|im_start|>assistant\n\nhidden\n\nanswer".as_slice()) + ); + } + + #[test] + fn completed_tool_history_without_active_tools_keeps_the_raw_key() { + let prompt = b"<|im_start|>assistant\n\n"; + let pieces = [ + b"hidden".as_slice(), + b"".as_slice(), + b"\n\nanswer".as_slice(), + b"<|im_end|>".as_slice(), + ]; + let retired = thinking_bank_retire_key( + prompt, + &[0, 1, 2, 3], + true, + false, + true, + false, + false, + ModelSyntax::Qwen4Exp, + ChatFormat::Qwen4Exp, + |token| pieces[token as usize].to_vec(), + ) + .unwrap(); + assert_eq!( + retired.text, + b"<|im_start|>assistant\n\nhidden\n\nanswer" + ); + assert!(retired.exact_text.is_none()); + } + + #[test] + fn interrupted_thinking_retire_is_rewind_only_after_a_committed_sample() { + let prompt = b"<|im_start|>assistant\n\n"; + let retired = thinking_bank_retire_key( + prompt, + &[0, 1], + false, + true, + false, + false, + false, + crate::render::ModelSyntax::Qwen4Exp, + crate::stream::ChatFormat::Qwen4Exp, + |token| [b"hidden".as_slice(), b" tail".as_slice()][token as usize].to_vec(), + ) + .unwrap(); + assert_eq!(retired.text, prompt); + assert!(retired.partial_only); + } + + #[test] + fn cont_stepper_retains_exact_tool_text_for_bank_checkpoint() { + let mut env = ParseEnv::default(); + env.default_effort = ThinkMode::None; + let parsed = parse_chat_request( + &env, + r#"{"messages":[{"role":"user","content":"call f"}],"tools":[{"type":"function","function":{"name":"f","parameters":{"type":"object"}}}],"thinking":{"type":"disabled"}}"#, + ) + .unwrap(); + let (mut stepper, _) = ContStepper::new( + &parsed, + 0, + "tool-bank", + 1, + false, + 16, + b"prompt".to_vec(), + 1, + 32, + ); + let exact = + "\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"f\">\n\n"; + assert!(stepper.feed(exact.as_bytes()).done); + let (_, outcome) = stepper.finalize(false, 0, 1, ReqTimings::default(), false); + assert_eq!(outcome.tool_ids.len(), 1); + let (calls, remembered) = stepper.take_tool_replay().unwrap(); + assert_eq!(calls[0].id, outcome.tool_ids[0]); + assert_eq!(remembered, exact); + } + + #[test] + fn warm_key_drops_the_uncommitted_last_sample() { + assert_eq!( + committed_key(b"prompt:", &[1, 2, 3], ModelSyntax::Motif3, |token| vec![ + b'0' + token as u8 + ]), + b"prompt:12" + ); + assert_eq!( + committed_key(b"prompt:", &[], ModelSyntax::Motif3, |_| unreachable!()), + b"prompt:" + ); + } + + #[test] + fn bank_retire_key_accepts_a_complete_tool_turn_from_the_snapshot() { + let token_text = |token| vec![b'0' + token as u8]; + assert_eq!( + bank_retire_key( + b"12", + &[1, 2, 3, 4], + &[], + false, + ModelSyntax::Motif3, + token_text + ), + None + ); + assert_eq!( + bank_retire_key( + b"12", + &[1, 2, 3, 4], + &[], + true, + ModelSyntax::Motif3, + token_text + ), + Some((b"123".to_vec(), false)) + ); + assert_eq!( + bank_retire_key( + b"1234", + &[1, 2, 3, 4], + &[], + false, + ModelSyntax::Motif3, + token_text + ), + Some((b"123".to_vec(), true)) + ); + assert_eq!( + bank_retire_key(b"12", &[], &[3, 4], false, ModelSyntax::Motif3, token_text), + Some((b"123".to_vec(), false)) + ); + assert_eq!( + bank_retire_key( + b"<|assistant|>", + &[], + &[1, 2], + false, + ModelSyntax::Motif3, + |token| match token { + 1 => b"I need".to_vec(), + 2 => b" it".to_vec(), + _ => Vec::new(), + } + ), + Some((b"<|assistant|>I need".to_vec(), false)) + ); + } + + #[test] + fn ling_keeps_the_empty_think_pair_its_template_replays() { + // Bailing V3 re-emits the pair before every history assistant turn, + // so a Motif-style strip would make the key stop being a prefix. + assert_eq!( + bank_retire_key( + b"ASSISTANT\n", + &[], + &[1, 2], + false, + ModelSyntax::Ling3Vl, + |token| match token { + 1 => b"LPDDR5X".to_vec(), + _ => Vec::new(), + } + ), + Some(( + b"ASSISTANT\nLPDDR5X".to_vec(), + false + )) + ); + } + + #[test] + fn warm_match_requires_a_strict_suffix_and_exact_snapshot_tokens() { + let warm = WarmBank { + record: Some(WarmRecord { + text: b"shared".to_vec(), + cache_text: None, + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation: 7, + ext_flags: 0, + trailer: Vec::new(), + }), + committed_tokens: 2, + stored_tokens: 0, + last_use: 0, + }; + assert!( + warm_admit_tokens(&warm, b"shared", None, &[10, 11], 7, 8, false, |_| vec![12]) + .is_none() + ); + assert!( + warm_admit_tokens(&warm, b"other", None, &[10, 11], 7, 8, false, |_| vec![12]) + .is_none() + ); + assert!(warm_admit_tokens( + &warm, + b"shared suffix", + None, + &[10, 11], + 8, + 8, + false, + |_| vec![12], + ) + .is_none()); + let (tokens, cached) = warm_admit_tokens( + &warm, + b"shared suffix", + None, + &[10, 11], + 7, + 8, + false, + |suffix| { + assert_eq!(suffix, b" suffix"); + vec![20, 21] + }, + ) + .unwrap(); + assert_eq!(tokens, vec![10, 11, 20, 21]); + assert_eq!(cached, 2); + assert!(warm_admit_tokens( + &warm, + b"shared suffix", + None, + &[10, 11], + 7, + 3, + false, + |_| vec![20, 21], + ) + .is_none()); + } + + #[test] + fn multi_bank_match_uses_the_longest_strict_prefix() { + let banks = vec![warm("shared", 3), warm("shared turn", 2), warm("other", 1)]; + assert_eq!( + warm_match_pick(&banks, b"shared turn suffix", None), + Some(WarmMatch { + bank: 1, + exact: false, + }) + ); + assert_eq!( + warm_match_pick(&banks, b"shared turn", None), + Some(WarmMatch { + bank: 0, + exact: false, + }) + ); + assert_eq!(warm_match_pick(&banks, b"unrelated", None), None); + } + + #[test] + fn warm_full_match_accepts_visible_and_exact_reasoning_keys() { + let banks = [WarmBank { + record: Some(WarmRecord { + text: b"visible".to_vec(), + cache_text: None, + exact_text: Some(b"raw reasoning".to_vec()), + exact_cache_text: None, + partial_only: false, + generation: 7, + ext_flags: 0, + trailer: Vec::new(), + }), + committed_tokens: 2, + stored_tokens: 0, + last_use: 0, + }]; + assert_eq!( + warm_match_pick(&banks, b"visible tail", None), + Some(WarmMatch { + bank: 0, + exact: false, + }) + ); + assert_eq!( + warm_match_pick(&banks, b"raw reasoning tail", None), + Some(WarmMatch { + bank: 0, + exact: true, + }) + ); + let (tokens, cached) = warm_admit_tokens( + &banks[0], + b"raw reasoning tail", + None, + &[10, 11], + 7, + 8, + true, + |suffix| { + assert_eq!(suffix, b" tail"); + vec![12] + }, + ) + .unwrap(); + assert_eq!(tokens, [10, 11, 12]); + assert_eq!(cached, 2); + } + + #[test] + fn partial_match_uses_the_longest_divergent_record() { + let mut banks = vec![ + warm("shared system alpha", 3), + warm("shared beta", 2), + warm("other", 1), + ]; + assert_eq!( + warm_partial_match_pick(&banks, b"shared system gamma", None, 5), + Some((0, b"shared system ".len())) + ); + assert_eq!( + warm_partial_match_pick(&banks, b"shared system gamma", None, 15), + None + ); + banks[0].record.as_mut().unwrap().partial_only = true; + assert_eq!( + warm_partial_match_pick(&banks, b"shared system gamma", None, 5), + Some((0, b"shared system ".len())) + ); + assert_eq!( + warm_match_pick(&banks, b"shared system alpha tail", None), + None + ); + } + + #[test] + fn rewind_only_record_cannot_supersede_an_exact_record() { + let mut banks = vec![warm("prefix", 1), warm("prefix plus tail", 2)]; + banks[1].record.as_mut().unwrap().partial_only = true; + assert!(!warm_record_superseded(&banks, 0)); + banks[0].record.as_mut().unwrap().partial_only = true; + banks[1].record.as_mut().unwrap().partial_only = false; + assert!(warm_record_superseded(&banks, 0)); + } + + #[test] + fn token_fallback_skips_image_records_and_non_stride_cuts() { + assert_eq!(solar_partial_stride(8192), 4096); + assert_eq!(solar_partial_stride(131072), 8192); + assert_eq!(solar_stride_floor(4096, 8192), Some(4096)); + assert_eq!(solar_stride_floor(4096, 131072), None); + assert_eq!(solar_stride_floor(9000, 131072), Some(8192)); + assert_eq!(solar_stride_floor(8191, 8192), Some(4096)); + assert_eq!(solar_stride_floor(8, 8192), None); + assert!(!warm_record_has_image(None)); + let text = WarmRecord { + text: b"prefix".to_vec(), + cache_text: None, + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation: 1, + ext_flags: 0, + trailer: Vec::new(), + }; + assert!(!warm_record_has_image(Some(&text))); + let pixels = WarmRecord { + cache_text: Some(b"img".to_vec()), + ext_flags: EXT_IMAGE_PIXELS_V2, + ..text + }; + assert!(warm_record_has_image(Some(&pixels))); + } + + #[test] + fn partial_cut_is_a_canonical_token_prefix_with_a_suffix_left_to_prefill() { + assert_eq!( + warm_partial_token_cut(&[10, 11, 12, 13], &[10, 11, 12, 20], 3, usize::MAX), + Some(3) + ); + assert_eq!( + warm_partial_token_cut(&[10, 11, 12, 13], &[10, 11, 12, 20], 4, usize::MAX), + None + ); + assert_eq!( + warm_partial_token_cut(&[10, 11, 12], &[10, 11, 12], 2, usize::MAX), + Some(2) + ); + } + + #[test] + fn qwen_image_cache_key_separates_pixels_and_caps_partial_reuse() { + let prompt = b"before<|image_pad|>after"; + let identity = |pixel_hash| ImageCacheIdentity { + pixel_hash, + grid_h: 16, + grid_w: 16, + token_count: 64, + token_offset: 17, + }; + let (red, spans) = qwen_image_cache_text_build(prompt, &[identity(1)]).unwrap(); + let (same, _) = qwen_image_cache_text_build(prompt, &[identity(1)]).unwrap(); + let (blue, _) = qwen_image_cache_text_build(prompt, &[identity(2)]).unwrap(); + assert_eq!( + red, + b"before<|image_pad|>\xffDS4IMG2:0000000000000001:00000010:00000010:00000040\xfeafter" + ); + assert_eq!(red, same); + assert_ne!(red, blue); + assert_eq!(qwen_image_cache_text_strip(&red).unwrap(), prompt); + let extended = + extended_image_cache_key(prompt, &red, b"before<|image_pad|>after answer").unwrap(); + assert_eq!(&extended[..red.len()], red); + assert_eq!(&extended[red.len()..], b" answer"); + let lcp = red.iter().zip(&blue).take_while(|(a, b)| a == b).count(); + assert_eq!(qwen_image_cache_token_cap(&spans, lcp), 17); + + let mut record_key = red.clone(); + record_key.extend_from_slice(b" answer"); + let banks = [WarmBank { + record: Some(WarmRecord { + text: b"before<|image_pad|>after answer".to_vec(), + cache_text: Some(record_key.clone()), + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation: 1, + ext_flags: EXT_IMAGE_PIXELS_V2, + trailer: Vec::new(), + }), + committed_tokens: 80, + stored_tokens: 0, + last_use: 1, + }]; + let mut same_request = record_key; + same_request.extend_from_slice(b" tail"); + let mut different_request = blue; + different_request.extend_from_slice(b" answer tail"); + assert_eq!( + warm_match_pick( + &banks, + b"before<|image_pad|>after answer tail", + Some(&same_request), + ), + Some(WarmMatch { + bank: 0, + exact: false, + }) + ); + assert_eq!( + warm_match_pick( + &banks, + b"before<|image_pad|>after answer tail", + Some(&different_request), + ), + None + ); + assert_eq!( + warm_match_pick(&banks, b"before<|image_pad|>after answer tail", None), + None + ); + } + + #[test] + fn partial_admit_uses_canonical_prompt_tokens_and_validates_generation() { + let warm = warm("shared system alpha", 7); + let (tokens, cached) = warm_partial_admit_tokens( + &warm, + &[10, 11, 12, 20, 21], + &[10, 11, 12, 13], + 1, + 3, + 8, + usize::MAX, + ) + .unwrap(); + assert_eq!(tokens, [10, 11, 12, 20, 21]); + assert_eq!(cached, 3); + assert!(warm_partial_admit_tokens( + &warm, + &[10, 11, 12, 20], + &[10, 11, 12, 13], + 2, + 3, + 8, + usize::MAX, + ) + .is_none()); + assert!(warm_partial_admit_tokens( + &warm, + &[10, 11, 12, 20], + &[10, 11, 12, 13], + 1, + 3, + 3, + usize::MAX, + ) + .is_none()); + } + + #[test] + fn cold_victim_matches_c_no_value_superseded_and_depth_tiers() { + let mut banks = vec![warm("trunk", 30), WarmBank::default(), warm("tenant", 10)]; + banks[0].committed_tokens = 70_000; + assert_eq!(warm_victim_pick(&banks, &[], None, 65_536, true), Some(1)); + + banks[1] = warm("trunk child", 20); + banks[1].committed_tokens = 70_000; + assert_eq!(warm_victim_pick(&banks, &[], None, 65_536, true), Some(0)); + + banks[0] = warm("alpha", 5); + banks[1] = warm("beta", 4); + banks[2] = warm("gamma", 10); + banks[0].committed_tokens = 70_000; + banks[1].committed_tokens = 70_000; + assert_eq!(warm_victim_pick(&banks, &[], None, 65_536, true), Some(2)); + banks[2].committed_tokens = 90_000; + assert_eq!(warm_victim_pick(&banks, &[], None, 65_536, true), Some(1)); + } + + #[test] + fn fork_target_never_spends_a_plain_tenant_record() { + let banks = vec![warm("source", 3), warm("tenant-a", 1), warm("tenant-b", 2)]; + assert_eq!(warm_victim_pick(&banks, &[], Some(0), 65_536, false), None); + + let mut spare = banks; + spare[2] = WarmBank::default(); + assert_eq!( + warm_victim_pick(&spare, &[], Some(0), 65_536, false), + Some(2) + ); + } + + #[test] + fn warm_fork_preserves_the_trunk_only_when_a_safe_target_exists() { + let mut with_spare = vec![warm("trunk", 3), warm("tenant", 1), WarmBank::default()]; + assert_eq!( + warm_placement(&with_spare, &[], 0, 10, 65_536, true), + Some(WarmPlacement { + source: 0, + target: 2, + fork: true, + }) + ); + + let no_spare = vec![warm("trunk", 3), warm("tenant-a", 1), warm("tenant-b", 2)]; + assert_eq!( + warm_placement(&no_spare, &[], 0, 10, 65_536, true), + Some(WarmPlacement { + source: 0, + target: 0, + fork: false, + }) + ); + with_spare[0].committed_tokens = 70_000; + assert!( + !warm_placement(&with_spare, &[], 0, 70_000, 65_536, true) + .unwrap() + .fork + ); + } + + #[test] + fn rolling_fork_reuses_prefix_without_spending_the_live_target() { + let mut banks = vec![warm("trunk", 3), WarmBank::default(), WarmBank::default()]; + let first = warm_placement(&banks, &[], 0, 10, 65_536, true).unwrap(); + assert!(first.fork && first.target == 1); + banks[1].record = None; + + let mut reserve = crate::serve_cont_roll::RollReserve::new(); + reserve.note_place(i32::try_from(first.target + 1).unwrap()); + let protected = reserve.protect(&[false, false, false]); + assert_eq!( + warm_victim_pick(&banks, &protected, None, 65_536, true), + Some(2) + ); + assert_eq!( + warm_placement(&banks, &protected, 0, 10, 65_536, true), + Some(WarmPlacement { + source: 0, + target: 2, + fork: true, + }) + ); + } + + #[test] + fn rolling_pin_prevents_evict_of_a_deep_fork_target() { + let mut banks = vec![warm("trunk", 3), warm("tenant", 1), WarmBank::default()]; + banks[1].committed_tokens = 70_000; + let mut reserve = crate::serve_cont_roll::RollReserve::new(); + reserve.note_place(3); + let protected = reserve.protect(&[false, false, false]); + assert_eq!( + warm_victim_pick(&banks, &protected, Some(0), 65_536, false), + None + ); + assert_eq!( + warm_placement(&banks, &protected, 0, 10, 65_536, true), + Some(WarmPlacement { + source: 0, + target: 0, + fork: false, + }) + ); + } + + #[test] + fn rolling_protected_saturation_refuses_when_hold_and_live_fill_the_table() { + let banks = vec![warm("trunk", 3), warm("tenant", 1)]; + let mut reserve = crate::serve_cont_roll::RollReserve::new(); + reserve.note_place(2); + let protected = reserve.protect(&[true, false]); + assert_eq!( + warm_victim_pick(&banks, &protected, None, 65_536, true), + None + ); + assert_eq!( + warm_placement(&banks, &protected, 0, 10, 65_536, true), + None + ); + } + + #[test] + fn protected_trunk_forks_safely_or_refuses_in_place_extension() { + let with_spare = vec![warm("trunk", 3), WarmBank::default()]; + assert!( + warm_placement(&with_spare, &[true, false], 0, 10, 65_536, true) + .is_some_and(|placement| placement.fork && placement.target == 1) + ); + + let no_spare = vec![warm("trunk", 3), warm("tenant", 2)]; + assert_eq!( + warm_placement(&no_spare, &[true, false], 0, 10, 65_536, true), + None + ); + } + + #[test] + fn victim_selection_skips_a_protected_bank_at_every_tier() { + let mut banks = vec![WarmBank::default(), warm("shallow", 1), warm("deep", 2)]; + banks[2].committed_tokens = 70_000; + assert_eq!( + warm_victim_pick(&banks, &[true, false, false], None, 65_536, true), + Some(1) + ); + + banks[0] = warm("trunk", 1); + banks[1] = warm("trunk child", 2); + assert_eq!( + warm_victim_pick(&banks, &[true, false, false], None, 65_536, false), + None + ); + + banks[0] = warm("shallow", 1); + banks[1].committed_tokens = 70_000; + assert_eq!( + warm_victim_pick(&banks, &[true, false, false], None, 65_536, true), + Some(1) + ); + + let all_protected = [true, true, true]; + assert_eq!( + warm_victim_pick(&banks, &all_protected, None, 65_536, true), + None + ); + } + + #[test] + fn disk_restore_replaces_only_empty_stale_or_shallow_live_banks() { + let empty = WarmBank::default(); + assert!(disk_restore_allowed(&empty, Some((7, 70_000)), 1, 65_536)); + + let live = WarmBank { + record: Some(WarmRecord { + text: b"live".to_vec(), + cache_text: None, + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation: 7, + ext_flags: 0, + trailer: Vec::new(), + }), + committed_tokens: 70_000, + stored_tokens: 0, + last_use: 0, + }; + assert!(disk_restore_allowed(&live, Some((8, 70_000)), 1, 65_536)); + assert!(disk_restore_allowed( + &live, + Some((7, 32_000)), + 70_000, + 65_536, + )); + assert!(!disk_restore_allowed( + &live, + Some((7, 70_000)), + 70_000, + 65_536, + )); + assert!(!disk_restore_allowed( + &live, + Some((7, 32_000)), + 32_000, + 65_536, + )); + } + + #[test] + fn disk_restore_may_replace_a_superseded_deep_record() { + let mut banks = vec![warm("trunk", 1), warm("trunk child", 2)]; + banks[0].committed_tokens = 70_000; + banks[1].committed_tokens = 70_000; + assert!(disk_restore_target_allowed(&banks, 0, 10, 65_536)); + assert!(!disk_restore_target_allowed(&banks, 1, 10, 65_536)); + } + + #[test] + fn bank_retires_only_after_native_done() { + assert!(bank_retire_allowed(true, true, true)); + assert!(!bank_retire_allowed(false, true, true)); + assert!(!bank_retire_allowed(true, false, true)); + assert!(!bank_retire_allowed(true, true, false)); + } + + #[test] + fn bank_retirement_uses_the_native_reported_bank() { + assert_eq!(retired_bank(true, true, true, Some(2), 3), Some(2)); + assert_eq!(retired_bank(true, true, true, None, 3), None); + assert_eq!(retired_bank(true, true, true, Some(3), 3), None); + } + + #[test] + fn bank_retire_keeps_prior_warm_ext_flags() { + use ds4_kv::{EXT_RESPONSES_VISIBLE, EXT_SESSION_TITLE}; + let prev = WarmRecord { + text: b"old".to_vec(), + cache_text: None, + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation: 1, + ext_flags: EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE, + trailer: b"ktm".to_vec(), + }; + let retired = retired_record(Some(&prev), b"new-key".to_vec(), None, 9).unwrap(); + assert_eq!(retired.text, b"new-key"); + assert_eq!(retired.generation, 9); + assert_eq!(retired.ext_flags, EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE); + assert!(retired_record(Some(&prev), Vec::new(), None, 9).is_none()); + assert_eq!( + retired_record(None, b"fresh".to_vec(), None, 2) + .unwrap() + .ext_flags, + 0 + ); + } + + #[test] + fn bank_save_stages_before_reuse_and_advances_marker_only_on_success() { + let (dir, mut store) = temp_store("save-order"); + let trailer = + crate::tool_memory::encode_ktm(&[(b"call_1", b"x")]).unwrap(); + let mut warm = WarmBank { + record: Some(WarmRecord { + text: b"shared prefix".to_vec(), + cache_text: None, + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation: 7, + ext_flags: 0, + trailer: trailer.clone(), + }), + committed_tokens: 3, + stored_tokens: 1, + last_use: 0, + }; + assert!(!save_bank_record( + &mut store, + &mut warm, + (0, 2, 8192), + 3, + 8, + Reason::BankShutdown, + |_| panic!("stale records must not stage payloads"), + ) + .unwrap()); + assert!(store.entries().is_empty()); + assert_eq!(warm.stored_tokens, 1); + assert!(save_bank_record( + &mut store, + &mut warm, + (0, 2, 8192), + 3, + 7, + Reason::BankShutdown, + |path| fs::write(path, b"opaque").map_err(|e| GenerateError::Engine(e.to_string())), + ) + .unwrap()); + assert_eq!(warm.stored_tokens, 3); + let path = store.entries()[0].path.clone(); + let record = store.read(&path).unwrap(); + assert_eq!(record.header.reason, Reason::BankShutdown); + assert_eq!(record.header.ext_flags, EXT_BANK_REPLAY_V1 | EXT_TOOL_MAP); + assert_eq!(record.header.tokens, 3); + assert_eq!(record.header.ctx_size, 8192); + assert_eq!(record.text, b"shared prefix"); + assert_eq!(record.payload, b"opaque"); + assert_eq!(record.trailer, trailer); + + warm.stored_tokens = 1; + let error = save_bank_record( + &mut store, + &mut warm, + (0, 2, 8192), + 3, + 7, + Reason::BankShutdown, + |_| Err(GenerateError::Engine("stage failed".into())), + ) + .unwrap_err(); + assert_eq!(error.to_string(), "stage failed"); + assert_eq!(warm.stored_tokens, 1); + assert_eq!(store.read(&path).unwrap().payload, b"opaque"); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn bank_save_skips_rewind_only_thinking_records() { + let (dir, mut store) = temp_store("partial-only"); + let mut warm = warm("prompt", 1); + warm.record.as_mut().unwrap().partial_only = true; + assert!(!save_bank_record( + &mut store, + &mut warm, + (6, 4, 262_144), + 2, + 1, + Reason::BankCheckpoint, + |_| panic!("rewind-only records must not be persisted"), + ) + .unwrap()); + assert!(store.entries().is_empty()); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn bank_save_persists_visible_key_without_the_in_memory_alias() { + let (dir, mut store) = temp_store("thinking-visible"); + let mut warm = warm("visible answer", 1); + warm.record.as_mut().unwrap().exact_text = Some(b"hidden reasoning answer".to_vec()); + assert!(save_bank_record( + &mut store, + &mut warm, + (6, 4, 262_144), + 3, + 1, + Reason::BankCheckpoint, + |path| fs::write(path, b"opaque").map_err(|e| GenerateError::Engine(e.to_string())), + ) + .unwrap()); + let record = store.read(&store.entries()[0].path).unwrap(); + assert_eq!(record.text, b"visible answer"); + assert_eq!(record.header.ext_flags, EXT_BANK_REPLAY_V1); + assert!(!record.text.windows(6).any(|window| window == b"hidden")); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn bank_save_persists_the_image_identity_key() { + let (dir, mut store) = temp_store("image-key"); + let visible = b"before<|image_pad|>after".to_vec(); + let (cache_key, _) = qwen_image_cache_text_build( + &visible, + &[ImageCacheIdentity { + pixel_hash: 7, + grid_h: 16, + grid_w: 16, + token_count: 64, + token_offset: 5, + }], + ) + .unwrap(); + let mut warm = WarmBank { + record: Some(WarmRecord { + text: visible.clone(), + cache_text: Some(cache_key.clone()), + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation: 3, + ext_flags: EXT_IMAGE_PIXELS_V2, + trailer: Vec::new(), + }), + committed_tokens: 64, + stored_tokens: 0, + last_use: 0, + }; + assert!(save_bank_record( + &mut store, + &mut warm, + (6, 2, 8192), + 64, + 3, + Reason::BankCheckpoint, + |path| fs::write(path, b"opaque").map_err(|e| GenerateError::Engine(e.to_string())), + ) + .unwrap()); + let record = store.read(&store.entries()[0].path).unwrap(); + assert_eq!(record.text, cache_key); + assert_eq!( + record.header.ext_flags, + EXT_BANK_REPLAY_V1 | EXT_IMAGE_PIXELS_V2 + ); + assert_eq!( + restored_record(record.text, 4, record.header.ext_flags) + .unwrap() + .text, + visible + ); + let _ = fs::remove_dir_all(dir); + } +} diff --git a/crates/ds4-server/src/serve_cont_evict.rs b/crates/ds4-server/src/serve_cont_evict.rs new file mode 100644 index 0000000000000000000000000000000000000000..2e3aa99981e749295754efde1edeb5c5eddb8744 --- /dev/null +++ b/crates/ds4-server/src/serve_cont_evict.rs @@ -0,0 +1,209 @@ +//! Live bank-evict persist: write `KvReason::BankEvict` before destroy. +//! +//! C `kv_cache_store_bank(..., "bank-evict", warm_pin_min)` then +//! `warm_rec_invalidate`. A continuation-protected (pinned) bank is never +//! a victim. +//! +//! The save path is the same `save_bank_record` ContLane::persist_bank uses; +//! only the reason is `BankEvict`. Live place_* call ContLane::evict_bank, +//! which is persist_bank(BankEvict) plus this pin-skip / drop contract. + +use std::path::Path; + +use ds4_kv::{Reason as KvReason, Store as KvStore}; + +use crate::generate::GenerateError; +#[cfg(test)] +use crate::serve_cont::WarmRecord; +use crate::serve_cont::{save_bank_record, WarmBank}; + +/// Persist a pin-tier bank with `BankEvict`, then drop its warm record. +/// +/// Returns `Ok(false)` and leaves the bank untouched when `pinned`. +pub(crate) fn persist_bank_evict( + store: &mut KvStore, + warm: &mut WarmBank, + pinned: bool, + identity: (u8, u8, u32), + committed: i32, + generation: u64, + pin_min: i32, + save_payload: impl FnOnce(&Path) -> Result<(), GenerateError>, +) -> Result { + if pinned { + return Ok(false); + } + if pin_min > 0 && committed >= pin_min { + if let Err(error) = save_bank_record( + store, + warm, + identity, + committed, + generation, + KvReason::BankEvict, + save_payload, + ) { + eprintln!("ds4-server-rs: bank checkpoint failed reason=bank-evict: {error}"); + } + } + warm.record = None; + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::*; + use ds4_kv::{Options, Reason, EXT_BANK_REPLAY_V1, EXT_RESPONSES_VISIBLE, EXT_SESSION_TITLE}; + use std::fs; + + fn temp_store(tag: &str) -> (std::path::PathBuf, KvStore) { + let dir = std::env::temp_dir().join(format!( + "ds4-server-bank-evict-{tag}-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&dir); + let store = KvStore::open(&dir, 16, false, Options::default()).unwrap(); + (dir, store) + } + + fn deep_warm(text: &str) -> WarmBank { + WarmBank { + record: Some(WarmRecord { + text: text.as_bytes().to_vec(), + cache_text: None, + exact_text: None, + exact_cache_text: None, + partial_only: false, + generation: 3, + ext_flags: 0, + trailer: Vec::new(), + }), + committed_tokens: 70_000, + stored_tokens: 0, + last_use: 1, + } + } + + #[test] + fn evict_writes_file_with_bank_evict_reason() { + // Given: an unpinned pin-tier bank with a matching generation + let (dir, mut store) = temp_store("write"); + let mut warm = deep_warm("evict-prefix"); + + // When: live evict persist runs at the pin threshold + let wrote = persist_bank_evict( + &mut store, + &mut warm, + false, + (1, 2, 8192), + 70_000, + 3, + 65_536, + |path| { + fs::write(path, b"bank-payload").map_err(|e| GenerateError::Engine(e.to_string())) + }, + ) + .unwrap(); + + // Then: a KVC file is stored with reason BankEvict and the record is gone + assert!(wrote); + assert!(warm.record.is_none()); + assert_eq!(store.entries().len(), 1); + let record = store.read(&store.entries()[0].path).unwrap(); + assert_eq!(record.header.reason, Reason::BankEvict); + assert_eq!(record.header.ext_flags, EXT_BANK_REPLAY_V1); + assert_eq!(record.header.tokens, 70_000); + assert_eq!(record.text, b"evict-prefix"); + assert_eq!(record.payload, b"bank-payload"); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn evict_skips_a_pinned_bank() { + // Given: a continuation-protected (pinned) pin-tier bank + let (dir, mut store) = temp_store("pinned"); + let mut warm = deep_warm("pinned-prefix"); + + // When: evict persist is asked to destroy that bank + let wrote = persist_bank_evict( + &mut store, + &mut warm, + true, + (1, 2, 8192), + 70_000, + 3, + 65_536, + |_| panic!("pinned banks must not stage a payload"), + ) + .unwrap(); + + // Then: no store write and the warm record stays + assert!(!wrote); + assert!(store.entries().is_empty()); + assert_eq!( + warm.record.as_ref().map(|record| record.text.as_slice()), + Some(b"pinned-prefix".as_slice()) + ); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn evict_persist_keeps_responses_visible_and_session_title() { + // Given: an unpinned bank whose live ext bits include Responses/title + let (dir, mut store) = temp_store("ext"); + let mut warm = deep_warm("ext-prefix"); + if let Some(record) = warm.record.as_mut() { + record.ext_flags = EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE; + } + + // When: evict persist writes the pin-tier snapshot + persist_bank_evict( + &mut store, + &mut warm, + false, + (1, 2, 8192), + 70_000, + 3, + 65_536, + |path| fs::write(path, b"p").map_err(|e| GenerateError::Engine(e.to_string())), + ) + .unwrap(); + + // Then: BankEvict is stored and those two bits are not clobbered + let record = store.read(&store.entries()[0].path).unwrap(); + assert_eq!(record.header.reason, Reason::BankEvict); + assert_eq!( + record.header.ext_flags, + EXT_BANK_REPLAY_V1 | EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE + ); + let _ = fs::remove_dir_all(dir); + } + + #[test] + fn evict_does_not_write_below_pin_min() { + // Given: an unpinned shallow bank (below warm_pin_min) + let (dir, mut store) = temp_store("shallow"); + let mut warm = deep_warm("shallow-prefix"); + warm.committed_tokens = 10; + + // When: evict persist runs with C's pin-tier min_committed + let wrote = persist_bank_evict( + &mut store, + &mut warm, + false, + (1, 2, 8192), + 10, + 3, + 65_536, + |_| panic!("shallow banks are not the pin-tier persist set"), + ) + .unwrap(); + + // Then: the record is still evicted, but no BankEvict file is written + assert!(wrote); + assert!(warm.record.is_none()); + assert!(store.entries().is_empty()); + let _ = KvReason::BankEvict; + let _ = fs::remove_dir_all(dir); + } +} diff --git a/crates/ds4-server/src/serve_cont_prefill.rs b/crates/ds4-server/src/serve_cont_prefill.rs new file mode 100644 index 0000000000000000000000000000000000000000..202bef49c9a60328675f2241a8378005cb64e97e --- /dev/null +++ b/crates/ds4-server/src/serve_cont_prefill.rs @@ -0,0 +1,236 @@ +//! C continuous prefill-chunk interleave (R4 overlap-lite) on the rolling +//! scheduler. Env names match C and must not change. + +use std::cell::Cell; +use std::ffi::OsStr; + +/// C `DS4_CONT_PREFILL_CHUNK`. Do not rename. +pub(crate) const ENV_PREFILL_CHUNK: &str = "DS4_CONT_PREFILL_CHUNK"; +/// C `DS4_CONT_PREFILL_CHUNK_LIVE`. Do not rename. +pub(crate) const ENV_PREFILL_CHUNK_LIVE: &str = "DS4_CONT_PREFILL_CHUNK_LIVE"; + +pub(crate) const DEFAULT_PREFILL_CHUNK: u32 = 4096; +pub(crate) const DEFAULT_PREFILL_CHUNK_LIVE: u32 = 512; +/// One definition with the serving plan, so `--print-plan` cannot advertise +/// a yield this policy then caps. +const PREFILL_FENCE: u32 = ds4_core::PREFILL_CHUNK_FENCE; + +/// C `bg_prefill_chunk_tokens` / `bg_prefill_chunk_live_tokens`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct PrefillChunkPolicy { + pub boot: u32, + pub live: u32, +} + +impl PrefillChunkPolicy { + pub(crate) fn from_env() -> Self { + let nofence = + std::env::var_os("DS4_CONT_PREFILL_NOFENCE").as_deref() == Some(OsStr::new("1")); + Self::from_os( + std::env::var_os(ENV_PREFILL_CHUNK).as_deref(), + std::env::var_os(ENV_PREFILL_CHUNK_LIVE).as_deref(), + nofence, + ) + } + + pub(crate) fn from_os(boot: Option<&OsStr>, live: Option<&OsStr>, nofence: bool) -> Self { + Self::from_raw( + parse_chunk(boot, DEFAULT_PREFILL_CHUNK), + parse_chunk(live, DEFAULT_PREFILL_CHUNK_LIVE), + nofence, + ) + } + + pub(crate) fn from_raw(boot: i32, live: i32, nofence: bool) -> Self { + let mut boot = boot.max(0) as u32; + let mut live = live.max(0) as u32; + if boot > PREFILL_FENCE && !nofence { + boot = PREFILL_FENCE; + } + if live > boot { + live = boot; + } + Self { boot, live } + } + + /// C: `W_boot != 0 && W_live != 0`. + pub(crate) const fn interleave(self) -> bool { + self.boot != 0 && self.live != 0 + } + + /// C `W_eff`: LIVE width only while someone is decoding; else boot. + /// `0` means one-shot (whole remaining suffix). + pub(crate) const fn chunk_width(self, live_decode: bool) -> u32 { + if self.boot == 0 { + 0 + } else if self.interleave() && live_decode { + self.live + } else { + self.boot + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RollPhase { + Prefill { remaining: u32 }, + Decode { remaining: u32 }, + Done, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TickOp { + Prefill { user: usize, tokens: u32 }, + Decode { user: usize }, +} + +/// One C outer-loop pass: advance pending prefills, then step live decode. +/// Interleave runs one chunk then falls through; LIVE=0 drains at boot width. +pub(crate) fn tick_roll_prefill( + policy: PrefillChunkPolicy, + jobs: &mut [(usize, RollPhase)], +) -> Vec { + let mut ops = Vec::new(); + let live_decode = jobs + .iter() + .any(|(_, phase)| matches!(phase, RollPhase::Decode { remaining } if *remaining > 0)); + let mut idx = 0; + while idx < jobs.len() { + let RollPhase::Prefill { remaining } = jobs[idx].1 else { + idx += 1; + continue; + }; + if remaining == 0 { + jobs[idx].1 = RollPhase::Done; + idx += 1; + continue; + } + let width = policy.chunk_width(live_decode); + let n = if width == 0 { + remaining + } else { + remaining.min(width) + }; + let user = jobs[idx].0; + let left = remaining - n; + jobs[idx].1 = if left == 0 { + RollPhase::Done + } else { + RollPhase::Prefill { remaining: left } + }; + ops.push(TickOp::Prefill { user, tokens: n }); + if policy.interleave() { + break; + } + if left > 0 { + continue; + } + idx += 1; + } + for (user, phase) in jobs.iter_mut() { + let RollPhase::Decode { remaining } = *phase else { + continue; + }; + if remaining == 0 { + *phase = RollPhase::Done; + continue; + } + let left = remaining - 1; + *phase = if left == 0 { + RollPhase::Done + } else { + RollPhase::Decode { remaining: left } + }; + ops.push(TickOp::Decode { user: *user }); + } + ops +} + +thread_local! { + static OWNER_TICK_CALLS: Cell = const { Cell::new(0) }; +} + +pub(crate) fn owner_tick_pair( + policy: PrefillChunkPolicy, + live_decode_remaining: u32, + prefill_remaining: u32, +) -> Vec { + OWNER_TICK_CALLS.with(|calls| calls.set(calls.get().saturating_add(1))); + let mut jobs = [ + ( + 1, + RollPhase::Decode { + remaining: live_decode_remaining, + }, + ), + ( + 2, + RollPhase::Prefill { + remaining: prefill_remaining, + }, + ), + ]; + tick_roll_prefill(policy, &mut jobs) +} + +#[cfg(test)] +pub(crate) fn owner_tick_call_count() -> u32 { + OWNER_TICK_CALLS.with(Cell::get) +} + +#[cfg(test)] +pub(crate) fn reset_owner_tick_call_count() { + OWNER_TICK_CALLS.with(|calls| calls.set(0)); +} + +fn parse_chunk(value: Option<&OsStr>, default: u32) -> i32 { + let Some(value) = value else { + return default as i32; + }; + let bytes = os_bytes(value); + if bytes.is_empty() { + return default as i32; + } + c_atoi(bytes) +} + +fn os_bytes(value: &OsStr) -> &[u8] { + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt; + value.as_bytes() + } + #[cfg(not(unix))] + { + value.as_encoded_bytes() + } +} + +fn c_atoi(bytes: &[u8]) -> i32 { + let mut i = 0; + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + let mut neg = false; + if i < bytes.len() && (bytes[i] == b'+' || bytes[i] == b'-') { + neg = bytes[i] == b'-'; + i += 1; + } + let mut n: i32 = 0; + let mut saw_digit = false; + while i < bytes.len() && bytes[i].is_ascii_digit() { + saw_digit = true; + n = n + .saturating_mul(10) + .saturating_add(i32::from(bytes[i] - b'0')); + i += 1; + } + if !saw_digit { + return 0; + } + if neg { + -n + } else { + n + } +} diff --git a/crates/ds4-server/src/serve_cont_prefill_test.rs b/crates/ds4-server/src/serve_cont_prefill_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..ada613213dd60d36466d44783bd1dac5cc89bd2d --- /dev/null +++ b/crates/ds4-server/src/serve_cont_prefill_test.rs @@ -0,0 +1,200 @@ +use std::ffi::OsStr; + +use crate::serve_cont_prefill::{ + owner_tick_call_count, owner_tick_pair, reset_owner_tick_call_count, tick_roll_prefill, + PrefillChunkPolicy, RollPhase, TickOp, DEFAULT_PREFILL_CHUNK, DEFAULT_PREFILL_CHUNK_LIVE, + ENV_PREFILL_CHUNK, ENV_PREFILL_CHUNK_LIVE, +}; +use crate::serve_cont_roll::ContRoll; + +#[test] +fn default_chunk_env_names_match_c() { + assert_eq!(ENV_PREFILL_CHUNK, "DS4_CONT_PREFILL_CHUNK"); + assert_eq!(ENV_PREFILL_CHUNK_LIVE, "DS4_CONT_PREFILL_CHUNK_LIVE"); + let policy = PrefillChunkPolicy::from_os(None, None, false); + assert_eq!(policy.boot, DEFAULT_PREFILL_CHUNK); + assert_eq!(policy.live, DEFAULT_PREFILL_CHUNK_LIVE); + assert!(policy.interleave()); + if std::env::var_os(ENV_PREFILL_CHUNK).is_none() + && std::env::var_os(ENV_PREFILL_CHUNK_LIVE).is_none() + { + assert_eq!(PrefillChunkPolicy::from_env(), policy); + } +} + +#[test] +fn live_decode_is_stepped_while_another_job_prefills_a_chunk() { + // Given: one live decode job and another still prefilling + let mut roll = ContRoll::new(); + roll.enqueue(1); + roll.enqueue(2); + let decode = roll.admit().expect("live decode admits"); + let prefill = roll + .admit() + .expect("prefill admits while decode is generating"); + let policy = PrefillChunkPolicy::from_raw(4096, 512, false); + let mut jobs = [ + (decode, RollPhase::Decode { remaining: 4 }), + (prefill, RollPhase::Prefill { remaining: 3000 }), + ]; + + // When: one C-loop pass + let ops = tick_roll_prefill(policy, &mut jobs); + + // Then: prefill advances one LIVE chunk and the live job is decode-stepped + assert_eq!( + ops, + vec![ + TickOp::Prefill { + user: prefill, + tokens: 512 + }, + TickOp::Decode { user: decode }, + ] + ); + assert_eq!(jobs[1].1, RollPhase::Prefill { remaining: 2488 }); + assert_eq!(jobs[0].1, RollPhase::Decode { remaining: 3 }); +} + +#[test] +fn live_chunk_env_zero_disables_live_shrink_as_c() { + // Given: LIVE=0 (C: interleave off; drain at boot width) + let policy = + PrefillChunkPolicy::from_os(Some(OsStr::new("4096")), Some(OsStr::new("0")), false); + assert!(!policy.interleave()); + assert_eq!(policy.chunk_width(true), 4096); + let mut jobs = [ + (1, RollPhase::Decode { remaining: 4 }), + (2, RollPhase::Prefill { remaining: 5000 }), + ]; + + // When: one C-loop pass + let ops = tick_roll_prefill(policy, &mut jobs); + + // Then: two boot-width chunks drain before any decode step + assert_eq!( + ops, + vec![ + TickOp::Prefill { + user: 2, + tokens: 4096 + }, + TickOp::Prefill { + user: 2, + tokens: 904 + }, + TickOp::Decode { user: 1 }, + ] + ); + assert_eq!(jobs[1].1, RollPhase::Done); + assert_eq!(jobs[0].1, RollPhase::Decode { remaining: 3 }); +} + +#[test] +fn idle_prefill_keeps_boot_width_when_nobody_is_decoding() { + // Given: interleave on, but no live decode + let policy = PrefillChunkPolicy::from_raw(4096, 512, false); + assert_eq!(policy.chunk_width(false), 4096); + let mut jobs = [(1, RollPhase::Prefill { remaining: 5000 })]; + + // When: one C-loop pass + let ops = tick_roll_prefill(policy, &mut jobs); + + // Then: idle prefill keeps the boot chunk, not LIVE shrink + assert_eq!( + ops, + vec![TickOp::Prefill { + user: 1, + tokens: 4096 + }] + ); + assert_eq!(jobs[0].1, RollPhase::Prefill { remaining: 904 }); +} + +#[test] +fn an_impossible_mix_does_not_bind() { + let mut req = ds4_core::ServingRequest::default(); + req.sched_chunk = Some(8192); + req.native_chunk = Some(1024); + let plan = ds4_core::resolve_plan( + &req, + Some(ds4_core::serving_caps( + ds4_core::ModelFamily::Qwen4Exp, + ds4_core::Variant::Qwen38FlashNext, + )), + &ds4_core::EngineFacts::default(), + ); + assert!(!plan.may_listen()); + let cfg = crate::ServerConfig::default(); + assert!(crate::listen_if_allowed(&cfg, &plan) + .expect("bind skipped") + .is_none()); +} + +#[test] +fn live_decode_uses_a_verified_native_capped_chunk() { + let mut req = ds4_core::ServingRequest::default(); + req.sched_chunk = Some(3000); + req.sched_chunk_live = Some(700); + req.native_chunk = Some(2048); + let plan = ds4_core::resolve_plan( + &req, + Some(ds4_core::serving_caps( + ds4_core::ModelFamily::Qwen4Exp, + ds4_core::Variant::Qwen38FlashNext, + )), + &ds4_core::EngineFacts::default(), + ); + let native = plan.effective.native_chunk.expect("native"); + assert!(ds4_core::VERIFIED_PREFILL_CHUNKS.contains(&plan.effective.sched_chunk)); + assert!(ds4_core::VERIFIED_PREFILL_CHUNKS.contains(&plan.effective.sched_chunk_live)); + assert!(plan.effective.sched_chunk <= native); + assert!(plan.effective.sched_chunk_live <= native); + assert_ne!(plan.effective.sched_chunk, 3000); + + let policy = PrefillChunkPolicy::from_raw( + plan.effective.sched_chunk as i32, + plan.effective.sched_chunk_live as i32, + false, + ); + assert!(policy.boot <= native); + assert!(policy.live <= native); + assert!(ds4_core::VERIFIED_PREFILL_CHUNKS.contains(&policy.boot)); + assert!(ds4_core::VERIFIED_PREFILL_CHUNKS.contains(&policy.live)); + + let mut jobs = [ + (1, RollPhase::Decode { remaining: 4 }), + (2, RollPhase::Prefill { remaining: 3000 }), + ]; + let ops = tick_roll_prefill(policy, &mut jobs); + assert_eq!( + ops, + vec![ + TickOp::Prefill { + user: 2, + tokens: policy.live + }, + TickOp::Decode { user: 1 }, + ] + ); +} + +#[test] +fn owner_tick_pair_steps_live_decode_while_peer_prefills_live_chunk() { + reset_owner_tick_call_count(); + let policy = PrefillChunkPolicy::from_raw(4096, 512, false); + + let ops = owner_tick_pair(policy, 4, 3000); + + assert_eq!(owner_tick_call_count(), 1); + assert_eq!( + ops, + vec![ + TickOp::Prefill { + user: 2, + tokens: 512 + }, + TickOp::Decode { user: 1 }, + ] + ); +} diff --git a/crates/ds4-server/src/serve_cont_roll.rs b/crates/ds4-server/src/serve_cont_roll.rs new file mode 100644 index 0000000000000000000000000000000000000000..e1417ddedf3a0b7e292ed13d3e0d9082c52d70c0 --- /dev/null +++ b/crates/ds4-server/src/serve_cont_roll.rs @@ -0,0 +1,467 @@ +//! Rolling continuous admit table. The engine's `ContDriver::admit` pulls +//! the next pending job while others are still live; one-at-a-time is not +//! the only path. + +use std::collections::{HashSet, VecDeque}; + +use crate::metrics::REJECT_REASON_NAMES; + +/// Existing serial-cont 503 body for an engine/governor admit refuse. +/// Rolling must reuse this string; do not invent a new HTTP body. +pub(crate) const CONT_ADMIT_REFUSED: &str = "continuous admission rejected; serial fallback"; + +/// C `ds4_gov_status` (ds4_mem_gov.h). ADMIT never reaches a tick site. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum GovStatus { + Admit = 0, + RefuseClass = 1, + RefuseLive = 2, + RetryObs = 3, + Unsupported = 4, + Fault = 5, +} + +/// C `DS4_REJECT_*` / `/metrics` `ds4_requests_rejected_total{reason=}`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum RejectReason { + ClassBudget = 0, + LiveHeadroom = 1, + ObsRetry = 2, + Unsupported = 3, + Fault = 4, + DeepPolicy = 5, + LaneDisabled = 6, +} + +impl RejectReason { + pub(crate) const fn name(self) -> &'static str { + REJECT_REASON_NAMES[self as usize] + } +} + +/// C `ds4_reject_reason_from_gov`: ADMIT is not a refusal; every other +/// governed status maps 1:1 onto the typed reject family. +pub(crate) const fn reject_reason_from_gov(status: GovStatus) -> Option { + match status { + GovStatus::Admit => None, + GovStatus::RefuseClass => Some(RejectReason::ClassBudget), + GovStatus::RefuseLive => Some(RejectReason::LiveHeadroom), + GovStatus::RetryObs => Some(RejectReason::ObsRetry), + GovStatus::Unsupported => Some(RejectReason::Unsupported), + GovStatus::Fault => Some(RejectReason::Fault), + } +} + +/// Host-side charge for one rolling admit. Production `AdmitAlways` lets +/// the engine's existing `cont_admit` governor decide; tests inject a +/// budget that refuses with a C status. +pub(crate) trait ContMemGov { + fn charge(&mut self, prompt_n: i32, max_new: i32) -> GovStatus; +} + +/// Production hook: do not second-guess the engine's fund check. +pub(crate) struct AdmitAlways; + +impl ContMemGov for AdmitAlways { + fn charge(&mut self, _prompt_n: i32, _max_new: i32) -> GovStatus { + GovStatus::Admit + } +} + +#[cfg(any(feature = "native", test))] +fn gov_status_from_core(status: ds4_core::GovStatus) -> GovStatus { + match status { + ds4_core::GovStatus::Admit => GovStatus::Admit, + ds4_core::GovStatus::RefuseClass => GovStatus::RefuseClass, + ds4_core::GovStatus::RefuseLive => GovStatus::RefuseLive, + ds4_core::GovStatus::RetryObs => GovStatus::RetryObs, + ds4_core::GovStatus::Unsupported => GovStatus::Unsupported, + ds4_core::GovStatus::Fault => GovStatus::Fault, + } +} + +/// Host D0b evaluator at the rolling-admit seam. `proposed_outstanding` +/// is an absolute lease replacement (sec 6.2), not a token count. Do not +/// invent a tokens-to-bytes formula here. +#[cfg(any(feature = "native", test))] +pub(crate) struct HostMemGov { + pub ledger: ds4_core::GovLedger, + pub obs: ds4_core::MemObservation, + pub claim: ds4_core::GovClaim, +} + +#[cfg(any(feature = "native", test))] +impl ContMemGov for HostMemGov { + fn charge(&mut self, _prompt_n: i32, _max_new: i32) -> GovStatus { + gov_status_from_core(ds4_core::gov_evaluate(&self.ledger, &self.obs, &self.claim).status()) + } +} + +/// Charge one rolling admit. `Ok` means place the job; `Err` is the same +/// typed reason C/serial-cont would tick on `/metrics`. +pub(crate) fn charge_roll_admit( + gov: &mut dyn ContMemGov, + prompt_n: i32, + max_new: i32, +) -> Result<(), RejectReason> { + match reject_reason_from_gov(gov.charge(prompt_n, max_new)) { + None => Ok(()), + Some(reason) => Err(reason), + } +} + +/// Banks already claimed by a prior rolling prepare. The next `prepare_slot` +/// ORs these onto the continuation-hold mask so fork/pin/evict cannot spend +/// a live target or drop protected saturation. +#[derive(Debug, Default)] +pub(crate) struct RollReserve { + banks: Vec, +} + +impl RollReserve { + pub(crate) fn new() -> Self { + Self::default() + } + + /// Record the 1-based `ContAdmit::place_bank` (0 = unset). + pub(crate) fn note_place(&mut self, place_bank: i32) { + let Some(bank) = (place_bank > 0) + .then(|| usize::try_from(place_bank - 1).ok()) + .flatten() + else { + return; + }; + if !self.banks.contains(&bank) { + self.banks.push(bank); + } + } + + pub(crate) fn contains(&self, bank: i32) -> bool { + usize::try_from(bank) + .ok() + .is_some_and(|bank| self.banks.contains(&bank)) + } + + pub(crate) fn protect(&self, hold: &[bool]) -> Vec { + let mut protected = hold.to_vec(); + for &bank in &self.banks { + if bank >= protected.len() { + protected.resize(bank + 1, false); + } + protected[bank] = true; + } + protected + } +} + +/// Host-side rolling job ids. `admit` may return another job while one is +/// already generating; `complete` retires a live job. +#[derive(Debug, Default)] +pub(crate) struct ContRoll { + pending: VecDeque, + live: HashSet, + done: Vec, +} + +impl ContRoll { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn enqueue(&mut self, user: usize) { + self.pending.push_back(user); + } + + /// Pull the next pending job into the live set. Returns `Some` even when + /// another job is already generating. + pub(crate) fn admit(&mut self) -> Option { + let user = self.pending.pop_front()?; + self.live.insert(user); + Some(user) + } + + pub(crate) fn live_count(&self) -> usize { + self.live.len() + } + + pub(crate) fn complete(&mut self, user: usize) { + if self.live.remove(&user) { + self.done.push(user); + } + } + + pub(crate) fn completed(&self) -> &[usize] { + &self.done + } +} + +#[cfg(test)] +mod tests { + use super::{ + charge_roll_admit, reject_reason_from_gov, AdmitAlways, ContMemGov, ContRoll, GovStatus, + RejectReason, CONT_ADMIT_REFUSED, + }; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + struct FakeJob { + id: usize, + } + + #[test] + fn admits_two_fake_jobs_and_both_complete() { + let mut roll = ContRoll::new(); + let first_job = FakeJob { id: 1 }; + let second_job = FakeJob { id: 2 }; + roll.enqueue(first_job.id); + roll.enqueue(second_job.id); + + let first = roll.admit().expect("first job admits"); + assert_eq!(first, first_job.id); + let second = roll + .admit() + .expect("second job admits while first is generating"); + assert_eq!(second, second_job.id); + assert_eq!(roll.live_count(), 2, "both jobs generate together"); + assert!(roll.admit().is_none()); + + roll.complete(first); + roll.complete(second); + assert_eq!(roll.completed(), &[first_job.id, second_job.id]); + } + + #[test] + fn rolling_reserve_protects_the_live_place_bank() { + // Given: first rolling admit placed on bank 2 (1-based place_bank = 3) + let mut reserve = super::RollReserve::new(); + reserve.note_place(3); + + // When: the second admit merges that reserve onto an empty hold mask + let protected = reserve.protect(&[false, false, false]); + + // Then: only the live target is protected + assert_eq!(protected, vec![false, false, true]); + } + + #[test] + fn rolling_reserve_keeps_hold_saturation() { + // Given: continuation hold already pins bank 0 + let mut reserve = super::RollReserve::new(); + reserve.note_place(2); + + // When: the second admit ORs reserved banks onto that hold + let protected = reserve.protect(&[true, false, false]); + + // Then: hold is kept and the live target is also protected + assert_eq!(protected, vec![true, true, false]); + } + + #[test] + fn rolling_reserve_ignores_unset_place_bank() { + let mut reserve = super::RollReserve::new(); + reserve.note_place(0); + assert_eq!(reserve.protect(&[false, false]), vec![false, false]); + } + + struct BudgetGov { + remaining: u32, + refuse: GovStatus, + } + + impl ContMemGov for BudgetGov { + fn charge(&mut self, _prompt_n: i32, _max_new: i32) -> GovStatus { + if self.remaining == 0 { + return self.refuse; + } + self.remaining -= 1; + GovStatus::Admit + } + } + + #[test] + fn reject_reason_from_gov_matches_c_serial_cont() { + // Given: C ds4_reject_reason_from_gov + // When: each governed status is mapped + // Then: ADMIT has no tick; refusals use the frozen /metrics names + assert_eq!(reject_reason_from_gov(GovStatus::Admit), None); + assert_eq!( + reject_reason_from_gov(GovStatus::RefuseClass).map(RejectReason::name), + Some("class_budget") + ); + assert_eq!( + reject_reason_from_gov(GovStatus::RefuseLive).map(RejectReason::name), + Some("live_headroom") + ); + assert_eq!( + reject_reason_from_gov(GovStatus::RetryObs).map(RejectReason::name), + Some("obs_retry") + ); + assert_eq!( + reject_reason_from_gov(GovStatus::Unsupported).map(RejectReason::name), + Some("unsupported") + ); + assert_eq!( + reject_reason_from_gov(GovStatus::Fault).map(RejectReason::name), + Some("fault") + ); + assert_eq!(RejectReason::DeepPolicy.name(), "deep_policy"); + assert_eq!(RejectReason::LaneDisabled.name(), "lane_disabled"); + assert_eq!(charge_roll_admit(&mut AdmitAlways, 1, 1), Ok(())); + } + + #[test] + fn rolling_admit_charges_when_budget_allows() { + // Given: governor still has one admission + let mut gov = BudgetGov { + remaining: 1, + refuse: GovStatus::RefuseLive, + }; + + // When: a rolling admit charges + let charged = charge_roll_admit(&mut gov, 128, 32); + + // Then: the job is admitted + assert_eq!(charged, Ok(())); + } + + #[test] + fn rolling_admit_typed_refuses_live_headroom_when_budget_exhausted() { + // Given: governor has no remaining budget + let mut gov = BudgetGov { + remaining: 0, + refuse: GovStatus::RefuseLive, + }; + + // When: a rolling admit charges + let refused = charge_roll_admit(&mut gov, 128, 32); + + // Then: typed refuse is live_headroom, HTTP body stays the serial-cont one + assert_eq!(refused, Err(RejectReason::LiveHeadroom)); + assert_eq!(RejectReason::LiveHeadroom.name(), "live_headroom"); + assert_eq!( + CONT_ADMIT_REFUSED, + "continuous admission rejected; serial fallback" + ); + } + + #[test] + fn rolling_admit_typed_refuses_class_budget() { + // Given: class-cap refuse (never retried) + let mut gov = BudgetGov { + remaining: 0, + refuse: GovStatus::RefuseClass, + }; + + // When: a rolling admit charges + let refused = charge_roll_admit(&mut gov, 4096, 32768); + + // Then: typed refuse is class_budget + assert_eq!(refused, Err(RejectReason::ClassBudget)); + assert_eq!(RejectReason::ClassBudget.name(), "class_budget"); + } + + #[test] + fn rolling_second_admit_typed_refuses_after_first_charges() { + // Given: budget for one live job + let mut gov = BudgetGov { + remaining: 1, + refuse: GovStatus::RefuseLive, + }; + + // When: two rolling admits charge + let first = charge_roll_admit(&mut gov, 64, 16); + let second = charge_roll_admit(&mut gov, 64, 16); + + // Then: first admits; second is the same live_headroom refuse + assert_eq!(first, Ok(())); + assert_eq!(second, Err(RejectReason::LiveHeadroom)); + } + + fn c_d0b_ledger() -> ds4_core::GovLedger { + let mut lg = ds4_core::GovLedger::default(); + let mut faults = 0; + assert!(ds4_core::gov_lease_publish( + &mut lg, + ds4_core::GovConsumer::EngineBoot as i32, + 10000, + 10000, + 0, + &mut faults + )); + assert!(ds4_core::gov_lease_publish( + &mut lg, + ds4_core::GovConsumer::Prewarm as i32, + 200, + 200, + 0, + &mut faults + )); + assert!(ds4_core::gov_lease_publish( + &mut lg, + ds4_core::GovConsumer::BatchBankPlan as i32, + 1200, + 1000, + 0, + &mut faults + )); + assert!(ds4_core::gov_lease_publish( + &mut lg, + ds4_core::GovConsumer::SerialSession as i32, + 0, + 0, + 200, + &mut faults + )); + lg.floor_bytes = 600; + lg.substrate_outstanding = 50; + lg + } + + fn bank_claim(proposed: u64, class_limit: u64) -> ds4_core::GovClaim { + ds4_core::GovClaim { + requester: ds4_core::GovConsumer::BatchBankPlan as i32, + memc: 12, + domain: 0, + proposed_outstanding: proposed, + operation_transient: 0, + class_limit, + } + } + + fn host_gov(free_bytes: u64, proposed: u64, class_limit: u64) -> super::HostMemGov { + super::HostMemGov { + ledger: c_d0b_ledger(), + obs: ds4_core::MemObservation { + status: ds4_core::MemObsStatus::Ok, + source: ds4_core::MemObsSource::MeminfoAvailable, + free_bytes, + total_bytes: 20000, + }, + claim: bank_claim(proposed, class_limit), + } + } + + #[test] + fn host_mem_gov_admits_c_d0b_exact_headroom() { + let mut gov = host_gov(1350, 1500, 2000); + assert_eq!(charge_roll_admit(&mut gov, 1, 1), Ok(())); + } + + #[test] + fn host_mem_gov_refuses_live_one_byte_short() { + let mut gov = host_gov(1349, 1500, 2000); + assert_eq!( + charge_roll_admit(&mut gov, 1, 1), + Err(RejectReason::LiveHeadroom) + ); + } + + #[test] + fn host_mem_gov_refuses_class_before_live() { + let mut gov = host_gov(u64::MAX / 2, 2500, 2000); + assert_eq!( + charge_roll_admit(&mut gov, 1, 1), + Err(RejectReason::ClassBudget) + ); + } +} diff --git a/crates/ds4-server/src/serve_owner_cont.rs b/crates/ds4-server/src/serve_owner_cont.rs new file mode 100644 index 0000000000000000000000000000000000000000..1dd95c3de81386ee65dd434e16f051415f9ca07b --- /dev/null +++ b/crates/ds4-server/src/serve_owner_cont.rs @@ -0,0 +1,442 @@ +use std::sync::mpsc::{Receiver, RecvTimeoutError, TryRecvError}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use super::*; +use crate::route::{route_decide, RouteEnv, LANE_CONTINUOUS}; +use crate::serve_cont::{ + ContExec, ContOwnedResult, ContOwnedWork, ContProbe, ContSource, ContWork, +}; +use crate::serve_cont_prefill::{owner_tick_pair, PrefillChunkPolicy}; + +pub(super) fn run_owner_maybe_roll( + cfg: &ServerConfig, + inner: &Arc>, + engine: &mut dyn DecodeIo, + exec: &mut dyn ContExec, + mut job: OwnerJob, + jobs_rx: &Receiver, +) -> Option { + let prompt_len = prepare_cont_prompt(&mut job.prepared, exec) + .map(|prepared| prepared.tokens.len() as i32) + .unwrap_or(0); + let env = roll_route_env(cfg, exec.seq_cap(), prompt_len); + let dec = route_decide(job.prepared.parsed.needs, job.prepared.surface, &env); + if dec.lane != LANE_CONTINUOUS { + run_owner_job(cfg, inner, engine, Some(exec), job); + return None; + } + if !resolve_bank_continuation(inner, &mut job.prepared, exec) { + run_owner_job(cfg, inner, engine, Some(exec), job); + return None; + } + let job = match try_serve_rolling(cfg, inner, engine, exec, job, jobs_rx) { + Ok(lookahead) => return lookahead, + Err(job) => job, + }; + let cap = usize::try_from(exec.max_seq().max(1)).unwrap_or(1); + if cap == 1 { + run_owner_job(cfg, inner, engine, Some(exec), job); + return None; + } + let wait = super::owner_static::coalesce_wait_from_env(); + let deadline = (!wait.is_zero()).then(|| Instant::now() + wait); + let mut jobs = vec![job]; + let mut lookahead = None; + while jobs.len() < cap { + if stop_requested(cfg) { + break; + } + let next = match jobs_rx.try_recv() { + Ok(job) => job, + Err(TryRecvError::Disconnected) => break, + Err(TryRecvError::Empty) => { + let Some(deadline) = deadline else { + break; + }; + let now = Instant::now(); + if now >= deadline { + break; + } + match jobs_rx.recv_timeout(deadline.saturating_duration_since(now)) { + Ok(job) => job, + Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => break, + } + } + }; + let mut next = next; + if cfg.max_queue_age_s > 0.0 + && next.prepared.arrived_at.elapsed().as_secs_f64() > cfg.max_queue_age_s + { + lookahead = Some(next); + break; + } + if next.sink.state.gone() || next.sink.state.observe_disconnect() { + finish_canceled_roll(next); + continue; + } + let prompt_len = prepare_cont_prompt(&mut next.prepared, exec) + .map(|prepared| prepared.tokens.len() as i32) + .unwrap_or(0); + let env = roll_route_env(cfg, exec.seq_cap(), prompt_len); + let dec = route_decide(next.prepared.parsed.needs, next.prepared.surface, &env); + if dec.lane != LANE_CONTINUOUS + || !resolve_bank_continuation(inner, &mut next.prepared, exec) + || next.prepared.parsed.directed_bank.is_some_and(|bank| { + jobs.iter() + .any(|job| job.prepared.parsed.directed_bank == Some(bank)) + }) + { + lookahead = Some(next); + break; + } + jobs.push(next); + } + if jobs.len() == 1 { + let job = jobs.pop().unwrap(); + run_owner_job(cfg, inner, engine, Some(exec), job); + } else { + serve_batch(cfg, inner, engine, exec, jobs); + } + lookahead +} + +struct OwnerRollSource<'a> { + cfg: &'a ServerConfig, + inner: &'a Arc>, + jobs_rx: &'a Receiver, + primed: Option, + lookahead: Option, + jobs: Vec>, + created: i64, +} + +impl ContSource for OwnerRollSource<'_> { + fn next(&mut self, probe: &dyn ContProbe) -> Option { + if self.lookahead.is_some() { + return None; + } + loop { + let primed = self.primed.is_some(); + if !primed && stop_requested(self.cfg) { + return None; + } + let mut job = match self.primed.take() { + Some(job) => job, + None => match self.jobs_rx.try_recv() { + Ok(job) => job, + Err(TryRecvError::Empty | TryRecvError::Disconnected) => return None, + }, + }; + if !primed + && self.cfg.max_queue_age_s > 0.0 + && job.prepared.arrived_at.elapsed().as_secs_f64() > self.cfg.max_queue_age_s + { + self.lookahead = Some(job); + return None; + } + if job.sink.state.gone() || job.sink.state.observe_disconnect() { + finish_canceled_roll(job); + continue; + } + if !primed { + if job.prepared.cont_prompt.is_none() { + job.prepared.cont_prompt = probe + .prompt_tokens(&mut job.prepared.parsed) + .ok() + .map(|(prompt, tokens)| crate::serve_cont::ContPreparedPrompt { + prompt, + tokens, + }); + } + let prompt_len = job + .prepared + .cont_prompt + .as_ref() + .map(|prepared| prepared.tokens.len() as i32) + .unwrap_or(0); + let env = roll_route_env(self.cfg, probe.seq_cap(), prompt_len); + let decision = route_decide(job.prepared.parsed.needs, job.prepared.surface, &env); + if decision.lane != LANE_CONTINUOUS + || !resolve_bank_continuation_with(self.inner, &mut job.prepared, |bank| { + probe.bank_live(bank) + }) + { + self.lookahead = Some(job); + return None; + } + } + job.lease.start(); + let id = next_job_id(&mut lock_inner(self.inner).admit, job.prepared.parsed.kind); + let key = self.jobs.len(); + let prepared = job.prepared.cont_prompt.take(); + let work = ContOwnedWork { + key, + parsed: job.prepared.parsed.clone(), + prepared, + job_id: id.clone(), + created: self.created, + cors: self.cfg.cors, + default_tokens: self.cfg.default_tokens, + t_arrive: job.prepared.arrived_at, + out: Box::new(job.sink.clone()), + }; + self.jobs.push(Some((id, job))); + return Some(work); + } + } + + fn publish(&mut self, key: usize, outcome: &GenerateOutcome) { + if let Some((_, job)) = self.jobs.get(key).and_then(Option::as_ref) { + publish_continuous_tool_turn(self.inner, job.prepared.parsed.api, outcome); + } + } + + fn settled(&mut self, key: usize, result: &Result) { + if !matches!(result, Ok(_) | Err(GenerateError::Io)) { + return; + } + let Some((_, job)) = self.jobs.get_mut(key).and_then(Option::take) else { + return; + }; + if let Ok(outcome) = result { + lock_inner(self.inner).record_generation(outcome); + } + let settlement = if job.sink.state.gone() || matches!(result, Err(GenerateError::Io)) { + Settlement::CANCELED + } else { + Settlement::COMPLETED + }; + finish_roll_lease(self.inner, job, settlement); + } +} + +fn try_serve_rolling( + cfg: &ServerConfig, + inner: &Arc>, + engine: &mut dyn DecodeIo, + exec: &mut dyn ContExec, + job: OwnerJob, + jobs_rx: &Receiver, +) -> Result, OwnerJob> { + let mut source = OwnerRollSource { + cfg, + inner, + jobs_rx, + primed: Some(job), + lookahead: None, + jobs: Vec::new(), + created: unix_now(), + }; + let mut bank_hold_retry = |bank, live| { + lock_inner(inner) + .creg + .bank_hold_retry(bank, live, monotonic_now()) + }; + let Some(results) = + exec.generate_rolling(&mut source, &mut bank_hold_retry, engine.kv_store_mut()) + else { + return Err(source + .primed + .take() + .expect("bounded executor must leave the primed job untouched")); + }; + let mut by_key: Vec>> = + (0..source.jobs.len()).map(|_| None).collect(); + for ContOwnedResult { key, result } in results { + if let Some(slot) = by_key.get_mut(key) { + *slot = Some(result); + } + } + let served = by_key + .iter() + .filter(|result| result.as_ref().is_some_and(Result::is_ok)) + .count(); + let fallback = by_key + .iter() + .filter(|result| matches!(result, Some(Err(GenerateError::Unsupported(_))))) + .count(); + eprintln!("ds4-server-rs: continuous rolling path=cont served={served} fallback={fallback}"); + for (key, item) in source.jobs.into_iter().enumerate() { + let Some((id, job)) = item else { + continue; + }; + let result = by_key[key].take().unwrap_or_else(|| { + Err(GenerateError::Engine( + "continuous rolling loop lost a job result".into(), + )) + }); + settle_roll_job(cfg, inner, engine, job, &id, result, false); + } + Ok(source.lookahead) +} + +fn roll_route_env(cfg: &ServerConfig, seq_cap: i32, prompt_len: i32) -> RouteEnv { + let (cont_tools_anthropic, cont_tools_responses) = process_cont_tools(); + RouteEnv { + coalesce: true, + have_cont: cfg.continuous, + cont_anthropic: parse_default_on(std::env::var_os("DS4_SERVER_CONT_ANTHROPIC").as_deref()), + cont_responses: parse_default_on(std::env::var_os("DS4_SERVER_CONT_RESPONSES").as_deref()), + cont_tools_anthropic, + cont_tools_responses, + seq_cap, + prompt_len, + } +} + +fn finish_canceled_roll(job: OwnerJob) { + let OwnerJob { + sink, + done, + mut lease, + .. + } = job; + lease.start(); + drop(sink); + lease.settlement = Settlement::CANCELED; + if let Err(err) = done.send(lease) { + let mut lease = err.0; + lease.settlement = lease.settlement.transport_gone(); + } +} + +fn serve_batch( + cfg: &ServerConfig, + inner: &Arc>, + engine: &mut dyn DecodeIo, + exec: &mut dyn ContExec, + mut jobs: Vec, +) { + for job in &mut jobs { + job.lease.start(); + } + let ids: Vec<_> = { + let mut inner = lock_inner(inner); + jobs.iter() + .map(|job| next_job_id(&mut inner.admit, job.prepared.parsed.kind)) + .collect() + }; + let mut bank_hold_retry = |bank, live| { + lock_inner(inner) + .creg + .bank_hold_retry(bank, live, monotonic_now()) + }; + if let Some(second) = jobs.get(1) { + let decode_remaining = + u32::try_from(jobs[0].prepared.parsed.max_tokens.max(1)).unwrap_or(1); + let prefill_remaining = second + .prepared + .cont_prompt + .as_ref() + .and_then(|prepared| u32::try_from(prepared.tokens.len()).ok()) + .unwrap_or(1) + .max(1); + let _ = owner_tick_pair( + PrefillChunkPolicy::from_env(), + decode_remaining, + prefill_remaining, + ); + } + let created = unix_now(); + let works = jobs + .iter_mut() + .zip(&ids) + .map(|(job, id)| ContWork { + parsed: &job.prepared.parsed, + prepared: job.prepared.cont_prompt.as_ref(), + job_id: id, + created, + cors: cfg.cors, + default_tokens: cfg.default_tokens, + t_arrive: job.prepared.arrived_at, + out: &mut job.sink, + }) + .collect(); + let results = exec.generate_batch(works, &mut bank_hold_retry, engine.kv_store_mut()); + let served = results.iter().filter(|result| result.is_ok()).count(); + let fallback = results + .iter() + .filter(|result| matches!(result, Err(GenerateError::Unsupported(_)))) + .count(); + eprintln!("ds4-server-rs: continuous batch path=cont served={served} fallback={fallback}"); + let mut results = results.into_iter(); + for (job, id) in jobs.into_iter().zip(ids) { + let result = results.next().unwrap_or_else(|| { + Err(GenerateError::Engine( + "continuous batch returned too few results".into(), + )) + }); + settle_roll_job(cfg, inner, engine, job, &id, result, true); + } +} + +fn settle_roll_job( + cfg: &ServerConfig, + inner: &Arc>, + engine: &mut dyn DecodeIo, + mut job: OwnerJob, + id: &str, + result: Result, + publish_outcome: bool, +) { + let arrived_at = job.prepared.arrived_at; + if let Ok(outcome) = &result { + lock_inner(inner).record_generation(outcome); + } + if publish_outcome { + if let Ok(outcome) = &result { + publish_continuous_tool_turn(inner, job.prepared.parsed.api, outcome); + } + } + let settlement = if job.sink.state.gone() { + Settlement::CANCELED + } else if job.prepared.parsed.needs & NEED_BANK_FRONTIER != 0 { + settle_bank_continuation(cfg, &job.prepared, result, &mut job.sink) + } else if matches!(result, Err(GenerateError::Unsupported(_))) { + let fallback = match &result { + Err(GenerateError::Unsupported(msg)) => Some((*msg).to_string()), + _ => None, + }; + run_serial( + cfg, + inner, + &job.prepared, + id, + engine, + None, + &mut job.sink, + arrived_at, + fallback, + ) + } else { + settle_generation_result(cfg, &job.prepared, result, &mut job.sink) + }; + finish_roll_lease(inner, job, settlement); +} + +fn finish_roll_lease( + inner: &Arc>, + mut job: OwnerJob, + mut settlement: Settlement, +) { + if job.sink.state.slow() { + settlement = settlement.slow_reader(); + } + drop(job.sink); + job.lease.settlement = settlement; + lock_inner(inner).metrics.record_route( + job.prepared.surface, + LANE_CONTINUOUS, + crate::route::REASON_CONT, + job.prepared.parsed.think_mode, + ); + if let Err(err) = job.done.send(job.lease) { + let mut lease = err.0; + lease.settlement = lease.settlement.transport_gone(); + } +} + +#[cfg(test)] +#[path = "serve_owner_cont_test.rs"] +mod tests; diff --git a/crates/ds4-server/src/serve_owner_cont_test.rs b/crates/ds4-server/src/serve_owner_cont_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..5844f749c5ce88b4ff74740e05c6b8cd02766803 --- /dev/null +++ b/crates/ds4-server/src/serve_owner_cont_test.rs @@ -0,0 +1,651 @@ +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::mpsc; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use super::super::*; +use super::run_owner_maybe_roll; +use crate::generate::{GenerateError, GenerateOutcome, ScriptedDecode}; +use crate::parse::{parse_request, ParseEnv}; +use crate::route::{ThinkMode, WireSurface, LANE_CONTINUOUS}; +use crate::serve_cont::{ContExec, ContOwnedResult, ContProbe, ContSource, ContStepper}; +use crate::serve_cont_prefill::{owner_tick_call_count, reset_owner_tick_call_count}; +use crate::stream::{ReqTimings, TAPE_PLAIN}; +use ds4_kv::Store as KvStore; + +struct RollSpy { + generate_calls: usize, + max_seq: i32, +} + +struct RollingProbe; + +impl ContProbe for RollingProbe { + fn prompt_tokens( + &self, + _parsed: &mut crate::parse::ParsedRequest, + ) -> Result<(Vec, Vec), GenerateError> { + Ok((b"prompt".to_vec(), vec![1, 2, 3])) + } + + fn seq_cap(&self) -> i32 { + 8192 + } + + fn bank_live(&self, _bank: i32) -> Option<(u64, i32)> { + None + } +} + +#[derive(Default)] +struct RefillSpy { + calls: usize, +} + +impl ContExec for RefillSpy { + fn model_id(&self) -> i32 { + 0 + } + + fn seq_cap(&self) -> i32 { + 8192 + } + + fn max_seq(&self) -> i32 { + 2 + } + + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + vec![1, 2, 3] + } + + fn encode_text(&self, _text: &str) -> Vec { + vec![1, 2, 3] + } + + fn generate( + &mut self, + _parsed: &crate::parse::ParsedRequest, + _job_id: &str, + _created: i64, + _cors: bool, + _default_tokens: i32, + _t_arrive: Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut KvStore>, + _out: &mut dyn Write, + ) -> Result { + panic!("rolling path must not fall back to generate") + } + + fn generate_rolling( + &mut self, + source: &mut dyn ContSource, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut KvStore>, + ) -> Option> { + let probe = RollingProbe; + let mut results = Vec::new(); + let first = source.next(&probe).expect("primed job"); + results.push(complete_owned(source, first)); + assert!(source.next(&probe).is_none(), "late job is not queued yet"); + std::thread::sleep(std::time::Duration::from_millis(60)); + while let Some(work) = source.next(&probe) { + results.push(complete_owned(source, work)); + } + self.calls = results.len(); + Some(results) + } +} + +fn complete_owned( + source: &mut dyn ContSource, + mut work: crate::serve_cont::ContOwnedWork, +) -> ContOwnedResult { + work.out + .write_all(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{}") + .unwrap(); + let outcome = GenerateOutcome { + finish: "stop".into(), + ..GenerateOutcome::default() + }; + source.publish(work.key, &outcome); + let result = Ok(outcome); + source.settled(work.key, &result); + ContOwnedResult { + key: work.key, + result, + } +} + +impl Default for RollSpy { + fn default() -> Self { + Self { + generate_calls: 0, + max_seq: 2, + } + } +} + +impl ContExec for RollSpy { + fn model_id(&self) -> i32 { + 0 + } + fn seq_cap(&self) -> i32 { + 8192 + } + fn max_seq(&self) -> i32 { + self.max_seq + } + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + vec![1, 2, 3] + } + fn encode_text(&self, _text: &str) -> Vec { + vec![1, 2, 3] + } + fn generate( + &mut self, + _parsed: &crate::parse::ParsedRequest, + _job_id: &str, + _created: i64, + _cors: bool, + _default_tokens: i32, + _t_arrive: Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut ds4_kv::Store>, + out: &mut dyn Write, + ) -> Result { + self.generate_calls += 1; + out.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{}") + .map_err(|_| GenerateError::Io)?; + Ok(GenerateOutcome { + generation: 1, + frontier: 1, + finish: "stop".into(), + ..GenerateOutcome::default() + }) + } +} + +#[test] +fn one_bank_leaves_the_second_request_queued_for_serial_execution() { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = cont_chat_job(&inner, "a"); + let (job_b, drain_b) = cont_chat_job(&inner, "b"); + let (tx, rx) = mpsc::channel(); + tx.send(job_b).unwrap(); + let mut spy = RollSpy { + max_seq: 1, + ..RollSpy::default() + }; + let mut engine = ScriptedDecode::from_pieces(&[b"serial-fallback"]); + + let leftover = run_owner_maybe_roll(&cfg, &inner, &mut engine, &mut spy, job_a, &rx); + + assert!(leftover.is_none()); + assert_eq!(spy.generate_calls, 1); + let job_b = rx.try_recv().expect("second request must remain queued"); + let leftover = run_owner_maybe_roll(&cfg, &inner, &mut engine, &mut spy, job_b, &rx); + assert!(leftover.is_none()); + assert_eq!(spy.generate_calls, 2); + drop(tx); + let _ = drain_a.done.recv(); + let _ = drain_b.done.recv(); +} + +fn cont_chat_job(inner: &Arc>, tag: &str) -> (OwnerJob, JobDrain) { + let env = ParseEnv { + default_model: "ds4".into(), + default_tokens: 8, + default_effort: ThinkMode::None, + default_temp: 0.0, + live_ids: Vec::new(), + engine_defaults: false, + }; + let body = format!( + r#"{{"messages":[{{"role":"user","content":"{tag}"}}],"thinking":{{"type":"disabled"}},"temperature":0}}"# + ); + let parsed = crate::parse::parse_request(WireSurface::OpenaiChat, &env, &body).unwrap(); + let prepared = PreparedJob { + parsed, + cont_prompt: None, + surface: WireSurface::OpenaiChat, + body_bytes: body.len() as u64, + arrived_at: Instant::now(), + }; + let mut g = lock_inner(inner); + assert_eq!(enqueue(&mut g.admit, prepared.body_bytes), EnqVerdict::Ok); + g.runtime.requests_started += 1; + g.runtime.requests_inflight += 1; + drop(g); + let lease = JobLease::new(Arc::clone(inner), prepared.body_bytes, None); + owner_job(prepared, lease) +} + +#[test] +fn owner_roll_pair_invokes_tick_roll_prefill() { + reset_owner_tick_call_count(); + let cfg = ServerConfig { + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = cont_chat_job(&inner, "a"); + let (job_b, drain_b) = cont_chat_job(&inner, "b"); + let (tx, rx) = mpsc::channel(); + tx.send(job_b).unwrap(); + drop(tx); + let mut spy = RollSpy::default(); + let mut engine = ScriptedDecode::from_pieces(&[b"serial-fallback"]); + + let leftover = run_owner_maybe_roll(&cfg, &inner, &mut engine, &mut spy, job_a, &rx); + + assert!(leftover.is_none()); + assert_eq!(spy.generate_calls, 2); + assert!( + owner_tick_call_count() >= 1, + "production rolling owner must call tick_roll_prefill" + ); + let _ = drain_a.done.recv(); + let _ = drain_b.done.recv(); +} + +#[test] +fn owner_roll_uses_every_configured_bank_without_a_two_job_ceiling() { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = cont_chat_job(&inner, "a"); + let (job_b, drain_b) = cont_chat_job(&inner, "b"); + let (job_c, drain_c) = cont_chat_job(&inner, "c"); + let (job_d, drain_d) = cont_chat_job(&inner, "d"); + let (tx, rx) = mpsc::channel(); + tx.send(job_b).unwrap(); + tx.send(job_c).unwrap(); + tx.send(job_d).unwrap(); + drop(tx); + let mut spy = RollSpy { + max_seq: 4, + ..RollSpy::default() + }; + let mut engine = ScriptedDecode::from_pieces(&[b"unused"]); + + assert!(run_owner_maybe_roll(&cfg, &inner, &mut engine, &mut spy, job_a, &rx).is_none()); + assert_eq!(spy.generate_calls, 4); + assert!(rx.try_recv().is_err()); + for drain in [drain_a, drain_b, drain_c, drain_d] { + drain + .done + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("batched job settled"); + } +} + +#[test] +fn late_roll_sibling_joins_when_coalesce_wait_is_set() { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = cont_chat_job(&inner, "a"); + let (job_b, drain_b) = cont_chat_job(&inner, "b"); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(30)); + tx.send(job_b).unwrap(); + }); + let mut spy = RollSpy::default(); + let mut engine = ScriptedDecode::from_pieces(&[b"serial-fallback"]); + std::env::set_var("DS4_SERVER_COALESCE_WAIT_MS", "100"); + let leftover = run_owner_maybe_roll(&cfg, &inner, &mut engine, &mut spy, job_a, &rx); + std::env::remove_var("DS4_SERVER_COALESCE_WAIT_MS"); + + assert!(leftover.is_none()); + assert_eq!(spy.generate_calls, 2); + let _ = drain_a.done.recv(); + let _ = drain_b.done.recv(); +} + +#[test] +fn rolling_epoch_refills_from_a_request_that_arrives_after_start() { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = cont_chat_job(&inner, "a"); + let (job_b, drain_b) = cont_chat_job(&inner, "b"); + let (tx, rx) = mpsc::channel(); + let sender = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(20)); + tx.send(job_b).unwrap(); + }); + let mut spy = RefillSpy::default(); + let mut engine = ScriptedDecode::from_pieces(&[b"unused"]); + + assert!(run_owner_maybe_roll(&cfg, &inner, &mut engine, &mut spy, job_a, &rx).is_none()); + sender.join().unwrap(); + assert_eq!(spy.calls, 2); + for drain in [drain_a, drain_b] { + drain + .done + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("rolling job settled"); + } +} + +struct StepperCont; + +impl ContExec for StepperCont { + fn model_id(&self) -> i32 { + 0 + } + fn seq_cap(&self) -> i32 { + 8192 + } + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + vec![1] + } + fn encode_text(&self, _text: &str) -> Vec { + vec![1] + } + fn generate( + &mut self, + parsed: &crate::parse::ParsedRequest, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + _t_arrive: Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut KvStore>, + out: &mut dyn Write, + ) -> Result { + let (mut stepper, head) = ContStepper::new( + parsed, + self.model_id(), + job_id, + created, + cors, + default_tokens, + b"prompt".to_vec(), + 1, + self.seq_cap(), + ); + out.write_all(&head).map_err(|_| GenerateError::Io)?; + for piece in TAPE_PLAIN { + let step = stepper.feed(piece.as_bytes()); + out.write_all(&step.bytes).map_err(|_| GenerateError::Io)?; + if step.done { + break; + } + } + let (tail, outcome) = stepper.finalize(true, 0, 1, ReqTimings::default(), cors); + out.write_all(&tail).map_err(|_| GenerateError::Io)?; + Ok(outcome) + } +} + +#[derive(Default)] +struct PublishingSpy { + calls: usize, +} + +impl ContExec for PublishingSpy { + fn model_id(&self) -> i32 { + 0 + } + fn seq_cap(&self) -> i32 { + 8192 + } + fn max_seq(&self) -> i32 { + 4 + } + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + vec![1] + } + fn encode_text(&self, _text: &str) -> Vec { + vec![1] + } + fn generate( + &mut self, + _parsed: &crate::parse::ParsedRequest, + _job_id: &str, + _created: i64, + cors: bool, + _default_tokens: i32, + _t_arrive: Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut KvStore>, + out: &mut dyn Write, + ) -> Result { + self.calls += 1; + out.write_all(&http_response_bytes( + 200, + Some("application/json"), + None, + cors, + "{}", + )) + .map_err(|_| GenerateError::Io)?; + Ok(GenerateOutcome { + tool_ids: vec![format!("toolu_pair{}", self.calls)], + bank: Some((self.calls - 1) as i32), + generation: 10 + self.calls as u64, + frontier: 100 + self.calls as i32, + finish: "tool_calls".into(), + ..GenerateOutcome::default() + }) + } +} + +fn anthropic_tool_job(inner: &Arc>, tag: &str) -> (OwnerJob, JobDrain) { + let env = parse_env(); + let body = format!( + r#"{{"messages":[{{"role":"user","content":"{tag}"}}],"max_tokens":8,"stream":true,"thinking":{{"type":"disabled"}},"tools":[{{"name":"bash","input_schema":{{"type":"object"}}}}]}}"# + ); + let parsed = parse_request(WireSurface::Anthropic, &env, &body).unwrap(); + let prepared = PreparedJob { + parsed, + cont_prompt: None, + surface: WireSurface::Anthropic, + body_bytes: body.len() as u64, + arrived_at: Instant::now(), + }; + let mut g = lock_inner(inner); + assert_eq!(enqueue(&mut g.admit, prepared.body_bytes), EnqVerdict::Ok); + g.runtime.requests_started += 1; + g.runtime.requests_inflight += 1; + drop(g); + let lease = JobLease::new(Arc::clone(inner), prepared.body_bytes, None); + owner_job(prepared, lease) +} + +#[test] +fn paired_tool_turns_publish_each_actual_bank() { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = anthropic_tool_job(&inner, "a"); + let (job_b, drain_b) = anthropic_tool_job(&inner, "b"); + let (tx, rx) = mpsc::channel(); + tx.send(job_b).unwrap(); + drop(tx); + let mut spy = PublishingSpy::default(); + let mut engine = ScriptedDecode::from_pieces(&[b"unused"]); + + assert!(run_owner_maybe_roll(&cfg, &inner, &mut engine, &mut spy, job_a, &rx).is_none()); + drain_a + .done + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("first pair job settled"); + drain_b + .done + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("second pair job settled"); + + let mut g = lock_inner(&inner); + assert_eq!( + g.creg + .bank_claim(Api::Anthropic, &["toolu_pair1".into()], monotonic_now()), + Some((0, 11, 101)) + ); + assert_eq!( + g.creg + .bank_claim(Api::Anthropic, &["toolu_pair2".into()], monotonic_now()), + Some((1, 12, 102)) + ); +} + +fn parse_env() -> ParseEnv { + ParseEnv { + default_model: "ds4".into(), + default_tokens: 16, + default_effort: ThinkMode::None, + default_temp: 0.0, + live_ids: Vec::new(), + engine_defaults: false, + } +} + +fn drive_cont_generate( + surface: WireSurface, + body: &str, +) -> (Result, String) { + let parsed = parse_request(surface, &parse_env(), body).unwrap(); + let mut cont = StepperCont; + let mut out = Vec::new(); + let mut hold = |_bank, _live| None; + let result = cont.generate( + &parsed, + "job-cont-stream", + 1, + false, + 16, + Instant::now(), + &mut hold, + None, + &mut out, + ); + (result, String::from_utf8(out).unwrap()) +} + +fn drive_streaming_http(path: &str, body: &str) -> (String, ServerInner) { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 16, + ..ServerConfig::default() + }; + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + write!( + client, + "POST {path} HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[]); + let mut cont = StepperCont; + handle_client_inner( + &cfg, + &inner, + &mut server, + Some(&mut engine), + Some(&mut cont), + ); + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + ( + String::from_utf8(response).unwrap(), + inner.into_inner().unwrap(), + ) +} + +#[test] +fn streaming_anthropic_cont_exec_does_not_return_unsupported() { + // Given: a streaming Anthropic no-tools greedy request + // When: ContExec::generate runs the continuous stepper + // Then: it must not return Unsupported (that is run_engine's serial fallback) + let (result, wire) = drive_cont_generate( + WireSurface::Anthropic, + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":8,"stream":true,"temperature":0,"thinking":{"type":"disabled"}}"#, + ); + assert!( + !matches!(result, Err(GenerateError::Unsupported(_))), + "{result:?}" + ); + let outcome = result.expect("continuous generate"); + assert_eq!(outcome.finish, "stop"); + assert!(wire.contains("event: message_start"), "{wire}"); +} + +#[test] +fn streaming_responses_cont_exec_does_not_return_unsupported() { + // Given: a streaming Responses no-tools greedy request + // When: ContExec::generate runs the continuous stepper + // Then: it must not return Unsupported (that is run_engine's serial fallback) + let (result, wire) = drive_cont_generate( + WireSurface::Responses, + r#"{"input":"hi","max_output_tokens":8,"stream":true,"temperature":0,"reasoning":{"effort":"none"}}"#, + ); + assert!( + !matches!(result, Err(GenerateError::Unsupported(_))), + "{result:?}" + ); + let outcome = result.expect("continuous generate"); + assert_eq!(outcome.finish, "stop"); + assert!(wire.contains("\"type\":\"response.created\""), "{wire}"); +} + +#[test] +fn streaming_anthropic_settles_continuous_with_message_start() { + let (response, inner) = drive_streaming_http( + "/v1/messages", + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":8,"stream":true,"temperature":0,"thinking":{"type":"disabled"}}"#, + ); + assert!(response.starts_with("HTTP/1.1 200 "), "{response}"); + assert!(response.contains("event: message_start"), "{response}"); + assert_eq!( + inner.metrics.route_requests[WireSurface::Anthropic as usize][LANE_CONTINUOUS as usize], + 1 + ); + assert_eq!(inner.runtime.requests_serial, 0); +} + +#[test] +fn streaming_responses_settles_continuous_with_response_created() { + let (response, inner) = drive_streaming_http( + "/v1/responses", + r#"{"input":"hi","max_output_tokens":8,"stream":true,"temperature":0,"reasoning":{"effort":"none"}}"#, + ); + assert!(response.starts_with("HTTP/1.1 200 "), "{response}"); + assert!( + response.contains("\"type\":\"response.created\""), + "{response}" + ); + assert_eq!( + inner.metrics.route_requests[WireSurface::Responses as usize][LANE_CONTINUOUS as usize], + 1 + ); + assert_eq!(inner.runtime.requests_serial, 0); +} diff --git a/crates/ds4-server/src/serve_owner_static.rs b/crates/ds4-server/src/serve_owner_static.rs new file mode 100644 index 0000000000000000000000000000000000000000..e075cbc7a05ed6e5a83ebc60cf6b05731eb55196 --- /dev/null +++ b/crates/ds4-server/src/serve_owner_static.rs @@ -0,0 +1,267 @@ +use std::sync::mpsc::{Receiver, RecvTimeoutError, TryRecvError}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use super::*; +use crate::generate::GenerateError; +use crate::route::{route_decide, RouteEnv, LANE_STATIC}; +use crate::serve_cont::ContExec; +use crate::serve_static::{ + coalesce_take, job_tok_footprint, run_static, static_peer_ok, CoalesceLimits, CoalescePeer, + StaticJob, StaticPeerSpec, StaticRow, +}; + +struct Member { + job: OwnerJob, + tokens: Vec, +} + +pub(super) fn run_owner_maybe_coalesce( + cfg: &ServerConfig, + inner: &Arc>, + engine: &mut dyn DecodeIo, + exec: &mut dyn ContExec, + mut job: OwnerJob, + jobs_rx: &Receiver, +) -> Option { + if exec.as_static().is_none() { + run_owner_job(cfg, inner, engine, Some(exec), job); + return None; + } + let tokens = prepare_cont_prompt(&mut job.prepared, exec) + .map(|prepared| prepared.tokens.clone()) + .unwrap_or_default(); + let env = static_route_env(cfg, exec, tokens.len() as i32); + let dec = route_decide(job.prepared.parsed.needs, job.prepared.surface, &env); + if dec.lane != LANE_STATIC { + return super::owner_cont::run_owner_maybe_roll(cfg, inner, engine, exec, job, jobs_rx); + } + let limits = exec + .as_static() + .map(|owner| owner.coalesce_limits()) + .unwrap_or_default(); + let mut batch = vec![Member { job, tokens }]; + let lookahead = gather_peers( + cfg, + exec, + &env, + limits, + &mut batch, + jobs_rx, + coalesce_wait_from_env(), + ); + if batch.len() < 2 { + let Member { job, .. } = batch.remove(0); + run_owner_job(cfg, inner, engine, Some(exec), job); + return lookahead; + } + serve_group(cfg, inner, engine, exec, dec.reason, batch); + lookahead +} + +fn static_route_env(cfg: &ServerConfig, exec: &dyn ContExec, prompt_len: i32) -> RouteEnv { + let (cont_tools_anthropic, cont_tools_responses) = process_cont_tools(); + RouteEnv { + coalesce: true, + have_cont: cfg.continuous, + cont_anthropic: parse_default_on(std::env::var_os("DS4_SERVER_CONT_ANTHROPIC").as_deref()), + cont_responses: parse_default_on(std::env::var_os("DS4_SERVER_CONT_RESPONSES").as_deref()), + cont_tools_anthropic, + cont_tools_responses, + seq_cap: exec.seq_cap(), + prompt_len, + } +} + +pub(super) fn coalesce_wait_from_env() -> Duration { + // Same knob as C coalesce_gather: default 0 (drain only). A small + // positive window lets two concurrent short HTTP jobs join n=2 + // instead of collapsing each to serial. + let ms = std::env::var("DS4_SERVER_COALESCE_WAIT_MS") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(0) + .max(0); + Duration::from_millis(ms as u64) +} + +fn gather_peers( + cfg: &ServerConfig, + exec: &dyn ContExec, + env: &RouteEnv, + limits: CoalesceLimits, + batch: &mut Vec, + jobs_rx: &Receiver, + wait: Duration, +) -> Option { + let deadline = (!wait.is_zero()).then(|| Instant::now() + wait); + loop { + if batch.len() >= limits.clamp().cap { + return None; + } + let mut next = match jobs_rx.try_recv() { + Ok(job) => job, + Err(TryRecvError::Disconnected) => return None, + Err(TryRecvError::Empty) => { + let Some(deadline) = deadline else { + return None; + }; + let now = Instant::now(); + if now >= deadline { + return None; + } + match jobs_rx.recv_timeout(deadline.saturating_duration_since(now)) { + Ok(job) => job, + Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => return None, + } + } + }; + if cfg.max_queue_age_s > 0.0 + && next.prepared.arrived_at.elapsed().as_secs_f64() > cfg.max_queue_age_s + { + return Some(next); + } + if next.sink.state.gone() || next.sink.state.observe_disconnect() { + finish_canceled(next); + continue; + } + let tokens = prepare_cont_prompt(&mut next.prepared, exec) + .map(|prepared| prepared.tokens.clone()) + .unwrap_or_default(); + let peer_ok = static_peer_ok(StaticPeerSpec { + needs: next.prepared.parsed.needs, + surface: next.prepared.surface, + cont_anthropic: env.cont_anthropic, + cont_responses: env.cont_responses, + }); + let tok_total: i64 = batch + .iter() + .map(|member| { + job_tok_footprint(member.tokens.len(), member.job.prepared.parsed.max_tokens) + }) + .sum(); + let take = coalesce_take( + tok_total, + &[CoalescePeer { + footprint: job_tok_footprint(tokens.len(), next.prepared.parsed.max_tokens), + peer_ok, + }], + limits, + ); + if take == 1 { + batch.push(Member { job: next, tokens }); + } else { + return Some(next); + } + } +} + +fn serve_group( + cfg: &ServerConfig, + inner: &Arc>, + engine: &mut dyn DecodeIo, + exec: &mut dyn ContExec, + reason: u8, + mut batch: Vec, +) { + for member in &mut batch { + member.job.lease.start(); + } + let result = { + let jobs: Vec> = batch + .iter() + .map(|member| StaticJob { + tokens: &member.tokens, + max_new_tokens: member.job.prepared.parsed.max_tokens, + eos: -1, + }) + .collect(); + match exec.as_static() { + Some(owner) => run_static(owner, &jobs), + None => Err(GenerateError::Unsupported("static owner is not attached")), + } + }; + for (index, member) in batch.into_iter().enumerate() { + settle_member(cfg, inner, engine, reason, member, index, &result); + } +} + +fn settle_member( + cfg: &ServerConfig, + inner: &Arc>, + engine: &dyn DecodeIo, + reason: u8, + member: Member, + index: usize, + result: &Result, GenerateError>, +) { + let Member { + job: + OwnerJob { + prepared, + mut sink, + done, + mut lease, + }, + tokens, + } = member; + let id = next_job_id(&mut lock_inner(inner).admit, prepared.parsed.kind); + lock_inner(inner).metrics.record_route( + prepared.surface, + LANE_STATIC, + reason, + prepared.parsed.think_mode, + ); + let one = match result { + Ok(rows) => Ok(rows.get(index).cloned().into_iter().collect()), + Err(GenerateError::Engine(msg)) => Err(GenerateError::Engine(msg.clone())), + Err(GenerateError::BadRequest(msg)) => Err(GenerateError::BadRequest(*msg)), + Err(GenerateError::Unsupported(msg)) => Err(GenerateError::Unsupported(*msg)), + Err(GenerateError::Streamed(msg)) => Err(GenerateError::Streamed(msg.clone())), + Err(GenerateError::Io) => Err(GenerateError::Io), + Err(GenerateError::ContinuationHold { retry_after }) => { + Err(GenerateError::ContinuationHold { + retry_after: *retry_after, + }) + } + }; + let mut settlement = settle_static_lane( + cfg, + inner.as_ref(), + &prepared, + &id, + engine, + i32::try_from(tokens.len()).unwrap_or(i32::MAX), + one, + &mut sink, + ); + if sink.state.slow() { + settlement = settlement.slow_reader(); + } + drop(sink); + lease.settlement = settlement; + if let Err(err) = done.send(lease) { + let mut lease = err.0; + lease.settlement = lease.settlement.transport_gone(); + } +} + +fn finish_canceled(job: OwnerJob) { + let OwnerJob { + sink, + done, + mut lease, + .. + } = job; + lease.start(); + drop(sink); + lease.settlement = Settlement::CANCELED; + if let Err(err) = done.send(lease) { + let mut lease = err.0; + lease.settlement = lease.settlement.transport_gone(); + } +} + +#[cfg(test)] +#[path = "serve_owner_static_test.rs"] +mod tests; diff --git a/crates/ds4-server/src/serve_owner_static_test.rs b/crates/ds4-server/src/serve_owner_static_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..c74c55fdd1c58a8a51dfed076a82bb759b050528 --- /dev/null +++ b/crates/ds4-server/src/serve_owner_static_test.rs @@ -0,0 +1,381 @@ +use std::cell::Cell; +use std::io::Write; +use std::sync::mpsc; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use super::super::*; +use super::run_owner_maybe_coalesce; +use crate::generate::{GenerateError, GenerateOutcome, ScriptedDecode}; +use crate::parse::ParseEnv; +use crate::route::{ThinkMode, WireSurface, LANE_SERIAL, LANE_STATIC}; +use crate::serve_cont::ContExec; +use crate::serve_static::{ + static_fallback_error, CoalesceLimits, StaticExec, StaticFinish, StaticJob, StaticRow, +}; + +#[derive(Default)] +struct OwnerSpy { + calls: usize, + ns: Vec, + seq_cap: i32, + encode_calls: Cell, + allow_cont: bool, + cont_calls: usize, +} + +impl StaticExec for OwnerSpy { + fn generate_static(&mut self, jobs: &[StaticJob<'_>]) -> Result, GenerateError> { + self.calls += 1; + self.ns.push(jobs.len()); + Ok(jobs + .iter() + .map(|job| StaticRow { + tokens: job.tokens.to_vec(), + finish: StaticFinish::Stop, + }) + .collect()) + } + + fn fallback_static(&mut self, err: GenerateError) -> Result, GenerateError> { + Err(static_fallback_error(err)) + } +} + +impl ContExec for OwnerSpy { + fn model_id(&self) -> i32 { + 0 + } + fn seq_cap(&self) -> i32 { + self.seq_cap + } + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + self.encode_calls.set(self.encode_calls.get() + 1); + vec![77] + } + fn encode_text(&self, _text: &str) -> Vec { + self.encode_calls.set(self.encode_calls.get() + 1); + vec![77] + } + fn generate( + &mut self, + _parsed: &crate::parse::ParsedRequest, + _job_id: &str, + _created: i64, + _cors: bool, + _default_tokens: i32, + _t_arrive: Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut ds4_kv::Store>, + out: &mut dyn Write, + ) -> Result { + assert!(self.allow_cont, "continuous generate on a static request"); + self.cont_calls += 1; + out.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{}") + .map_err(|_| GenerateError::Io)?; + Ok(GenerateOutcome { + finish: "stop".into(), + ..GenerateOutcome::default() + }) + } + fn as_static(&mut self) -> Option<&mut dyn StaticExec> { + Some(self) + } +} + +#[test] +fn continuous_owner_tokenizes_a_long_request_once_before_generation() { + let cfg = ServerConfig { + continuous: true, + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job, drain) = static_chat_job(&inner, "long-session"); + let (_tx, rx) = mpsc::channel(); + let mut spy = OwnerSpy { + seq_cap: 8192, + allow_cont: true, + ..OwnerSpy::default() + }; + let mut engine = ScriptedDecode::from_pieces(&[]); + + assert!(run_owner_maybe_coalesce(&cfg, &inner, &mut engine, &mut spy, job, &rx).is_none()); + + assert_eq!(spy.cont_calls, 1); + assert_eq!(spy.encode_calls.get(), 1); + let _ = drain.done.recv(); +} + +fn static_chat_job(inner: &Arc>, tag: &str) -> (OwnerJob, JobDrain) { + let env = ParseEnv { + default_model: "ds4".into(), + default_tokens: 8, + default_effort: ThinkMode::None, + default_temp: 0.0, + live_ids: Vec::new(), + engine_defaults: false, + }; + let body = format!( + r#"{{"messages":[{{"role":"user","content":"{tag}"}}],"thinking":{{"type":"disabled"}},"temperature":0}}"# + ); + let parsed = crate::parse::parse_request(WireSurface::OpenaiChat, &env, &body).unwrap(); + let prepared = PreparedJob { + parsed, + cont_prompt: None, + surface: WireSurface::OpenaiChat, + body_bytes: body.len() as u64, + arrived_at: Instant::now(), + }; + let mut g = lock_inner(inner); + assert_eq!(enqueue(&mut g.admit, prepared.body_bytes), EnqVerdict::Ok); + g.runtime.requests_started += 1; + g.runtime.requests_inflight += 1; + drop(g); + let lease = JobLease::new(Arc::clone(inner), prepared.body_bytes, None); + owner_job(prepared, lease) +} + +#[test] +fn late_sibling_joins_when_coalesce_wait_is_set() { + let cfg = ServerConfig { + continuous: false, + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = static_chat_job(&inner, "a"); + let (job_b, drain_b) = static_chat_job(&inner, "b"); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(30)); + tx.send(job_b).unwrap(); + }); + let mut spy = OwnerSpy::default(); + let mut engine = ScriptedDecode::from_pieces(&[b"serial-fallback"]); + std::env::set_var("DS4_SERVER_COALESCE_WAIT_MS", "100"); + let leftover = run_owner_maybe_coalesce(&cfg, &inner, &mut engine, &mut spy, job_a, &rx); + std::env::remove_var("DS4_SERVER_COALESCE_WAIT_MS"); + + assert!(leftover.is_none()); + assert_eq!(spy.ns, vec![2]); + let _ = drain_a.done.recv(); + let _ = drain_b.done.recv(); +} + +#[test] +fn two_queued_static_jobs_coalesce_on_owner_fifo() { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = static_chat_job(&inner, "a"); + let (job_b, drain_b) = static_chat_job(&inner, "b"); + let (tx, rx) = mpsc::channel(); + tx.send(job_b).unwrap(); + drop(tx); + let mut spy = OwnerSpy::default(); + let mut engine = ScriptedDecode::from_pieces(&[b"serial-fallback"]); + + let leftover = run_owner_maybe_coalesce(&cfg, &inner, &mut engine, &mut spy, job_a, &rx); + + assert!(leftover.is_none()); + assert_eq!(spy.calls, 1); + assert_eq!(spy.ns, vec![2]); + let text_a = String::from_utf8(drain_a.state.take()).unwrap(); + let text_b = String::from_utf8(drain_b.state.take()).unwrap(); + let _ = drain_a.done.recv(); + let _ = drain_b.done.recv(); + assert!(text_a.starts_with("HTTP/1.1 200 OK"), "{text_a}"); + assert!(text_b.starts_with("HTTP/1.1 200 OK"), "{text_b}"); + assert!(!text_a.contains("serial-fallback"), "{text_a}"); + assert!(!text_b.contains("serial-fallback"), "{text_b}"); + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_serial, 0); + assert_eq!( + g.metrics.route_requests[WireSurface::OpenaiChat as usize][LANE_STATIC as usize], + 2 + ); +} + +#[test] +fn continuous_zero_keeps_the_batch_ctx_for_two_short_static_jobs() { + let cfg = ServerConfig { + continuous: false, + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = static_chat_job(&inner, "a"); + let (job_b, drain_b) = static_chat_job(&inner, "b"); + let (tx, rx) = mpsc::channel(); + tx.send(job_b).unwrap(); + drop(tx); + let mut spy = OwnerSpy { + seq_cap: 8, + ..OwnerSpy::default() + }; + let mut engine = ScriptedDecode::from_pieces(&[b"serial-fallback"]); + + let leftover = run_owner_maybe_coalesce(&cfg, &inner, &mut engine, &mut spy, job_a, &rx); + + assert!(leftover.is_none()); + assert_eq!(spy.calls, 1); + assert_eq!(spy.ns, vec![2]); + let text_a = String::from_utf8(drain_a.state.take()).unwrap(); + let text_b = String::from_utf8(drain_b.state.take()).unwrap(); + let _ = drain_a.done.recv(); + let _ = drain_b.done.recv(); + assert!(text_a.starts_with("HTTP/1.1 200 OK"), "{text_a}"); + assert!(text_b.starts_with("HTTP/1.1 200 OK"), "{text_b}"); + assert!(!text_a.contains("serial-fallback"), "{text_a}"); + assert!(!text_b.contains("serial-fallback"), "{text_b}"); + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_serial, 0); + assert_eq!( + g.metrics.route_requests[WireSurface::OpenaiChat as usize][LANE_STATIC as usize], + 2 + ); +} + +#[test] +fn continuous_zero_n1_collapses_static_gather_to_serial() { + // Given: CONTINUOUS=0 so route_decide picks STATIC; owner FIFO has one job + let cfg = ServerConfig { + continuous: false, + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = static_chat_job(&inner, "a"); + let (_tx, rx) = mpsc::channel(); + let mut spy = OwnerSpy { + seq_cap: 8, + ..OwnerSpy::default() + }; + let mut engine = ScriptedDecode::from_pieces(&[b"serial-fallback"]); + + // When: coalesce_gather width is 1 + let leftover = run_owner_maybe_coalesce(&cfg, &inner, &mut engine, &mut spy, job_a, &rx); + + // Then: C worker_main n==1 → run_job_single. generate_static stays cold. + assert!(leftover.is_none()); + assert_eq!(spy.calls, 0); + let text = String::from_utf8(drain_a.state.take()).unwrap(); + let _ = drain_a.done.recv(); + assert!(text.starts_with("HTTP/1.1 200 OK"), "{text}"); + assert!( + !text.contains(crate::serve_static::STATIC_WIDTH_ERR), + "HTTP n=1 must not refuse width: {text}" + ); + assert!(text.contains("serial-fallback"), "{text}"); + let g = lock_inner(&inner); + assert_eq!(g.runtime.requests_serial, 1); + assert_eq!( + g.metrics.route_requests[WireSurface::OpenaiChat as usize][LANE_SERIAL as usize], + 1 + ); + assert_eq!( + g.metrics.route_requests[WireSurface::OpenaiChat as usize][LANE_STATIC as usize], + 0 + ); +} + +#[test] +fn owner_ctx_overflow_uses_c_fallback_not_serial() { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Arc::new(Mutex::new(ServerInner::from_cfg(&cfg))); + let (job_a, drain_a) = static_chat_job(&inner, "a"); + let (job_b, drain_b) = static_chat_job(&inner, "b"); + let (tx, rx) = mpsc::channel(); + tx.send(job_b).unwrap(); + drop(tx); + let mut spy = OverflowSpy { + inner: OwnerSpy::default(), + }; + let mut engine = ScriptedDecode::from_pieces(&[b"serial-fallback"]); + + let leftover = run_owner_maybe_coalesce(&cfg, &inner, &mut engine, &mut spy, job_a, &rx); + + assert!(leftover.is_none()); + assert_eq!(spy.inner.calls, 0); + let text_a = String::from_utf8(drain_a.state.take()).unwrap(); + let text_b = String::from_utf8(drain_b.state.take()).unwrap(); + let _ = drain_a.done.recv(); + let _ = drain_b.done.recv(); + assert!(text_a.contains("out of memory"), "{text_a}"); + assert!(text_b.contains("out of memory"), "{text_b}"); + assert!(!text_a.contains("serial-fallback"), "{text_a}"); + assert_eq!(lock_inner(&inner).runtime.requests_serial, 0); +} + +struct OverflowSpy { + inner: OwnerSpy, +} + +impl StaticExec for OverflowSpy { + fn generate_static(&mut self, jobs: &[StaticJob<'_>]) -> Result, GenerateError> { + self.inner.generate_static(jobs) + } + fn fallback_static(&mut self, err: GenerateError) -> Result, GenerateError> { + self.inner.fallback_static(err) + } + fn ctx_max_seq(&self) -> i32 { + 1 + } + fn coalesce_limits(&self) -> CoalesceLimits { + CoalesceLimits::UNBOUNDED + } +} + +impl ContExec for OverflowSpy { + fn model_id(&self) -> i32 { + 0 + } + fn seq_cap(&self) -> i32 { + 0 + } + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + vec![77] + } + fn encode_text(&self, _text: &str) -> Vec { + vec![77] + } + fn generate( + &mut self, + parsed: &crate::parse::ParsedRequest, + job_id: &str, + created: i64, + cors: bool, + default_tokens: i32, + t_arrive: Instant, + bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + store: Option<&mut ds4_kv::Store>, + out: &mut dyn Write, + ) -> Result { + self.inner.generate( + parsed, + job_id, + created, + cors, + default_tokens, + t_arrive, + bank_hold_retry, + store, + out, + ) + } + fn as_static(&mut self) -> Option<&mut dyn StaticExec> { + Some(self) + } +} diff --git a/crates/ds4-server/src/serve_serial_fit.rs b/crates/ds4-server/src/serve_serial_fit.rs new file mode 100644 index 0000000000000000000000000000000000000000..a7633c4665716f8f91eae8a5aa5dc0ed0144f759 --- /dev/null +++ b/crates/ds4-server/src/serve_serial_fit.rs @@ -0,0 +1,215 @@ +//! C `serial_session_ensure_fit` decision logic (`ds4_server.c` v0.5.2 inc1). +//! +//! The serial session is created at server `-c` and its S6 lazy graph is +//! sized by that ctx, not the request. On a bank-holding boot the full +//! graph can never fit and every serial request dies at the alloc. A serial +//! request needs only prompt+budget positions, so when the full graph +//! cannot fit the session is re-created at the largest ctx the fit gate +//! passes, bounded by what the request could use. When even a minimal +//! output window cannot fit, refuse 503 (retryable capacity), never the +//! doomed alloc. Pure arithmetic lives here; the session swap driver is in +//! `serve.rs` (`ensure_serial_session_fit`). + +/// C `serial_frame_kind`. `RequiredMiss` never reaches the host driver — +/// `run_serial` answers the protocol-native 409 before generation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SerialFrameKind { + Canonical, + ResolvedLive, + RequiredMiss, +} + +/// C `serial_fit_plan`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SerialFitPlan { + Reuse, + Resize, + RefusePreserve, + PassNative, +} + +/// C `serial_session_reuse_ok`: a graphless boot-shape session has no state +/// to preserve; do not let a tiny first serial request commit a deep +/// full-`-c` graph merely because it fits before the batch banks grow. +pub(crate) fn serial_session_reuse_ok( + cur_ctx: i64, + graph_pending: bool, + need_min: i64, + request_cap: i64, + current_graph_fits: bool, +) -> bool { + if cur_ctx < need_min { + return false; + } + if !graph_pending { + return true; + } + cur_ctx <= request_cap && current_graph_fits +} + +/// C `serial_session_fit_plan`. +pub(crate) fn serial_session_fit_plan( + cur_ctx: i64, + graph_pending: bool, + need_min: i64, + request_cap: i64, + current_graph_fits: bool, + frame_kind: SerialFrameKind, +) -> SerialFitPlan { + if frame_kind == SerialFrameKind::RequiredMiss { + return SerialFitPlan::PassNative; + } + if serial_session_reuse_ok( + cur_ctx, + graph_pending, + need_min, + request_cap, + current_graph_fits, + ) { + return SerialFitPlan::Reuse; + } + if frame_kind == SerialFrameKind::ResolvedLive { + return SerialFitPlan::RefusePreserve; + } + SerialFitPlan::Resize +} + +/// C `serial_session_ensure_fit` request bounds. `live_need_min` is uncapped +/// (the live-preserve check compares it against the retained session ctx); +/// the rightsize bounds are capped at boot `-c`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SerialFitBounds { + pub live_need_min: i64, + pub need_min: i64, + pub request_cap: i64, +} + +/// C arithmetic: budget floors at 1; the minimal output window is +/// `min(budget, 1024)`; the search cap grants 32768 continuation headroom so +/// a serial conversation grows for many turns inside one session. +pub(crate) fn serial_fit_bounds(prompt_len: i64, budget: i64, boot_ctx: i64) -> SerialFitBounds { + let budget = budget.max(1); + let live_need_min = prompt_len + budget.min(1024); + let need_full = (prompt_len + budget).min(boot_ctx); + let need_min = live_need_min.min(boot_ctx); + let request_cap = (need_full + 32768).min(boot_ctx); + SerialFitBounds { + live_need_min, + need_min, + request_cap, + } +} + +/// C `serial_live_capacity_refuse` client message. +pub(crate) fn serial_live_preserve_msg(effective_prompt_len: i64, cur_ctx: i64) -> String { + format!( + "Server cannot extend the retained {effective_prompt_len}-token continuation inside \ + its {cur_ctx}-token serial context; retry with a smaller output budget \ + or replay the full history" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + // C test_serial_pending_session_respects_request_cap vectors. + #[test] + fn reuse_ok_matches_c_vectors() { + assert!(!serial_session_reuse_ok(262144, true, 1050, 33818, true)); + assert!(serial_session_reuse_ok(16384, true, 1050, 16384, true)); + assert!(serial_session_reuse_ok(32768, true, 1050, 33818, true)); + assert!(!serial_session_reuse_ok(32768, true, 1050, 33818, false)); + assert!(serial_session_reuse_ok(262144, false, 1050, 33818, false)); + assert!(!serial_session_reuse_ok(1024, false, 1050, 33818, true)); + } + + #[test] + fn fit_plan_matches_c_vectors() { + assert_eq!( + serial_session_fit_plan( + 32066, + false, + 251024, + 262144, + false, + SerialFrameKind::ResolvedLive + ), + SerialFitPlan::RefusePreserve + ); + assert_eq!( + serial_session_fit_plan( + 262144, + false, + 251024, + 262144, + false, + SerialFrameKind::ResolvedLive + ), + SerialFitPlan::Reuse + ); + assert_eq!( + serial_session_fit_plan( + 32066, + false, + 64, + 32800, + false, + SerialFrameKind::RequiredMiss + ), + SerialFitPlan::PassNative + ); + assert_eq!( + serial_session_fit_plan(32066, false, 1024, 33792, false, SerialFrameKind::Canonical), + SerialFitPlan::Reuse + ); + } + + #[test] + fn undersized_canonical_session_resizes() { + assert_eq!( + serial_session_fit_plan(2048, true, 5000, 40000, true, SerialFrameKind::Canonical), + SerialFitPlan::Resize + ); + // Pending full--c session whose graph cannot alloc beside the banks. + assert_eq!( + serial_session_fit_plan(250000, true, 1050, 33818, false, SerialFrameKind::Canonical), + SerialFitPlan::Resize + ); + } + + #[test] + fn bounds_follow_c_arithmetic() { + // Default output budget degenerates to -c for budget-omitting clients. + let b = serial_fit_bounds(26, 393216, 250000); + assert_eq!(b.live_need_min, 26 + 1024); + assert_eq!(b.need_min, 1050); + assert_eq!(b.request_cap, 250000); + + let b = serial_fit_bounds(26, 384, 250000); + assert_eq!(b.live_need_min, 26 + 384); + assert_eq!(b.need_min, 410); + assert_eq!(b.request_cap, 26 + 384 + 32768); + + // Zero/explicit-zero budgets floor at 1 like C; + // request_cap = min(need_full + 32768, boot). + let b = serial_fit_bounds(100, 0, 4096); + assert_eq!(b.live_need_min, 101); + assert_eq!(b.need_min, 101); + assert_eq!(b.request_cap, 4096); + + // Prompt past boot ctx clamps everywhere. + let b = serial_fit_bounds(300000, 1024, 250000); + assert_eq!(b.live_need_min, 301024); + assert_eq!(b.need_min, 250000); + assert_eq!(b.request_cap, 250000); + } + + #[test] + fn live_preserve_msg_matches_c_shape() { + let msg = serial_live_preserve_msg(251000, 32066); + assert!(msg.starts_with("Server cannot extend the retained 251000-token continuation")); + assert!(msg.contains("its 32066-token serial context")); + assert!(msg.ends_with("or replay the full history")); + } +} diff --git a/crates/ds4-server/src/serve_serial_reclaim.rs b/crates/ds4-server/src/serve_serial_reclaim.rs new file mode 100644 index 0000000000000000000000000000000000000000..06cdf127465dacb1dc8c1c69867dd346ecfd7704 --- /dev/null +++ b/crates/ds4-server/src/serve_serial_reclaim.rs @@ -0,0 +1,291 @@ +//! C serial emergency reclaim (`serial_session_ensure_fit` / +//! `serial_reclaim_rank` / `serial_reclaim_gate`). +//! +//! Before a typed serial refuse, collect idle banks. Never admit a spend +//! that would leave the box under `--mem-floor-gb` (`DS4_MEM_FLOOR_GB`). + +use crate::serve_cont_roll::RejectReason; + +/// C `ds4_mem_floor_bytes` default: `4ull << 30`. +pub(crate) const DEFAULT_MEM_FLOOR_GB: u64 = 4; + +/// C `ds4_mem_floor_bytes` / `--mem-floor-gb` live line, in bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct MemFloor(u64); + +/// Observed free bytes at the fit quote. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct AvailBytes(u64); + +/// Serial graph bytes the request still needs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct NeedBytes(u64); + +/// Idle-bank pages the ranker may return. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ReclaimableBytes(u64); + +/// C `fq.headroom_bytes` slack added to the reclaim want. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct HeadroomBytes(u64); + +impl MemFloor { + pub(crate) const fn from_gb(gb: u64) -> Self { + Self(gb << 30) + } + + pub(crate) const fn bytes(self) -> u64 { + self.0 + } + + pub(crate) const fn gb(self) -> u64 { + self.0 >> 30 + } + + /// C `ds4_mem_floor_bytes`: default 4 GiB; `DS4_MEM_FLOOR_GB` via atol. + pub(crate) fn from_env_gb(raw: Option<&[u8]>) -> Self { + let Some(raw) = raw else { + return Self::from_gb(DEFAULT_MEM_FLOOR_GB); + }; + if raw.is_empty() { + return Self::from_gb(DEFAULT_MEM_FLOOR_GB); + } + let fv = ds4_sys::libc_atoi(raw); + if fv < 0 { + Self::from_gb(DEFAULT_MEM_FLOOR_GB) + } else { + Self::from_gb(fv as u64) + } + } + + /// CLI `--mem-floor-gb` wins over `DS4_MEM_FLOOR_GB`; both use C atol. + pub(crate) fn from_cli_or_env(cli: Option<&[u8]>, env: Option<&[u8]>) -> Self { + match cli { + Some(cli) => Self::from_env_gb(Some(cli)), + None => Self::from_env_gb(env), + } + } +} + +impl AvailBytes { + pub(crate) const fn from_raw(bytes: u64) -> Self { + Self(bytes) + } + + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +impl NeedBytes { + pub(crate) const fn from_raw(bytes: u64) -> Self { + Self(bytes) + } + + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +impl ReclaimableBytes { + pub(crate) const fn from_raw(bytes: u64) -> Self { + Self(bytes) + } + + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +impl HeadroomBytes { + pub(crate) const fn from_raw(bytes: u64) -> Self { + Self(bytes) + } + + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +/// Quoted avail/need for the live serial path. +/// C `ds4_session_graph_fit_quote` fields we can host without native trim. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SerialFitQuote { + pub avail: AvailBytes, + pub need: NeedBytes, + pub reclaimable: ReclaimableBytes, + pub headroom: HeadroomBytes, +} + +impl SerialFitQuote { + pub(crate) const fn ask(self, floor: MemFloor) -> SerialReclaimAsk { + SerialReclaimAsk { + avail: self.avail, + need: self.need, + floor, + reclaimable: self.reclaimable, + headroom: self.headroom, + } + } +} + +/// No native graph-fit quote yet. need=0 so the gate still runs and +/// cannot invent a harder refuse than C (margin, not floor). +pub(crate) const fn unquoted_serial_fit() -> SerialFitQuote { + SerialFitQuote { + avail: AvailBytes::from_raw(u64::MAX), + need: NeedBytes::from_raw(0), + reclaimable: ReclaimableBytes::from_raw(0), + headroom: HeadroomBytes::from_raw(0), + } +} + +/// Map C `ds4_session_graph_fit_quote` bytes. `fail_open` means the +/// probe had no budget numbers; treat that as quote failure so the +/// caller keeps the unquoted margin instead of inventing a refuse. +pub(crate) fn serial_fit_from_native( + need_bytes: u64, + avail_bytes: u64, + headroom_bytes: u64, + reclaimable_bytes: u64, + fail_open: bool, +) -> Option { + if fail_open { + return None; + } + Some(SerialFitQuote { + avail: AvailBytes::from_raw(avail_bytes), + need: NeedBytes::from_raw(need_bytes), + reclaimable: ReclaimableBytes::from_raw(reclaimable_bytes), + headroom: HeadroomBytes::from_raw(headroom_bytes), + }) +} + +/// Test override, then a live quote, else the unquoted margin. +pub(crate) fn resolve_serial_fit( + configured: Option, + live: Option, +) -> SerialFitQuote { + configured.or(live).unwrap_or_else(unquoted_serial_fit) +} + +/// One serial fit-quote plus what idle banks can still return. +#[derive(Clone, Copy, Debug)] +pub(crate) struct SerialReclaimAsk { + pub avail: AvailBytes, + pub need: NeedBytes, + pub floor: MemFloor, + pub reclaimable: ReclaimableBytes, + pub headroom: HeadroomBytes, +} + +/// C `serial_reclaim_gate` outcome. Reclaim is attempted before Refuse. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SerialReclaimOutcome { + Admit { + reclaimed: u64, + }, + Refuse { + reclaimed: u64, + reason: RejectReason, + }, +} + +impl SerialReclaimOutcome { + pub(crate) const fn reclaimed(self) -> u64 { + match self { + Self::Admit { reclaimed } | Self::Refuse { reclaimed, .. } => reclaimed, + } + } + + pub(crate) const fn admitted(self) -> bool { + matches!(self, Self::Admit { .. }) + } +} + +/// C `serial_session_ensure_fit` 503 after reclaim still cannot fund the graph. +pub(crate) fn serial_capacity_refuse_msg(prompt_n: i32) -> String { + format!( + "Server is temporarily at capacity for a {prompt_n}-token serial request \ + (no session graph fits beside the batch banks); retry shortly" + ) +} + +/// One idle bank for C `serial_reclaim_rank`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct ReclaimBank { + pub id: u32, + pub valid: bool, + pub last_use: u64, + pub committed: i32, + pub protected: bool, + pub superseded: bool, +} + +const fn fits_above_floor(avail: AvailBytes, need: NeedBytes, floor: MemFloor) -> bool { + avail.raw() >= floor.bytes() && avail.raw() - floor.bytes() >= need.raw() +} + +const fn serial_deficit(ask: SerialReclaimAsk) -> u64 { + ask.need + .raw() + .saturating_add(ask.floor.bytes()) + .saturating_sub(ask.avail.raw()) +} + +/// C `serial_session_ensure_fit` collect want: deficit + headroom. +pub(crate) fn serial_reclaim_want(ask: SerialReclaimAsk) -> u64 { + if fits_above_floor(ask.avail, ask.need, ask.floor) { + return 0; + } + serial_deficit(ask).saturating_add(ask.headroom.raw()) +} + +/// C `serial_reclaim_gate`: reclaim idle pages before a typed refuse. +/// Usable = avail − floor; the floor is never spent. +pub(crate) fn serial_reclaim_gate(ask: SerialReclaimAsk) -> SerialReclaimOutcome { + if fits_above_floor(ask.avail, ask.need, ask.floor) { + return SerialReclaimOutcome::Admit { reclaimed: 0 }; + } + let want = serial_reclaim_want(ask); + let reclaimed = want.min(ask.reclaimable.raw()); + let after = AvailBytes::from_raw(ask.avail.raw().saturating_add(reclaimed)); + if fits_above_floor(after, ask.need, ask.floor) { + SerialReclaimOutcome::Admit { reclaimed } + } else { + SerialReclaimOutcome::Refuse { + reclaimed, + reason: RejectReason::LiveHeadroom, + } + } +} + +/// C `serial_reclaim_rank`: cheapest warm value first. Hard-excludes +/// grace/pin-protected banks and deep records (`committed >= warm_pin_min`). +pub(crate) fn serial_reclaim_rank(banks: &[ReclaimBank], warm_pin_min: i32) -> Vec { + let mut no_value = Vec::new(); + let mut superseded = Vec::new(); + let mut plain = Vec::new(); + for bank in banks { + if bank.protected { + continue; + } + if !bank.valid { + no_value.push(bank.id); + continue; + } + if warm_pin_min > 0 && bank.committed >= warm_pin_min { + continue; + } + if bank.superseded { + superseded.push(*bank); + } else { + plain.push(*bank); + } + } + superseded.sort_by_key(|bank| bank.last_use); + plain.sort_by_key(|bank| bank.last_use); + no_value.extend(superseded.iter().map(|bank| bank.id)); + no_value.extend(plain.iter().map(|bank| bank.id)); + no_value +} diff --git a/crates/ds4-server/src/serve_serial_reclaim_test.rs b/crates/ds4-server/src/serve_serial_reclaim_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..63e88e502df65f789c26e652a37896f1810a0b5b --- /dev/null +++ b/crates/ds4-server/src/serve_serial_reclaim_test.rs @@ -0,0 +1,348 @@ +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::Mutex; + +use crate::generate::ScriptedDecode; +use crate::serve::{handle_client_inner, ServerConfig, ServerInner}; +use crate::serve_cont_roll::RejectReason; +use crate::serve_serial_reclaim::{ + resolve_serial_fit, serial_capacity_refuse_msg, serial_fit_from_native, serial_reclaim_gate, + serial_reclaim_rank, serial_reclaim_want, unquoted_serial_fit, AvailBytes, HeadroomBytes, + MemFloor, NeedBytes, ReclaimBank, ReclaimableBytes, SerialFitQuote, SerialReclaimAsk, + SerialReclaimOutcome, +}; + +const GIB: u64 = 1 << 30; + +fn ask(avail_gib: u64, need_gib: u64, floor_gb: u64, reclaim_gib: u64) -> SerialReclaimAsk { + SerialReclaimAsk { + avail: AvailBytes::from_raw(avail_gib * GIB), + need: NeedBytes::from_raw(need_gib * GIB), + floor: MemFloor::from_gb(floor_gb), + reclaimable: ReclaimableBytes::from_raw(reclaim_gib * GIB), + headroom: HeadroomBytes::from_raw(0), + } +} + +fn remaining_after(ask: SerialReclaimAsk, out: SerialReclaimOutcome) -> u64 { + let after = ask.avail.raw().saturating_add(out.reclaimed()); + if out.admitted() { + after.saturating_sub(ask.need.raw()) + } else { + after + } +} + +#[test] +fn mem_floor_default_is_c_four_gib() { + // Given: no DS4_MEM_FLOOR_GB + // When: parse the C floor helper + let floor = MemFloor::from_env_gb(None); + + // Then: 4 GiB, same as ds4_mem_floor_bytes + assert_eq!(floor, MemFloor::from_gb(4)); + assert_eq!(floor.bytes(), 4 * GIB); +} + +#[test] +fn mem_floor_env_zero_is_c_kill_switch() { + // Given: DS4_MEM_FLOOR_GB=0 + // When: parse + let floor = MemFloor::from_env_gb(Some(b"0")); + + // Then: floor is 0 (C A/B kill switch) + assert_eq!(floor.bytes(), 0); +} + +#[test] +fn reclaim_then_admit_when_idle_banks_cover_deficit() { + // Given: 5 GiB free, 4 GiB floor, 2 GiB serial need, 3 GiB idle + let ask = ask(5, 2, 4, 3); + + // When: the serial reclaim gate runs + let out = serial_reclaim_gate(ask); + + // Then: reclaim the 1 GiB deficit, then admit; floor intact + assert_eq!(out, SerialReclaimOutcome::Admit { reclaimed: GIB }); + assert!(remaining_after(ask, out) >= ask.floor.bytes()); +} + +#[test] +fn reclaim_runs_before_typed_refuse_when_still_short() { + // Given: reclaim cannot cover need above the floor + let ask = ask(5, 3, 4, 1); + + // When: the serial reclaim gate runs + let out = serial_reclaim_gate(ask); + + // Then: reclaim still ran; typed refuse is C live_headroom + assert_eq!( + out, + SerialReclaimOutcome::Refuse { + reclaimed: GIB, + reason: RejectReason::LiveHeadroom, + } + ); + assert_eq!(RejectReason::LiveHeadroom.name(), "live_headroom"); + assert!(remaining_after(ask, out) >= ask.floor.bytes()); +} + +#[test] +fn refuse_does_not_admit_below_mem_floor() { + // Given: already at the floor, reclaim cannot fund the graph + let ask = ask(4, 2, 4, 1); + + // When: the serial reclaim gate runs + let out = serial_reclaim_gate(ask); + + // Then: still refuse; never admit a spend that crosses the floor + assert!(!out.admitted()); + match out { + SerialReclaimOutcome::Refuse { reason, reclaimed } => { + assert_eq!(reason, RejectReason::LiveHeadroom); + assert_eq!(reclaimed, GIB); + } + SerialReclaimOutcome::Admit { .. } => panic!("admitted below --mem-floor-gb"), + } + assert!(remaining_after(ask, out) >= ask.floor.bytes()); +} + +#[test] +fn already_fits_does_not_reclaim() { + // Given: free already covers need + floor + let ask = ask(7, 2, 4, 8); + + // When: the serial reclaim gate runs + let out = serial_reclaim_gate(ask); + + // Then: admit with no reclaim + assert_eq!(out, SerialReclaimOutcome::Admit { reclaimed: 0 }); +} + +#[test] +fn reclaim_restores_floor() { + for (need, reclaim) in [(0, 1), (2, 3)] { + let short = ask(1, need, 2, 0); + assert_eq!(serial_reclaim_want(short), reclaim * GIB); + assert!(!serial_reclaim_gate(short).admitted()); + + let funded = ask(1, need, 2, reclaim); + let outcome = serial_reclaim_gate(funded); + assert_eq!( + outcome, + SerialReclaimOutcome::Admit { + reclaimed: reclaim * GIB + } + ); + assert_eq!(remaining_after(funded, outcome), funded.floor.bytes()); + } +} + +#[test] +fn mem_floor_cli_wins_over_env() { + let floor = MemFloor::from_cli_or_env(Some(b"2"), Some(b"8")); + assert_eq!(floor, MemFloor::from_gb(2)); +} + +#[test] +fn unquoted_live_fit_admits_without_harder_refuse() { + let out = serial_reclaim_gate(unquoted_serial_fit().ask(MemFloor::from_gb(4))); + assert_eq!(out, SerialReclaimOutcome::Admit { reclaimed: 0 }); +} + +#[test] +fn reclaim_want_is_zero_when_the_quote_already_fits() { + assert_eq!( + serial_reclaim_want(unquoted_serial_fit().ask(MemFloor::from_gb(4))), + 0 + ); + let short = ask(5, 3, 4, 1); + assert!(serial_reclaim_want(short) > 0); +} + +#[test] +fn fail_open_quote_falls_back_to_unquoted_margin() { + assert!(serial_fit_from_native(7_090_000_000, 5_460_000_000, 0, 0, true).is_none()); + let quote = resolve_serial_fit(None, None); + assert_eq!( + serial_reclaim_gate(quote.ask(MemFloor::from_gb(4))), + SerialReclaimOutcome::Admit { reclaimed: 0 } + ); +} + +#[test] +fn live_quote_numbers_drive_the_gate_without_raising_the_floor() { + let quote = serial_fit_from_native(7_090_000_000, 5_460_000_000, 256, 6_000_000_000, false) + .expect("numbered quote"); + let out = serial_reclaim_gate(quote.ask(MemFloor::from_gb(4))); + assert!(out.admitted(), "{out:?}"); + assert_eq!( + resolve_serial_fit(Some(unquoted_serial_fit()), Some(quote)), + unquoted_serial_fit() + ); +} + +fn drive_serial_completion(cfg: &ServerConfig) -> Vec { + let inner = Mutex::new(ServerInner::from_cfg(cfg)); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let body = r#"{"prompt":"hello","max_tokens":0}"#; + write!( + client, + "POST /v1/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[]); + handle_client_inner(cfg, &inner, &mut server, Some(&mut engine), None); + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + response +} + +#[test] +fn serial_path_refuses_with_c_body_when_gate_refuses() { + let mut cfg = ServerConfig { + have_engine: true, + mem_floor_gb: 4, + ..ServerConfig::default() + }; + cfg.serial_fit = Some(SerialFitQuote { + avail: AvailBytes::from_raw(5 * GIB), + need: NeedBytes::from_raw(3 * GIB), + reclaimable: ReclaimableBytes::from_raw(GIB), + headroom: HeadroomBytes::from_raw(0), + }); + + let bytes = drive_serial_completion(&cfg); + let response = String::from_utf8_lossy(&bytes); + + assert!(response.starts_with("HTTP/1.1 503"), "{response}"); + assert!( + response.contains(&serial_capacity_refuse_msg(1)), + "{response}" + ); +} + +#[test] +fn serial_path_admits_after_reclaim_covers_deficit() { + let mut cfg = ServerConfig { + have_engine: true, + mem_floor_gb: 4, + ..ServerConfig::default() + }; + cfg.serial_fit = Some(SerialFitQuote { + avail: AvailBytes::from_raw(5 * GIB), + need: NeedBytes::from_raw(2 * GIB), + reclaimable: ReclaimableBytes::from_raw(3 * GIB), + headroom: HeadroomBytes::from_raw(0), + }); + + let bytes = drive_serial_completion(&cfg); + let response = String::from_utf8_lossy(&bytes); + + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!( + !response.contains("no session graph fits beside the batch banks"), + "{response}" + ); +} + +#[test] +fn refuse_body_matches_c_serial_capacity() { + // Given: C serial_session_ensure_fit 503 + // When: format the refuse + let msg = serial_capacity_refuse_msg(26); + + // Then: exact C body + assert_eq!( + msg, + "Server is temporarily at capacity for a 26-token serial request \ + (no session graph fits beside the batch banks); retry shortly" + ); +} + +#[test] +fn rank_skips_protected_and_deep_pin() { + // Given: C serial_reclaim_rank hard exclusions + let banks = [ + ReclaimBank { + id: 0, + valid: false, + last_use: 10, + committed: 0, + protected: true, + superseded: false, + }, + ReclaimBank { + id: 1, + valid: false, + last_use: 1, + committed: 0, + protected: false, + superseded: false, + }, + ReclaimBank { + id: 2, + valid: true, + last_use: 2, + committed: 40, + protected: false, + superseded: true, + }, + ReclaimBank { + id: 3, + valid: true, + last_use: 3, + committed: 50, + protected: false, + superseded: false, + }, + ReclaimBank { + id: 4, + valid: true, + last_use: 4, + committed: 192, + protected: false, + superseded: false, + }, + ]; + + // When: rank with warm_pin_min=192 + let ranked = serial_reclaim_rank(&banks, 192); + + // Then: protected 0 and deep 4 stay intact; cheapest first + assert_eq!(ranked, vec![1, 2, 3]); +} + +#[test] +fn rank_orders_superseded_by_lru() { + // Given: two shallow superseded banks + let banks = [ + ReclaimBank { + id: 8, + valid: true, + last_use: 30, + committed: 10, + protected: false, + superseded: true, + }, + ReclaimBank { + id: 7, + valid: true, + last_use: 10, + committed: 10, + protected: false, + superseded: true, + }, + ]; + + // When: rank + let ranked = serial_reclaim_rank(&banks, 192); + + // Then: older last_use first (C insertion sort) + assert_eq!(ranked, vec![7, 8]); +} diff --git a/crates/ds4-server/src/serve_static.rs b/crates/ds4-server/src/serve_static.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb1f7b1dc7ac348e99d0f15455c633845729af5b --- /dev/null +++ b/crates/ds4-server/src/serve_static.rs @@ -0,0 +1,257 @@ +//! Static-lane owner around `BatchCtx::generate_static` (C `n>=2`). + +use crate::generate::GenerateError; + +#[path = "serve_static_coalesce.rs"] +mod coalesce; +pub use coalesce::{ + coalesce_take, job_tok_footprint, static_ctx_overflow, static_peer_ok, CoalesceLimits, + CoalescePeer, StaticPeerSpec, COALESCE_HARD_MAX, +}; + +/// C `ds4_bridge_batch_ctx_generate_static` when `n` is not a legal width. +pub const STATIC_WIDTH_ERR: &str = "static batch request count is out of range"; + +/// C `generate_batch_jobs` when the batched `err` buffer is empty. +pub const STATIC_FALLBACK_ERR: &str = "out of memory"; + +/// C `generate_batch_jobs` admission: coalesced group only. +pub const STATIC_N_MIN: usize = 2; + +/// One greedy static row. Tokens are borrowed only for the call. +#[derive(Clone, Copy)] +pub struct StaticJob<'a> { + pub tokens: &'a [i32], + pub max_new_tokens: i32, + pub eos: i32, +} + +/// Owned sibling waiting to coalesce with the next static-routed request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OwnedStaticJob { + pub tokens: Vec, + pub max_new_tokens: i32, + pub eos: i32, +} + +/// C `ds4_batch_gen_result.finish`: 1 = EOS → `"stop"`, 0 = budget → `"length"`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StaticFinish { + Length, + Stop, +} + +impl StaticFinish { + pub const fn reason(self) -> &'static str { + match self { + Self::Length => "length", + Self::Stop => "stop", + } + } +} + +/// C `write_batch_completion`: clamp to decode budget and force `"length"`. +pub fn settle_static_finish( + finish: StaticFinish, + n_tokens: usize, + budget: i32, +) -> (&'static str, usize) { + let budget = budget.max(0) as usize; + if n_tokens > budget { + ("length", budget) + } else { + (finish.reason(), n_tokens) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StaticRow { + pub tokens: Vec, + pub finish: StaticFinish, +} + +/// Trait seam so tests can spy on `generate_static` without a GGUF. +pub trait StaticExec { + fn generate_static(&mut self, jobs: &[StaticJob<'_>]) -> Result, GenerateError>; + + /// Extra rows already waiting (coalesce). Default: none. + fn pending_siblings(&self) -> &[OwnedStaticJob] { + &[] + } + + fn coalesce_limits(&self) -> CoalesceLimits { + CoalesceLimits::UNBOUNDED + } + + fn ctx_max_seq(&self) -> i32 { + i32::MAX + } + + fn ctx_max_tokens(&self) -> i32 { + i32::MAX + } + + /// C per-call fallback after `generate_static` fails. Default: C err text. + fn fallback_static(&mut self, err: GenerateError) -> Result, GenerateError> { + Err(static_fallback_error(err)) + } +} + +/// Map a failed `generate_static` onto C `generate_batch_jobs` err text. +pub fn static_fallback_error(err: GenerateError) -> GenerateError { + match err { + GenerateError::Engine(msg) if !msg.is_empty() => GenerateError::Engine(msg), + GenerateError::Engine(_) + | GenerateError::BadRequest(_) + | GenerateError::Unsupported(_) + | GenerateError::Streamed(_) + | GenerateError::Io + | GenerateError::ContinuationHold { .. } => { + GenerateError::Engine(STATIC_FALLBACK_ERR.to_string()) + } + } +} + +/// Production owner: `BatchCtx::generate_static` with no GGUF in this crate. +#[cfg(feature = "native")] +pub struct BatchStatic<'a, 'm> { + ctx: &'a mut ds4_core::BatchCtx<'m>, +} + +#[cfg(feature = "native")] +impl<'a, 'm> BatchStatic<'a, 'm> { + pub fn new(ctx: &'a mut ds4_core::BatchCtx<'m>) -> Self { + Self { ctx } + } +} + +#[cfg(feature = "native")] +impl StaticExec for BatchStatic<'_, '_> { + fn generate_static(&mut self, jobs: &[StaticJob<'_>]) -> Result, GenerateError> { + let requests: Vec> = jobs + .iter() + .map(|job| ds4_core::StaticBatchRequest { + tokens: job.tokens, + max_new_tokens: job.max_new_tokens, + eos: job.eos, + }) + .collect(); + self.ctx + .generate_static(&requests) + .map(|rows| { + rows.into_iter() + .map(|row| StaticRow { + tokens: row.tokens, + finish: match row.finish { + ds4_core::StaticBatchFinish::Budget => StaticFinish::Length, + ds4_core::StaticBatchFinish::Eos => StaticFinish::Stop, + }, + }) + .collect() + }) + .map_err(|err| GenerateError::Engine(err.message)) + } + + fn ctx_max_seq(&self) -> i32 { + self.ctx.max_seq() + } + + fn coalesce_limits(&self) -> CoalesceLimits { + CoalesceLimits { + cap: self.ctx.max_seq().max(1) as usize, + max_tok_total: 0, + } + } +} + +/// Used when the route is static but no owner is attached (n<2 still refuses). +pub struct DetachedStatic; + +impl StaticExec for DetachedStatic { + fn generate_static( + &mut self, + _jobs: &[StaticJob<'_>], + ) -> Result, GenerateError> { + Err(GenerateError::Unsupported("static owner is not attached")) + } +} + +/// `None` means `n` is admitted at the owner boundary. +pub const fn static_width_error(n: usize) -> Option<&'static str> { + if n < STATIC_N_MIN { + Some(STATIC_WIDTH_ERR) + } else { + None + } +} + +/// Owner entry: refuse `n<2` with the C width string; otherwise call +/// [`StaticExec::generate_static`]. On err, C fallback text — never serial. +pub fn run_static( + exec: &mut dyn StaticExec, + jobs: &[StaticJob<'_>], +) -> Result, GenerateError> { + if let Some(msg) = static_width_error(jobs.len()) { + return Err(GenerateError::Engine(msg.to_string())); + } + let packed: i64 = jobs.iter().map(|job| job.tokens.len() as i64).sum(); + if static_ctx_overflow( + jobs.len(), + packed, + exec.ctx_max_seq(), + exec.ctx_max_tokens(), + ) { + return exec.fallback_static(GenerateError::Engine(STATIC_FALLBACK_ERR.to_string())); + } + match exec.generate_static(jobs) { + Ok(rows) => Ok(rows), + Err(err) => exec.fallback_static(err), + } +} + +/// Routed owner: current request plus C-gathered [`StaticExec::pending_siblings`]. +pub fn run_static_routed( + exec: &mut dyn StaticExec, + current: StaticJob<'_>, +) -> Result, GenerateError> { + let siblings = exec.pending_siblings().to_vec(); + let limits = exec.coalesce_limits(); + let peers: Vec = siblings + .iter() + .map(|sibling| CoalescePeer { + footprint: job_tok_footprint(sibling.tokens.len(), sibling.max_new_tokens), + peer_ok: true, + }) + .collect(); + let take = coalesce_take( + job_tok_footprint(current.tokens.len(), current.max_new_tokens), + &peers, + limits, + ); + let mut jobs = Vec::with_capacity(take + 1); + for sibling in siblings.iter().take(take) { + jobs.push(StaticJob { + tokens: &sibling.tokens, + max_new_tokens: sibling.max_new_tokens, + eos: sibling.eos, + }); + } + jobs.push(current); + run_static(exec, &jobs) +} + +#[path = "serve_static_settle.rs"] +mod settle; +pub use settle::{write_static_completion, StaticSettle}; + +#[cfg(test)] +#[path = "serve_static_harness.rs"] +mod harness; + +#[cfg(test)] +#[path = "serve_static_test.rs"] +mod tests; + +#[cfg(test)] +#[path = "serve_static_fallback_test.rs"] +mod fallback_tests; diff --git a/crates/ds4-server/src/serve_static_coalesce.rs b/crates/ds4-server/src/serve_static_coalesce.rs new file mode 100644 index 0000000000000000000000000000000000000000..746ebdcb7cbd5aa5f1fcd47182013ce79b8c25dc --- /dev/null +++ b/crates/ds4-server/src/serve_static_coalesce.rs @@ -0,0 +1,116 @@ +//! C `coalesce_gather` + StaticBatchContext overflow, without a new scheduler. + +use crate::route::WireSurface; + +/// C `DS4_COALESCE_HARD_MAX`. +pub const COALESCE_HARD_MAX: usize = 64; + +/// C worker gather bounds: `cmax` and `cmaxtok` (`0` = unbounded tokens). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CoalesceLimits { + pub cap: usize, + pub max_tok_total: i64, +} + +impl CoalesceLimits { + pub const UNBOUNDED: Self = Self { + cap: COALESCE_HARD_MAX, + max_tok_total: 0, + }; + + pub const fn clamp(self) -> Self { + let cap = if self.cap < 1 { + 1 + } else if self.cap > COALESCE_HARD_MAX { + COALESCE_HARD_MAX + } else { + self.cap + }; + Self { + cap, + max_tok_total: if self.max_tok_total < 0 { + 0 + } else { + self.max_tok_total + }, + } + } +} + +impl Default for CoalesceLimits { + fn default() -> Self { + Self::UNBOUNDED + } +} + +/// C `job_tok_footprint`: prompt tokens + requested decode budget. +pub const fn job_tok_footprint(prompt_len: usize, max_new_tokens: i32) -> i64 { + let budget = if max_new_tokens > 0 { + max_new_tokens as i64 + } else { + 0 + }; + prompt_len as i64 + budget +} + +/// One FIFO candidate in front of `coalesce_gather`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CoalescePeer { + pub footprint: i64, + pub peer_ok: bool, +} + +/// C `job_static_peer_ok`: needs-free on a static-servable surface. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StaticPeerSpec { + pub needs: u32, + pub surface: WireSurface, + pub cont_anthropic: bool, + pub cont_responses: bool, +} + +pub const fn static_peer_ok(spec: StaticPeerSpec) -> bool { + if spec.needs != 0 { + return false; + } + match spec.surface { + WireSurface::OpenaiChat | WireSurface::OpenaiCompletion => true, + WireSurface::Anthropic => spec.cont_anthropic, + WireSurface::Responses => spec.cont_responses, + } +} + +/// Extra jobs taken from the front of `queued`. Head is already counted as 1. +pub fn coalesce_take( + head_footprint: i64, + queued: &[CoalescePeer], + limits: CoalesceLimits, +) -> usize { + let limits = limits.clamp(); + let mut n = 1usize; + let mut tok_total = head_footprint; + for peer in queued { + if n >= limits.cap { + break; + } + if !peer.peer_ok { + break; + } + if limits.max_tok_total > 0 && tok_total + peer.footprint > limits.max_tok_total { + break; + } + tok_total += peer.footprint; + n += 1; + } + n - 1 +} + +/// C `use_ctx` negation: group does not fit the persistent StaticBatchContext. +pub const fn static_ctx_overflow( + n: usize, + packed_prompt: i64, + max_seq: i32, + max_tokens: i32, +) -> bool { + max_seq <= 0 || n > max_seq as usize || packed_prompt > max_tokens as i64 +} diff --git a/crates/ds4-server/src/serve_static_fallback_test.rs b/crates/ds4-server/src/serve_static_fallback_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..029c4afbf22c96d0c731b75aad006dac979282c9 --- /dev/null +++ b/crates/ds4-server/src/serve_static_fallback_test.rs @@ -0,0 +1,83 @@ +use super::harness::{drive_static_chat_with, pair_jobs, FailKind, SpyStatic}; +use super::*; +use crate::generate::GenerateError; + +#[test] +fn run_static_takes_fallback_when_generate_static_returns_err() { + // Given: n=2 and generate_static returns a C bridge error + let first = [10, 20]; + let second = [30]; + let jobs = pair_jobs(&first, &second); + let mut spy = SpyStatic { + fail: Some(FailKind::Engine("static batch native result is invalid")), + ..Default::default() + }; + + // When + let err = run_static(&mut spy, &jobs).unwrap_err(); + + // Then: fallback path, same C text, no new envelope + assert_eq!(spy.calls, 1); + assert_eq!(spy.fallbacks, 1); + match err { + GenerateError::Engine(msg) => { + assert_eq!(msg, "static batch native result is invalid"); + } + other => panic!("expected Engine, got {other:?}"), + } +} + +#[test] +fn run_static_fallback_uses_c_oom_text_when_generate_static_has_no_engine_msg() { + // Given: n=2 and generate_static returns a non-Engine miss + let first = [10, 20]; + let second = [30]; + let jobs = pair_jobs(&first, &second); + let mut spy = SpyStatic { + fail: Some(FailKind::Unsupported("static owner is not attached")), + ..Default::default() + }; + + // When + let err = run_static(&mut spy, &jobs).unwrap_err(); + + // Then: C generate_batch_jobs empty-err default + assert_eq!(spy.fallbacks, 1); + match err { + GenerateError::Engine(msg) => assert_eq!(msg, STATIC_FALLBACK_ERR), + other => panic!("expected Engine, got {other:?}"), + } + assert_eq!(STATIC_FALLBACK_ERR, "out of memory"); +} + +#[test] +fn static_routed_request_uses_c_error_not_serial_when_generate_static_fails() { + // Given: LANE_STATIC n=2 and generate_static fails + let mut spy = SpyStatic { + siblings: vec![OwnedStaticJob { + tokens: vec![30], + max_new_tokens: 1, + eos: -1, + }], + fail: Some(FailKind::Engine("static batch native result is invalid")), + ..Default::default() + }; + + // When + let (response, inner) = drive_static_chat_with(&mut spy); + let text = String::from_utf8_lossy(&response); + + // Then: fallback taken, C text, serial stays cold + assert_eq!(spy.calls, 1); + assert_eq!(spy.fallbacks, 1); + assert!(text.starts_with("HTTP/1.1 500"), "{text}"); + assert!( + text.contains("static batch native result is invalid"), + "missing C error text: {text}" + ); + assert!( + !text.contains("serial-fallback"), + "serial decode must not run: {text}" + ); + assert_eq!(inner.runtime.requests_serial, 0); +} diff --git a/crates/ds4-server/src/serve_static_harness.rs b/crates/ds4-server/src/serve_static_harness.rs new file mode 100644 index 0000000000000000000000000000000000000000..5579a6a62261a4637922d2b66e3a3365261b19cb --- /dev/null +++ b/crates/ds4-server/src/serve_static_harness.rs @@ -0,0 +1,192 @@ +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::Mutex; + +use crate::generate::{GenerateError, GenerateOutcome, ScriptedDecode}; +use crate::parse::ParsedRequest; +use crate::serve::{handle_client_inner, ServerConfig, ServerInner}; +use crate::serve_cont::ContExec; +use crate::serve_static::{ + static_fallback_error, CoalesceLimits, OwnedStaticJob, StaticExec, StaticFinish, StaticJob, + StaticRow, +}; + +#[derive(Clone, Copy)] +pub(super) enum FailKind { + Engine(&'static str), + Unsupported(&'static str), +} + +pub(super) struct SpyStatic { + pub calls: usize, + pub fallbacks: usize, + pub ns: Vec, + pub siblings: Vec, + pub fail: Option, + pub ctx_max_seq: i32, + pub ctx_max_tokens: i32, + pub limits: CoalesceLimits, +} + +impl Default for SpyStatic { + fn default() -> Self { + Self { + calls: 0, + fallbacks: 0, + ns: Vec::new(), + siblings: Vec::new(), + fail: None, + ctx_max_seq: i32::MAX, + ctx_max_tokens: i32::MAX, + limits: CoalesceLimits::UNBOUNDED, + } + } +} + +impl StaticExec for SpyStatic { + fn generate_static(&mut self, jobs: &[StaticJob<'_>]) -> Result, GenerateError> { + self.calls += 1; + self.ns.push(jobs.len()); + match self.fail { + Some(FailKind::Engine(msg)) => Err(GenerateError::Engine(msg.to_string())), + Some(FailKind::Unsupported(msg)) => Err(GenerateError::Unsupported(msg)), + None => Ok(jobs + .iter() + .map(|job| StaticRow { + tokens: job.tokens.to_vec(), + finish: StaticFinish::Stop, + }) + .collect()), + } + } + + fn fallback_static(&mut self, err: GenerateError) -> Result, GenerateError> { + self.fallbacks += 1; + Err(static_fallback_error(err)) + } + + fn pending_siblings(&self) -> &[OwnedStaticJob] { + &self.siblings + } + + fn coalesce_limits(&self) -> CoalesceLimits { + self.limits + } + + fn ctx_max_seq(&self) -> i32 { + self.ctx_max_seq + } + + fn ctx_max_tokens(&self) -> i32 { + self.ctx_max_tokens + } +} + +pub(super) fn pair_jobs<'a>(first: &'a [i32], second: &'a [i32]) -> [StaticJob<'a>; 2] { + [ + StaticJob { + tokens: first, + max_new_tokens: 8, + eos: 99, + }, + StaticJob { + tokens: second, + max_new_tokens: 1, + eos: -1, + }, + ] +} + +pub(super) struct ForceStaticCont; + +impl ContExec for ForceStaticCont { + fn model_id(&self) -> i32 { + 0 + } + fn seq_cap(&self) -> i32 { + 0 + } + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + vec![77] + } + fn encode_text(&self, _text: &str) -> Vec { + vec![77] + } + fn generate( + &mut self, + _parsed: &ParsedRequest, + _job_id: &str, + _created: i64, + _cors: bool, + _default_tokens: i32, + _t_arrive: std::time::Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut ds4_kv::Store>, + _out: &mut dyn Write, + ) -> Result { + panic!("continuous generate must not run on a static-routed request"); + } +} + +impl ContExec for SpyStatic { + fn model_id(&self) -> i32 { + 0 + } + fn seq_cap(&self) -> i32 { + 0 + } + fn encode_chat(&self, _rendered: &[u8]) -> Vec { + vec![77] + } + fn encode_text(&self, _text: &str) -> Vec { + vec![77] + } + fn generate( + &mut self, + _parsed: &ParsedRequest, + _job_id: &str, + _created: i64, + _cors: bool, + _default_tokens: i32, + _t_arrive: std::time::Instant, + _bank_hold_retry: &mut dyn FnMut(i32, Option<(u64, i32)>) -> Option, + _store: Option<&mut ds4_kv::Store>, + _out: &mut dyn Write, + ) -> Result { + panic!("continuous generate must not run on a static-routed request"); + } + fn as_static(&mut self) -> Option<&mut dyn StaticExec> { + Some(self) + } +} + +pub(super) fn drive_static_chat() -> (Vec, ServerInner) { + let mut cont = ForceStaticCont; + drive_static_chat_with(&mut cont) +} + +pub(super) fn drive_static_chat_with(cont: &mut dyn ContExec) -> (Vec, ServerInner) { + let cfg = ServerConfig { + have_engine: true, + default_tokens: 8, + ..ServerConfig::default() + }; + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let mut client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let body = r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"},"temperature":0}"#; + write!( + client, + "POST /v1/chat/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + client.shutdown(std::net::Shutdown::Write).unwrap(); + let (mut server, _) = listener.accept().unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[b"serial-fallback"]); + handle_client_inner(&cfg, &inner, &mut server, Some(&mut engine), Some(cont)); + drop(server); + let mut response = Vec::new(); + client.read_to_end(&mut response).unwrap(); + (response, inner.into_inner().unwrap()) +} diff --git a/crates/ds4-server/src/serve_static_settle.rs b/crates/ds4-server/src/serve_static_settle.rs new file mode 100644 index 0000000000000000000000000000000000000000..1ac63d1970f9cdc80cff96d729de76e6bd49a536 --- /dev/null +++ b/crates/ds4-server/src/serve_static_settle.rs @@ -0,0 +1,136 @@ +use super::{settle_static_finish, StaticRow}; +use crate::generate::{chat_format_for_syntax, responses_ids, stream_req_from_parsed}; +use crate::parse::ParsedRequest; +use crate::render::syntax_for_model_id; +use crate::route::{decode_budget, think_mode_enabled, Api, ReqKind}; +use crate::stream::{ + anthropic_final_response, final_response, responses_final_response, ThinkBlock, +}; +use crate::tools::parse_generated_for_response; + +pub struct StaticSettle<'a> { + pub parsed: &'a ParsedRequest, + pub job_id: &'a str, + pub created: i64, + pub cors: bool, + pub default_tokens: i32, + pub model_id: i32, + pub prompt_n: i32, + pub row: &'a StaticRow, +} + +pub fn detok_static_row( + row: &StaticRow, + budget: i32, + is_stop: impl Fn(i32) -> bool, + text_of: impl Fn(i32) -> Vec, +) -> (Vec, &'static str, i32) { + let (finish, n) = settle_static_finish(row.finish, row.tokens.len(), budget); + let mut text = Vec::new(); + let mut emitted = 0i32; + for &token in row.tokens.iter().take(n) { + if is_stop(token) { + break; + } + text.extend(text_of(token)); + emitted += 1; + } + (text, finish, emitted) +} + +pub fn write_static_completion( + spec: StaticSettle<'_>, + is_stop: impl Fn(i32) -> bool, + text_of: impl Fn(i32) -> Vec, +) -> Vec { + let budget = decode_budget( + spec.parsed.max_tokens_set, + spec.parsed.max_tokens, + spec.default_tokens, + ); + let (text, finish, emitted) = detok_static_row(spec.row, budget, is_stop, text_of); + let req = stream_req_from_parsed(spec.parsed, spec.model_id); + match spec.parsed.kind { + ReqKind::Completion => final_response( + &req, + spec.job_id, + &text, + None, + finish, + spec.prompt_n, + emitted, + spec.created, + spec.cors, + &[], + &[], + ), + ReqKind::Chat => write_static_chat(spec, &req, &text, finish, emitted), + } +} + +fn write_static_chat( + spec: StaticSettle<'_>, + req: &crate::stream::StreamReq, + text: &[u8], + finish: &str, + emitted: i32, +) -> Vec { + let syntax = syntax_for_model_id(spec.model_id); + let format = chat_format_for_syntax(syntax); + let (parsed_gen, finish) = parse_generated_for_response( + syntax, + text, + false, + false, + think_mode_enabled(spec.parsed.think_mode), + format, + &spec.parsed.tool_orders, + finish, + ); + match spec.parsed.api { + Api::Anthropic => anthropic_final_response( + req, + spec.job_id, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + finish, + None, + spec.prompt_n, + emitted, + spec.cors, + &parsed_gen.calls, + ), + Api::Responses => { + let (rid, rsid, mid) = responses_ids(spec.job_id); + responses_final_response( + req, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + finish, + ThinkBlock::Closed, + spec.prompt_n, + emitted, + 0, + spec.created, + spec.cors, + &rid, + &rsid, + &mid, + &parsed_gen.calls, + ) + } + Api::Openai => final_response( + req, + spec.job_id, + &parsed_gen.content, + Some(&parsed_gen.reasoning), + finish, + spec.prompt_n, + emitted, + spec.created, + spec.cors, + &parsed_gen.calls, + &[], + ), + } +} diff --git a/crates/ds4-server/src/serve_static_test.rs b/crates/ds4-server/src/serve_static_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..157cb6e040059a1e0549fcc68c2ad0b9aba8adca --- /dev/null +++ b/crates/ds4-server/src/serve_static_test.rs @@ -0,0 +1,306 @@ +use super::harness::{drive_static_chat, drive_static_chat_with, pair_jobs, SpyStatic}; +use super::*; +use crate::generate::GenerateError; +use crate::route::{WireSurface, LANE_SERIAL, LANE_STATIC, NEED_STREAMING}; + +fn peer(footprint: i64, peer_ok: bool) -> CoalescePeer { + CoalescePeer { footprint, peer_ok } +} + +#[test] +fn static_width_error_matches_c_bridge_string_when_n_lt_2() { + // Given: C generate_batch_jobs is n>=2 + // When: owner sees n=1 + // Then: C bridge width string + assert_eq!(static_width_error(0), Some(STATIC_WIDTH_ERR)); + assert_eq!(static_width_error(1), Some(STATIC_WIDTH_ERR)); + assert_eq!( + STATIC_WIDTH_ERR, + "static batch request count is out of range" + ); +} + +#[test] +fn static_width_admits_n_eq_2() { + // Given: a coalesced pair + // When: owner checks width + // Then: admitted + assert_eq!(static_width_error(2), None); + assert_eq!(static_width_error(3), None); +} + +#[test] +fn run_static_returns_c_error_string_when_n_eq_1() { + // Given: one fixture row + let tokens = [10, 20]; + let jobs = [StaticJob { + tokens: &tokens, + max_new_tokens: 8, + eos: 99, + }]; + let mut spy = SpyStatic::default(); + + // When + let err = run_static(&mut spy, &jobs).unwrap_err(); + + // Then: C string, generate_static not called + match err { + GenerateError::Engine(msg) => assert_eq!(msg, STATIC_WIDTH_ERR), + other => panic!("expected Engine, got {other:?}"), + } + assert_eq!(spy.calls, 0); +} + +#[test] +fn run_static_invokes_generate_static_when_n_eq_2() { + // Given: two fixture rows (no GGUF) + let first = [10, 20]; + let second = [30]; + let jobs = pair_jobs(&first, &second); + let mut spy = SpyStatic::default(); + + // When + let rows = run_static(&mut spy, &jobs).unwrap(); + + // Then: per-row lengths stay ragged; fallback stays cold + assert_eq!(spy.calls, 1); + assert_eq!(spy.fallbacks, 0); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].tokens, first); + assert_eq!(rows[1].tokens, second); + assert_eq!([rows[0].tokens.len(), rows[1].tokens.len()], [2, 1]); +} + +#[test] +fn static_routed_n1_collapses_to_serial() { + // Given: route_decide picked LANE_STATIC (seq_cap=0, needs=0) + // When: a single static-routed request is served (n=1, no coalesce) + let (response, inner) = drive_static_chat(); + let text = String::from_utf8_lossy(&response); + + // Then: C worker_main n==1 → run_job_single. generate_static stays + // n>=2 (see run_static_returns_c_error_string_when_n_eq_1). Decision + // reason stays static; the entry lane is serial. + assert!(text.starts_with("HTTP/1.1 200 OK"), "{text}"); + assert!( + !text.contains(STATIC_WIDTH_ERR), + "HTTP n=1 must not refuse width: {text}" + ); + assert!( + text.contains("serial-fallback"), + "serial decode must run: {text}" + ); + assert_eq!(inner.runtime.requests_serial, 1); + assert_eq!( + inner.metrics.route_requests[WireSurface::OpenaiChat as usize][LANE_SERIAL as usize], + 1 + ); + assert_eq!( + inner.metrics.route_requests[WireSurface::OpenaiChat as usize][LANE_STATIC as usize], + 0 + ); +} + +#[test] +fn static_routed_request_invokes_generate_static_when_n_eq_2() { + // Given: route_decide picked LANE_STATIC and one sibling is waiting + let mut spy = SpyStatic { + siblings: vec![OwnedStaticJob { + tokens: vec![30], + max_new_tokens: 1, + eos: -1, + }], + ..Default::default() + }; + + // When: the HTTP request is served through handle_client_inner + let (response, inner) = drive_static_chat_with(&mut spy); + let text = String::from_utf8_lossy(&response); + + // Then: generate_static ran; serial session path stayed cold + assert_eq!(spy.calls, 1); + assert!(text.starts_with("HTTP/1.1 200 OK"), "{text}"); + assert!( + !text.contains(STATIC_WIDTH_ERR), + "width refuse must not fire at n=2: {text}" + ); + assert!( + !text.contains("serial-fallback"), + "serial decode must not run: {text}" + ); + assert_eq!(inner.runtime.requests_serial, 0); + assert_eq!( + inner.metrics.route_requests[WireSurface::OpenaiChat as usize][LANE_STATIC as usize], + 1 + ); +} + +#[test] +fn settle_static_finish_maps_c_eos_and_budget() { + // Given: C r->finish ? "stop" : "length" + // When: EOS / budget / over-budget + // Then: wire strings stay stop|length; over-budget clamps + assert_eq!(settle_static_finish(StaticFinish::Stop, 3, 8), ("stop", 3)); + assert_eq!( + settle_static_finish(StaticFinish::Length, 3, 8), + ("length", 3) + ); + assert_eq!( + settle_static_finish(StaticFinish::Stop, 9, 8), + ("length", 8) + ); +} + +#[test] +fn static_routed_request_settles_c_chat_terminal_when_n_eq_2() { + // Given: LANE_STATIC n=2 success (todos 11-12 wrote "{}") + let mut spy = SpyStatic { + siblings: vec![OwnedStaticJob { + tokens: vec![30], + max_new_tokens: 1, + eos: -1, + }], + ..Default::default() + }; + + // When + let (response, inner) = drive_static_chat_with(&mut spy); + let text = String::from_utf8_lossy(&response); + + // Then: C write_batch_completion shape; no invented timings/IDs + assert_eq!(spy.calls, 1); + assert!(text.starts_with("HTTP/1.1 200 OK"), "{text}"); + assert!( + text.contains("\"object\":\"chat.completion\""), + "missing C object: {text}" + ); + assert!( + text.contains("\"finish_reason\":\"stop\""), + "missing C finish_reason: {text}" + ); + assert!( + !text.contains("\"timings\""), + "C static omits timings: {text}" + ); + assert!( + !text.contains("serial-fallback"), + "serial decode must not run: {text}" + ); + assert_eq!(inner.runtime.requests_serial, 0); +} + +#[test] +fn coalesce_take_joins_two_queued_static_jobs_when_c_would() { + // Given: C coalesce_gather head + one static-peer-ok queued job under cap/budget + let queued = [peer(8, true)]; + let limits = CoalesceLimits { + cap: 8, + max_tok_total: 4096, + }; + + // When + let take = coalesce_take(8, &queued, limits); + + // Then: n=2 group (head kept, one extra taken) + assert_eq!(take, 1); +} + +#[test] +fn coalesce_take_stops_at_non_peer_to_preserve_fifo() { + // Given: next queued job is not static-peer-ok, a later one is + let queued = [peer(8, false), peer(8, true)]; + + // When + let take = coalesce_take(8, &queued, CoalesceLimits::UNBOUNDED); + + // Then: C breaks on the non-batchable head; later peer stays queued + assert_eq!(take, 0); +} + +#[test] +fn coalesce_take_leaves_token_overflow_peer_queued() { + // Given: adding the next footprint would exceed max_tok_total + let queued = [peer(100, true)]; + let limits = CoalesceLimits { + cap: 8, + max_tok_total: 150, + }; + + // When + let take = coalesce_take(100, &queued, limits); + + // Then: C splits into another batch; head stays alone + assert_eq!(take, 0); +} + +#[test] +fn run_static_uses_c_fallback_when_ctx_overflows() { + // Given: n=2 fits gather but not StaticBatchContext width + let first = [10, 20]; + let second = [30]; + let jobs = pair_jobs(&first, &second); + let mut spy = SpyStatic { + ctx_max_seq: 1, + ..Default::default() + }; + + // When + let err = run_static(&mut spy, &jobs).unwrap_err(); + + // Then: C fallback text, generate_static skipped, never serial + assert_eq!(spy.calls, 0); + assert_eq!(spy.fallbacks, 1); + match err { + GenerateError::Engine(msg) => assert_eq!(msg, STATIC_FALLBACK_ERR), + other => panic!("expected Engine, got {other:?}"), + } +} + +#[test] +fn run_static_routed_coalesces_queued_sibling_when_c_would() { + // Given: one queued sibling C would join (peer-ok, under budget) + let current_tokens = [10, 20]; + let current = StaticJob { + tokens: ¤t_tokens, + max_new_tokens: 8, + eos: -1, + }; + let mut spy = SpyStatic { + siblings: vec![OwnedStaticJob { + tokens: vec![30], + max_new_tokens: 1, + eos: -1, + }], + ..Default::default() + }; + + // When + let rows = run_static_routed(&mut spy, current).unwrap(); + + // Then: one generate_static call at n=2 + assert_eq!(spy.calls, 1); + assert_eq!(spy.ns, vec![2]); + assert_eq!(spy.fallbacks, 0); + assert_eq!(rows.len(), 2); +} + +#[test] +fn static_peer_ok_matches_c_needs_free_openai() { + // Given: C job_static_peer_ok / needs==0 on OpenAI chat + let ok = StaticPeerSpec { + needs: 0, + surface: WireSurface::OpenaiChat, + cont_anthropic: false, + cont_responses: false, + }; + let streaming = StaticPeerSpec { + needs: NEED_STREAMING, + surface: WireSurface::OpenaiChat, + cont_anthropic: false, + cont_responses: false, + }; + + // When / Then + assert!(static_peer_ok(ok)); + assert!(!static_peer_ok(streaming)); +} diff --git a/crates/ds4-server/src/stream.rs b/crates/ds4-server/src/stream.rs new file mode 100644 index 0000000000000000000000000000000000000000..c01898fda33c2f9956971df07b0386e830762e42 --- /dev/null +++ b/crates/ds4-server/src/stream.rs @@ -0,0 +1,2777 @@ +//! Four-surface stream projectors and buffered finals from `ds4_server.c` +//! at v0.6.5-dfm. Incremental tool projection is host-owned +//! (`tool_stream`); GPU decode still native. + +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use crate::error::{cors_headers, http_response_bytes, wire_stream_error_bytes}; +use crate::json::{json_args_parse, json_escape_bytes}; +use crate::parse::{ToolCall, ToolSchemaOrder}; +use crate::render::{ModelSyntax, QWEN_TOOL_CALL_START}; +use crate::route::{think_mode_enabled, Api, ReqKind, ThinkMode}; +use crate::tool_stream::{DsmlToolStream, ToolSink}; +use crate::tools::find_tool_start; + +mod channels; +use channels::{Channel, Channels, Flush}; +mod iquest; + +/// Frozen epoch used by the C stream oracle and tape tests. +pub const CREATED_TEST: i64 = 1_767_225_600; +const ITEM_IN_PROGRESS: &str = "in_progress"; + +pub const TEST_RESP_ID: &str = "resp_aaaaaaaaaaaaaaaaaaaaaaaa"; +pub const TEST_RS_ID: &str = "rs_aaaaaaaaaaaaaaaaaaaaaaaa"; +pub const TEST_MSG_ID: &str = "msg_aaaaaaaaaaaaaaaaaaaaaaaa"; + +pub const TAPE_PLAIN: &[&str] = &["Hel", "lo", " wor", "ld."]; +pub const TAPE_THINKING: &[&str] = &["plan", "", "Answer", " done."]; +pub const TAPE_UTF8: &[&[u8]] = &[b"caf", b"\xc3", b"\xa9", b" ok"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChatFormat { + DeepSeek, + SolarOpen2, + Exaone, + Qwen4Exp, + K2Horizon, + Inkling, + IQuestQ1, + Beam, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ToolHead { + Wait, + Present, + Absent, +} + +fn qwen_tool_head(r: &StreamReq, raw: &[u8], flush: Flush) -> ToolHead { + if !r.has_tools || r.syntax != ModelSyntax::Mimo2 { + return ToolHead::Absent; + } + let marker = QWEN_TOOL_CALL_START.as_bytes(); + if raw.starts_with(marker) { + return ToolHead::Present; + } + if flush == Flush::More && marker.starts_with(raw) { + return ToolHead::Wait; + } + ToolHead::Absent +} + +pub fn think_start(fmt: ChatFormat) -> &'static str { + match fmt { + ChatFormat::SolarOpen2 => "<|think:start|>", + ChatFormat::K2Horizon => crate::render::K2_THINK_START, + ChatFormat::Inkling => crate::render::inkling::THINK, + ChatFormat::Beam => crate::tools::beam::ANALYSIS, + _ => "", + } +} + +pub fn think_end(fmt: ChatFormat) -> &'static str { + match fmt { + ChatFormat::SolarOpen2 => "<|think:end|>", + ChatFormat::K2Horizon => crate::render::K2_THINK_END, + ChatFormat::Inkling => crate::render::inkling::END, + ChatFormat::Beam => crate::tools::beam::END, + _ => "", + } +} + +pub fn unix_now() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +pub fn utf8_expected_len(c: u8) -> usize { + if c < 0x80 { + 1 + } else if (0xc2..=0xdf).contains(&c) { + 2 + } else if (0xe0..=0xef).contains(&c) { + 3 + } else if (0xf0..=0xf4).contains(&c) { + 4 + } else { + 1 + } +} + +/// Hold a trailing incomplete UTF-8 character while the next token can still +/// complete it. Once nothing can (the generation is final, or the byte after +/// `limit` continues no character) it is released and json_escape_bytes writes +/// U+FFFD; C ignores `final` and loses those bytes. +pub fn utf8_stream_safe_len(s: &[u8], start: usize, limit: usize, final_: bool) -> usize { + if limit <= start || final_ || s.get(limit).is_some_and(|&c| c & 0xc0 != 0x80) { + return limit; + } + let mut p = limit; + let mut cont = 0; + while p > start && cont < 4 && (s[p - 1] & 0xc0) == 0x80 { + p -= 1; + cont += 1; + } + if p == limit { + return if utf8_expected_len(s[limit - 1]) > 1 { + limit - 1 + } else { + limit + }; + } + if p == start && (s[p] & 0xc0) == 0x80 { + return start; + } + let lead = p - 1; + let need = utf8_expected_len(s[lead]); + if limit - lead < need { + lead + } else { + limit + } +} + +/// A buffered body as a string. json_escape_bytes already writes U+FFFD for +/// model bytes that form no character; the lossy decode keeps any other stray +/// byte from panicking the request into a 500. +fn utf8_body(b: Vec) -> String { + String::from_utf8(b).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()) +} + +pub fn append_json_object_or_empty(out: &mut Vec, json: &str) { + let Some(args) = json_args_parse(json) else { + out.extend_from_slice(b"{}"); + return; + }; + out.push(b'{'); + for (i, a) in args.iter().enumerate() { + if i > 0 { + out.push(b','); + } + out.extend(json_escape_bytes(a.key.as_bytes())); + out.push(b':'); + if a.is_string { + out.extend(json_escape_bytes(a.value.as_bytes())); + } else { + out.extend_from_slice(a.value.as_bytes()); + } + } + out.push(b'}'); +} + +pub fn append_json_object_string(out: &mut Vec, json: &str) { + let mut tmp = Vec::new(); + append_json_object_or_empty(&mut tmp, json); + out.extend(json_escape_bytes(&tmp)); +} + +pub fn append_tool_calls_json(out: &mut Vec, calls: &[ToolCall], id_prefix: &str) { + out.push(b'['); + for (i, tc) in calls.iter().enumerate() { + if i > 0 { + out.push(b','); + } + let fallback = format!("{id_prefix}_tool_{i}"); + let id = if tc.id.is_empty() { + fallback.as_str() + } else { + tc.id.as_str() + }; + out.extend_from_slice(b"{\"id\":"); + out.extend(json_escape_bytes(id.as_bytes())); + out.extend_from_slice(b",\"type\":\"function\",\"function\":{\"name\":"); + out.extend(json_escape_bytes(tc.name.as_bytes())); + out.extend_from_slice(b",\"arguments\":"); + append_json_object_string(out, &tc.arguments); + out.extend_from_slice(b"}}"); + } + out.push(b']'); +} + +pub fn append_tool_call_deltas_json(out: &mut Vec, calls: &[ToolCall], id_prefix: &str) { + out.push(b'['); + for (i, tc) in calls.iter().enumerate() { + if i > 0 { + out.push(b','); + } + append_tool_call_delta_entry(out, i, tc, id_prefix); + } + out.push(b']'); +} + +fn append_tool_call_delta_entry(out: &mut Vec, index: usize, tc: &ToolCall, id_prefix: &str) { + let fallback = format!("{id_prefix}_tool_{index}"); + let id = if tc.id.is_empty() { + fallback.as_str() + } else { + tc.id.as_str() + }; + out.extend_from_slice(format!("{{\"index\":{index},\"id\":").as_bytes()); + out.extend(json_escape_bytes(id.as_bytes())); + out.extend_from_slice(b",\"type\":\"function\",\"function\":{\"name\":"); + out.extend(json_escape_bytes(tc.name.as_bytes())); + out.extend_from_slice(b",\"arguments\":"); + append_json_object_string(out, &tc.arguments); + out.extend_from_slice(b"}}"); +} + +fn tool_schema_orders_find<'a>( + orders: &'a [ToolSchemaOrder], + name: &str, +) -> Option<&'a ToolSchemaOrder> { + orders.iter().find(|o| o.name == name) +} + +fn responses_tool_call_is_tool_search(tc: &ToolCall, order: Option<&ToolSchemaOrder>) -> bool { + tc.name == "tool_search" && order.map(|o| o.responses_tool_search).unwrap_or(true) +} + +fn responses_tool_wire_name<'a>(tc: &'a ToolCall, order: Option<&'a ToolSchemaOrder>) -> &'a str { + order + .and_then(|o| (!o.wire_name.is_empty()).then_some(o.wire_name.as_str())) + .unwrap_or(tc.name.as_str()) +} + +pub fn append_anthropic_tool_use(out: &mut Vec, tc: &ToolCall, id_prefix: &str, i: usize) { + let fallback = format!("toolu_{id_prefix}_{i}"); + let id = if tc.id.is_empty() { + fallback.as_str() + } else { + tc.id.as_str() + }; + out.extend_from_slice(b"{\"type\":\"tool_use\",\"id\":"); + out.extend(json_escape_bytes(id.as_bytes())); + out.extend_from_slice(b",\"name\":"); + out.extend(json_escape_bytes(tc.name.as_bytes())); + out.extend_from_slice(b",\"input\":"); + append_json_object_or_empty(out, &tc.arguments); + out.push(b'}'); +} + +fn append_responses_function_call_item( + out: &mut Vec, + tc: &ToolCall, + fc_id: &str, + call_id: &str, + item_status: &str, + orders: &[ToolSchemaOrder], +) { + let order = tool_schema_orders_find(orders, &tc.name); + if responses_tool_call_is_tool_search(tc, order) { + out.extend_from_slice( + format!( + "{{\"id\":\"{fc_id}\",\"type\":\"tool_search_call\",\"status\":\"{item_status}\",\"call_id\":" + ) + .as_bytes(), + ); + out.extend(json_escape_bytes(call_id.as_bytes())); + out.extend_from_slice(b",\"execution\":\"client\",\"arguments\":"); + append_json_object_or_empty(out, &tc.arguments); + out.push(b'}'); + return; + } + let name = responses_tool_wire_name(tc, order); + out.extend_from_slice( + format!( + "{{\"id\":\"{fc_id}\",\"type\":\"function_call\",\"status\":\"{item_status}\",\"name\":" + ) + .as_bytes(), + ); + out.extend(json_escape_bytes(name.as_bytes())); + if let Some(ns) = order + .map(|o| o.namespace.as_str()) + .filter(|s| !s.is_empty()) + { + out.extend_from_slice(b",\"namespace\":"); + out.extend(json_escape_bytes(ns.as_bytes())); + } + out.extend_from_slice(b",\"call_id\":"); + out.extend(json_escape_bytes(call_id.as_bytes())); + out.extend_from_slice(b",\"arguments\":"); + // Clients append argument deltas to the added item's initial value. + if item_status == ITEM_IN_PROGRESS { + out.extend_from_slice(b"\"\""); + } else { + append_json_object_string(out, &tc.arguments); + } + out.push(b'}'); +} + +fn trim_tool_separator_ws(raw: &[u8], start: usize, mut limit: usize) -> usize { + while limit > start && raw[limit - 1].is_ascii_whitespace() { + limit -= 1; + } + limit +} + +pub fn text_stream_safe_limit( + raw: &[u8], + start: usize, + raw_len: usize, + has_tools: bool, + final_: bool, + format: ChatFormat, + orders: &[ToolSchemaOrder], +) -> usize { + if raw_len <= start { + return raw_len; + } + let mut raw_len = raw_len; + if !final_ { + // The tail may be the first half of the close marker cut between two + // tokens: hold it back until the next token proves it (f302be7, + // server_anthropic_stream.c:100-108). + let lit = think_end(format).as_bytes(); + for k in (1..lit.len()).rev() { + if raw_len - start >= k && raw[raw_len - k..raw_len] == lit[..k] { + raw_len -= k; + break; + } + } + if raw_len <= start { + return start; + } + } + let mut limit = raw_len; + if has_tools { + if let Some(rel) = find_tool_start(&raw[start..], format, orders) { + limit = trim_tool_separator_ws(raw, start, start + rel); + return utf8_stream_safe_len(raw, start, limit, true); + } + if !final_ { + while limit > start && raw[limit - 1].is_ascii_whitespace() { + limit -= 1; + } + let max_marker = 80; + let scan = if raw_len - start > max_marker { + raw_len - max_marker + } else { + start + }; + for i in (scan + 1..=raw_len).rev() { + if raw[i - 1] == b'<' { + if i - 1 < limit { + limit = i - 1; + } + break; + } + } + limit = trim_tool_separator_ws(raw, start, limit); + } + } + utf8_stream_safe_len(raw, start, limit, final_) +} + +/// A stray close marker in the body (f302be7, server_anthropic_stream.c:86-97): +/// in non-thinking mode the prompt already closed thinking, yet the artifact +/// sometimes samples one more close marker ahead of its narration. The final +/// parse moves that narration into reasoning; a stream cannot retract bytes it +/// already pushed, so it never pushes the marker itself. A marker after a +/// tool start belongs to the tool block and does not count. +fn text_stream_stray( + raw: &[u8], + pos: usize, + has_tools: bool, + format: ChatFormat, + orders: &[ToolSchemaOrder], +) -> Option { + let rest = raw.get(pos..)?; + let rel = find_substr(rest, 0, think_end(format).as_bytes())?; + if has_tools { + if let Some(trel) = find_tool_start(rest, format, orders) { + if trel < rel { + return None; + } + } + } + Some(pos + rel) +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct ReqTimings { + pub valid: bool, + pub ttft_ms: f64, + pub prefill_ms: f64, + pub decode_ms: f64, + pub prefill_tokens: i32, + pub prefill_cached: i32, + pub decode_tokens: i32, + pub decode_steps: i32, +} + +impl ReqTimings { + pub fn json_suffix(self) -> String { + if !self.valid { + return String::new(); + } + let mut b = format!( + ",\"timings\":{{\"ttft_ms\":{:.1},\"prefill_tokens\":{},\"prefill_cached_tokens\":{}", + self.ttft_ms, self.prefill_tokens, self.prefill_cached + ); + if self.prefill_tokens > 0 && self.prefill_ms > 0.0 { + b.push_str(&format!( + ",\"prefill_tok_s\":{:.1}", + self.prefill_tokens as f64 / (self.prefill_ms / 1e3) + )); + } + if self.decode_tokens > 1 && self.decode_ms > 0.0 { + b.push_str(&format!( + ",\"decode_tok_s\":{:.1}", + (self.decode_tokens - 1) as f64 / (self.decode_ms / 1e3) + )); + } + if self.decode_steps > 0 { + b.push_str(&format!( + ",\"tok_per_step\":{:.2}", + self.decode_tokens as f64 / self.decode_steps as f64 + )); + } + b.push('}'); + b + } +} + +#[derive(Debug, Clone)] +pub struct StreamReq { + pub kind: ReqKind, + pub api: Api, + pub model: String, + pub think_mode: ThinkMode, + pub has_tools: bool, + pub stream: bool, + pub stream_include_usage: bool, + pub reasoning_summary_emit: bool, + pub chat_format: ChatFormat, + pub syntax: ModelSyntax, + pub cache_read_tokens: i32, + pub cache_write_tokens: i32, + pub timings: ReqTimings, + pub tool_orders: Vec, +} + +impl Default for StreamReq { + fn default() -> Self { + Self { + kind: ReqKind::Chat, + api: Api::Openai, + model: "deepseek-v4-flash".into(), + think_mode: ThinkMode::Low, + has_tools: false, + stream: true, + stream_include_usage: false, + reasoning_summary_emit: false, + chat_format: ChatFormat::DeepSeek, + syntax: ModelSyntax::DeepSeek, + cache_read_tokens: 0, + cache_write_tokens: 0, + timings: ReqTimings::default(), + tool_orders: Vec::new(), + } + } +} + +pub struct Writer { + pub created: i64, + pub out: Vec, +} + +impl Writer { + pub fn new(created: i64) -> Self { + Self { + created, + out: Vec::new(), + } + } + + fn put(&mut self, s: &[u8]) { + self.out.extend_from_slice(s); + } + + fn put_str(&mut self, s: &str) { + self.put(s.as_bytes()); + } + + fn esc(&mut self, s: &[u8]) { + self.out.extend(json_escape_bytes(s)); + } + + fn esc_str(&mut self, s: &str) { + self.esc(s.as_bytes()); + } +} + +fn clamp_usage(value: i32, max: i32) -> i32 { + if value < 0 { + 0 + } else if max >= 0 && value > max { + max + } else { + value + } +} + +fn openai_usage(r: &StreamReq, prompt: i32, completion: i32) -> String { + let cached = clamp_usage(r.cache_read_tokens, prompt); + let write = clamp_usage(r.cache_write_tokens, prompt - cached); + format!( + "{{\"prompt_tokens\":{prompt},\"completion_tokens\":{completion},\"total_tokens\":{},\"prompt_tokens_details\":{{\"cached_tokens\":{cached},\"cache_write_tokens\":{write}}}}}", + prompt + completion + ) +} + +fn anthropic_usage(r: &StreamReq, prompt: i32, completion: i32) -> String { + let cache_read = clamp_usage(r.cache_read_tokens, prompt); + let cache_write = clamp_usage(r.cache_write_tokens, prompt - cache_read); + let mut input = prompt - cache_read - cache_write; + if input < 0 { + input = 0; + } + format!( + "{{\"input_tokens\":{input},\"output_tokens\":{completion},\"cache_read_input_tokens\":{cache_read},\"cache_creation_input_tokens\":{cache_write}}}" + ) +} + +fn responses_usage(r: &StreamReq, input: i32, output: i32, reasoning: i32) -> String { + let cached = clamp_usage(r.cache_read_tokens, input); + let write = clamp_usage(r.cache_write_tokens, input - cached); + let reasoning = clamp_usage(reasoning, output); + format!( + "{{\"input_tokens\":{input},\"input_tokens_details\":{{\"cached_tokens\":{cached},\"cache_write_tokens\":{write}}},\"output_tokens\":{output},\"output_tokens_details\":{{\"reasoning_tokens\":{reasoning}}},\"total_tokens\":{}}}", + input + output + ) +} + +pub fn sse_headers(cors: bool) -> Vec { + let mut h = + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nCache-Control: no-cache\r\n" + .to_vec(); + if cors { + h.extend_from_slice(cors_headers().as_bytes()); + } + h.extend_from_slice(b"Connection: close\r\n\r\n"); + h +} + +pub fn sse_chunk( + w: &mut Writer, + r: &StreamReq, + id: &str, + text: Option<&[u8]>, + finish: Option<&str>, +) { + if r.kind == ReqKind::Chat { + w.put_str(&format!( + "data: {{\"id\":\"{id}\",\"object\":\"chat.completion.chunk\",\"created\":{},\"model\":", + w.created + )); + w.esc_str(&r.model); + w.put_str(",\"choices\":[{\"index\":0,\"delta\":"); + if let Some(t) = text { + w.put_str("{\"content\":"); + w.esc(t); + w.put_str("}"); + } else if finish.is_some() { + w.put_str("{}"); + } else { + w.put_str("{\"role\":\"assistant\"}"); + } + w.put_str(",\"finish_reason\":"); + if let Some(f) = finish { + w.esc_str(f); + } else { + w.put_str("null"); + } + w.put_str("}]}\n\n"); + } else { + w.put_str(&format!( + "data: {{\"id\":\"{id}\",\"object\":\"text_completion\",\"created\":{},\"model\":", + w.created + )); + w.esc_str(&r.model); + w.put_str(",\"choices\":[{\"text\":"); + w.esc(text.unwrap_or(b"")); + w.put_str(",\"index\":0,\"finish_reason\":"); + if let Some(f) = finish { + w.esc_str(f); + } else { + w.put_str("null"); + } + w.put_str("}]}\n\n"); + } +} + +fn sse_usage_chunk(w: &mut Writer, r: &StreamReq, id: &str, prompt: i32, completion: i32) { + if !r.stream_include_usage { + return; + } + let obj = if r.kind == ReqKind::Chat { + "chat.completion.chunk" + } else { + "text_completion" + }; + w.put_str(&format!( + "data: {{\"id\":\"{id}\",\"object\":\"{obj}\",\"created\":{},\"model\":", + w.created + )); + w.esc_str(&r.model); + w.put_str(",\"choices\":[],\"usage\":"); + w.put_str(&openai_usage(r, prompt, completion)); + w.put_str("}\n\n"); +} + +pub fn sse_done(w: &mut Writer, r: &StreamReq, id: &str, prompt: i32, completion: i32) { + sse_usage_chunk(w, r, id, prompt, completion); + w.put_str("data: [DONE]\n\n"); +} + +fn sse_event(w: &mut Writer, event: &str, data: &[u8]) { + w.put_str("event: "); + w.put_str(event); + w.put_str("\ndata: "); + w.put(data); + w.put_str("\n\n"); +} + +fn sse_chat_delta_n(w: &mut Writer, r: &StreamReq, id: &str, field: &str, text: &[u8]) { + if text.is_empty() { + return; + } + w.put_str(&format!( + "data: {{\"id\":\"{id}\",\"object\":\"chat.completion.chunk\",\"created\":{},\"model\":", + w.created + )); + w.esc_str(&r.model); + w.put_str(",\"choices\":[{\"index\":0,\"delta\":{"); + w.esc_str(field); + w.put_str(":"); + w.esc(text); + w.put_str("},\"finish_reason\":null}]}\n\n"); +} + +struct OpenaiToolSink<'a> { + w: &'a mut Writer, + r: &'a StreamReq, + job: &'a str, +} + +impl ToolSink for OpenaiToolSink<'_> { + fn start_invoke(&mut self, index: i32, id: &str, name: &[u8]) -> bool { + self.w.put_str(&format!( + "data: {{\"id\":\"{}\",\"object\":\"chat.completion.chunk\",\"created\":{},\"model\":", + self.job, self.w.created + )); + self.w.esc_str(&self.r.model); + self.w.put_str(&format!( + ",\"choices\":[{{\"index\":0,\"delta\":{{\"tool_calls\":[{{\"index\":{index},\"id\":" + )); + self.w.esc_str(id); + self.w + .put_str(",\"type\":\"function\",\"function\":{\"name\":"); + self.w.esc(name); + self.w + .put_str(",\"arguments\":\"\"}}]},\"finish_reason\":null}]}\n\n"); + true + } + + fn args_fragment(&mut self, index: i32, text: &[u8]) -> bool { + if text.is_empty() { + return true; + } + self.w.put_str(&format!( + "data: {{\"id\":\"{}\",\"object\":\"chat.completion.chunk\",\"created\":{},\"model\":", + self.job, self.w.created + )); + self.w.esc_str(&self.r.model); + self.w.put_str(&format!( + ",\"choices\":[{{\"index\":0,\"delta\":{{\"tool_calls\":[{{\"index\":{index},\"function\":{{\"arguments\":" + )); + self.w.esc(text); + self.w.put_str("}}]},\"finish_reason\":null}]}\n\n"); + true + } + + fn close_invoke(&mut self, _index: i32) -> bool { + true + } +} + +struct AnthropicToolSink<'a> { + w: &'a mut Writer, + next_index: &'a mut i32, + open_block: &'a mut AnthBlock, + job: &'a str, +} + +impl ToolSink for AnthropicToolSink<'_> { + fn start_invoke(&mut self, _index: i32, id: &str, name: &[u8]) -> bool { + if *self.open_block == AnthBlock::Tool { + return true; + } + if *self.open_block != AnthBlock::None { + return false; + } + let mut start = format!( + "{{\"type\":\"content_block_start\",\"index\":{},\"content_block\":{{\"type\":\"tool_use\",\"id\":", + *self.next_index + ) + .into_bytes(); + start.extend(json_escape_bytes(id.as_bytes())); + start.extend_from_slice(b",\"name\":"); + start.extend(json_escape_bytes(name)); + start.extend_from_slice(b",\"input\":{}}}"); + sse_event(self.w, "content_block_start", &start); + *self.open_block = AnthBlock::Tool; + true + } + + fn args_fragment(&mut self, _index: i32, text: &[u8]) -> bool { + if text.is_empty() { + return true; + } + let mut b = format!( + "{{\"type\":\"content_block_delta\",\"index\":{},\"delta\":{{\"type\":\"input_json_delta\",\"partial_json\":", + *self.next_index + ) + .into_bytes(); + b.extend(json_escape_bytes(text)); + b.extend_from_slice(b"}}"); + sse_event(self.w, "content_block_delta", &b); + true + } + + fn close_invoke(&mut self, _index: i32) -> bool { + if *self.open_block == AnthBlock::None { + return true; + } + if *self.open_block == AnthBlock::Thinking { + let mut b = format!( + "{{\"type\":\"content_block_delta\",\"index\":{},\"delta\":{{\"type\":\"signature_delta\",\"signature\":", + *self.next_index + ) + .into_bytes(); + b.extend(json_escape_bytes(self.job.as_bytes())); + b.extend_from_slice(b"}}"); + sse_event(self.w, "content_block_delta", &b); + } + let stop = format!( + "{{\"type\":\"content_block_stop\",\"index\":{}}}", + *self.next_index + ); + sse_event(self.w, "content_block_stop", stop.as_bytes()); + *self.open_block = AnthBlock::None; + *self.next_index += 1; + true + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OpenaiMode { + Thinking, + Text, + Tool, + Suppress, +} + +#[derive(Debug)] +pub struct OpenaiStream { + channels: Option, + iquest: Option, + pub mode: OpenaiMode, + pub emit_pos: usize, + pub active: bool, + pub checked_think_prefix: bool, + pub tool: DsmlToolStream, +} + +pub fn openai_stream_start(r: &StreamReq) -> OpenaiStream { + OpenaiStream { + channels: Channels::for_format(r.chat_format), + iquest: (r.chat_format == ChatFormat::IQuestQ1 && r.has_tools) + .then(iquest::Channels::default), + mode: if think_mode_enabled(r.think_mode) { + OpenaiMode::Thinking + } else { + OpenaiMode::Text + }, + emit_pos: 0, + active: true, + checked_think_prefix: false, + tool: DsmlToolStream::with_prefix("call_"), + } +} + +pub fn openai_sse_stream_update( + w: &mut Writer, + r: &StreamReq, + id: &str, + st: &mut OpenaiStream, + raw: &[u8], + final_: bool, +) -> bool { + if !st.active { + return true; + } + if let Some(channels) = &mut st.channels { + let flush = if final_ { Flush::Final } else { Flush::More }; + while let Some(fragment) = channels.advance(raw, flush) { + let field = match fragment.channel { + Channel::Text => "content", + Channel::Thinking => "reasoning_content", + _ => continue, + }; + if !fragment.body.is_empty() { + sse_chat_delta_n(w, r, id, field, &raw[fragment.body]); + } + } + return true; + } + if st.mode == OpenaiMode::Thinking && !st.checked_think_prefix { + match qwen_tool_head(r, raw, if final_ { Flush::Final } else { Flush::More }) { + ToolHead::Wait => return true, + ToolHead::Present => st.mode = OpenaiMode::Text, + ToolHead::Absent => {} + } + } + if st.mode == OpenaiMode::Thinking { + if !st.checked_think_prefix { + let open = think_start(r.chat_format).as_bytes(); + if raw.len() < open.len() && raw == &open[..raw.len()] && !final_ { + return true; + } + if raw.len() >= open.len() && raw.starts_with(open) { + st.emit_pos = open.len(); + } + st.checked_think_prefix = true; + } + let close_s = think_end(r.chat_format); + let close = find_substr(raw, st.emit_pos, close_s.as_bytes()); + let limit = if let Some(c) = close { + c + } else if final_ { + utf8_stream_safe_len(raw, st.emit_pos, raw.len(), true) + } else { + let hold = close_s.len() - 1; + let lim = if raw.len() > hold { + raw.len() - hold + } else { + st.emit_pos + }; + utf8_stream_safe_len(raw, st.emit_pos, lim, false) + }; + if limit > st.emit_pos { + sse_chat_delta_n(w, r, id, "reasoning_content", &raw[st.emit_pos..limit]); + st.emit_pos = limit; + } + if let Some(c) = close { + st.emit_pos = c + close_s.len(); + st.mode = OpenaiMode::Text; + } else if final_ { + st.mode = OpenaiMode::Suppress; + return true; + } else { + return true; + } + } + if st.mode == OpenaiMode::Text && st.iquest.is_some() { + let channels = st.iquest.as_mut().unwrap(); + channels.pos = channels.pos.max(st.emit_pos); + while let Some(fragment) = channels.advance(raw, final_, &r.tool_orders) { + match fragment { + iquest::Fragment::Text(span) => sse_chat_delta_n(w, r, id, "content", &raw[span]), + iquest::Fragment::Call(index, call) => { + let call_id = st.tool.id_at(index); + let mut sink = OpenaiToolSink { w, r, job: id }; + if !sink.start_invoke(index, &call_id, call.name.as_bytes()) + || !sink.args_fragment(index, call.arguments.as_bytes()) + || !sink.close_invoke(index) + { + return false; + } + st.tool.emitted_any = true; + st.tool.index = channels.calls; + } + } + } + st.emit_pos = channels.pos; + return true; + } + if st.mode == OpenaiMode::Text { + // Never push a stray close marker; push the narration before it + // (f302be7, server_anthropic_stream.c:200-206). + while let Some(cut) = + text_stream_stray(raw, st.emit_pos, r.has_tools, r.chat_format, &r.tool_orders) + { + if cut > st.emit_pos { + sse_chat_delta_n(w, r, id, "content", &raw[st.emit_pos..cut]); + } + st.emit_pos = cut + think_end(r.chat_format).len(); + } + let tool = if r.has_tools { + find_tool_start(&raw[st.emit_pos..], r.chat_format, &r.tool_orders) + .map(|rel| st.emit_pos + rel) + } else { + None + }; + let limit = text_stream_safe_limit( + raw, + st.emit_pos, + raw.len(), + r.has_tools, + final_, + r.chat_format, + &r.tool_orders, + ); + if limit > st.emit_pos { + sse_chat_delta_n(w, r, id, "content", &raw[st.emit_pos..limit]); + st.emit_pos = limit; + } + if let Some(pos) = tool { + st.emit_pos = pos; + if st.tool.init(raw, st.emit_pos) { + st.mode = OpenaiMode::Tool; + } else { + st.mode = OpenaiMode::Suppress; + } + } else if final_ { + st.mode = OpenaiMode::Suppress; + } + } + if st.mode == OpenaiMode::Tool { + let mut sink = OpenaiToolSink { w, r, job: id }; + if !st.tool.update(&mut sink, raw) { + return false; + } + if !st.tool.active { + st.mode = OpenaiMode::Suppress; + } + } + true +} + +pub fn openai_sse_finish_live( + w: &mut Writer, + r: &StreamReq, + id: &str, + st: &mut OpenaiStream, + raw: &[u8], + finish: &str, + prompt: i32, + completion: i32, + calls: &[ToolCall], +) -> bool { + if !openai_sse_stream_update(w, r, id, st, raw, true) { + return false; + } + if !calls.is_empty() && !st.tool.emitted_any { + w.put_str(&format!( + "data: {{\"id\":\"{id}\",\"object\":\"chat.completion.chunk\",\"created\":{},\"model\":", + w.created + )); + w.esc_str(&r.model); + w.put_str(",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":"); + let mut deltas = Vec::new(); + append_tool_call_deltas_json(&mut deltas, calls, id); + w.put(&deltas); + w.put_str("},\"finish_reason\":null}]}\n\n"); + } else if let Some(partial) = st.tool.partial_invoke_index() { + // The live stream put this call's name on the wire and then failed + // on the V4.1 artifact's mixed spellings, before its arguments + // (docs 6.23). Complete that index with an arguments-only delta + // (the client merges deltas by index) and emit any later calls in + // full; the earlier ones were streamed complete. + let idx = partial.max(0) as usize; + if let Some(first) = calls.get(idx) { + let mut deltas = Vec::new(); + deltas.push(b'['); + deltas.extend_from_slice( + format!("{{\"index\":{idx},\"function\":{{\"arguments\":").as_bytes(), + ); + append_json_object_string(&mut deltas, &first.arguments); + deltas.extend_from_slice(b"}}"); + for (i, tc) in calls.iter().enumerate().skip(idx + 1) { + deltas.push(b','); + append_tool_call_delta_entry(&mut deltas, i, tc, id); + } + deltas.push(b']'); + w.put_str(&format!( + "data: {{\"id\":\"{id}\",\"object\":\"chat.completion.chunk\",\"created\":{},\"model\":", + w.created + )); + w.esc_str(&r.model); + w.put_str(",\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":"); + w.put(&deltas); + w.put_str("},\"finish_reason\":null}]}\n\n"); + } + } + w.put_str(&format!( + "data: {{\"id\":\"{id}\",\"object\":\"chat.completion.chunk\",\"created\":{},\"model\":", + w.created + )); + w.esc_str(&r.model); + w.put_str(",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":"); + w.esc_str(finish); + w.put_str("}]}\n\n"); + sse_done(w, r, id, prompt, completion); + true +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AnthMode { + Thinking, + Text, + Tool, + Suppress, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AnthBlock { + None, + Thinking, + Text, + Tool, +} + +#[derive(Debug)] +pub struct AnthropicStream { + channels: Option, + iquest: Option, + mode: AnthMode, + open_block: AnthBlock, + next_index: i32, + emit_pos: usize, + active: bool, + checked_think_prefix: bool, + sent_thinking: bool, + sent_text: bool, + pub tool: DsmlToolStream, +} + +fn anthropic_sse_open_block(w: &mut Writer, st: &mut AnthropicStream, ty: AnthBlock) -> bool { + if st.open_block == ty { + return true; + } + if st.open_block != AnthBlock::None { + return false; + } + let body = if ty == AnthBlock::Thinking { + format!( + "{{\"type\":\"content_block_start\",\"index\":{},\"content_block\":{{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}}}", + st.next_index + ) + } else { + format!( + "{{\"type\":\"content_block_start\",\"index\":{},\"content_block\":{{\"type\":\"text\",\"text\":\"\"}}}}", + st.next_index + ) + }; + sse_event(w, "content_block_start", body.as_bytes()); + st.open_block = ty; + true +} + +fn anthropic_sse_delta_live(w: &mut Writer, st: &AnthropicStream, ty: AnthBlock, text: &[u8]) { + if text.is_empty() { + return; + } + let mut b = if ty == AnthBlock::Thinking { + format!( + "{{\"type\":\"content_block_delta\",\"index\":{},\"delta\":{{\"type\":\"thinking_delta\",\"thinking\":", + st.next_index + ) + .into_bytes() + } else { + format!( + "{{\"type\":\"content_block_delta\",\"index\":{},\"delta\":{{\"type\":\"text_delta\",\"text\":", + st.next_index + ) + .into_bytes() + }; + b.extend(json_escape_bytes(text)); + b.extend_from_slice(b"}}"); + sse_event(w, "content_block_delta", &b); +} + +fn anthropic_sse_close_block_live(w: &mut Writer, id: &str, st: &mut AnthropicStream) -> bool { + if st.open_block == AnthBlock::None { + return true; + } + if st.open_block == AnthBlock::Thinking { + let mut b = format!( + "{{\"type\":\"content_block_delta\",\"index\":{},\"delta\":{{\"type\":\"signature_delta\",\"signature\":", + st.next_index + ) + .into_bytes(); + b.extend(json_escape_bytes(id.as_bytes())); + b.extend_from_slice(b"}}"); + sse_event(w, "content_block_delta", &b); + } + let stop = format!( + "{{\"type\":\"content_block_stop\",\"index\":{}}}", + st.next_index + ); + sse_event(w, "content_block_stop", stop.as_bytes()); + st.open_block = AnthBlock::None; + st.next_index += 1; + true +} + +pub fn anthropic_sse_start_live( + w: &mut Writer, + r: &StreamReq, + id: &str, + prompt_tokens: i32, +) -> AnthropicStream { + let mut msg = format!( + "{{\"type\":\"message_start\",\"message\":{{\"id\":\"{id}\",\"type\":\"message\",\"role\":\"assistant\",\"model\":" + ) + .into_bytes(); + msg.extend(json_escape_bytes(r.model.as_bytes())); + msg.extend_from_slice( + b",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":", + ); + msg.extend_from_slice(anthropic_usage(r, prompt_tokens, 0).as_bytes()); + msg.extend_from_slice(b"}}"); + sse_event(w, "message_start", &msg); + AnthropicStream { + channels: Channels::for_format(r.chat_format), + iquest: (r.chat_format == ChatFormat::IQuestQ1 && r.has_tools) + .then(iquest::Channels::default), + mode: if think_mode_enabled(r.think_mode) { + AnthMode::Thinking + } else { + AnthMode::Text + }, + open_block: AnthBlock::None, + next_index: 0, + emit_pos: 0, + active: true, + checked_think_prefix: false, + sent_thinking: false, + sent_text: false, + tool: DsmlToolStream::with_prefix("toolu_"), + } +} + +pub fn anthropic_sse_stream_update( + w: &mut Writer, + r: &StreamReq, + id: &str, + st: &mut AnthropicStream, + raw: &[u8], + final_: bool, +) -> bool { + if !st.active { + return true; + } + if st.channels.is_some() { + let flush = if final_ { Flush::Final } else { Flush::More }; + while let Some(fragment) = st.channels.as_mut().unwrap().advance(raw, flush) { + let block = match fragment.channel { + Channel::Thinking => AnthBlock::Thinking, + Channel::Text => AnthBlock::Text, + _ => AnthBlock::None, + }; + if st.open_block != block && !anthropic_sse_close_block_live(w, id, st) { + return false; + } + if block != AnthBlock::None && !fragment.body.is_empty() { + if !anthropic_sse_open_block(w, st, block) { + return false; + } + anthropic_sse_delta_live(w, st, block, &raw[fragment.body]); + st.sent_thinking |= block == AnthBlock::Thinking; + st.sent_text |= block == AnthBlock::Text; + } + if (fragment.closed || final_) && !anthropic_sse_close_block_live(w, id, st) { + return false; + } + } + if final_ && !anthropic_sse_close_block_live(w, id, st) { + return false; + } + return true; + } + if st.mode == AnthMode::Thinking && !st.checked_think_prefix { + match qwen_tool_head(r, raw, if final_ { Flush::Final } else { Flush::More }) { + ToolHead::Wait => return true, + ToolHead::Present => st.mode = AnthMode::Text, + ToolHead::Absent => {} + } + } + if st.mode == AnthMode::Thinking { + if !st.checked_think_prefix { + let open = think_start(r.chat_format).as_bytes(); + if raw.len() < open.len() && raw == &open[..raw.len()] && !final_ { + return true; + } + if raw.len() >= open.len() && raw.starts_with(open) { + st.emit_pos = open.len(); + } + st.checked_think_prefix = true; + } + let close_s = think_end(r.chat_format); + let close = find_substr(raw, st.emit_pos, close_s.as_bytes()); + let limit = if let Some(c) = close { + c + } else if final_ { + utf8_stream_safe_len(raw, st.emit_pos, raw.len(), true) + } else { + let hold = close_s.len() - 1; + let lim = if raw.len() > hold { + raw.len() - hold + } else { + st.emit_pos + }; + utf8_stream_safe_len(raw, st.emit_pos, lim, false) + }; + if limit > st.emit_pos { + if !anthropic_sse_open_block(w, st, AnthBlock::Thinking) { + return false; + } + anthropic_sse_delta_live(w, st, AnthBlock::Thinking, &raw[st.emit_pos..limit]); + st.sent_thinking = true; + st.emit_pos = limit; + } + if close.is_some() || final_ { + if !anthropic_sse_close_block_live(w, id, st) { + return false; + } + if let Some(c) = close { + st.emit_pos = c + close_s.len(); + st.mode = AnthMode::Text; + } else { + st.mode = AnthMode::Suppress; + return true; + } + } else { + return true; + } + } + if st.mode == AnthMode::Text && st.iquest.is_some() { + let mut channels = st.iquest.take().unwrap(); + channels.pos = channels.pos.max(st.emit_pos); + while let Some(fragment) = channels.advance(raw, final_, &r.tool_orders) { + match fragment { + iquest::Fragment::Text(span) => { + if !anthropic_sse_open_block(w, st, AnthBlock::Text) { + return false; + } + anthropic_sse_delta_live(w, st, AnthBlock::Text, &raw[span]); + st.sent_text = true; + } + iquest::Fragment::Call(index, call) => { + if !anthropic_sse_close_block_live(w, id, st) { + return false; + } + let call_id = st.tool.id_at(index); + let mut sink = AnthropicToolSink { + w, + next_index: &mut st.next_index, + open_block: &mut st.open_block, + job: id, + }; + if !sink.start_invoke(index, &call_id, call.name.as_bytes()) + || !sink.args_fragment(index, call.arguments.as_bytes()) + || !sink.close_invoke(index) + { + return false; + } + st.tool.emitted_any = true; + st.tool.index = channels.calls; + } + } + } + st.emit_pos = channels.pos; + st.iquest = Some(channels); + if final_ && !anthropic_sse_close_block_live(w, id, st) { + return false; + } + return true; + } + if st.mode == AnthMode::Text { + // Never push a stray close marker; push the narration before it + // (f302be7, server_anthropic_stream.c:200-206). + while let Some(cut) = + text_stream_stray(raw, st.emit_pos, r.has_tools, r.chat_format, &r.tool_orders) + { + if cut > st.emit_pos { + if !anthropic_sse_open_block(w, st, AnthBlock::Text) { + return false; + } + anthropic_sse_delta_live(w, st, AnthBlock::Text, &raw[st.emit_pos..cut]); + st.sent_text = true; + } + st.emit_pos = cut + think_end(r.chat_format).len(); + } + let tool = if r.has_tools { + find_tool_start(&raw[st.emit_pos..], r.chat_format, &r.tool_orders) + .map(|rel| st.emit_pos + rel) + } else { + None + }; + let limit = text_stream_safe_limit( + raw, + st.emit_pos, + raw.len(), + r.has_tools, + final_, + r.chat_format, + &r.tool_orders, + ); + if limit > st.emit_pos { + if !anthropic_sse_open_block(w, st, AnthBlock::Text) { + return false; + } + anthropic_sse_delta_live(w, st, AnthBlock::Text, &raw[st.emit_pos..limit]); + st.sent_text = true; + st.emit_pos = limit; + } + if let Some(pos) = tool { + if !anthropic_sse_close_block_live(w, id, st) { + return false; + } + st.emit_pos = pos; + if !final_ && st.tool.init(raw, st.emit_pos) { + st.mode = AnthMode::Tool; + } else { + st.mode = AnthMode::Suppress; + } + } else if final_ { + if !anthropic_sse_close_block_live(w, id, st) { + return false; + } + st.mode = AnthMode::Suppress; + } + } + if st.mode == AnthMode::Tool { + let mut sink = AnthropicToolSink { + w, + next_index: &mut st.next_index, + open_block: &mut st.open_block, + job: id, + }; + if !st.tool.update(&mut sink, raw) { + return false; + } + if !st.tool.active { + st.mode = AnthMode::Suppress; + } + } + true +} + +fn anthropic_stop_reason(finish: &str, matched_stop: Option<&str>) -> &'static str { + if finish == "tool_calls" { + "tool_use" + } else if finish == "length" { + "max_tokens" + } else if matched_stop.map(|s| !s.is_empty()).unwrap_or(false) && finish == "stop" { + "stop_sequence" + } else { + "end_turn" + } +} + +fn append_anthropic_stop_fields(out: &mut Vec, finish: &str, matched_stop: Option<&str>) { + let reason = anthropic_stop_reason(finish, matched_stop); + out.extend_from_slice(b"\"stop_reason\":"); + out.extend(json_escape_bytes(reason.as_bytes())); + out.extend_from_slice(b",\"stop_sequence\":"); + if reason == "stop_sequence" { + out.extend(json_escape_bytes(matched_stop.unwrap_or("").as_bytes())); + } else { + out.extend_from_slice(b"null"); + } +} + +fn anthropic_sse_stop_live( + w: &mut Writer, + finish: &str, + matched_stop: Option<&str>, + completion: i32, +) { + let mut b = b"{\"type\":\"message_delta\",\"delta\":{".to_vec(); + append_anthropic_stop_fields(&mut b, finish, matched_stop); + b.extend_from_slice(format!("}},\"usage\":{{\"output_tokens\":{completion}}}}}").as_bytes()); + sse_event(w, "message_delta", &b); + sse_event(w, "message_stop", b"{\"type\":\"message_stop\"}"); +} + +pub fn anthropic_sse_finish_live( + w: &mut Writer, + r: &StreamReq, + id: &str, + st: &mut AnthropicStream, + raw: &[u8], + finish: &str, + matched_stop: Option<&str>, + completion: i32, + calls: &[ToolCall], +) -> bool { + if !anthropic_sse_stream_update(w, r, id, st, raw, true) { + return false; + } + if st.sent_thinking && !st.sent_text && calls.is_empty() { + if !anthropic_sse_open_block(w, st, AnthBlock::Text) { + return false; + } + if !anthropic_sse_close_block_live(w, id, st) { + return false; + } + } + anthropic_sse_tool_blocks_live(w, id, st, calls); + anthropic_sse_stop_live(w, finish, matched_stop, completion); + true +} + +fn anthropic_sse_tool_blocks_live( + w: &mut Writer, + id: &str, + st: &mut AnthropicStream, + calls: &[ToolCall], +) { + let already = if st.tool.emitted_any { + st.tool.index.max(0) as usize + } else { + 0 + }; + for (i, tc) in calls.iter().enumerate().skip(already) { + let fallback = format!("toolu_{id}_{i}"); + let tool_id = if tc.id.is_empty() { + fallback.as_str() + } else { + tc.id.as_str() + }; + let mut start = format!( + "{{\"type\":\"content_block_start\",\"index\":{},\"content_block\":{{\"type\":\"tool_use\",\"id\":", + st.next_index + ) + .into_bytes(); + start.extend(json_escape_bytes(tool_id.as_bytes())); + start.extend_from_slice(b",\"name\":"); + start.extend(json_escape_bytes(tc.name.as_bytes())); + start.extend_from_slice(b",\"input\":{}}}"); + sse_event(w, "content_block_start", &start); + + let mut delta = format!( + "{{\"type\":\"content_block_delta\",\"index\":{},\"delta\":{{\"type\":\"input_json_delta\",\"partial_json\":", + st.next_index + ) + .into_bytes(); + append_json_object_string(&mut delta, &tc.arguments); + delta.extend_from_slice(b"}}"); + sse_event(w, "content_block_delta", &delta); + + let stop = format!( + "{{\"type\":\"content_block_stop\",\"index\":{}}}", + st.next_index + ); + sse_event(w, "content_block_stop", stop.as_bytes()); + st.next_index += 1; + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RespMode { + Thinking, + Text, + Suppress, +} + +#[derive(Debug)] +pub struct ResponsesStream { + channels: Option, + iquest: Option, + mode: RespMode, + emit_pos: usize, + active: bool, + checked_think_prefix: bool, + reasoning_item_opened: bool, + reasoning_item_closed: bool, + reasoning_summary_started: bool, + reasoning_closed_naturally: bool, + message_item_opened: bool, + message_text_part_open: bool, + message_item_closed: bool, + reasoning_emitted_any: bool, + message_emitted_any: bool, + reasoning_start: usize, + reasoning_end: usize, + message_start: usize, + message_end: usize, + message_tail_start: usize, + message_tail_end: usize, + response_id: String, + reasoning_id: String, + message_id: String, + reasoning_index: i32, + message_index: i32, + next_output_index: i32, + sequence: i32, + created_at: i64, +} + +pub fn responses_stream_init( + r: &StreamReq, + response_id: &str, + reasoning_id: &str, + message_id: &str, +) -> ResponsesStream { + ResponsesStream { + channels: Channels::for_format(r.chat_format), + iquest: (r.chat_format == ChatFormat::IQuestQ1 && r.has_tools) + .then(iquest::Channels::default), + mode: if think_mode_enabled(r.think_mode) { + RespMode::Thinking + } else { + RespMode::Text + }, + emit_pos: 0, + active: false, + checked_think_prefix: false, + reasoning_item_opened: false, + reasoning_item_closed: false, + reasoning_summary_started: false, + reasoning_closed_naturally: false, + message_item_opened: false, + message_text_part_open: false, + message_item_closed: false, + reasoning_emitted_any: false, + message_emitted_any: false, + reasoning_start: 0, + reasoning_end: 0, + message_start: 0, + message_end: 0, + message_tail_start: 0, + message_tail_end: 0, + response_id: response_id.into(), + reasoning_id: reasoning_id.into(), + message_id: message_id.into(), + reasoning_index: -1, + message_index: -1, + next_output_index: 0, + sequence: 0, + created_at: 0, + } +} + +fn responses_sse_emit_event(w: &mut Writer, st: &mut ResponsesStream, body: &[u8]) { + w.put_str("data: "); + if let Some(end) = find_type_close(body) { + w.put(&body[..end]); + w.put_str(&format!(",\"sequence_number\":{}", st.sequence)); + st.sequence += 1; + w.put(&body[end..]); + } else { + w.put(body); + } + w.put_str("\n\n"); +} + +pub(crate) const STREAM_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); + +fn responses_sse_in_progress(w: &mut Writer, r: &StreamReq, st: &mut ResponsesStream) { + let mut b = format!( + "{{\"type\":\"response.in_progress\",\"response\":{{\"id\":\"{}\",\"object\":\"response\",\"created_at\":{},\"status\":\"in_progress\",\"model\":", + st.response_id, st.created_at + ) + .into_bytes(); + b.extend(json_escape_bytes(r.model.as_bytes())); + b.extend_from_slice(b",\"output\":[]}}"); + responses_sse_emit_event(w, st, &b); +} + +pub(crate) fn stream_heartbeat_if_due( + w: &mut Writer, + r: &StreamReq, + responses: Option<&mut ResponsesStream>, + last: &mut Instant, + now: Instant, + comment: &str, +) -> bool { + if !r.stream + || now + .checked_duration_since(*last) + .is_none_or(|elapsed| elapsed < STREAM_HEARTBEAT_INTERVAL) + { + return false; + } + match r.api { + Api::Anthropic => w.put_str("event: ping\ndata: {\"type\": \"ping\"}\n\n"), + Api::Responses => match responses { + Some(st) if st.created_at != 0 => responses_sse_in_progress(w, r, st), + _ => w.put_str(comment), + }, + Api::Openai => w.put_str(comment), + } + *last = now; + true +} + +fn find_type_close(body: &[u8]) -> Option { + if body.first() != Some(&b'{') { + return None; + } + let p = &body[1..]; + if !p.starts_with(b"\"type\":\"") { + return None; + } + let mut q = 1 + 8; + while q < body.len() { + if body[q] == b'\\' && q + 1 < body.len() { + q += 2; + continue; + } + if body[q] == b'"' { + return Some(q + 1); + } + q += 1; + } + None +} + +pub fn responses_sse_created( + w: &mut Writer, + r: &StreamReq, + st: &mut ResponsesStream, + created_at: i64, +) { + st.active = true; + let mut b = format!( + "{{\"type\":\"response.created\",\"response\":{{\"id\":\"{}\",\"object\":\"response\",\"created_at\":{created_at},\"status\":\"in_progress\",\"model\":", + st.response_id + ) + .into_bytes(); + b.extend(json_escape_bytes(r.model.as_bytes())); + b.extend_from_slice(b",\"output\":[]}}"); + responses_sse_emit_event(w, st, &b); + st.created_at = created_at; +} + +pub fn stream_error( + w: &mut Writer, + r: &StreamReq, + responses: Option<&mut ResponsesStream>, + msg: &str, +) { + let sequence = if r.api == Api::Responses { + responses.map_or(0, |st| { + let sequence = st.sequence; + st.sequence += 1; + sequence + }) + } else { + 0 + }; + let surface = match r.api { + Api::Anthropic => crate::route::WireSurface::Anthropic, + Api::Responses => crate::route::WireSurface::Responses, + Api::Openai if r.kind == ReqKind::Completion => crate::route::WireSurface::OpenaiCompletion, + Api::Openai => crate::route::WireSurface::OpenaiChat, + }; + w.put(&wire_stream_error_bytes(surface, msg, sequence)); +} + +fn responses_sse_reasoning_added(w: &mut Writer, st: &mut ResponsesStream) { + let b = format!( + "{{\"type\":\"response.output_item.added\",\"output_index\":{},\"item\":{{\"id\":\"{}\",\"type\":\"reasoning\",\"status\":\"in_progress\",\"summary\":[]}}}}", + st.reasoning_index, st.reasoning_id + ); + responses_sse_emit_event(w, st, b.as_bytes()); +} + +fn responses_sse_reasoning_summary_part_added(w: &mut Writer, st: &mut ResponsesStream) { + let b = format!( + "{{\"type\":\"response.reasoning_summary_part.added\",\"item_id\":\"{}\",\"output_index\":{},\"summary_index\":0,\"part\":{{\"type\":\"summary_text\",\"text\":\"\"}}}}", + st.reasoning_id, st.reasoning_index + ); + responses_sse_emit_event(w, st, b.as_bytes()); +} + +fn responses_sse_reasoning_delta(w: &mut Writer, st: &mut ResponsesStream, text: &[u8]) { + if text.is_empty() { + return; + } + let mut b = format!( + "{{\"type\":\"response.reasoning_summary_text.delta\",\"item_id\":\"{}\",\"output_index\":{},\"summary_index\":0,\"delta\":", + st.reasoning_id, st.reasoning_index + ) + .into_bytes(); + b.extend(json_escape_bytes(text)); + b.push(b'}'); + responses_sse_emit_event(w, st, &b); +} + +fn responses_item_status_for_finish(finish: &str) -> &'static str { + if finish == "length" || finish == "error" { + "incomplete" + } else { + "completed" + } +} + +fn responses_status_for_finish(finish: &str) -> &'static str { + if finish == "length" { + "incomplete" + } else if finish == "error" { + "failed" + } else { + "completed" + } +} + +fn responses_sse_reasoning_done( + w: &mut Writer, + st: &mut ResponsesStream, + raw: &[u8], + _finish: &str, +) -> bool { + let item_status = if st.reasoning_closed_naturally { + "completed" + } else { + "incomplete" + }; + let channel_text = st.channels.as_ref().map(|c| c.text(raw, Channel::Thinking)); + let rtext = if let Some(text) = &channel_text { + text.as_slice() + } else if st.reasoning_end > st.reasoning_start { + &raw[st.reasoning_start..st.reasoning_end] + } else { + b"" + }; + let mut b = format!( + "{{\"type\":\"response.reasoning_summary_text.done\",\"item_id\":\"{}\",\"output_index\":{},\"summary_index\":0,\"text\":", + st.reasoning_id, st.reasoning_index + ) + .into_bytes(); + b.extend(json_escape_bytes(rtext)); + b.push(b'}'); + responses_sse_emit_event(w, st, &b); + if st.reasoning_summary_started { + let mut b = format!( + "{{\"type\":\"response.reasoning_summary_part.done\",\"item_id\":\"{}\",\"output_index\":{},\"summary_index\":0,\"part\":{{\"type\":\"summary_text\",\"text\":", + st.reasoning_id, st.reasoning_index + ) + .into_bytes(); + b.extend(json_escape_bytes(rtext)); + b.extend_from_slice(b"}}"); + responses_sse_emit_event(w, st, &b); + } + let mut b = format!( + "{{\"type\":\"response.output_item.done\",\"output_index\":{},\"item\":{{\"id\":\"{}\",\"type\":\"reasoning\",\"status\":\"{item_status}\",\"summary\":[", + st.reasoning_index, st.reasoning_id + ) + .into_bytes(); + if !rtext.is_empty() { + b.extend_from_slice(b"{\"type\":\"summary_text\",\"text\":"); + b.extend(json_escape_bytes(rtext)); + b.push(b'}'); + } + b.extend_from_slice(b"]}}"); + responses_sse_emit_event(w, st, &b); + true +} + +fn responses_sse_message_added(w: &mut Writer, st: &mut ResponsesStream) { + let b = format!( + "{{\"type\":\"response.output_item.added\",\"output_index\":{},\"item\":{{\"id\":\"{}\",\"type\":\"message\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[]}}}}", + st.message_index, st.message_id + ); + responses_sse_emit_event(w, st, b.as_bytes()); +} + +fn responses_sse_message_text_part_added(w: &mut Writer, st: &mut ResponsesStream) { + let b = format!( + "{{\"type\":\"response.content_part.added\",\"item_id\":\"{}\",\"output_index\":{},\"content_index\":0,\"part\":{{\"type\":\"output_text\",\"text\":\"\",\"annotations\":[]}}}}", + st.message_id, st.message_index + ); + responses_sse_emit_event(w, st, b.as_bytes()); +} + +fn responses_sse_output_text_delta(w: &mut Writer, st: &mut ResponsesStream, text: &[u8]) { + if text.is_empty() { + return; + } + let mut b = format!( + "{{\"type\":\"response.output_text.delta\",\"item_id\":\"{}\",\"output_index\":{},\"content_index\":0,\"delta\":", + st.message_id, st.message_index + ) + .into_bytes(); + b.extend(json_escape_bytes(text)); + b.push(b'}'); + responses_sse_emit_event(w, st, &b); +} + +fn json_escape_fragment(s: &[u8]) -> Vec { + let full = json_escape_bytes(s); + full[1..full.len() - 1].to_vec() +} + +fn responses_message_text_escape_fixed(st: &ResponsesStream, raw: &[u8]) -> Vec { + let mut b = vec![b'"']; + if let Some(channels) = &st.iquest { + b.extend(json_escape_fragment(&channels.text)); + } else if let Some(channels) = &st.channels { + b.extend(json_escape_fragment(&channels.text(raw, Channel::Text))); + } else if st.message_end > st.message_start { + b.extend(json_escape_fragment(&raw[st.message_start..st.message_end])); + } + if st.message_tail_end > st.message_tail_start { + b.extend(json_escape_fragment( + &raw[st.message_tail_start..st.message_tail_end], + )); + } + b.push(b'"'); + b +} + +fn responses_sse_message_done( + w: &mut Writer, + st: &mut ResponsesStream, + raw: &[u8], + finish: &str, +) -> bool { + let item_status = responses_item_status_for_finish(finish); + let text = responses_message_text_escape_fixed(st, raw); + let mut b = format!( + "{{\"type\":\"response.output_text.done\",\"item_id\":\"{}\",\"output_index\":{},\"content_index\":0,\"text\":", + st.message_id, st.message_index + ) + .into_bytes(); + b.extend(&text); + b.push(b'}'); + responses_sse_emit_event(w, st, &b); + + let mut b = format!( + "{{\"type\":\"response.content_part.done\",\"item_id\":\"{}\",\"output_index\":{},\"content_index\":0,\"part\":{{\"type\":\"output_text\",\"text\":", + st.message_id, st.message_index + ) + .into_bytes(); + b.extend(&text); + b.extend_from_slice(b",\"annotations\":[]}}"); + responses_sse_emit_event(w, st, &b); + + let mut b = format!( + "{{\"type\":\"response.output_item.done\",\"output_index\":{},\"item\":{{\"id\":\"{}\",\"type\":\"message\",\"status\":\"{item_status}\",\"role\":\"assistant\",\"content\":[{{\"type\":\"output_text\",\"text\":", + st.message_index, st.message_id + ) + .into_bytes(); + b.extend(&text); + b.extend_from_slice(b",\"annotations\":[]}]}}"); + responses_sse_emit_event(w, st, &b); + true +} + +fn responses_sse_function_calls( + w: &mut Writer, + r: &StreamReq, + st: &mut ResponsesStream, + calls: &[ToolCall], + item_status: &str, +) { + for (i, tc) in calls.iter().enumerate() { + let output_index = st.next_output_index; + st.next_output_index += 1; + let item_id = format!("fc_{}_{i}", st.response_id); + let call_id = if tc.id.is_empty() { + format!("{}_tool_{i}", st.response_id) + } else { + tc.id.clone() + }; + let order = tool_schema_orders_find(&r.tool_orders, &tc.name); + + let mut added = format!( + "{{\"type\":\"response.output_item.added\",\"output_index\":{output_index},\"item\":" + ) + .into_bytes(); + append_responses_function_call_item( + &mut added, + tc, + &item_id, + &call_id, + ITEM_IN_PROGRESS, + &r.tool_orders, + ); + added.push(b'}'); + responses_sse_emit_event(w, st, &added); + + if !responses_tool_call_is_tool_search(tc, order) { + let mut delta = format!( + "{{\"type\":\"response.function_call_arguments.delta\",\"item_id\":\"{item_id}\",\"output_index\":{output_index},\"delta\":" + ) + .into_bytes(); + append_json_object_string(&mut delta, &tc.arguments); + delta.push(b'}'); + responses_sse_emit_event(w, st, &delta); + + let mut done = format!( + "{{\"type\":\"response.function_call_arguments.done\",\"item_id\":\"{item_id}\",\"output_index\":{output_index},\"name\":" + ) + .into_bytes(); + done.extend(json_escape_bytes( + responses_tool_wire_name(tc, order).as_bytes(), + )); + done.extend_from_slice(b",\"arguments\":"); + append_json_object_string(&mut done, &tc.arguments); + done.push(b'}'); + responses_sse_emit_event(w, st, &done); + } + + let mut done = format!( + "{{\"type\":\"response.output_item.done\",\"output_index\":{output_index},\"item\":" + ) + .into_bytes(); + append_responses_function_call_item( + &mut done, + tc, + &item_id, + &call_id, + item_status, + &r.tool_orders, + ); + done.push(b'}'); + responses_sse_emit_event(w, st, &done); + } +} + +fn responses_sse_completed( + w: &mut Writer, + r: &StreamReq, + st: &mut ResponsesStream, + raw: &[u8], + finish: &str, + prompt: i32, + completion: i32, + reasoning_tokens: i32, + created_at: i64, + calls: &[ToolCall], +) { + let event_type = if finish == "error" { + "response.failed" + } else if finish == "length" { + "response.incomplete" + } else { + "response.completed" + }; + let status = responses_status_for_finish(finish); + let item_status = responses_item_status_for_finish(finish); + let mut b = format!( + "{{\"type\":\"{event_type}\",\"response\":{{\"id\":\"{}\",\"object\":\"response\",\"created_at\":{created_at},\"status\":\"{status}\",\"model\":", + st.response_id + ) + .into_bytes(); + b.extend(json_escape_bytes(r.model.as_bytes())); + if event_type == "response.failed" { + b.extend_from_slice( + b",\"error\":{\"code\":\"server_error\",\"message\":\"generation failed\"}", + ); + } else if event_type == "response.incomplete" { + b.extend_from_slice(b",\"incomplete_details\":{\"reason\":\"max_output_tokens\"}"); + } + b.extend_from_slice(b",\"output\":["); + let mut wrote = false; + if st.reasoning_emitted_any { + let reasoning_status = if st.reasoning_closed_naturally { + "completed" + } else { + "incomplete" + }; + let channel_text = st.channels.as_ref().map(|c| c.text(raw, Channel::Thinking)); + let rtext = if let Some(text) = &channel_text { + text.as_slice() + } else if st.reasoning_end > st.reasoning_start { + &raw[st.reasoning_start..st.reasoning_end] + } else { + b"" + }; + b.extend_from_slice( + format!( + "{{\"id\":\"{}\",\"type\":\"reasoning\",\"status\":\"{reasoning_status}\",\"summary\":[", + st.reasoning_id + ) + .as_bytes(), + ); + if !rtext.is_empty() { + b.extend_from_slice(b"{\"type\":\"summary_text\",\"text\":"); + b.extend(json_escape_bytes(rtext)); + b.push(b'}'); + } + b.extend_from_slice(b"]}"); + wrote = true; + } + if st.message_emitted_any { + if wrote { + b.push(b','); + } + b.extend_from_slice( + format!( + "{{\"id\":\"{}\",\"type\":\"message\",\"status\":\"{item_status}\",\"role\":\"assistant\",\"content\":[{{\"type\":\"output_text\",\"text\":", + st.message_id + ) + .as_bytes(), + ); + b.extend(responses_message_text_escape_fixed(st, raw)); + b.extend_from_slice(b",\"annotations\":[]}]}"); + wrote = true; + } + for (i, tc) in calls.iter().enumerate() { + if wrote { + b.push(b','); + } + let fc_id = format!("fc_{}_{i}", st.response_id); + let call_id = if tc.id.is_empty() { + format!("{}_tool_{i}", st.response_id) + } else { + tc.id.clone() + }; + append_responses_function_call_item( + &mut b, + tc, + &fc_id, + &call_id, + item_status, + &r.tool_orders, + ); + wrote = true; + } + b.push(b']'); + b.extend_from_slice(b",\"usage\":"); + b.extend_from_slice(responses_usage(r, prompt, completion, reasoning_tokens).as_bytes()); + b.extend_from_slice(b"}}"); + responses_sse_emit_event(w, st, &b); +} + +pub fn responses_sse_stream_update( + w: &mut Writer, + r: &StreamReq, + st: &mut ResponsesStream, + raw: &[u8], + final_: bool, +) -> bool { + if !st.active { + return true; + } + if st.channels.is_some() { + let flush = if final_ { Flush::Final } else { Flush::More }; + while let Some(fragment) = st.channels.as_mut().unwrap().advance(raw, flush) { + if fragment.channel == Channel::Thinking { + st.reasoning_closed_naturally |= fragment.closed; + } + if fragment.body.is_empty() { + continue; + } + match fragment.channel { + Channel::Thinking if r.reasoning_summary_emit => { + if !st.reasoning_item_opened { + st.reasoning_index = st.next_output_index; + st.next_output_index += 1; + responses_sse_reasoning_added(w, st); + st.reasoning_item_opened = true; + } + if !st.reasoning_summary_started { + responses_sse_reasoning_summary_part_added(w, st); + st.reasoning_summary_started = true; + } + responses_sse_reasoning_delta(w, st, &raw[fragment.body]); + st.reasoning_emitted_any = true; + } + Channel::Text => { + if !st.message_item_opened { + st.message_index = st.next_output_index; + st.next_output_index += 1; + responses_sse_message_added(w, st); + st.message_item_opened = true; + } + if !st.message_text_part_open { + responses_sse_message_text_part_added(w, st); + st.message_text_part_open = true; + } + responses_sse_output_text_delta(w, st, &raw[fragment.body]); + st.message_emitted_any = true; + } + _ => {} + } + } + return true; + } + if st.mode == RespMode::Thinking && !st.checked_think_prefix { + match qwen_tool_head(r, raw, if final_ { Flush::Final } else { Flush::More }) { + ToolHead::Wait => return true, + ToolHead::Present => st.mode = RespMode::Text, + ToolHead::Absent => {} + } + } + let emit_reasoning = r.reasoning_summary_emit; + if st.mode == RespMode::Thinking { + if !st.checked_think_prefix { + let open = think_start(r.chat_format).as_bytes(); + if raw.len() < open.len() && raw == &open[..raw.len()] && !final_ { + return true; + } + if raw.len() >= open.len() && raw.starts_with(open) { + st.emit_pos = open.len(); + } + st.checked_think_prefix = true; + } + let close_s = think_end(r.chat_format); + let close = find_substr(raw, st.emit_pos, close_s.as_bytes()); + let limit = if let Some(c) = close { + c + } else if final_ { + utf8_stream_safe_len(raw, st.emit_pos, raw.len(), true) + } else { + let hold = close_s.len() - 1; + let lim = if raw.len() > hold { + raw.len() - hold + } else { + st.emit_pos + }; + utf8_stream_safe_len(raw, st.emit_pos, lim, false) + }; + if limit > st.emit_pos { + if emit_reasoning { + if !st.reasoning_item_opened { + st.reasoning_index = st.next_output_index; + st.next_output_index += 1; + responses_sse_reasoning_added(w, st); + st.reasoning_item_opened = true; + } + if !st.reasoning_summary_started { + responses_sse_reasoning_summary_part_added(w, st); + st.reasoning_summary_started = true; + } + responses_sse_reasoning_delta(w, st, &raw[st.emit_pos..limit]); + if !st.reasoning_emitted_any { + st.reasoning_start = st.emit_pos; + } + st.reasoning_end = limit; + st.reasoning_emitted_any = true; + } + st.emit_pos = limit; + } + if let Some(c) = close { + st.emit_pos = c + close_s.len(); + st.mode = RespMode::Text; + st.reasoning_closed_naturally = true; + } else if final_ { + st.mode = RespMode::Suppress; + return true; + } else { + return true; + } + } + if st.mode == RespMode::Text && st.iquest.is_some() { + let mut channels = st.iquest.take().unwrap(); + channels.pos = channels.pos.max(st.emit_pos); + while let Some(fragment) = channels.advance(raw, final_, &r.tool_orders) { + let iquest::Fragment::Text(span) = fragment else { + continue; + }; + if !st.message_item_opened { + st.message_index = st.next_output_index; + st.next_output_index += 1; + responses_sse_message_added(w, st); + st.message_item_opened = true; + } + if !st.message_text_part_open { + responses_sse_message_text_part_added(w, st); + st.message_text_part_open = true; + } + responses_sse_output_text_delta(w, st, &raw[span]); + st.message_emitted_any = true; + } + st.emit_pos = channels.pos; + st.iquest = Some(channels); + return true; + } + if st.mode == RespMode::Text { + let limit = text_stream_safe_limit( + raw, + st.emit_pos, + raw.len(), + r.has_tools, + final_, + r.chat_format, + &r.tool_orders, + ); + if limit > st.emit_pos { + if !st.message_item_opened { + st.message_index = st.next_output_index; + st.next_output_index += 1; + responses_sse_message_added(w, st); + st.message_item_opened = true; + } + if !st.message_text_part_open { + responses_sse_message_text_part_added(w, st); + st.message_text_part_open = true; + } + responses_sse_output_text_delta(w, st, &raw[st.emit_pos..limit]); + if !st.message_emitted_any { + st.message_start = st.emit_pos; + } + st.message_end = limit; + st.message_emitted_any = true; + st.emit_pos = limit; + } + if final_ { + st.mode = RespMode::Suppress; + } + } + true +} + +pub fn responses_sse_finish_live( + w: &mut Writer, + r: &StreamReq, + st: &mut ResponsesStream, + raw: &[u8], + finish: &str, + prompt: i32, + completion: i32, + reasoning_tokens: i32, + created_at: i64, + calls: &[ToolCall], +) -> bool { + if !responses_sse_stream_update(w, r, st, raw, true) { + return false; + } + if st.reasoning_end > raw.len() { + st.reasoning_end = raw.len(); + } + if st.reasoning_start > st.reasoning_end { + st.reasoning_start = st.reasoning_end; + } + if st.message_end > raw.len() { + st.message_end = raw.len(); + } + if st.message_start > st.message_end { + st.message_start = st.message_end; + } + if st.reasoning_item_opened && !st.reasoning_item_closed { + if !responses_sse_reasoning_done(w, st, raw, finish) { + return false; + } + st.reasoning_item_closed = true; + } + if st.message_item_opened && !st.message_item_closed { + if !responses_sse_message_done(w, st, raw, finish) { + return false; + } + st.message_item_closed = true; + } + responses_sse_function_calls(w, r, st, calls, responses_item_status_for_finish(finish)); + responses_sse_completed( + w, + r, + st, + raw, + finish, + prompt, + completion, + reasoning_tokens, + created_at, + calls, + ); + true +} + +fn append_anthropic_content( + out: &mut Vec, + text: &[u8], + reasoning: &[u8], + calls: &[ToolCall], + id_prefix: &str, +) { + out.push(b'['); + let mut wrote = false; + let mut wrote_after_thinking = false; + if !reasoning.is_empty() { + out.extend_from_slice(b"{\"type\":\"thinking\",\"thinking\":"); + out.extend(json_escape_bytes(reasoning)); + out.extend_from_slice(b",\"signature\":\"\"}"); + wrote = true; + } + if !text.is_empty() { + if wrote { + out.push(b','); + } + out.extend_from_slice(b"{\"type\":\"text\",\"text\":"); + out.extend(json_escape_bytes(text)); + out.push(b'}'); + wrote = true; + wrote_after_thinking = true; + } + for (i, tc) in calls.iter().enumerate() { + if wrote { + out.push(b','); + } + append_anthropic_tool_use(out, tc, id_prefix, i); + wrote = true; + wrote_after_thinking = true; + } + if !wrote || (!reasoning.is_empty() && !wrote_after_thinking) { + if wrote { + out.push(b','); + } + out.extend_from_slice(b"{\"type\":\"text\",\"text\":\"\"}"); + } + out.push(b']'); +} + +fn append_token_ids(out: &mut Vec, ids: &[i32]) { + if ids.is_empty() { + return; + } + out.extend_from_slice(b",\"token_ids\":["); + for (index, id) in ids.iter().enumerate() { + if index > 0 { + out.push(b','); + } + out.extend(id.to_string().as_bytes()); + } + out.push(b']'); +} + +pub fn final_response( + r: &StreamReq, + id: &str, + text: &[u8], + reasoning: Option<&[u8]>, + finish: &str, + prompt: i32, + completion: i32, + created: i64, + cors: bool, + calls: &[ToolCall], + token_ids: &[i32], +) -> Vec { + let reasoning = reasoning.unwrap_or(b""); + let mut b = Vec::new(); + if r.kind == ReqKind::Chat { + b.extend( + format!( + "{{\"id\":\"{id}\",\"object\":\"chat.completion\",\"created\":{created},\"model\":" + ) + .as_bytes(), + ); + b.extend(json_escape_bytes(r.model.as_bytes())); + b.extend_from_slice( + b",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":", + ); + b.extend(json_escape_bytes(text)); + if !reasoning.is_empty() { + b.extend_from_slice(b",\"reasoning_content\":"); + b.extend(json_escape_bytes(reasoning)); + } + if !calls.is_empty() { + b.extend_from_slice(b",\"tool_calls\":"); + append_tool_calls_json(&mut b, calls, id); + } + b.extend_from_slice(b"},\"finish_reason\":"); + b.extend(json_escape_bytes(finish.as_bytes())); + b.extend_from_slice(b"}]"); + append_token_ids(&mut b, token_ids); + b.extend_from_slice(b",\"usage\":"); + } else { + b.extend( + format!( + "{{\"id\":\"{id}\",\"object\":\"text_completion\",\"created\":{created},\"model\":" + ) + .as_bytes(), + ); + b.extend(json_escape_bytes(r.model.as_bytes())); + b.extend_from_slice(b",\"choices\":[{\"text\":"); + b.extend(json_escape_bytes(text)); + b.extend_from_slice(b",\"index\":0,\"finish_reason\":"); + b.extend(json_escape_bytes(finish.as_bytes())); + b.extend_from_slice(b"}]"); + append_token_ids(&mut b, token_ids); + b.extend_from_slice(b",\"usage\":"); + } + b.extend_from_slice(openai_usage(r, prompt, completion).as_bytes()); + b.extend_from_slice(r.timings.json_suffix().as_bytes()); + b.extend_from_slice(b"}\n"); + let body = utf8_body(b); + http_response_bytes(200, Some("application/json"), None, cors, &body) +} + +pub fn anthropic_final_response( + r: &StreamReq, + id: &str, + text: &[u8], + reasoning: Option<&[u8]>, + finish: &str, + matched_stop: Option<&str>, + prompt: i32, + completion: i32, + cors: bool, + calls: &[ToolCall], +) -> Vec { + let reasoning = reasoning.unwrap_or(b""); + let mut b = format!("{{\"id\":\"{id}\",\"type\":\"message\",\"role\":\"assistant\",\"model\":") + .into_bytes(); + b.extend(json_escape_bytes(r.model.as_bytes())); + b.extend_from_slice(b",\"content\":"); + append_anthropic_content(&mut b, text, reasoning, calls, id); + b.push(b','); + append_anthropic_stop_fields(&mut b, finish, matched_stop); + b.extend_from_slice(b",\"usage\":"); + b.extend_from_slice(anthropic_usage(r, prompt, completion).as_bytes()); + b.extend_from_slice(r.timings.json_suffix().as_bytes()); + b.extend_from_slice(b"}\n"); + let body = utf8_body(b); + http_response_bytes(200, Some("application/json"), None, cors, &body) +} + +/// Think-block completeness. Separate from the finish cause. +pub enum ThinkBlock { + Open, + Closed, +} + +pub fn responses_final_response( + r: &StreamReq, + text: &[u8], + reasoning: Option<&[u8]>, + finish: &str, + think: ThinkBlock, + prompt: i32, + completion: i32, + reasoning_tokens: i32, + created: i64, + cors: bool, + response_id: &str, + reasoning_id: &str, + message_id: &str, + calls: &[ToolCall], +) -> Vec { + let reasoning = reasoning.unwrap_or(b""); + let status = responses_status_for_finish(finish); + let item_status = responses_item_status_for_finish(finish); + // An open block stays incomplete even when the cause is stop. + let reasoning_status = match think { + ThinkBlock::Open => "incomplete", + ThinkBlock::Closed => item_status, + }; + let mut b = format!( + "{{\"id\":\"{response_id}\",\"object\":\"response\",\"created_at\":{created},\"status\":\"{status}\",\"model\":" + ) + .into_bytes(); + b.extend(json_escape_bytes(r.model.as_bytes())); + if finish == "error" { + b.extend_from_slice( + b",\"error\":{\"code\":\"server_error\",\"message\":\"generation failed\"}", + ); + } else if finish == "length" { + b.extend_from_slice(b",\"incomplete_details\":{\"reason\":\"max_output_tokens\"}"); + } + b.extend_from_slice(b",\"output\":["); + let mut wrote = false; + if !reasoning.is_empty() && r.reasoning_summary_emit { + b.extend_from_slice( + format!( + "{{\"id\":\"{reasoning_id}\",\"type\":\"reasoning\",\"status\":\"{reasoning_status}\",\"summary\":[{{\"type\":\"summary_text\",\"text\":" + ) + .as_bytes(), + ); + b.extend(json_escape_bytes(reasoning)); + b.extend_from_slice(b"}]}"); + wrote = true; + } + if !text.is_empty() { + if wrote { + b.push(b','); + } + b.extend_from_slice( + format!( + "{{\"id\":\"{message_id}\",\"type\":\"message\",\"status\":\"{item_status}\",\"role\":\"assistant\",\"content\":[{{\"type\":\"output_text\",\"text\":" + ) + .as_bytes(), + ); + b.extend(json_escape_bytes(text)); + b.extend_from_slice(b",\"annotations\":[]}]}"); + wrote = true; + } + for (i, tc) in calls.iter().enumerate() { + if wrote { + b.push(b','); + } + let fc_id = format!("fc_{response_id}_{i}"); + let call_id = if tc.id.is_empty() { + format!("{response_id}_tool_{i}") + } else { + tc.id.clone() + }; + append_responses_function_call_item( + &mut b, + tc, + &fc_id, + &call_id, + item_status, + &r.tool_orders, + ); + wrote = true; + } + b.push(b']'); + b.extend_from_slice(b",\"usage\":"); + b.extend_from_slice(responses_usage(r, prompt, completion, reasoning_tokens).as_bytes()); + b.extend_from_slice(r.timings.json_suffix().as_bytes()); + b.push(b'}'); + let body = utf8_body(b); + http_response_bytes(200, Some("application/json"), None, cors, &body) +} + +fn find_substr(hay: &[u8], start: usize, needle: &[u8]) -> Option { + if start > hay.len() { + return None; + } + hay[start..] + .windows(needle.len()) + .position(|w| w == needle) + .map(|i| start + i) +} + +pub fn project_openai_chat_thinking(created: i64) -> Vec { + let mut r = StreamReq::default(); + r.api = Api::Openai; + r.kind = ReqKind::Chat; + r.stream = true; + r.think_mode = ThinkMode::Low; + let mut w = Writer::new(created); + w.put(&sse_headers(false)); + sse_chunk(&mut w, &r, "chatcmpl_tape", None, None); + let mut st = openai_stream_start(&r); + let mut raw = Vec::new(); + for piece in TAPE_THINKING { + raw.extend_from_slice(piece.as_bytes()); + openai_sse_stream_update(&mut w, &r, "chatcmpl_tape", &mut st, &raw, false); + } + openai_sse_finish_live( + &mut w, + &r, + "chatcmpl_tape", + &mut st, + &raw, + "stop", + 7, + 4, + &[], + ); + w.out +} + +pub fn project_openai_chat_utf8(created: i64) -> Vec { + let mut r = StreamReq::default(); + r.think_mode = ThinkMode::None; + let mut w = Writer::new(created); + w.put(&sse_headers(false)); + let mut st = openai_stream_start(&r); + let mut raw = Vec::new(); + for piece in TAPE_UTF8 { + raw.extend_from_slice(piece); + openai_sse_stream_update(&mut w, &r, "chatcmpl_tape8", &mut st, &raw, false); + } + openai_sse_finish_live( + &mut w, + &r, + "chatcmpl_tape8", + &mut st, + &raw, + "stop", + 4, + 4, + &[], + ); + w.out +} + +pub fn project_openai_completion(created: i64) -> Vec { + let mut r = StreamReq::default(); + r.kind = ReqKind::Completion; + r.api = Api::Openai; + r.stream = true; + let mut w = Writer::new(created); + w.put(&sse_headers(false)); + for piece in TAPE_PLAIN { + sse_chunk(&mut w, &r, "cmpl_tape", Some(piece.as_bytes()), None); + } + sse_chunk(&mut w, &r, "cmpl_tape", None, Some("stop")); + sse_done(&mut w, &r, "cmpl_tape", 4, 4); + w.out +} + +pub fn project_anthropic_thinking(created: i64) -> Vec { + let mut r = StreamReq::default(); + r.api = Api::Anthropic; + r.stream = true; + r.think_mode = ThinkMode::Low; + let mut w = Writer::new(created); + let mut st = anthropic_sse_start_live(&mut w, &r, "msg_tape", 7); + let mut raw = Vec::new(); + for piece in TAPE_THINKING { + raw.extend_from_slice(piece.as_bytes()); + anthropic_sse_stream_update(&mut w, &r, "msg_tape", &mut st, &raw, false); + } + anthropic_sse_finish_live(&mut w, &r, "msg_tape", &mut st, &raw, "stop", None, 4, &[]); + w.out +} + +pub fn project_responses_thinking(created: i64) -> Vec { + let mut r = StreamReq::default(); + r.api = Api::Responses; + r.stream = true; + r.think_mode = ThinkMode::Low; + r.reasoning_summary_emit = true; + let mut w = Writer::new(created); + let mut st = responses_stream_init(&r, TEST_RESP_ID, TEST_RS_ID, TEST_MSG_ID); + st.active = true; + responses_sse_created(&mut w, &r, &mut st, created); + let mut raw = Vec::new(); + for piece in TAPE_THINKING { + raw.extend_from_slice(piece.as_bytes()); + responses_sse_stream_update(&mut w, &r, &mut st, &raw, false); + } + responses_sse_finish_live(&mut w, &r, &mut st, &raw, "stop", 7, 4, 1, created, &[]); + w.out +} + +#[cfg(test)] +mod stream_failure_tests { + use super::*; + use std::time::{Duration, Instant}; + + #[test] + fn responses_stream_error_continues_the_live_sequence() { + let mut req = StreamReq { + api: Api::Responses, + ..StreamReq::default() + }; + req.stream = true; + let mut state = responses_stream_init(&req, "resp_test", "rs_test", "msg_test"); + let mut writer = Writer::new(7); + responses_sse_created(&mut writer, &req, &mut state, 7); + writer.out.clear(); + + stream_error(&mut writer, &req, Some(&mut state), "boom"); + + assert_eq!( + writer.out, + b"data: {\"type\":\"error\",\"sequence_number\":1,\"code\":\"server_error\",\"message\":\"boom\",\"param\":null}\n\n" + ); + assert_eq!(state.sequence, 2); + } + + #[test] + fn heartbeat_uses_native_surface_and_preserves_responses_sequence() { + let now = Instant::now(); + let mut last = now; + let mut writer = Writer::new(7); + let mut req = StreamReq { + stream: true, + model: "model".into(), + ..StreamReq::default() + }; + + assert!(!stream_heartbeat_if_due( + &mut writer, + &req, + None, + &mut last, + now + Duration::from_secs(4), + ": decode\n\n", + )); + assert!(writer.out.is_empty()); + + assert!(stream_heartbeat_if_due( + &mut writer, + &req, + None, + &mut last, + now + Duration::from_secs(5), + ": decode\n\n", + )); + assert_eq!(writer.out, b": decode\n\n"); + + writer.out.clear(); + req.api = Api::Anthropic; + assert!(stream_heartbeat_if_due( + &mut writer, + &req, + None, + &mut last, + now + Duration::from_secs(10), + ": decode\n\n", + )); + assert_eq!(writer.out, b"event: ping\ndata: {\"type\": \"ping\"}\n\n"); + + writer.out.clear(); + req.api = Api::Responses; + let mut state = responses_stream_init(&req, "resp_test", "rs_test", "msg_test"); + responses_sse_created(&mut writer, &req, &mut state, 7); + writer.out.clear(); + assert!(stream_heartbeat_if_due( + &mut writer, + &req, + Some(&mut state), + &mut last, + now + Duration::from_secs(15), + ": decode\n\n", + )); + assert_eq!( + writer.out, + b"data: {\"type\":\"response.in_progress\",\"sequence_number\":1,\"response\":{\"id\":\"resp_test\",\"object\":\"response\",\"created_at\":7,\"status\":\"in_progress\",\"model\":\"model\",\"output\":[]}}\n\n" + ); + assert_eq!(state.sequence, 2); + } + + #[test] + fn responses_function_calls_emit_item_and_argument_lifecycle() { + let req = StreamReq { + api: Api::Responses, + stream: true, + model: "model".into(), + ..StreamReq::default() + }; + let mut state = responses_stream_init(&req, "resp_test", "rs_test", "msg_test"); + let mut writer = Writer::new(7); + responses_sse_created(&mut writer, &req, &mut state, 7); + let calls = [ToolCall { + id: "call_test".into(), + name: "bash".into(), + arguments: r#"{"command":"true"}"#.into(), + }]; + + responses_sse_finish_live( + &mut writer, + &req, + &mut state, + b"", + "tool_calls", + 3, + 1, + 0, + 7, + &calls, + ); + + let wire = String::from_utf8(writer.out).unwrap(); + let added = wire + .find("\"type\":\"response.output_item.added\"") + .unwrap(); + let delta = wire + .find("\"type\":\"response.function_call_arguments.delta\"") + .unwrap(); + let done = wire + .find("\"type\":\"response.function_call_arguments.done\"") + .unwrap(); + let item_done = wire.find("\"type\":\"response.output_item.done\"").unwrap(); + let completed = wire.find("\"type\":\"response.completed\"").unwrap(); + assert!(added < delta && delta < done && done < item_done && item_done < completed); + assert!(wire.contains("\"item_id\":\"fc_resp_test_0\""), "{wire}"); + assert!(wire.contains("\"call_id\":\"call_test\""), "{wire}"); + } +} diff --git a/crates/ds4-server/src/stream/channels.rs b/crates/ds4-server/src/stream/channels.rs new file mode 100644 index 0000000000000000000000000000000000000000..6ab56b397f34bded88f987e8d24693fc51e71a1b --- /dev/null +++ b/crates/ds4-server/src/stream/channels.rs @@ -0,0 +1,242 @@ +//! Incremental channel framing over the accumulator's append-only raw bytes, +//! for the channel-framed families: Inkling and Beam's Harmony. + +use std::ops::Range; + +use super::ChatFormat; +use crate::render::inkling; +use crate::tools::beam; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) enum Channel { + #[default] + Header, + Text, + Thinking, + Tool, + Done, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum Flush { + More, + Final, +} + +pub(super) struct Fragment { + pub(super) channel: Channel, + pub(super) body: Range, + pub(super) closed: bool, +} + +/// A family's message framing. Only a tool header may follow text (its +/// recipient). A closer in `ends` reopens a header (a closer equal to `lead` +/// stays to lead it); one in `last` ends the generation. +#[derive(Debug)] +pub(super) struct Grammar { + // Skipped before a header: Inkling's model role, Beam's assistant start. + lead: &'static str, + // At a header: the generation is over. + done: &'static [&'static str], + headers: [(&'static str, Channel); 3], + ends: &'static [&'static str], + last: &'static [&'static str], +} + +static INKLING: Grammar = Grammar { + lead: inkling::MODEL, + done: &[inkling::EOS], + headers: [ + (inkling::TEXT, Channel::Text), + (inkling::THINK, Channel::Thinking), + (inkling::INVOKE, Channel::Tool), + ], + ends: &[inkling::END], + last: &[], +}; + +// The template also closes analysis at a bare `<|start|>assistant`. +static BEAM: Grammar = Grammar { + lead: beam::ASSISTANT, + done: &[beam::RETURN, beam::CALL], + headers: [ + (beam::FINAL, Channel::Text), + (beam::ANALYSIS, Channel::Thinking), + (beam::TOOL, Channel::Tool), + ], + ends: &[beam::END, beam::ASSISTANT], + last: &[beam::RETURN, beam::CALL], +}; + +#[derive(Debug)] +pub(super) struct Channels { + grammar: &'static Grammar, + pos: usize, + channel: Channel, + text: Vec>, + thinking: Vec>, +} + +fn find(hay: &[u8], needle: &str) -> Option { + hay.windows(needle.len()) + .position(|s| s == needle.as_bytes()) +} + +impl Channels { + pub(super) fn for_format(format: ChatFormat) -> Option { + let grammar = match format { + ChatFormat::Inkling => &INKLING, + ChatFormat::Beam => &BEAM, + _ => return None, + }; + Some(Self { + grammar, + pos: 0, + channel: Channel::Header, + text: Vec::new(), + thinking: Vec::new(), + }) + } + + pub(super) fn advance(&mut self, raw: &[u8], flush: Flush) -> Option { + let g = self.grammar; + while self.pos < raw.len() && self.channel != Channel::Done { + if self.channel == Channel::Header { + if raw[self.pos..].starts_with(g.lead.as_bytes()) { + self.pos += g.lead.len(); + } + if g.done + .iter() + .any(|d| raw[self.pos..].starts_with(d.as_bytes())) + { + self.channel = Channel::Done; + return None; + } + let (at, marker, channel) = g + .headers + .iter() + .filter_map(|&(marker, channel)| { + find(&raw[self.pos..], marker).map(|at| (self.pos + at, marker, channel)) + }) + .min_by_key(|(at, _, _)| *at)?; + if channel != Channel::Tool && at != self.pos { + self.channel = Channel::Done; + return None; + } + self.pos = at + marker.len(); + self.channel = channel; + continue; + } + let closers = || g.ends.iter().chain(g.last); + let end = closers() + .filter_map(|&m| find(&raw[self.pos..], m).map(|at| (self.pos + at, m))) + .min_by_key(|&(at, _)| at); + let mut limit = end.map_or(raw.len(), |(at, _)| at); + if end.is_none() && flush == Flush::More { + // Hold only a suffix that could complete a closer. + limit -= closers() + .filter_map(|m| { + (1..m.len().min(raw.len() - self.pos + 1)) + .rev() + .find(|&held| raw.ends_with(&m.as_bytes()[..held])) + }) + .max() + .unwrap_or(0); + } + limit = super::utf8_stream_safe_len(raw, self.pos, limit, flush == Flush::Final); + let body = self.pos..limit; + let channel = self.channel; + if body.is_empty() && end.is_none() && flush == Flush::More { + return None; + } + let ranges = match channel { + Channel::Text => Some(&mut self.text), + Channel::Thinking => Some(&mut self.thinking), + _ => None, + }; + if let Some(ranges) = ranges.filter(|_| !body.is_empty()) { + if let Some(last) = ranges.last_mut().filter(|r| r.end == body.start) { + last.end = body.end; + } else { + ranges.push(body.clone()); + } + } + self.pos = limit; + if let Some((at, marker)) = end { + self.pos = if marker == g.lead { + at + } else { + at + marker.len() + }; + self.channel = if g.last.contains(&marker) { + Channel::Done + } else { + Channel::Header + }; + } else if flush == Flush::Final { + self.channel = Channel::Done; + } + return Some(Fragment { + channel, + body, + closed: end.is_some(), + }); + } + None + } + + pub(super) fn text(&self, raw: &[u8], channel: Channel) -> Vec { + let ranges = if channel == Channel::Thinking { + &self.thinking + } else { + &self.text + }; + ranges + .iter() + .flat_map(|r| &raw[r.start.min(raw.len())..r.end.min(raw.len())]) + .copied() + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frames(raw: &[u8], split: usize) -> (Vec, Vec, usize) { + let mut ch = Channels::for_format(ChatFormat::Beam).unwrap(); + let (mut thinking, mut text, mut tools) = (Vec::new(), Vec::new(), 0); + for (cut, flush) in [(split, Flush::More), (raw.len(), Flush::Final)] { + while let Some(f) = ch.advance(&raw[..cut], flush) { + match f.channel { + Channel::Thinking => thinking.extend_from_slice(&raw[f.body]), + Channel::Text => text.extend_from_slice(&raw[f.body]), + Channel::Tool => tools += usize::from(f.closed), + _ => {} + } + } + } + (thinking, text, tools) + } + + #[test] + fn beam_frames_every_split() { + let raw = b"<|channel|>analysis<|message|>plan it<|end|><|start|>assistant<|channel|>final<|message|>answer"; + for split in 0..=raw.len() { + assert_eq!( + frames(raw, split), + (b"plan it".to_vec(), b"answer".to_vec(), 0), + "split {split}" + ); + } + let raw = "<|channel|>analysis<|message|>p<|end|><|start|>assistant to=f<|channel|>analysis \ + <|constrain|>xml<|message|>\n1\n<|call|>"; + for split in 0..=raw.len() { + assert_eq!( + frames(raw.as_bytes(), split), + (b"p".to_vec(), Vec::new(), 1), + "split {split}" + ); + } + } +} diff --git a/crates/ds4-server/src/stream/iquest.rs b/crates/ds4-server/src/stream/iquest.rs new file mode 100644 index 0000000000000000000000000000000000000000..9a2742c5babb4be5f4c47eb19de6580922d64a95 --- /dev/null +++ b/crates/ds4-server/src/stream/iquest.rs @@ -0,0 +1,76 @@ +//! Buffer each XML call until complete; emit ordinary text around calls. +use std::ops::Range; + +use crate::parse::{ToolCall, ToolSchemaOrder}; +use crate::render::{IQUEST_TOOL_CALL_END, IQUEST_TOOL_CALL_START}; +use crate::tools::iquest::call_body; + +use super::utf8_stream_safe_len; + +#[derive(Debug)] +pub(super) enum Fragment { + Text(Range), + Call(i32, ToolCall), +} + +#[derive(Debug, Default)] +pub(super) struct Channels { + pub(super) pos: usize, + pub(super) calls: i32, + pub(super) text: Vec, +} + +impl Channels { + pub(super) fn advance( + &mut self, + raw: &[u8], + final_: bool, + orders: &[ToolSchemaOrder], + ) -> Option { + if self.pos >= raw.len() { + return None; + } + let start = IQUEST_TOOL_CALL_START.as_bytes(); + let end = IQUEST_TOOL_CALL_END.as_bytes(); + let tail = &raw[self.pos..]; + let marker = tail.windows(start.len()).position(|x| x == start); + let limit = if let Some(rel) = marker { + if rel == 0 { + let first = self.pos + start.len(); + if let Some(rel) = raw[first..].windows(end.len()).position(|x| x == end) { + let close = first + rel; + let next = close + end.len(); + if let Some(call) = call_body(&raw[first..close], orders) { + self.pos = next; + let index = self.calls; + self.calls += 1; + return Some(Fragment::Call(index, call)); + } + next + } else if final_ { + raw.len() + } else { + return None; + } + } else { + self.pos + rel + } + } else if final_ { + raw.len() + } else { + let keep = (1..start.len().min(tail.len() + 1)) + .rev() + .find(|&n| tail.ends_with(&start[..n])) + .unwrap_or(0); + raw.len() - keep + }; + let limit = utf8_stream_safe_len(raw, self.pos, limit, final_); + if limit == self.pos { + return None; + } + let span = self.pos..limit; + self.text.extend_from_slice(&raw[span.clone()]); + self.pos = limit; + Some(Fragment::Text(span)) + } +} diff --git a/crates/ds4-server/src/tool_memory.rs b/crates/ds4-server/src/tool_memory.rs new file mode 100644 index 0000000000000000000000000000000000000000..1e534a431acc79942d6ae54286561dbf6e99e85e --- /dev/null +++ b/crates/ds4-server/src/tool_memory.rs @@ -0,0 +1,1098 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::Arc; + +use ds4_kv::{Store as KvStore, EXT_TOOL_MAP}; + +use crate::parse::{ChatMsg, ToolCall}; +use crate::render::{MOTIF_TOOL_CALLS, SOLAR_TOOL_CALLS, SOLAR_TOOL_CALL_END}; +use crate::tools::{ + release_tool_id, reserve_tool_id, DSML_TOOL_CALLS_END, DSML_TOOL_CALLS_END_SHORT, + DSML_TOOL_CALLS_START, DSML_TOOL_CALLS_START_SHORT, +}; + +const KTM_HEADER: &[u8; 4] = b"KTM\x01"; +const DEFAULT_MAX_IDS: usize = 100_000; +const MAX_ID_BYTES: usize = 256; +const MAX_DSML_BYTES: usize = 512 * 1024 * 1024; + +fn wire_lengths(id: &[u8], dsml: &[u8]) -> Option<(u32, u32)> { + if id.is_empty() || dsml.is_empty() || id.contains(&0) || dsml.contains(&0) { + return None; + } + Some(( + u32::try_from(id.len()).ok()?, + u32::try_from(dsml.len()).ok()?, + )) +} + +fn encode_ktm_bounded(entries: &[(&[u8], &[u8])], max_bytes: usize) -> Option> { + let mut count = 0u32; + let mut bytes = 8u64; + for &(id, dsml) in entries { + let Some((id_len, dsml_len)) = wire_lengths(id, dsml) else { + continue; + }; + count = count.checked_add(1)?; + bytes = bytes + .checked_add(8)? + .checked_add(u64::from(id_len))? + .checked_add(u64::from(dsml_len))?; + } + if count == 0 { + return Some(Vec::new()); + } + if bytes > u64::try_from(max_bytes).unwrap_or(u64::MAX) { + return None; + } + + let mut out = Vec::new(); + out.try_reserve_exact(usize::try_from(bytes).ok()?).ok()?; + out.extend_from_slice(KTM_HEADER); + out.extend_from_slice(&count.to_le_bytes()); + for &(id, dsml) in entries { + let Some((id_len, dsml_len)) = wire_lengths(id, dsml) else { + continue; + }; + out.extend_from_slice(&id_len.to_le_bytes()); + out.extend_from_slice(&dsml_len.to_le_bytes()); + out.extend_from_slice(id); + out.extend_from_slice(dsml); + } + Some(out) +} + +#[cfg(test)] +pub(crate) fn encode_ktm(entries: &[(&[u8], &[u8])]) -> Option> { + encode_ktm_bounded(entries, usize::MAX) +} + +pub(crate) fn decode_ktm(bytes: &[u8], max_ids: usize, mut accept: F) -> usize +where + F: FnMut(&[u8], &[u8]) -> bool, +{ + if bytes.len() < 8 || &bytes[..4] != KTM_HEADER { + return 0; + } + let count = u32::from_le_bytes(bytes[4..8].try_into().unwrap()); + let max_ids = if max_ids == 0 { + DEFAULT_MAX_IDS + } else { + max_ids + }; + if u64::from(count) > u64::try_from(max_ids).unwrap_or(u64::MAX).saturating_mul(4) { + return 0; + } + + let mut loaded = 0; + let mut pos = 8usize; + for _ in 0..count { + let Some(lens_end) = pos.checked_add(8).filter(|&end| end <= bytes.len()) else { + return loaded; + }; + let id_len = u32::from_le_bytes(bytes[pos..pos + 4].try_into().unwrap()) as usize; + let dsml_len = u32::from_le_bytes(bytes[pos + 4..lens_end].try_into().unwrap()) as usize; + if id_len == 0 || id_len > MAX_ID_BYTES || dsml_len == 0 || dsml_len > MAX_DSML_BYTES { + return loaded; + } + pos = lens_end; + let Some(id_end) = pos.checked_add(id_len).filter(|&end| end <= bytes.len()) else { + return loaded; + }; + let Some(dsml_end) = id_end + .checked_add(dsml_len) + .filter(|&end| end <= bytes.len()) + else { + return loaded; + }; + let id = &bytes[pos..id_end]; + let id = &id[..id.iter().position(|&byte| byte == 0).unwrap_or(id.len())]; + let dsml = &bytes[id_end..dsml_end]; + let dsml = &dsml[..dsml + .iter() + .position(|&byte| byte == 0) + .unwrap_or(dsml.len())]; + if accept(id, dsml) { + loaded += 1; + } + pos = dsml_end; + } + loaded +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Source { + Disk, + Ram, +} + +/// The reference's `--disable-exact-dsml-tool-replay` (`ds4_help.c:378`, parsed +/// `ds4_server.c:15701`). Every tool-memory entry point checks the switch for +/// itself (`:10635` has_id, `:10654` remember, `:10666` put, `:10682` attach, +/// `:10976`/`:10987` trailer write, `:11041` trailer load, `:11095` restore), +/// so a disabled server neither stores nor replays sampled bytes and falls back +/// to the canonical JSON rendering. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ExactReplay { + On, + Off, +} + +/// The boot values the server builds its one tool map from. The reference keeps +/// both on the server struct (`ds4_server.c:15435-15436`) and hands the server +/// to every entry point. +#[derive(Clone, Copy, Debug)] +pub struct ToolMemoryConfig { + /// `--tool-memory-max-ids N` (`ds4_help.c:379`), default 100000 + /// (`DS4_TOOL_MEMORY_DEFAULT_MAX_IDS`, `ds4_server.c:10264`). + pub max_ids: usize, + pub replay: ExactReplay, +} + +impl Default for ToolMemoryConfig { + fn default() -> Self { + Self { + max_ids: DEFAULT_MAX_IDS, + replay: ExactReplay::On, + } + } +} + +struct Entry { + block: Arc, + source: Source, + stamp: u64, +} + +pub(crate) struct ToolMemory { + by_id: HashMap, + blocks: HashMap, VecDeque>, + clock: u64, + bytes: usize, + max_ids: usize, + max_bytes: usize, + replay: ExactReplay, +} + +impl Default for ToolMemory { + fn default() -> Self { + Self::new(DEFAULT_MAX_IDS, MAX_DSML_BYTES) + } +} + +impl Drop for ToolMemory { + fn drop(&mut self) { + for id in self.by_id.keys() { + release_tool_id(id); + } + } +} + +impl ToolMemory { + pub(crate) fn new(max_ids: usize, max_bytes: usize) -> Self { + Self::with_replay(max_ids, max_bytes, ExactReplay::On) + } + + /// The boot config's map: the one place the two switches enter the server. + /// Built by the native engines' `with_tool_memory`, so a model-free test + /// build has no caller for it. + #[cfg_attr(not(feature = "native"), allow(dead_code))] + pub(crate) fn from_config(config: ToolMemoryConfig) -> Self { + Self::with_replay(config.max_ids, MAX_DSML_BYTES, config.replay) + } + + fn with_replay(max_ids: usize, max_bytes: usize, replay: ExactReplay) -> Self { + Self { + by_id: HashMap::new(), + blocks: HashMap::new(), + clock: 0, + bytes: 0, + max_ids: if max_ids == 0 { + DEFAULT_MAX_IDS + } else { + max_ids + }, + max_bytes: if max_bytes == 0 { + MAX_DSML_BYTES + } else { + max_bytes + }, + replay, + } + } + + #[cfg(test)] + pub(crate) fn contains_id(&self, id: &str) -> bool { + self.by_id.contains_key(id) + } + + pub(crate) fn put(&mut self, id: &str, dsml: &str, source: Source) { + // `tool_memory_put_source` (ds4_server.c:10666). + if self.replay == ExactReplay::Off { + return; + } + if id.is_empty() + || id.len() > MAX_ID_BYTES + || dsml.is_empty() + || dsml.len() > MAX_DSML_BYTES + || id.as_bytes().contains(&0) + || dsml.as_bytes().contains(&0) + { + return; + } + + self.clock = self.clock.wrapping_add(1); + if let Some(entry) = self.by_id.get_mut(id) { + if entry.block.as_ref() == dsml { + entry.stamp = self.clock; + if source == Source::Ram { + entry.source = Source::Ram; + } + self.prune(); + return; + } + } + self.remove(id); + + let block = self + .blocks + .get_key_value(dsml) + .map(|(block, _)| Arc::clone(block)) + .unwrap_or_else(|| { + let block: Arc = Arc::from(dsml); + self.bytes = self.bytes.saturating_add(block.len() + 1); + self.blocks.insert(Arc::clone(&block), VecDeque::new()); + block + }); + self.blocks + .get_mut(block.as_ref()) + .unwrap() + .push_front(id.to_string()); + self.bytes = self.bytes.saturating_add(id.len() + 1); + self.by_id.insert( + id.to_string(), + Entry { + block, + source, + stamp: self.clock, + }, + ); + reserve_tool_id(id); + self.prune(); + } + + pub(crate) fn remember(&mut self, calls: &[ToolCall], raw_dsml: &str) -> usize { + // `tool_memory_remember` (ds4_server.c:10654). + if raw_dsml.is_empty() || self.replay == ExactReplay::Off { + return 0; + } + let mut remembered = 0; + for call in calls { + self.put(&call.id, raw_dsml, Source::Ram); + if self + .by_id + .get(&call.id) + .map(|entry| entry.block.as_ref() == raw_dsml) + .unwrap_or(false) + { + remembered += 1; + } + } + remembered + } + + pub(crate) fn attach(&mut self, messages: &mut [ChatMsg]) -> usize { + let mut attached = 0; + // `tool_memory_attach_to_messages` (ds4_server.c:10682): a disabled + // server declines to attach; the reference still counts what it could + // not attach in the replay stats (`:10684-10692`), which the port does + // not keep per request. + if self.replay == ExactReplay::Off { + return attached; + } + for message in messages { + if message.calls.is_empty() + || !message.raw_dsml.is_empty() + || !message.raw_tool_text.is_empty() + { + continue; + } + let mut matched: Option> = None; + let mut exact = true; + for call in &message.calls { + let Some(block) = self.lookup(&call.id) else { + exact = false; + continue; + }; + if let Some(current) = &matched { + if !Arc::ptr_eq(current, &block) { + exact = false; + } + } else { + matched = Some(block); + } + } + if exact { + if let Some(block) = matched { + message.raw_dsml = block.to_string(); + message.raw_tool_text = block.to_string(); + attached += 1; + } + } + } + attached + } + + pub(crate) fn checkpoint(&self, text: &[u8]) -> Option> { + // `kv_tool_map_measure_locked` (ds4_server.c:10976) and the serializer + // (`:10987`) both return early when replay is off, leaving the + // checkpoint itself untouched: this line changes no KV behaviour. + if self.replay == ExactReplay::Off { + return Some(Vec::new()); + } + let mut entries = Vec::new(); + let mut seen: HashSet> = HashSet::new(); + let mut pos = 0; + while let Some((start, end)) = next_tool_block(text, pos) { + if let Ok(raw) = std::str::from_utf8(&text[start..end]) { + if let Some((block, ids)) = self.blocks.get_key_value(raw) { + if seen.insert(Arc::clone(block)) { + for id in ids { + entries.push((id.as_bytes(), block.as_bytes())); + } + } + } + } + pos = end; + } + // A continuous bank deliberately omits its last sampled token, which + // can cut the current tool block inside the closing marker. Complete + // blocks use the hash lookup above; only this bounded LRU tail needs a + // scan, and equal-prefix ambiguity fails closed. + let mut tail = None; + let mut tail_len = 0; + let mut ambiguous = false; + for (block, ids) in &self.blocks { + if seen.contains(block) { + continue; + } + let matched = tool_block_tail_prefix_len(text, block.as_bytes()); + if matched > tail_len { + tail = Some((block, ids)); + tail_len = matched; + ambiguous = false; + } else if matched != 0 && matched == tail_len { + ambiguous = true; + } + } + if !ambiguous { + if let Some((block, ids)) = tail { + for id in ids { + entries.push((id.as_bytes(), block.as_bytes())); + } + } + } + encode_ktm_bounded(&entries, self.max_bytes) + } + + pub(crate) fn load_trailer(&mut self, trailer: &[u8], wanted: &HashSet) -> usize { + // `kv_tool_map_load_from_pos` (ds4_server.c:11041). + if self.replay == ExactReplay::Off { + return 0; + } + let max_ids = self.max_ids; + decode_ktm(trailer, max_ids, |id, dsml| { + let Ok(id) = std::str::from_utf8(id) else { + return false; + }; + if !wanted.contains(id) { + return false; + } + let Ok(dsml) = std::str::from_utf8(dsml) else { + return false; + }; + self.put(id, dsml, Source::Disk); + true + }) + } + + pub(crate) fn wanted_ids(messages: &[ChatMsg]) -> HashSet { + let mut wanted = HashSet::new(); + for message in messages { + if !message.tool_call_id.is_empty() { + wanted.insert(message.tool_call_id.clone()); + } + wanted.extend( + message + .tool_call_ids + .iter() + .filter(|id| !id.is_empty()) + .cloned(), + ); + wanted.extend( + message + .calls + .iter() + .map(|call| &call.id) + .filter(|id| !id.is_empty()) + .cloned(), + ); + } + wanted + } + + pub(crate) fn restore_store( + &mut self, + store: &KvStore, + model_id: u8, + messages: &[ChatMsg], + ) -> usize { + // `kv_cache_restore_tool_memory_for_messages` (ds4_server.c:11095). + if self.replay == ExactReplay::Off { + return 0; + } + let mut missing = Self::wanted_ids(messages); + missing.retain(|id| !self.by_id.contains_key(id)); + if missing.is_empty() { + return 0; + } + let paths: Vec<_> = store + .entries() + .iter() + .filter(|entry| { + entry.header.model_id == model_id && entry.header.ext_flags & EXT_TOOL_MAP != 0 + }) + .map(|entry| entry.path.clone()) + .collect(); + let mut loaded = 0; + for path in paths { + let Ok((header, trailer)) = store.read_trailer(&path, MAX_DSML_BYTES as u64) else { + continue; + }; + if header.model_id == model_id && header.ext_flags & EXT_TOOL_MAP != 0 { + loaded += self.load_trailer(&trailer, &missing); + missing.retain(|id| !self.by_id.contains_key(id)); + if missing.is_empty() { + break; + } + } + } + loaded + } + + fn lookup(&mut self, id: &str) -> Option> { + self.clock = self.clock.wrapping_add(1); + let entry = self.by_id.get_mut(id)?; + entry.stamp = self.clock; + Some(Arc::clone(&entry.block)) + } + + fn prune(&mut self) { + // ponytail: tool calls are rare; replace the O(n) oldest scan only if profiling says so. + while self.by_id.len() > self.max_ids || self.bytes > self.max_bytes { + let Some(oldest) = self + .by_id + .iter() + .min_by_key(|(_, entry)| entry.stamp) + .map(|(id, _)| id.clone()) + else { + break; + }; + self.remove(&oldest); + } + } + + fn remove(&mut self, id: &str) { + let Some(entry) = self.by_id.remove(id) else { + return; + }; + release_tool_id(id); + self.bytes = self.bytes.saturating_sub(id.len() + 1); + let empty = if let Some(ids) = self.blocks.get_mut(entry.block.as_ref()) { + ids.retain(|entry_id| entry_id != id); + ids.is_empty() + } else { + false + }; + if empty { + self.blocks.remove(entry.block.as_ref()); + self.bytes = self.bytes.saturating_sub(entry.block.len() + 1); + } + } +} + +fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn tool_block_forms() -> [(&'static [u8], &'static [u8], Option<&'static [u8]>); 9] { + [ + ( + b"\n\n<\xef\xbd\x9cDSML\xef\xbd\x9ctool_calls>", + DSML_TOOL_CALLS_END.as_bytes(), + None, + ), + ( + DSML_TOOL_CALLS_START.as_bytes(), + DSML_TOOL_CALLS_END.as_bytes(), + None, + ), + ( + b"\n\n", + DSML_TOOL_CALLS_END_SHORT.as_bytes(), + None, + ), + ( + DSML_TOOL_CALLS_START_SHORT.as_bytes(), + DSML_TOOL_CALLS_END_SHORT.as_bytes(), + None, + ), + (b"\n\n", b"", None), + (b"", b"", None), + ( + SOLAR_TOOL_CALLS.as_bytes(), + SOLAR_TOOL_CALL_END.as_bytes(), + Some(SOLAR_TOOL_CALLS.as_bytes()), + ), + ( + b"\n", + b"", + Some(MOTIF_TOOL_CALLS.as_bytes()), + ), + ( + MOTIF_TOOL_CALLS.as_bytes(), + b"", + Some(MOTIF_TOOL_CALLS.as_bytes()), + ), + ] +} + +fn tool_block_tail_prefix_len(text: &[u8], block: &[u8]) -> usize { + if text.is_empty() || block.is_empty() { + return 0; + } + let mut best = 0; + for (start_marker, _, _) in tool_block_forms() { + let mut block_from = 0; + while block_from < block.len() { + let Some(block_rel) = find_bytes(&block[block_from..], start_marker) else { + break; + }; + let block_marker = block_from + block_rel; + let min_candidate = text.len().saturating_sub(block.len()); + let mut text_from = min_candidate.saturating_add(block_marker).min(text.len()); + while text_from < text.len() { + let Some(text_rel) = find_bytes(&text[text_from..], start_marker) else { + break; + }; + let text_marker = text_from + text_rel; + if text_marker >= block_marker { + let candidate = text_marker - block_marker; + let suffix = &text[candidate..]; + if suffix.len() <= block.len() + && suffix.len() >= block_marker + start_marker.len() + && block.starts_with(suffix) + { + best = best.max(suffix.len()); + } + } + text_from = text_marker + 1; + } + block_from = block_marker + 1; + } + } + best +} + +fn next_tool_block(text: &[u8], from: usize) -> Option<(usize, usize)> { + let mut best: Option<(usize, usize, Option<&[u8]>, &[u8])> = None; + for (start_marker, end_marker, repeat_start) in tool_block_forms() { + let Some(start_rel) = find_bytes(&text[from..], start_marker) else { + continue; + }; + let start = from + start_rel; + let body = start + start_marker.len(); + let Some(end_rel) = find_bytes(&text[body..], end_marker) else { + continue; + }; + let end = body + end_rel + end_marker.len(); + if best.map(|current| start < current.0).unwrap_or(true) { + best = Some((start, end, repeat_start, end_marker)); + } + } + let (start, mut end, repeat_start, repeat_end) = best?; + if let Some(repeat_start) = repeat_start { + loop { + let next = text[end..] + .iter() + .position(|byte| !byte.is_ascii_whitespace()) + .map(|offset| end + offset) + .unwrap_or(text.len()); + if !text[next..].starts_with(repeat_start) { + break; + } + let body = next + repeat_start.len(); + let Some(end_rel) = find_bytes(&text[body..], repeat_end) else { + break; + }; + end = body + end_rel + repeat_end.len(); + } + } + Some((start, end)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::{HashMap, HashSet}; + use std::path::PathBuf; + use std::process::Command; + + fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() + } + + fn oracle() -> PathBuf { + std::env::var("DS4_KV_C_ORACLE") + .map(PathBuf::from) + .unwrap_or_else(|_| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/kv_c_oracle") + }) + } + + #[test] + fn ktm_wire_uses_binary_version_and_little_endian_lengths() { + let got = encode_ktm(&[(b"call_a", b"")]).unwrap(); + let mut expected = b"KTM\x01\x01\0\0\0\x06\0\0\0\x06\0\0\0".to_vec(); + expected.extend_from_slice(b"call_a"); + assert_eq!(got, expected); + assert_eq!(encode_ktm(&[]).unwrap(), Vec::::new()); + } + + #[test] + fn ktm_codec_matches_c_oracle() { + let entries = [ + (b"call_a".as_slice(), b"one".as_slice()), + (b"call_b".as_slice(), b"two".as_slice()), + ]; + let rust = encode_ktm(&entries).unwrap(); + let output = Command::new(oracle()) + .args([ + "ktm-encode", + &hex(entries[0].0), + &hex(entries[0].1), + &hex(entries[1].0), + &hex(entries[1].1), + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8(output.stdout).unwrap().trim(), hex(&rust)); + + let output = Command::new(oracle()) + .args(["ktm-decode", &hex(&rust), "100000"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + format!( + "{}:{}\n{}:{}\nloaded=2\n", + hex(entries[0].0), + hex(entries[0].1), + hex(entries[1].0), + hex(entries[1].1) + ) + ); + } + + #[test] + fn ktm_decode_keeps_valid_prefix_and_filters_wanted() { + let mut bytes = encode_ktm(&[(b"keep", b"one"), (b"want", b"two")]).unwrap(); + bytes.pop(); + let mut got = Vec::new(); + assert_eq!( + decode_ktm(&bytes, 100_000, |id, dsml| { + got.push((id.to_vec(), dsml.to_vec())); + true + }), + 1 + ); + assert_eq!(got, vec![(b"keep".to_vec(), b"one".to_vec())]); + + let bytes = encode_ktm(&[(b"skip", b"one"), (b"want", b"two")]).unwrap(); + let wanted = HashSet::from([b"want".as_slice()]); + let mut got = Vec::new(); + assert_eq!( + decode_ktm(&bytes, 100_000, |id, dsml| { + if !wanted.contains(id) { + return false; + } + got.push((id.to_vec(), dsml.to_vec())); + true + }), + 1 + ); + assert_eq!(got, vec![(b"want".to_vec(), b"two".to_vec())]); + } + + #[test] + fn ktm_decode_rejects_bad_header_count_and_lengths() { + assert_eq!(decode_ktm(b"KTM1\0\0\0\0", 100_000, |_, _| true), 0); + + let mut over_count = b"KTM\x01".to_vec(); + over_count.extend_from_slice(&400_001u32.to_le_bytes()); + assert_eq!(decode_ktm(&over_count, 0, |_, _| true), 0); + + let mut zero_id = b"KTM\x01\x01\0\0\0".to_vec(); + zero_id.extend_from_slice(&0u32.to_le_bytes()); + zero_id.extend_from_slice(&1u32.to_le_bytes()); + zero_id.push(b'x'); + assert_eq!(decode_ktm(&zero_id, 100_000, |_, _| true), 0); + + let mut over_id = b"KTM\x01\x01\0\0\0".to_vec(); + over_id.extend_from_slice(&257u32.to_le_bytes()); + over_id.extend_from_slice(&1u32.to_le_bytes()); + assert_eq!(decode_ktm(&over_id, 100_000, |_, _| true), 0); + + let mut over_dsml = b"KTM\x01\x01\0\0\0".to_vec(); + over_dsml.extend_from_slice(&1u32.to_le_bytes()); + over_dsml.extend_from_slice(&(512u32 * 1024 * 1024 + 1).to_le_bytes()); + assert_eq!(decode_ktm(&over_dsml, 100_000, |_, _| true), 0); + } + + #[test] + fn ktm_decode_preserves_order_for_duplicate_last_wins() { + let bytes = encode_ktm(&[(b"same", b"old"), (b"same", b"new")]).unwrap(); + let mut by_id = HashMap::new(); + assert_eq!( + decode_ktm(&bytes, 100_000, |id, dsml| { + by_id.insert(id.to_vec(), dsml.to_vec()); + true + }), + 2 + ); + assert_eq!( + by_id.get(b"same".as_slice()).map(Vec::as_slice), + Some(b"new".as_slice()) + ); + } + + #[test] + fn ktm_decode_matches_c_string_nul_truncation() { + let bytes = b"KTM\x01\x01\0\0\0\x03\0\0\0\x03\0\0\0a\0bx\0y"; + let mut got = None; + assert_eq!( + decode_ktm(bytes, 100_000, |id, dsml| { + got = Some((id.to_vec(), dsml.to_vec())); + true + }), + 1 + ); + assert_eq!(got, Some((b"a".to_vec(), b"x".to_vec()))); + + let output = Command::new(oracle()) + .args(["ktm-decode", &hex(bytes), "100000"]) + .output() + .unwrap(); + assert!(output.status.success()); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + "61:78\nloaded=1\n" + ); + } + + fn dsml(name: &str) -> String { + format!( + "\n\n{DSML_TOOL_CALLS_START}\n<|DSML|invoke name=\"{name}\">\n\n{DSML_TOOL_CALLS_END}" + ) + } + + fn calls(ids: &[&str]) -> Vec { + ids.iter() + .map(|id| ToolCall { + id: (*id).into(), + ..Default::default() + }) + .collect() + } + + /// `--disable-exact-dsml-tool-replay` (`ds4_help.c:378`): the reference + /// gates each entry point for itself (`ds4_server.c:10635`, `:10654`, + /// `:10666`, `:10682`, `:10976`, `:11041`, `:11095`), so a disabled server + /// must neither store nor replay and must leave the checkpoint intact. + #[test] + fn disabled_replay_stores_nothing_replays_nothing_and_keeps_the_checkpoint() { + let off = ToolMemoryConfig { + max_ids: 10, + replay: ExactReplay::Off, + }; + let raw = dsml("bash"); + let mut memory = ToolMemory::from_config(off); + + assert_eq!(memory.remember(&calls(&["call_a"]), &raw), 0); + assert!(!memory.contains_id("call_a")); + + memory.put("call_b", &raw, Source::Ram); + assert!(!memory.contains_id("call_b")); + + let mut messages = vec![ChatMsg { + calls: calls(&["call_a"]), + ..Default::default() + }]; + assert_eq!(memory.attach(&mut messages), 0); + assert!(messages[0].raw_dsml.is_empty()); + + // Both halves of the C's early return: the measurement (`:10976`) and + // the serializer (`:10987`). Some(empty) keeps the checkpoint write + // itself untouched, which is the C's behaviour too. + assert_eq!(memory.checkpoint(raw.as_bytes()), Some(Vec::new())); + assert_eq!(memory.load_trailer(raw.as_bytes(), &HashSet::new()), 0); + } + + /// `--tool-memory-max-ids N` (`ds4_help.c:379`) reaches the map through the + /// boot config, the same value `ds4_server.c:15436` carries into + /// `tool_mem.max_entries`. + #[test] + fn config_max_ids_bounds_the_map() { + let mut memory = ToolMemory::from_config(ToolMemoryConfig { + max_ids: 2, + replay: ExactReplay::On, + }); + for id in ["call_a", "call_b", "call_c"] { + memory.put(id, &dsml(id), Source::Ram); + } + assert_eq!(memory.by_id.len(), 2); + } + + #[test] + fn memory_shares_blocks_preserves_id_order_and_upgrades_source() { + let raw = dsml("bash"); + let mut memory = ToolMemory::new(10, 4096); + memory.put("call_a", &raw, Source::Disk); + memory.put("call_b", &raw, Source::Disk); + let shared_bytes = memory.bytes; + assert_eq!(memory.remember(&calls(&["call_a"]), &raw), 1); + assert_eq!(memory.bytes, shared_bytes); + assert_eq!(memory.blocks.len(), 1); + assert_eq!(memory.by_id["call_a"].source, Source::Ram); + + let trailer = memory.checkpoint(raw.as_bytes()).unwrap(); + let mut ids = Vec::new(); + assert_eq!( + decode_ktm(&trailer, 10, |id, _| { + ids.push(String::from_utf8(id.to_vec()).unwrap()); + true + }), + 2 + ); + assert_eq!(ids, ["call_b", "call_a"]); + } + + #[test] + fn attach_is_all_or_nothing_for_each_assistant_block() { + let first = dsml("first"); + let second = dsml("second"); + let mut memory = ToolMemory::new(10, 4096); + memory.put("call_a", &first, Source::Ram); + memory.put("call_b", &first, Source::Ram); + memory.put("call_other", &second, Source::Ram); + + let mut messages = vec![ + ChatMsg { + calls: calls(&["call_a", "call_b"]), + ..Default::default() + }, + ChatMsg { + calls: calls(&["call_a", "missing"]), + ..Default::default() + }, + ChatMsg { + calls: calls(&["call_a", "call_other"]), + ..Default::default() + }, + ]; + assert_eq!(memory.attach(&mut messages), 1); + assert_eq!(messages[0].raw_dsml, first); + assert!(messages[1].raw_dsml.is_empty()); + assert!(messages[2].raw_dsml.is_empty()); + } + + #[test] + fn checkpoint_filters_complete_blocks_and_deduplicates_solar_span() { + let exact = dsml("exact"); + let absent = dsml("absent"); + let solar = format!( + "{SOLAR_TOOL_CALLS}one{SOLAR_TOOL_CALL_END} \n {SOLAR_TOOL_CALLS}two{SOLAR_TOOL_CALL_END}" + ); + let mut memory = ToolMemory::new(10, 8192); + memory.put("call_exact", &exact, Source::Ram); + memory.put("call_absent", &absent, Source::Ram); + memory.put("call_solar", &solar, Source::Ram); + + let text = format!("prefix{exact} middle {exact} tail {solar}"); + let trailer = memory.checkpoint(text.as_bytes()).unwrap(); + let mut got = Vec::new(); + assert_eq!( + decode_ktm(&trailer, 10, |id, dsml| { + got.push((id.to_vec(), dsml.to_vec())); + true + }), + 2 + ); + assert_eq!(got[0], (b"call_exact".to_vec(), exact.into_bytes())); + assert_eq!(got[1], (b"call_solar".to_vec(), solar.into_bytes())); + } + + #[test] + fn checkpoint_keeps_a_tool_turn_cut_inside_its_close_marker() { + let raw = dsml("bank-cut"); + let mut memory = ToolMemory::new(10, 4096); + memory.put("call_cut", &raw, Source::Ram); + let cut = raw.len() - 5; + let text = format!("prompt{}", &raw[..cut]); + + let trailer = memory.checkpoint(text.as_bytes()).unwrap(); + let mut got = Vec::new(); + assert_eq!( + decode_ktm(&trailer, 10, |id, dsml| { + got.push((id.to_vec(), dsml.to_vec())); + true + }), + 1 + ); + assert_eq!(got, vec![(b"call_cut".to_vec(), raw.into_bytes())]); + } + + #[test] + fn checkpoint_restores_motif_exact_tool_text_from_a_partial_bank_key() { + let raw = + "\n{\"name\":\"pair_values\",\"arguments\":{\"a\":1,\"b\":2}}"; + let mut source = ToolMemory::new(10, 4096); + source.put("call_motif", raw, Source::Ram); + let mut text = b"prompt".to_vec(); + text.extend_from_slice(&raw.as_bytes()[..raw.len() - 5]); + let trailer = source.checkpoint(&text).unwrap(); + + let mut restored = ToolMemory::new(10, 4096); + let wanted = HashSet::from(["call_motif".to_string()]); + assert_eq!(restored.load_trailer(&trailer, &wanted), 1); + let mut messages = vec![ChatMsg { + role: "assistant".into(), + calls: calls(&["call_motif"]), + ..Default::default() + }]; + assert_eq!(restored.attach(&mut messages), 1); + assert_eq!(messages[0].raw_tool_text, raw); + } + + #[test] + fn memory_prunes_lru_and_counts_shared_dsml_once() { + let raw = dsml("shared"); + let budget = raw.len() + 1 + ("call_a".len() + 1) * 2; + let mut memory = ToolMemory::new(2, budget); + memory.put("call_a", &raw, Source::Ram); + memory.put("call_b", &raw, Source::Ram); + assert_eq!(memory.blocks.len(), 1); + assert_eq!(memory.bytes, budget); + assert!(memory.lookup("call_a").is_some()); + memory.put("call_c", &raw, Source::Ram); + assert!(memory.contains_id("call_a")); + assert!(!memory.contains_id("call_b")); + assert!(memory.contains_id("call_c")); + assert_eq!(memory.bytes, budget); + } + + #[test] + fn checkpoint_rejects_shared_block_expansion_over_memory_budget() { + let raw = dsml("shared-expansion"); + let budget = raw.len() + 1 + ("call_a".len() + 1) * 3; + let mut memory = ToolMemory::new(3, budget); + memory.put("call_a", &raw, Source::Ram); + memory.put("call_b", &raw, Source::Ram); + memory.put("call_c", &raw, Source::Ram); + + assert_eq!(memory.by_id.len(), 3); + assert!(memory.checkpoint(raw.as_bytes()).is_none()); + } + + #[test] + fn trailer_load_filters_wanted_ids() { + let trailer = encode_ktm(&[(b"skip", b"one"), (b"want", b"two")]).unwrap(); + let mut memory = ToolMemory::new(10, 4096); + assert_eq!( + memory.load_trailer(&trailer, &HashSet::from(["want".to_string()])), + 1 + ); + assert!(!memory.contains_id("skip")); + assert!(memory.contains_id("want")); + } + + #[test] + fn store_restore_filters_model_and_requested_ids_before_attach() { + use ds4_kv::{Header, Options, Reason, Record, Store, EXT_TOOL_MAP}; + use std::fs; + + let dir = + std::env::temp_dir().join(format!("ds4-tool-memory-restore-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + let mut store = Store::open(&dir, 16, true, Options::default()).unwrap(); + let raw = dsml("restore"); + let trailer = encode_ktm(&[ + (b"call_want", raw.as_bytes()), + (b"call_skip", b"not requested"), + ]) + .unwrap(); + store + .write(Record { + header: Header { + quant_bits: 2, + reason: Reason::Evict, + ext_flags: EXT_TOOL_MAP, + model_id: 2, + tokens: 1, + hits: 0, + ctx_size: 4096, + created_at: 1, + last_used: 1, + payload_bytes: 0, + text_bytes: 0, + }, + text: raw.as_bytes().to_vec(), + payload: b"opaque".to_vec(), + trailer, + }) + .unwrap(); + let mut messages = vec![ChatMsg { + calls: calls(&["call_want"]), + ..Default::default() + }]; + assert_eq!(ToolMemory::wanted_ids(&messages).len(), 1); + let mut memory = ToolMemory::new(10, 4096); + assert_eq!(memory.restore_store(&store, 1, &messages), 0); + assert_eq!(memory.restore_store(&store, 2, &messages), 1); + assert!(!memory.contains_id("call_skip")); + assert_eq!(memory.attach(&mut messages), 1); + assert_eq!(messages[0].raw_dsml, raw); + + let fresh = dsml("fresh"); + memory.put("call_want", &fresh, Source::Ram); + assert_eq!(memory.restore_store(&store, 2, &messages), 0); + let mut replay = vec![ChatMsg { + calls: calls(&["call_want"]), + ..Default::default() + }]; + assert_eq!(memory.attach(&mut replay), 1); + assert_eq!(replay[0].raw_dsml, fresh); + let _ = fs::remove_dir_all(dir); + } +} diff --git a/crates/ds4-server/src/tool_stream.rs b/crates/ds4-server/src/tool_stream.rs new file mode 100644 index 0000000000000000000000000000000000000000..36e29c64a395c9cd0230fb7a7e41ea6e5b1edbc2 --- /dev/null +++ b/crates/ds4-server/src/tool_stream.rs @@ -0,0 +1,873 @@ +//! Incremental DSML → OpenAI / Anthropic live tool projection. +//! Copied from `ds4_server.c` at v0.6.3-dfm. Final parse stays authoritative. + +use crate::json::json_escape_bytes; +use crate::parse::ToolCall; +use crate::stream::utf8_stream_safe_len; +use crate::tools::{ + dsml_attr, mint_tool_id, DSML_INVOKE_END, DSML_INVOKE_END_SHORT, DSML_INVOKE_END_SPACED, + DSML_INVOKE_START, DSML_INVOKE_START_SHORT, DSML_INVOKE_START_SPACED, DSML_PARAM_END, + DSML_PARAM_END_SHORT, DSML_PARAM_END_SPACED, DSML_PARAM_START, DSML_PARAM_START_SHORT, + DSML_PARAM_START_SPACED, DSML_TOOL_CALLS_END, DSML_TOOL_CALLS_END_SHORT, + DSML_TOOL_CALLS_END_SPACED, DSML_TOOL_CALLS_START, DSML_TOOL_CALLS_START_SHORT, + DSML_TOOL_CALLS_START_SPACED, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DsmlToolState { + BetweenInvokes, + BetweenParams, + ParamValue, + Done, + Error, +} + +#[derive(Clone, Copy, Debug)] +pub struct DsmlSyntax { + pub tool_calls_start: &'static [u8], + pub tool_calls_end: &'static [u8], + pub invoke_start: &'static [u8], + pub invoke_end: &'static [u8], + pub param_start: &'static [u8], + pub param_end: &'static [u8], + /// Old generations write `&`/`<` in string values as XML entities (the + /// stream holds back a chunk tail that would split one); the official + /// spaced generation writes values raw and its only escape is the render + /// guard (engine server_dsml_stream.c dsml_value_unescape / :384-406). + pub entity_values: bool, +} + +pub const DSML_SYNTAXES: [DsmlSyntax; 4] = [ + DsmlSyntax { + tool_calls_start: DSML_TOOL_CALLS_START.as_bytes(), + tool_calls_end: DSML_TOOL_CALLS_END.as_bytes(), + invoke_start: DSML_INVOKE_START.as_bytes(), + invoke_end: DSML_INVOKE_END.as_bytes(), + param_start: DSML_PARAM_START.as_bytes(), + param_end: DSML_PARAM_END.as_bytes(), + entity_values: true, + }, + DsmlSyntax { + tool_calls_start: DSML_TOOL_CALLS_START_SHORT.as_bytes(), + tool_calls_end: DSML_TOOL_CALLS_END_SHORT.as_bytes(), + invoke_start: DSML_INVOKE_START_SHORT.as_bytes(), + invoke_end: DSML_INVOKE_END_SHORT.as_bytes(), + param_start: DSML_PARAM_START_SHORT.as_bytes(), + param_end: DSML_PARAM_END_SHORT.as_bytes(), + entity_values: true, + }, + DsmlSyntax { + tool_calls_start: b"", + tool_calls_end: b"", + invoke_start: b"", + param_start: b"", + entity_values: true, + }, + // The official V4.1 encoding.py spelling (4294136). + DsmlSyntax { + tool_calls_start: DSML_TOOL_CALLS_START_SPACED.as_bytes(), + tool_calls_end: DSML_TOOL_CALLS_END_SPACED.as_bytes(), + invoke_start: DSML_INVOKE_START_SPACED.as_bytes(), + invoke_end: DSML_INVOKE_END_SPACED.as_bytes(), + param_start: DSML_PARAM_START_SPACED.as_bytes(), + param_end: DSML_PARAM_END_SPACED.as_bytes(), + entity_values: false, + }, +]; + +#[derive(Debug, Clone)] +pub struct DsmlToolStream { + pub state: DsmlToolState, + pub syn: Option, + pub parse_pos: usize, + pub index: i32, + pub active: bool, + pub emitted_any: bool, + pub args_open: bool, + first_param: bool, + param_is_string: bool, + ids: Vec, + id_prefix: &'static str, + random_ids: bool, +} + +impl Default for DsmlToolStream { + fn default() -> Self { + Self { + state: DsmlToolState::BetweenInvokes, + syn: None, + parse_pos: 0, + index: 0, + active: false, + emitted_any: false, + args_open: false, + first_param: false, + param_is_string: false, + ids: Vec::new(), + id_prefix: "call_", + random_ids: false, + } + } +} + +impl DsmlToolStream { + pub fn with_prefix(prefix: &'static str) -> Self { + Self { + id_prefix: prefix, + ..Self::default() + } + } + + pub(crate) fn use_random_ids(&mut self) { + self.random_ids = true; + } + + pub fn init(&mut self, raw: &[u8], pos: usize) -> bool { + let prefix = self.id_prefix; + let random_ids = self.random_ids; + *self = Self::with_prefix(prefix); + self.random_ids = random_ids; + self.active = true; + self.state = DsmlToolState::BetweenInvokes; + for syn in DSML_SYNTAXES { + if raw_full_lit(raw, pos, syn.tool_calls_start) { + self.syn = Some(syn); + self.parse_pos = pos + syn.tool_calls_start.len(); + return true; + } + } + self.active = false; + self.state = DsmlToolState::Error; + false + } + + pub fn id_at(&mut self, index: i32) -> String { + if index < 0 { + return String::new(); + } + let i = index as usize; + while self.ids.len() <= i { + self.ids.push(String::new()); + } + if self.ids[i].is_empty() { + self.ids[i] = if self.random_ids { + mint_tool_id(self.id_prefix) + } else { + format!("{}{index}", self.id_prefix) + }; + } + self.ids[i].clone() + } + + pub fn apply_ids(&self, calls: &mut [ToolCall]) { + let n = calls.len().min(self.ids.len()); + for i in 0..n { + if calls[i].id.is_empty() && !self.ids[i].is_empty() { + calls[i].id = self.ids[i].clone(); + } + } + } + + /// A call whose name went on the wire but whose arguments did not: the + /// live stream started this invoke and then failed mid-block on the + /// V4.1 artifact's mixed tag spellings (docs 6.23). The finish + /// completes this index with an arguments-only delta -- the client + /// merges deltas by index. None when the live stream finished + /// cleanly, emitted no call, or already streamed part of the + /// arguments (nothing safe to append). + pub fn partial_invoke_index(&self) -> Option { + if !self.emitted_any || self.state == DsmlToolState::Done || self.args_open { + return None; + } + Some(self.index) + } + + fn fail(&mut self) -> bool { + self.active = false; + self.state = DsmlToolState::Error; + true + } + + pub fn update(&mut self, sink: &mut S, raw: &[u8]) -> bool { + let Some(syn) = self.syn else { + return true; + }; + while self.active && self.parse_pos < raw.len() { + match self.state { + DsmlToolState::BetweenInvokes => { + while self.parse_pos < raw.len() && raw[self.parse_pos].is_ascii_whitespace() { + self.parse_pos += 1; + } + if self.parse_pos >= raw.len() { + return true; + } + if raw_full_lit(raw, self.parse_pos, syn.tool_calls_end) { + self.parse_pos += syn.tool_calls_end.len(); + self.active = false; + self.state = DsmlToolState::Done; + return true; + } + if raw_partial_any(raw, self.parse_pos, syn.tool_calls_end, syn.invoke_start) { + return true; + } + if raw_full_lit(raw, self.parse_pos, syn.invoke_start) { + let before = self.parse_pos; + let before_state = self.state; + if !self.start_invoke(sink, raw) { + return false; + } + if self.parse_pos == before && self.state == before_state { + return true; + } + continue; + } + return self.fail(); + } + DsmlToolState::BetweenParams => { + while self.parse_pos < raw.len() && raw[self.parse_pos].is_ascii_whitespace() { + self.parse_pos += 1; + } + if self.parse_pos >= raw.len() { + return true; + } + if raw_full_lit(raw, self.parse_pos, syn.invoke_end) { + if self.args_open && !sink.args_fragment(self.index, b"}") { + return false; + } + self.args_open = false; + if !sink.close_invoke(self.index) { + return false; + } + self.parse_pos += syn.invoke_end.len(); + self.index += 1; + self.state = DsmlToolState::BetweenInvokes; + continue; + } + if raw_partial_lit(raw, self.parse_pos, syn.invoke_end) + || param_open_partial(&syn, raw, self.parse_pos) + { + return true; + } + if param_open_full(&syn, raw, self.parse_pos) { + let before = self.parse_pos; + let before_state = self.state; + if !self.start_param(sink, raw) { + return false; + } + if self.parse_pos == before && self.state == before_state { + return true; + } + continue; + } + return self.fail(); + } + DsmlToolState::ParamValue => { + if let Some((end, end_len)) = param_close_find(&syn, raw, self.parse_pos) { + if !self.finish_param(sink, raw, self.parse_pos + end, end_len) { + return false; + } + continue; + } + let limit = tool_param_value_stream_safe_len( + raw, + self.parse_pos, + syn.param_end, + bare_param_alias(&syn).then_some(BARE_PARAM_END), + self.param_is_string, + syn.entity_values, + ); + if limit > self.parse_pos { + let ok = if self.param_is_string { + emit_string_value(sink, self.index, &syn, &raw[self.parse_pos..limit]) + } else { + sink.args_fragment(self.index, &raw[self.parse_pos..limit]) + }; + if !ok { + return false; + } + self.parse_pos = limit; + } + return true; + } + DsmlToolState::Done | DsmlToolState::Error => return true, + } + } + true + } + + fn start_invoke(&mut self, sink: &mut S, raw: &[u8]) -> bool { + let Some(rel) = raw[self.parse_pos..].iter().position(|&c| c == b'>') else { + return true; + }; + let tag = &raw[self.parse_pos..self.parse_pos + rel + 1]; + let Some(name) = dsml_attr(tag, "name") else { + return self.fail(); + }; + let id = self.id_at(self.index); + if !sink.start_invoke(self.index, &id, &name) { + return false; + } + if !sink.args_fragment(self.index, b"{") { + return false; + } + self.emitted_any = true; + self.args_open = true; + self.first_param = true; + self.parse_pos = self.parse_pos + rel + 1; + self.state = DsmlToolState::BetweenParams; + true + } + + fn start_param(&mut self, sink: &mut S, raw: &[u8]) -> bool { + let Some(rel) = raw[self.parse_pos..].iter().position(|&c| c == b'>') else { + return true; + }; + let tag = &raw[self.parse_pos..self.parse_pos + rel + 1]; + let Some(name) = dsml_attr(tag, "name") else { + return self.fail(); + }; + // A bare parameter carries no string attribute; the engine defaults + // it to string, matching the final parser (f302be7). + let string_value = dsml_attr(tag, "string") + .map(|v| v == b"true") + .unwrap_or(true); + if !emit_param_prefix(sink, self.index, &name, string_value, self.first_param) { + return false; + } + self.first_param = false; + self.param_is_string = string_value; + self.parse_pos = self.parse_pos + rel + 1; + self.state = DsmlToolState::ParamValue; + true + } + + fn finish_param( + &mut self, + sink: &mut S, + raw: &[u8], + value_end: usize, + end_len: usize, + ) -> bool { + let syn = self.syn.unwrap(); + if value_end > self.parse_pos { + let ok = if self.param_is_string { + emit_string_value(sink, self.index, &syn, &raw[self.parse_pos..value_end]) + } else { + sink.args_fragment(self.index, &raw[self.parse_pos..value_end]) + }; + if !ok { + return false; + } + } + if self.param_is_string && !sink.args_fragment(self.index, b"\"") { + return false; + } + self.parse_pos = value_end + end_len; + self.state = DsmlToolState::BetweenParams; + true + } +} + +pub trait ToolSink { + fn start_invoke(&mut self, index: i32, id: &str, name: &[u8]) -> bool; + fn args_fragment(&mut self, index: i32, text: &[u8]) -> bool; + fn close_invoke(&mut self, index: i32) -> bool; +} + +fn emit_param_prefix( + sink: &mut S, + index: i32, + name: &[u8], + is_string: bool, + first: bool, +) -> bool { + let mut frag = Vec::new(); + if !first { + frag.push(b','); + } + frag.extend(json_escape_bytes(name)); + frag.push(b':'); + if is_string { + frag.push(b'"'); + } + sink.args_fragment(index, &frag) +} + +fn emit_string_value(sink: &mut S, index: i32, syn: &DsmlSyntax, text: &[u8]) -> bool { + if text.is_empty() { + return true; + } + let unescaped = + crate::tools::dsml_value_unescape_syntax(syn.entity_values, syn.param_end, text); + let frag = json_escape_fragment(&unescaped); + sink.args_fragment(index, &frag) +} + +pub fn json_escape_fragment(s: &[u8]) -> Vec { + let full = json_escape_bytes(s); + if full.len() >= 2 { + full[1..full.len() - 1].to_vec() + } else { + Vec::new() + } +} + +fn raw_full_lit(raw: &[u8], pos: usize, lit: &[u8]) -> bool { + pos <= raw.len() && raw.len() - pos >= lit.len() && raw[pos..].starts_with(lit) +} + +fn raw_partial_lit(raw: &[u8], pos: usize, lit: &[u8]) -> bool { + if pos > raw.len() || raw.len() - pos >= lit.len() { + return false; + } + raw[pos..] == lit[..raw.len() - pos] +} + +fn raw_partial_any(raw: &[u8], pos: usize, a: &[u8], b: &[u8]) -> bool { + raw_partial_lit(raw, pos, a) || raw_partial_lit(raw, pos, b) +} + +/// The engine's bare-alias rule (f302be7, server_dsml_parse.c +/// dsml_param_open_len/close_len): the quantized artifact, with a system +/// text and the tool instructions in context, writes the parameter tags +/// bare -- `v` -- while the wrapper and +/// invoke opens stay DSML (fable5.md, 10-11/10-12; measured 5/5 with a +/// Chinese system, 0/5 without). A syntax whose own spelling is not +/// already the bare one accepts the bare form as an alias. The open alias +/// keeps the trailing space: `` is not an open tag. The +/// engine routed the final parse, the live streams, and the KV syntax +/// tracking through the same entry points; these are those entry points. +const BARE_PARAM_START: &[u8] = b""; + +fn bare_param_alias(syn: &DsmlSyntax) -> bool { + syn.param_start != &b" bool { + raw_full_lit(raw, pos, syn.param_start) + || (bare_param_alias(syn) && raw_full_lit(raw, pos, BARE_PARAM_START)) +} + +fn param_open_partial(syn: &DsmlSyntax, raw: &[u8], pos: usize) -> bool { + raw_partial_lit(raw, pos, syn.param_start) + || (bare_param_alias(syn) && raw_partial_lit(raw, pos, BARE_PARAM_START)) +} + +/// The earliest close from `from` in either spelling: (offset, length) +/// (f302be7's dsml_param_close_find takes the earlier of the two). +fn param_close_find(syn: &DsmlSyntax, raw: &[u8], from: usize) -> Option<(usize, usize)> { + let own = find_lit_bounded(&raw[from..], syn.param_end).map(|r| (r, syn.param_end.len())); + if !bare_param_alias(syn) { + return own; + } + let bare = find_lit_bounded(&raw[from..], BARE_PARAM_END).map(|r| (r, BARE_PARAM_END.len())); + match (own, bare) { + (Some(a), Some(b)) => Some(if b.0 < a.0 { b } else { a }), + (a, b) => a.or(b), + } +} + +fn find_lit_bounded(s: &[u8], lit: &[u8]) -> Option { + if lit.is_empty() { + return Some(0); + } + s.windows(lit.len()).position(|w| w == lit) +} + +/// The engine's `dsml_entity_stream_safe_len` (server_dsml_stream.c:384-406): +/// a chunk tail that would split an escape sequence is held back. The old +/// generations' escapes are the XML entities; the spaced generation writes +/// values raw and its only escape is the render guard "<" + the closing +/// tag minus its leading '<' (dsml_value_unescape builds the same sentinel). +fn dsml_entity_stream_safe_len( + raw: &[u8], + start: usize, + limit: usize, + param_end: &[u8], + entity_values: bool, +) -> usize { + const ENTS: &[&[u8]] = &[b"&", b"<", b">", b""", b"'"]; + let sentinel = [b"<", ¶m_end[1..]].concat(); + let (list, max_ent): (&[&[u8]], usize) = if entity_values { + (ENTS, 6) + } else { + (&[sentinel.as_slice()], sentinel.len()) + }; + let scan = if limit > start + max_ent { + limit - max_ent + } else { + start + }; + for i in (scan + 1..=limit).rev() { + if raw[i - 1] != b'&' { + continue; + } + let amp = i - 1; + let tail = limit - amp; + for ent in list { + if tail < ent.len() && raw[amp..].starts_with(&ent[..tail]) { + return amp; + } + } + break; + } + limit +} + +/// A chunk tail that is a proper prefix of `close` is held back; the hold +/// runs once per accepted spelling (f302be7's close_partial). +fn partial_close_hold(raw: &[u8], start: usize, close: &[u8]) -> usize { + let raw_len = raw.len(); + let end_len = close.len(); + let scan = if raw_len > start + end_len { + raw_len - end_len + } else { + start + }; + for i in (scan + 1..=raw_len).rev() { + if raw[i - 1] != b'<' { + continue; + } + let marker = i - 1; + let tail = raw_len - marker; + if tail < end_len && raw[marker..].starts_with(&close[..tail]) { + return marker; + } + break; + } + raw_len +} + +fn tool_param_value_stream_safe_len( + raw: &[u8], + start: usize, + param_end: &[u8], + param_end_alias: Option<&[u8]>, + is_string: bool, + entity_values: bool, +) -> usize { + let mut limit = partial_close_hold(raw, start, param_end); + if let Some(alias) = param_end_alias { + limit = limit.min(partial_close_hold(raw, start, alias)); + } + if is_string { + limit = dsml_entity_stream_safe_len(raw, start, limit, param_end, entity_values); + } + utf8_stream_safe_len(raw, start, limit, false) +} + +/// Tape dumps matching `tests/parity/tool_stream_c_oracle`. +pub fn dump_script(name: &str) -> Vec { + use crate::parse::ToolCall; + use crate::route::{Api, ReqKind, ThinkMode}; + use crate::stream::{ + anthropic_sse_finish_live, anthropic_sse_start_live, anthropic_sse_stream_update, + openai_sse_finish_live, openai_sse_stream_update, openai_stream_start, sse_chunk, + ChatFormat, StreamReq, Writer, CREATED_TEST, + }; + + let swapped_bash = [ToolCall { + name: "bash".into(), + arguments: "{\"description\":\"list files\",\"command\":\"ls -la\",\"timeout\":10}".into(), + ..Default::default() + }]; + + let mut r = StreamReq { + kind: ReqKind::Chat, + api: Api::Openai, + model: "deepseek-v4-flash".into(), + think_mode: ThinkMode::None, + has_tools: true, + stream: true, + chat_format: ChatFormat::DeepSeek, + ..StreamReq::default() + }; + + match name { + "openai-partial" => { + let mut w = Writer::new(CREATED_TEST); + sse_chunk(&mut w, &r, "chatcmpl_partial_tool", None, None); + let mut st = openai_stream_start(&r); + let raw1 = format!( + "Before.\n\n{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">echo partial", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ); + openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_partial_tool", + &mut st, + raw1.as_bytes(), + false, + ); + let raw2 = format!( + "Before.\n\n{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">echo partial done{}\n{}\n{}", + DSML_TOOL_CALLS_START, + DSML_INVOKE_START, + DSML_PARAM_START, + DSML_PARAM_END, + DSML_INVOKE_END, + DSML_TOOL_CALLS_END + ); + openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_partial_tool", + &mut st, + raw2.as_bytes(), + false, + ); + openai_sse_finish_live( + &mut w, + &r, + "chatcmpl_partial_tool", + &mut st, + raw2.as_bytes(), + "tool_calls", + 10, + 4, + &[], + ); + w.out + } + "openai-raw" => { + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + let raw = format!( + "{}\n{} name=\"edit\">\n{} name=\"edits\" string=\"false\">[1,2,3", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ); + openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_raw_tool", + &mut st, + raw.as_bytes(), + false, + ); + w.out + } + "openai-wait-tag" => { + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + let raw1 = format!("{}\n{}", DSML_TOOL_CALLS_START, DSML_INVOKE_START); + openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_incomplete_tool", + &mut st, + raw1.as_bytes(), + false, + ); + let raw2 = format!( + "{}\n{} name=\"bash\">\n{}", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ); + openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_incomplete_tool", + &mut st, + raw2.as_bytes(), + false, + ); + w.out + } + "openai-entity" => { + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + let raw1 = format!( + "{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">echo &", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ); + openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_entity_tool", + &mut st, + raw1.as_bytes(), + false, + ); + let raw2 = format!( + "{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">echo & done{}\n{}\n{}", + DSML_TOOL_CALLS_START, + DSML_INVOKE_START, + DSML_PARAM_START, + DSML_PARAM_END, + DSML_INVOKE_END, + DSML_TOOL_CALLS_END + ); + openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_entity_tool", + &mut st, + raw2.as_bytes(), + false, + ); + w.out + } + "openai-think-tool" => { + r.think_mode = ThinkMode::Low; + let mut w = Writer::new(CREATED_TEST); + sse_chunk(&mut w, &r, "chatcmpl_test", None, None); + let mut st = openai_stream_start(&r); + let raw1 = "need a toolHello.\n\n"; + openai_sse_stream_update(&mut w, &r, "chatcmpl_test", &mut st, raw1.as_bytes(), false); + let raw2 = format!( + "need a toolHello.\n\n{}\n", + DSML_TOOL_CALLS_START + ); + openai_sse_stream_update(&mut w, &r, "chatcmpl_test", &mut st, raw2.as_bytes(), false); + openai_sse_finish_live( + &mut w, + &r, + "chatcmpl_test", + &mut st, + raw2.as_bytes(), + "tool_calls", + 10, + 8, + &swapped_bash, + ); + w.out + } + "anthropic-partial" => { + r.api = Api::Anthropic; + let mut w = Writer::new(CREATED_TEST); + let mut st = anthropic_sse_start_live(&mut w, &r, "msg_tool", 7); + let raw1 = format!( + "Before.\n\n{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">echo partial", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ); + anthropic_sse_stream_update(&mut w, &r, "msg_tool", &mut st, raw1.as_bytes(), false); + let raw2 = format!( + "Before.\n\n{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">echo partial done{}\n{}\n{}", + DSML_TOOL_CALLS_START, + DSML_INVOKE_START, + DSML_PARAM_START, + DSML_PARAM_END, + DSML_INVOKE_END, + DSML_TOOL_CALLS_END + ); + anthropic_sse_stream_update(&mut w, &r, "msg_tool", &mut st, raw2.as_bytes(), false); + anthropic_sse_finish_live( + &mut w, + &r, + "msg_tool", + &mut st, + raw2.as_bytes(), + "tool_calls", + None, + 5, + &[], + ); + w.out + } + "anthropic-think-tool" => { + r.api = Api::Anthropic; + r.think_mode = ThinkMode::Low; + let mut w = Writer::new(CREATED_TEST); + let mut st = anthropic_sse_start_live(&mut w, &r, "msg_test", 10); + let raw1 = "need a toolHello.\n\n"; + anthropic_sse_stream_update(&mut w, &r, "msg_test", &mut st, raw1.as_bytes(), false); + let raw2 = format!("need a toolHello.\n\n{}\n", DSML_TOOL_CALLS_START); + anthropic_sse_stream_update(&mut w, &r, "msg_test", &mut st, raw2.as_bytes(), false); + anthropic_sse_finish_live( + &mut w, + &r, + "msg_test", + &mut st, + raw2.as_bytes(), + "tool_calls", + None, + 8, + &swapped_bash, + ); + w.out + } + "openai-utf8" => { + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + let mut raw1 = format!( + "{}\n{} name=\"write\">\n{} name=\"content\" string=\"true\">flag ", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ) + .into_bytes(); + raw1.extend_from_slice(&[0xf0, 0x9f]); + openai_sse_stream_update(&mut w, &r, "chatcmpl_utf8_tool", &mut st, &raw1, false); + let mut raw2 = format!( + "{}\n{} name=\"write\">\n{} name=\"content\" string=\"true\">flag ", + DSML_TOOL_CALLS_START, DSML_INVOKE_START, DSML_PARAM_START + ) + .into_bytes(); + raw2.extend_from_slice(&[0xf0, 0x9f, 0x9a, 0xa9]); + raw2.extend_from_slice( + format!( + " done{}\n{}\n{}", + DSML_PARAM_END, DSML_INVOKE_END, DSML_TOOL_CALLS_END + ) + .as_bytes(), + ); + openai_sse_stream_update(&mut w, &r, "chatcmpl_utf8_tool", &mut st, &raw2, false); + w.out + } + "openai-multi" => { + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + let raw = format!( + "{}\n{} name=\"read\">\n{} name=\"path\" string=\"true\">a.c{}\n{}\n{} name=\"bash\">\n{} name=\"command\" string=\"true\">wc -l a.c{}\n{}\n{}", + DSML_TOOL_CALLS_START, + DSML_INVOKE_START, + DSML_PARAM_START, + DSML_PARAM_END, + DSML_INVOKE_END, + DSML_INVOKE_START, + DSML_PARAM_START, + DSML_PARAM_END, + DSML_INVOKE_END, + DSML_TOOL_CALLS_END + ); + openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_multi_tool", + &mut st, + raw.as_bytes(), + false, + ); + w.out + } + _ => b"ERROR unknown-script\n".to_vec(), + } +} + +#[cfg(test)] +mod id_tests { + use super::{DsmlToolStream, DSML_TOOL_CALLS_START}; + use crate::parse::ToolCall; + + #[test] + fn random_stream_ids_are_unique_and_inherited_by_final_calls() { + let mut first = DsmlToolStream::with_prefix("call_"); + first.use_random_ids(); + assert!(first.init(DSML_TOOL_CALLS_START.as_bytes(), 0)); + let id = first.id_at(0); + assert_eq!(id.len(), 37); + assert!(id.starts_with("call_")); + assert!(id[5..] + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))); + + let mut calls = vec![ToolCall::default()]; + first.apply_ids(&mut calls); + assert_eq!(calls[0].id, id); + + let mut second = DsmlToolStream::with_prefix("call_"); + second.use_random_ids(); + assert_ne!(second.id_at(0), id); + } +} diff --git a/crates/ds4-server/src/tools.rs b/crates/ds4-server/src/tools.rs new file mode 100644 index 0000000000000000000000000000000000000000..2d085bfae4f519eb0ba1fab4c5afa90a40447d98 --- /dev/null +++ b/crates/ds4-server/src/tools.rs @@ -0,0 +1,2832 @@ +//! Generated-message tool parse + serial semantic accumulator from +//! `ds4_server.c` at v0.6.5-dfm. DSML greedy / required-prefix sampling +//! and corrective retry (`retry`) are host-owned. + +use crate::dsml::{ + dsml_decode_state_is_syntax, DsmlDecodeState, DsmlDecodeTracker, SampleOverride, SamplePolicy, + ToolSyntax, +}; +use crate::json::{json_escape_bytes, json_minify_raw_value, json_raw_value, json_string, Json}; +use crate::parse::{ToolCall, ToolSchemaOrder}; +use crate::render::{ + syntax_for_model_id, ModelSyntax, GLM_TOOL_CALL_END, GLM_TOOL_CALL_START, QWEN_TOOL_CALL_END, + QWEN_TOOL_CALL_START, SOLAR_THINK_END, SOLAR_THINK_START, SOLAR_TOOL_ARG_END, + SOLAR_TOOL_ARG_START, SOLAR_TOOL_ARG_VALUE, SOLAR_TOOL_CALLS, SOLAR_TOOL_CALL_END, +}; + +pub(crate) mod beam; +pub(crate) mod iquest; +use crate::stream::{think_end, think_start, ChatFormat}; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::Read as _; +use std::sync::{Mutex, OnceLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +pub const DSML_TOOL_CALLS_START: &str = "<|DSML|tool_calls>"; +pub const DSML_TOOL_CALLS_END: &str = ""; +pub const DSML_INVOKE_START: &str = "<|DSML|invoke"; +pub const DSML_INVOKE_END: &str = ""; +pub const DSML_PARAM_START: &str = "<|DSML|parameter"; +pub const DSML_PARAM_END: &str = ""; +pub const DSML_TOOL_CALLS_START_SHORT: &str = ""; +pub const DSML_TOOL_CALLS_END_SHORT: &str = ""; +pub const DSML_INVOKE_START_SHORT: &str = "): the block name and the tag names all open with a space -- +/// `<| calls>`, `<| invoke`, `<| parameter` (engine server_types.h +/// DS4_TOOL_CALLS_START_SPACED, 4294136). String values are written raw in +/// this generation and are NOT entity-unescaped on the way back +/// (server_dsml_stream.c dsml_value_unescape; only the render-side `<` +/// guard before the closing tag is restored). +pub const DSML_TOOL_CALLS_START_SPACED: &str = "<|DSML| calls>"; +pub const DSML_TOOL_CALLS_END_SPACED: &str = ""; +pub const DSML_INVOKE_START_SPACED: &str = "<|DSML| invoke"; +pub const DSML_INVOKE_END_SPACED: &str = ""; +pub const DSML_PARAM_START_SPACED: &str = "<|DSML| parameter"; +pub const DSML_PARAM_END_SPACED: &str = ""; + +#[derive(Debug, Clone, Default)] +pub struct ParsedGenerated { + pub content: Vec, + pub reasoning: Vec, + pub calls: Vec, + pub raw_dsml: String, + pub raw_tool_text: String, + pub ok: bool, + pub recovered: bool, +} + +fn is_c_space(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c) +} + +fn find_substr(hay: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + hay.windows(needle.len()).position(|w| w == needle) +} + +fn find_last_substr(hay: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + hay.windows(needle.len()) + .enumerate() + .rev() + .find(|(_, w)| *w == needle) + .map(|(i, _)| i) +} + +fn skip_ws(s: &[u8], mut i: usize) -> usize { + while i < s.len() && is_c_space(s[i]) { + i += 1; + } + i +} + +fn trim_tool_separator_ws(raw: &[u8], start: usize, mut limit: usize) -> usize { + while limit > start && is_c_space(raw[limit - 1]) { + limit -= 1; + } + limit +} + +fn trim_ascii_span(s: &[u8]) -> &[u8] { + let mut a = 0; + let mut b = s.len(); + while a < b && is_c_space(s[a]) { + a += 1; + } + while b > a && is_c_space(s[b - 1]) { + b -= 1; + } + &s[a..b] +} + +pub(crate) fn dsml_unescape_text(s: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while i < s.len() { + if s[i] != b'&' { + out.push(s[i]); + i += 1; + continue; + } + if s[i..].starts_with(b"&") { + out.push(b'&'); + i += 5; + } else if s[i..].starts_with(b"<") { + out.push(b'<'); + i += 4; + } else if s[i..].starts_with(b">") { + out.push(b'>'); + i += 4; + } else if s[i..].starts_with(b""") { + out.push(b'"'); + i += 6; + } else if s[i..].starts_with(b"'") { + out.push(b'\''); + i += 6; + } else { + out.push(b'&'); + i += 1; + } + } + out +} + +pub(crate) fn dsml_attr(tag: &[u8], name: &str) -> Option> { + let mut pat = name.as_bytes().to_vec(); + pat.extend_from_slice(b"=\""); + let p = find_substr(tag, &pat)?; + let start = p + pat.len(); + let rel = tag[start..].iter().position(|&c| c == b'"')?; + Some(dsml_unescape_text(&tag[start..start + rel])) +} + +fn tool_call_json_args_add(args: &mut Vec, name: &[u8], value: &[u8], is_string: bool) { + if !args.is_empty() { + args.extend_from_slice(b", "); + } + args.extend(json_escape_bytes(name)); + args.extend_from_slice(b": "); + if is_string { + args.extend(json_escape_bytes(value)); + } else { + let min = json_minify_raw_value(&String::from_utf8_lossy(value)); + if min.is_empty() { + args.extend_from_slice(b"null"); + } else { + args.extend_from_slice(min.as_bytes()); + } + } +} + +fn split_reasoning_content(text: &[u8], n: usize, format: ChatFormat) -> (Vec, Vec) { + let s = &text[..n.min(text.len())]; + let start = think_start(format).as_bytes(); + let end = think_end(format).as_bytes(); + let body = if s.starts_with(start) { + &s[start.len()..] + } else { + s + }; + if let Some(i) = find_substr(body, end) { + (body[i + end.len()..].to_vec(), body[..i].to_vec()) + } else { + (s.to_vec(), Vec::new()) + } +} + +fn unterminated_reasoning(text: &[u8]) -> (Vec, Vec) { + let body = if text.starts_with(b"") { + &text[7..] + } else { + text + }; + (Vec::new(), body.to_vec()) +} + +fn unterminated_reasoning_before_tool(text: &[u8], mut prefix_len: usize) -> (Vec, Vec) { + prefix_len = prefix_len.min(text.len()); + let (body, plen) = if prefix_len >= 7 && text.starts_with(b"") { + (&text[7..], prefix_len - 7) + } else { + (text, prefix_len) + }; + (Vec::new(), body[..plen.min(body.len())].to_vec()) +} + +fn dsml_parse_leaf_param( + s: &[u8], + i: &mut usize, + param_start: &[u8], + param_end: &[u8], + out: &mut Vec, +) -> bool { + if !s[*i..].starts_with(param_start) { + return false; + } + let Some(gt) = s[*i..].iter().position(|&c| c == b'>') else { + return false; + }; + let tag_end = *i + gt; + let tag = &s[*i..=tag_end]; + let Some(name) = dsml_attr(tag, "name") else { + return false; + }; + let is_string_attr = dsml_attr(tag, "string"); + let value_start = tag_end + 1; + let Some(rel) = find_substr(&s[value_start..], param_end) else { + return false; + }; + let raw = &s[value_start..value_start + rel]; + let is_string = is_string_attr + .as_deref() + .map(|v| v == b"true") + .unwrap_or(true); + let value = if is_string { + dsml_unescape_text(raw) + } else { + raw.to_vec() + }; + tool_call_json_args_add(out, &name, &value, is_string); + *i = value_start + rel + param_end.len(); + true +} + +fn dsml_parse_nested_params_object( + s: &[u8], + i: &mut usize, + param_start: &[u8], + param_end: &[u8], +) -> Option> { + let mut members = Vec::new(); + let mut any = false; + loop { + *i = skip_ws(s, *i); + if !s[*i..].starts_with(param_start) { + break; + } + if !dsml_parse_leaf_param(s, i, param_start, param_end, &mut members) { + return None; + } + any = true; + } + if !any { + return None; + } + let mut out = vec![b'{']; + out.extend_from_slice(&members); + out.push(b'}'); + Some(out) +} + +#[derive(Clone, Copy)] +struct DsmlStyle { + tool_calls_start: &'static [u8], + tool_calls_end: &'static [u8], + invoke_start: &'static [u8], + invoke_end: &'static [u8], + param_start: &'static [u8], + param_end: &'static [u8], + /// The old generations write `&`/`<` in string values as XML entities and + /// the parse reverses them; the official spaced generation writes values + /// raw (engine server_dsml_stream.c dsml_value_unescape, 4294136). + entity_values: bool, +} + +/// The engine's four spellings in its table's order (server_dsml_stream.c:5, +/// dsml_syntaxes): long, short, plain, spaced. Only the spaced generation +/// differs on values (entity_values = false). +const DSML_STYLES: [DsmlStyle; 4] = [ + DsmlStyle { + tool_calls_start: DSML_TOOL_CALLS_START.as_bytes(), + tool_calls_end: DSML_TOOL_CALLS_END.as_bytes(), + invoke_start: DSML_INVOKE_START.as_bytes(), + invoke_end: DSML_INVOKE_END.as_bytes(), + param_start: DSML_PARAM_START.as_bytes(), + param_end: DSML_PARAM_END.as_bytes(), + entity_values: true, + }, + DsmlStyle { + tool_calls_start: DSML_TOOL_CALLS_START_SHORT.as_bytes(), + tool_calls_end: DSML_TOOL_CALLS_END_SHORT.as_bytes(), + invoke_start: DSML_INVOKE_START_SHORT.as_bytes(), + invoke_end: DSML_INVOKE_END_SHORT.as_bytes(), + param_start: DSML_PARAM_START_SHORT.as_bytes(), + param_end: DSML_PARAM_END_SHORT.as_bytes(), + entity_values: true, + }, + DsmlStyle { + tool_calls_start: b"", + tool_calls_end: b"", + invoke_start: b"", + param_start: b"", + entity_values: true, + }, + DsmlStyle { + tool_calls_start: DSML_TOOL_CALLS_START_SPACED.as_bytes(), + tool_calls_end: DSML_TOOL_CALLS_END_SPACED.as_bytes(), + invoke_start: DSML_INVOKE_START_SPACED.as_bytes(), + invoke_end: DSML_INVOKE_END_SPACED.as_bytes(), + param_start: DSML_PARAM_START_SPACED.as_bytes(), + param_end: DSML_PARAM_END_SPACED.as_bytes(), + entity_values: false, + }, +]; + +/// The engine's `find_tool_block` (server_dsml_parse.c:66-82): among all four +/// spellings take the earliest occurrence; the first pass only accepts a block +/// preceded by "\n\n" (the template's separator) and returns the separator's +/// position, the second accepts a bare marker -- prose quoting a tag mid-text +/// must not steal the real tool block. +fn dsml_find_block(search: &[u8], need_sep: bool) -> Option<(usize, usize)> { + let mut best: Option<(usize, usize)> = None; + for (si, sty) in DSML_STYLES.iter().enumerate() { + let mut from = 0; + while let Some(rel) = find_substr(&search[from..], sty.tool_calls_start) { + let pos = from + rel; + if need_sep && !(pos >= 2 && &search[pos - 2..pos] == b"\n\n") { + from = pos + 1; + continue; + } + let at = if need_sep { pos - 2 } else { pos }; + if best.map_or(true, |(b, _)| at < b) { + best = Some((at, si)); + } + break; + } + } + best +} + +/// The engine's `dsml_value_unescape` (server_dsml_stream.c:37-53): the old +/// generations reverse full XML entities; the spaced generation writes values +/// raw and restores only the `<` the renderer adds before a closing tag +/// inside a value, so a value that genuinely contains "<" keeps it. +pub(crate) fn dsml_value_unescape_syntax( + entity_values: bool, + param_end: &[u8], + s: &[u8], +) -> Vec { + if entity_values { + return dsml_unescape_text(s); + } + let mut out = Vec::new(); + let mut i = 0; + while i < s.len() { + if s[i..].starts_with(b"<") && s[i + 4..].starts_with(¶m_end[1..]) { + out.push(b'<'); + i += 4; + } else { + out.push(s[i]); + i += 1; + } + } + out +} + +fn parse_dsml_generated(text: &[u8], require_thinking_closed: bool) -> Option { + let mut tool_search = 0usize; + if require_thinking_closed { + match find_last_substr(text, b"") { + None => { + let (content, reasoning) = unterminated_reasoning(text); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + } + Some(i) => tool_search = i + 8, + } + } + let search = &text[tool_search..]; + let Some((rel, style_idx)) = + dsml_find_block(search, true).or_else(|| dsml_find_block(search, false)) + else { + let (content, reasoning) = split_reasoning_content(text, text.len(), ChatFormat::DeepSeek); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + }; + let start = tool_search + rel; + let content_len = trim_tool_separator_ws(text, 0, start); + let sty = DSML_STYLES[style_idx]; + let Some(p0) = find_substr(&text[start..], sty.tool_calls_start) else { + return None; + }; + let mut p = start + p0 + sty.tool_calls_start.len(); + let mut calls = Vec::new(); + loop { + p = skip_ws(text, p); + if text[p..].starts_with(sty.tool_calls_end) { + let raw_end = p + sty.tool_calls_end.len(); + let (content, reasoning) = + split_reasoning_content(text, content_len, ChatFormat::DeepSeek); + return Some(ParsedGenerated { + content, + reasoning, + calls, + raw_dsml: String::from_utf8_lossy(&text[start..raw_end]).into_owned(), + ok: true, + ..Default::default() + }); + } + if !text[p..].starts_with(sty.invoke_start) { + return None; + } + let Some(gt) = text[p..].iter().position(|&c| c == b'>') else { + return None; + }; + let tag = &text[p..=p + gt]; + let Some(name) = dsml_attr(tag, "name") else { + return None; + }; + p = p + gt + 1; + let mut args = Vec::new(); + loop { + p = skip_ws(text, p); + if text[p..].starts_with(sty.invoke_end) { + p += sty.invoke_end.len(); + break; + } + if !text[p..].starts_with(sty.param_start) { + return None; + } + let Some(gt) = text[p..].iter().position(|&c| c == b'>') else { + return None; + }; + let tag = &text[p..=p + gt]; + let Some(param_name) = dsml_attr(tag, "name") else { + return None; + }; + let param_is_string = dsml_attr(tag, "string"); + let value_start = p + gt + 1; + let after_ws = skip_ws(text, value_start); + if param_is_string.is_none() && text[after_ws..].starts_with(sty.param_start) { + let mut nested_p = value_start; + let Some(nested) = dsml_parse_nested_params_object( + text, + &mut nested_p, + sty.param_start, + sty.param_end, + ) else { + return None; + }; + tool_call_json_args_add(&mut args, ¶m_name, &nested, false); + p = skip_ws(text, nested_p); + if text[p..].starts_with(sty.param_end) { + p += sty.param_end.len(); + } + continue; + } + let Some(rel) = find_substr(&text[value_start..], sty.param_end) else { + return None; + }; + let raw = &text[value_start..value_start + rel]; + let is_string = param_is_string + .as_deref() + .map(|v| v == b"true") + .unwrap_or(true); + let value = if is_string { + dsml_value_unescape_syntax(sty.entity_values, sty.param_end, raw) + } else { + raw.to_vec() + }; + tool_call_json_args_add(&mut args, ¶m_name, &value, is_string); + p = value_start + rel + sty.param_end.len(); + } + let mut wrapped = vec![b'{']; + wrapped.extend_from_slice(&args); + wrapped.push(b'}'); + calls.push(ToolCall { + name: String::from_utf8_lossy(&name).into_owned(), + arguments: String::from_utf8_lossy(&wrapped).into_owned(), + ..Default::default() + }); + } +} + +/// Collapse every decorated DSML spelling to the plain tag set, keeping a +/// per-byte map back into the original text. +/// +/// The V4.1 artifact decorates the wrapper and invoke opens with U+FF5C +/// (|, EF BD 9C) but leaves the parameter tags plain inside the same block +/// (measured 2026-10-10; docs/deepseek41-port-plan.md 6.23), so the strict +/// one-style parse rejects a call the model considers complete. The six +/// replacements mirror the tool shim's `normalize` +/// (`~/youngai/tool-shim/shim.py:53-58`), written against this artifact's +/// live output on 2026-10-07. +fn dsml_normalize(text: &[u8]) -> (Vec, Vec) { + let pats: [(&[u8], &[u8]); 8] = [ + // The official spaced spelling folds onto the plain one -- `<\u{ff5c} calls>` + // -> `` (engine server_types.h DS4_*_SPACED); the bar+space pair + // never appears in the old generations, so the order is unambiguous. + (b"<\xef\xbd\x9cDSML\xef\xbd\x9c ", b"<"), + (b" String { + String::from_utf8_lossy(raw) + .trim() + .trim_start_matches('$') + .to_string() +} + +/// Tolerant DSML scan for the V4.1 route (docs/deepseek41-port-plan.md 6.23). +/// +/// Mirrors the tool shim's scanner (`~/youngai/tool-shim/shim.py:44-190`), +/// built against this artifact's live output: normalize the decorated +/// spellings, key the scan on ``, tolerate a missing closer or a +/// bare ``, skip undeclared tool names, and coerce each +/// value to the type the request's schema declares (the model regularly +/// emits a number as a quoted string). Runs only as a fallback when the +/// strict style match found no calls, so every other family's bytes stay +/// identical; returns None when no calls can be reconstructed. +pub fn parse_dsml_tolerant( + text: &[u8], + orders: &[ToolSchemaOrder], + require_thinking_closed: bool, +) -> Option { + let (norm, map) = dsml_normalize(text); + // With thinking enabled the engine searches for a block only after the + // LAST think close: DSML inside reasoning is model text, never + // executable (server_dsml_parse.c:291-316), and a missing close means + // no tool search at all. Mirror parse_dsml_generated's scope. + let search_from = if require_thinking_closed { + let end = think_end(ChatFormat::DeepSeek).as_bytes(); + let last = find_last_substr(&norm, end)?; + last + end.len() + } else { + 0 + }; + let start = [ + find_substr(&norm[search_from..], b"").map(|r| search_from + r), + find_substr(&norm[search_from..], b"')?; + p += gt + 1; + } + let mut calls = Vec::new(); + loop { + p = skip_ws(&norm, p); + if p >= norm.len() + || norm[p..].starts_with(b"') else { + break; + }; + let tag = &norm[p..=p + gt]; + let Some(raw_name) = dsml_attr(tag, "name") else { + break; + }; + let name = dsml_clean_name(&raw_name); + p += gt + 1; + let mut args = Vec::new(); + let mut complete = true; + loop { + p = skip_ws(&norm, p); + if p >= norm.len() + || norm[p..].starts_with(b"') { + p += gt + 1; + } + } + if !complete { + break; // a truncated argument: drop this call and stop + } + // An undeclared name is skipped, declared siblings survive (the + // shim's `allowed` check). + let declared = + orders.is_empty() || orders.iter().any(|o| o.name == name || o.wire_name == name); + if !declared { + continue; + } + let mut wrapped = vec![b'{']; + wrapped.extend_from_slice(&args); + wrapped.push(b'}'); + calls.push(ToolCall { + name, + arguments: String::from_utf8_lossy(&wrapped).into_owned(), + ..Default::default() + }); + } + if calls.is_empty() { + return None; + } + // The reference's raw span ends at the closing wrapper + // (`ds4_server.c:6218-6219`: `raw_block_end = p + strlen(tool_calls_end)`), + // and exact replay re-emits these bytes verbatim, so a span that stopped at + // the last invocation would put an unclosed tool block back in the prompt. + // This scan leaves `p` ON the close marker; step over it. + if norm[p..].starts_with(b"') { + p += gt + 1; + } + } + let orig_start = map[start]; + let content_len = trim_tool_separator_ws(text, 0, orig_start); + let (content, reasoning) = split_reasoning_content(text, content_len, ChatFormat::DeepSeek); + let orig_end = if p < norm.len() { map[p] } else { text.len() }; + Some(ParsedGenerated { + content, + reasoning, + calls, + raw_dsml: String::from_utf8_lossy(&text[orig_start..orig_end.max(orig_start)]).into_owned(), + ok: true, + ..Default::default() + }) +} + +/// The artifact's tool-name-root shape (Spark session 01a13136, 2026-10-12, +/// bytes captured from the open-grok transcript): the call opens with the +/// TOOL NAME as the root element and the parameters as plain children -- +/// `\n..\n8\n` -- +/// with no wrapper and no invoke anywhere. The engine keys every arm on +/// the wrapper/invoke opens (server_dsml_parse.c find_tool_block) and the +/// shim's scanner on the same starts (shim.py:492-494), so the whole block +/// falls back to assistant text and the calls never run. Extension of the +/// same recovery (plan 6.23, the named deviation): a root is bound only +/// when its tag names a tool the request's schema declares, every child +/// goes through the bare-child binding (declared property, matching close, +/// schema coercion), and the close must match the root; every other shape +/// keeps the pre-existing behavior. The root tag body may also carry the +/// single stray quote the served turn of 2026-10-18 shows (``, +/// open-grok session 01a14f5d); see the body check below for the measurement. +fn dsml_tolerant_tool_root_scan( + text: &[u8], + norm: &[u8], + map: &[usize], + orders: &[ToolSchemaOrder], + from: usize, +) -> Option { + let mut calls = Vec::new(); + let mut p = from; + let mut first = None; + let mut last = 0; + while p < norm.len() { + // The earliest declared-tool root tag from here wins; a tag naming + // no declared tool is model text and stays untouched. + let mut best: Option<(usize, &ToolSchemaOrder, &str, usize)> = None; + for o in orders { + for n in [o.name.as_str(), o.wire_name.as_str()] { + if n.is_empty() { + continue; + } + let mut open = Vec::with_capacity(n.len() + 1); + open.extend_from_slice(b"<"); + open.extend_from_slice(n.as_bytes()); + let Some(rel) = find_substr(&norm[p..], &open) else { + continue; + }; + let at = p + rel; + let Some(gt) = norm[at + open.len()..].iter().position(|&c| c == b'>') else { + continue; + }; + // The tag body is empty (``) or one stray quote + // (``): the artifact keeps the closing quote of the + // `name="` attribute the reference's own prompt teaches + // (ds4_server.c:2580-2589) while dropping the attribute name, + // and its closes carry the decorated DS41 forms + // (ds4_server.c:679-683). Measured 2026-10-18 on the served + // Spark turn (open-grok session 01a14f5d): with that one byte + // present the scan recovered nothing and no tool ran; removed, + // both turns recover four calls each. Any other body is not + // this rule's shape. + let body = &norm[at + open.len()..at + open.len() + gt]; + let stray_quote: &[u8] = b"\""; + if !body.is_empty() && body != stray_quote { + continue; + } + if best.as_ref().map_or(true, |(b, _, _, _)| at < *b) { + best = Some((at, o, n, at + open.len() + gt + 1)); + } + } + } + let Some((at, o, n, tag_end)) = best else { + break; + }; + let mut q = tag_end; + let mut args = Vec::new(); + loop { + q = skip_ws(norm, q); + if q >= norm.len() || norm[q..].starts_with(b"'); + if !norm[q..].starts_with(&close) { + // No matching close: not a complete call; the bytes stay model + // text and the scan resumes after this tag. + p = at + 1; + continue; + } + q += close.len(); + if first.is_none() { + first = Some(at); + } + last = q; + let mut wrapped = vec![b'{']; + wrapped.extend_from_slice(&args); + wrapped.push(b'}'); + calls.push(ToolCall { + name: o.name.clone(), + arguments: String::from_utf8_lossy(&wrapped).into_owned(), + ..Default::default() + }); + p = q; + } + if calls.is_empty() { + return None; + } + let orig_start = map[first?]; + let content_len = trim_tool_separator_ws(text, 0, orig_start); + let (content, reasoning) = split_reasoning_content(text, content_len, ChatFormat::DeepSeek); + let orig_end = if last < norm.len() { + map[last] + } else { + text.len() + }; + Some(ParsedGenerated { + content, + reasoning, + calls, + raw_dsml: String::from_utf8_lossy(&text[orig_start..orig_end.max(orig_start)]).into_owned(), + ok: true, + ..Default::default() + }) +} + +/// One `` of the tolerant scan, with the shim's coercion rules. +/// The value goes through the declared schema type first (integer, number, +/// boolean, object, array); a string= hint or a plain string is the +/// fallback. Nested trees keep the strict parser's rule: no string +/// attribute and the value starts with a nested `, +) -> bool { + let Some(gt) = norm[*p..].iter().position(|&c| c == b'>') else { + return false; + }; + let tag = &norm[*p..=*p + gt]; + let Some(raw_name) = dsml_attr(tag, "name") else { + return false; + }; + let pname = dsml_clean_name(&raw_name); + let string_attr = dsml_attr(tag, "string"); + let value_start = *p + gt + 1; + + let after_ws = skip_ws(norm, value_start); + if string_attr.is_none() && norm[after_ws..].starts_with(b"") + { + np = skip_ws(norm, np); + if norm[np..].starts_with(b"") { + np += b"".len(); + } + *p = np; + tool_call_json_args_add(out, pname.as_bytes(), &obj, false); + return true; + } + } + + let (raw, next) = match find_substr(&norm[value_start..], b"") { + Some(rel) => ( + &norm[value_start..value_start + rel], + value_start + rel + b"".len(), + ), + None => { + // Missing closer: the value runs to the end of the invoke body + // (the shim's lookahead: ||EOF). + // A body that ends at EOF may be a truncated stream rather than + // a model omission; drop the call instead of serving half an + // argument. + let end = [ + b", value: &[u8]) -> Option> { + match declared { + Some("integer") => std::str::from_utf8(value) + .ok() + .and_then(|s| s.trim().parse::().ok()) + .map(|n| n.to_string().into_bytes()), + Some("number") => std::str::from_utf8(value) + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|f| f.is_finite()) + .map(|f| f.to_string().into_bytes()), + Some("boolean") if value == b"true" || value == b"false" => Some(value.to_vec()), + Some("object") | Some("array") => { + json_value_is_complete(&String::from_utf8_lossy(value)).map(|min| min.into_bytes()) + } + _ => None, + } +} + +/// One bare child element of an invoke body (the model occasionally writes +/// `Tokyo` instead of the parameter spelling; Spark tool-call +/// gate requests 8/23, 2026-10-11). The engine rejects the whole block on +/// the first non-parameter token (server_dsml_parse.c:365-369) and the +/// shim's scanner binds only parameter tags (shim.py:59-61), so binding a +/// child is an extension of the tolerant recovery (plan 6.23): it fires +/// only when the tag names a property the request's schema declares for +/// this tool, and the value is coerced like a parameter. Any other child +/// keeps the pre-existing behavior (stray bytes end the invoke body). +fn dsml_tolerant_child_param( + norm: &[u8], + p: &mut usize, + tool: &str, + orders: &[ToolSchemaOrder], + out: &mut Vec, +) -> bool { + let rest = &norm[*p..]; + if !rest.starts_with(b"<") || rest.starts_with(b"') else { + return false; + }; + let tag = &rest[1..gt]; + // A bare child carries only its tag name; anything else is stray bytes. + if tag.is_empty() || tag.iter().any(|&c| !c.is_ascii_alphanumeric() && c != b'_') { + return false; + } + let pname = String::from_utf8_lossy(tag).into_owned(); + let Some(declared) = tool_schema_order_prop_type(tool_schema_orders_find(orders, tool), &pname) + else { + return false; // undeclared child: keep the pre-existing recovery + }; + let mut close = Vec::with_capacity(tag.len() + 3); + close.extend_from_slice(b"'); + let value_start = *p + gt + 1; + let Some(rel) = find_substr(&norm[value_start..], &close) else { + return false; + }; + let raw = &norm[value_start..value_start + rel]; + if raw.contains(&b'<') { + return false; // a nested tag is not a value this rule binds + } + *p = value_start + rel + close.len(); + let unescaped = dsml_unescape_text(raw); + let value = trim_ascii_span(&unescaped); + match dsml_coerce_declared(Some(declared), value) { + Some(v) => tool_call_json_args_add(out, pname.as_bytes(), &v, false), + None => tool_call_json_args_add(out, pname.as_bytes(), value, true), + } + true +} + +fn parse_hermes_tool_call_json(json: &str) -> Option { + let mut p = Json::new(json); + p.ws(); + if p.bump() != Some(b'{') { + return None; + } + p.ws(); + let mut tc = ToolCall::default(); + while p.peek().is_some() && p.peek() != Some(b'}') { + let key = json_string(&mut p)?; + p.ws(); + if p.bump() != Some(b':') { + return None; + } + if key == "name" { + tc.name = json_string(&mut p)?; + } else if key == "arguments" { + tc.arguments = json_raw_value(&mut p)?; + } else if key == "id" { + tc.id = json_string(&mut p)?; + } else if !crate::json::json_skip_value(&mut p) { + return None; + } + p.ws(); + if p.peek() == Some(b',') { + p.i += 1; + } + p.ws(); + } + if p.bump() != Some(b'}') { + return None; + } + p.ws(); + if p.peek().is_some() || tc.name.is_empty() { + return None; + } + if tc.arguments.is_empty() { + tc.arguments = "null".into(); + } + Some(tc) +} + +fn parse_hermes_generated(text: &[u8], require_thinking_closed: bool) -> Option { + let tool_start = b""; + let tool_end = b""; + let mut tool_search = 0usize; + let mut recovered = false; + if require_thinking_closed { + match find_last_substr(text, b"") { + None => { + let candidate = find_substr(text, tool_start); + if candidate.is_none() + || find_substr(&text[candidate.unwrap()..], tool_end).is_none() + { + let (content, reasoning) = unterminated_reasoning(text); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + } + tool_search = candidate.unwrap(); + recovered = true; + } + Some(i) => tool_search = i + 8, + } + } + let Some(rel) = find_substr(&text[tool_search..], tool_start) else { + let (content, reasoning) = split_reasoning_content(text, text.len(), ChatFormat::DeepSeek); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + }; + let start = tool_search + rel; + let raw_block_start = if start > 0 && text[start - 1] == b'\n' { + start - 1 + } else { + start + }; + let content_len = trim_tool_separator_ws(text, 0, raw_block_start); + let mut p = start; + let mut calls = Vec::new(); + loop { + p = skip_ws(text, p); + if !text[p..].starts_with(tool_start) { + break; + } + p += tool_start.len(); + let Some(rel) = find_substr(&text[p..], tool_end) else { + return None; + }; + let json = String::from_utf8_lossy(&text[p..p + rel]).into_owned(); + calls.push(parse_hermes_tool_call_json(&json)?); + p = p + rel + tool_end.len(); + } + if calls.is_empty() { + return None; + } + let (content, reasoning) = if recovered { + unterminated_reasoning_before_tool(text, content_len) + } else { + split_reasoning_content(text, content_len, ChatFormat::DeepSeek) + }; + Some(ParsedGenerated { + content, + reasoning, + calls, + raw_tool_text: String::from_utf8_lossy(&text[raw_block_start..p]).into_owned(), + ok: true, + ..Default::default() + }) +} + +fn parse_dots3_invoke(body: &[u8]) -> Option { + let name_open = b""; + let p0 = find_substr(body, name_open)?; + let name_s = p0 + name_open.len(); + let name_e = body[name_s..].iter().position(|&c| c == b'"')?; + let mut tc = ToolCall { + name: String::from_utf8_lossy(&body[name_s..name_s + name_e]).into_owned(), + ..Default::default() + }; + let mut p = name_s + name_e + 1; + if p < body.len() && body[p] == b'>' { + p += 1; + } + let mut args = vec![b'{']; + let mut n_args = 0; + loop { + let Some(open) = find_substr(&body[p..], param_open) else { + break; + }; + let key_s = p + open + param_open.len(); + let Some(key_e) = body[key_s..].iter().position(|&c| c == b'"') else { + break; + }; + let mut value = key_s + key_e + 1; + if value < body.len() && body[value] == b'>' { + value += 1; + } + if value < body.len() && body[value] == b'\n' { + value += 1; + } + let Some(close) = find_substr(&body[value..], param_close) else { + break; + }; + let mut value_end = value + close; + if value_end > value && body[value_end - 1] == b'\n' { + value_end -= 1; + } + if n_args > 0 { + args.extend_from_slice(b", "); + } + args.extend(json_escape_bytes(&body[key_s..key_s + key_e])); + args.extend_from_slice(b": "); + let raw = &body[value..value_end]; + let raw_s = String::from_utf8_lossy(raw); + let mut probe = Json::new(&raw_s); + probe.ws(); + if let Some(v) = json_raw_value(&mut probe) { + probe.ws(); + if probe.peek().is_none() { + args.extend_from_slice(v.as_bytes()); + } else { + args.extend(json_escape_bytes(raw)); + } + } else { + args.extend(json_escape_bytes(raw)); + } + n_args += 1; + p = value + close + param_close.len(); + } + args.push(b'}'); + if tc.name.is_empty() { + return None; + } + tc.arguments = String::from_utf8_lossy(&args).into_owned(); + Some(tc) +} + +fn parse_dots3_generated(text: &[u8], require_thinking_closed: bool) -> Option { + let tool_start = b""; + let tool_end = b""; + let mut tool_search = 0usize; + let mut recovered = false; + if require_thinking_closed { + match find_last_substr(text, b"") { + None => { + let candidate = find_substr(text, tool_start); + if candidate.is_none() + || find_substr(&text[candidate.unwrap()..], tool_end).is_none() + { + let (content, reasoning) = unterminated_reasoning(text); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + } + tool_search = candidate.unwrap(); + recovered = true; + } + Some(i) => tool_search = i + 8, + } + } + let Some(rel) = find_substr(&text[tool_search..], tool_start) else { + let (content, reasoning) = split_reasoning_content(text, text.len(), ChatFormat::DeepSeek); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + }; + let start = tool_search + rel; + let raw_block_start = if start > 0 && text[start - 1] == b'\n' { + start - 1 + } else { + start + }; + let content_len = trim_tool_separator_ws(text, 0, raw_block_start); + let mut p = start; + let mut calls = Vec::new(); + loop { + p = skip_ws(text, p); + if !text[p..].starts_with(tool_start) { + break; + } + p += tool_start.len(); + let Some(rel) = find_substr(&text[p..], tool_end) else { + return None; + }; + let close = p + rel; + let mut invoke = p; + while invoke < close { + let Some(next) = find_substr(&text[invoke..close], b"") + .map(|i| next + i) + .unwrap_or(close); + calls.push(parse_dots3_invoke(&text[next..invoke_end])?); + invoke = invoke_end; + } + p = close + tool_end.len(); + } + if calls.is_empty() { + return None; + } + let (content, reasoning) = if recovered { + unterminated_reasoning_before_tool(text, content_len) + } else { + split_reasoning_content(text, content_len, ChatFormat::DeepSeek) + }; + Some(ParsedGenerated { + content, + reasoning, + calls, + raw_tool_text: String::from_utf8_lossy(&text[raw_block_start..p]).into_owned(), + ok: true, + ..Default::default() + }) +} + +fn json_value_is_complete(value: &str) -> Option { + let mut p = Json::new(value); + p.ws(); + if p.peek().is_none() { + return None; + } + let raw = json_raw_value(&mut p)?; + p.ws(); + if p.peek().is_some() { + return None; + } + Some(json_minify_raw_value(&raw)) +} + +fn solar_tool_arg_json_add( + args: &mut Vec, + name: &[u8], + raw: &[u8], + expected_type: Option<&str>, +) { + let trimmed = trim_ascii_span(raw); + if !args.is_empty() { + args.extend_from_slice(b", "); + } + args.extend(json_escape_bytes(name)); + args.extend_from_slice(b": "); + if expected_type == Some("string") { + args.extend(json_escape_bytes(raw)); + } else if let Some(min) = json_value_is_complete(&String::from_utf8_lossy(trimmed)) { + args.extend_from_slice(min.as_bytes()); + } else { + args.extend(json_escape_bytes(raw)); + } +} + +fn tool_schema_orders_find<'a>( + orders: &'a [ToolSchemaOrder], + name: &str, +) -> Option<&'a ToolSchemaOrder> { + orders.iter().find(|o| o.name == name) +} + +fn tool_schema_order_prop_type<'a>( + order: Option<&'a ToolSchemaOrder>, + name: &str, +) -> Option<&'a str> { + let order = order?; + order + .prop + .iter() + .zip(order.prop_type.iter()) + .find(|(p, _)| p.as_str() == name) + .map(|(_, t)| t.as_str()) +} + +/// The declared type when the client wrote it as a list (`Option` under +/// schemars). Empty entries mean the client declared nothing usable, so the +/// caller falls back to the reference's own rule for GLM: text. +fn tool_schema_order_prop_type_first<'a>( + order: Option<&'a ToolSchemaOrder>, + name: &str, +) -> Option<&'a str> { + let order = order?; + order + .prop + .iter() + .zip(order.prop_type_first.iter()) + .find(|(p, _)| p.as_str() == name) + .map(|(_, t)| t.as_str()) + .filter(|t| !t.is_empty()) +} + +fn parse_solar_generated( + text: &[u8], + require_thinking_closed: bool, + orders: &[ToolSchemaOrder], +) -> Option { + let think_end = SOLAR_THINK_END.as_bytes(); + let think_start = SOLAR_THINK_START.as_bytes(); + let mut tool_search = 0usize; + if require_thinking_closed { + match find_last_substr(text, think_end) { + None => { + let body = if text.starts_with(think_start) { + &text[think_start.len()..] + } else { + text + }; + return Some(ParsedGenerated { + content: Vec::new(), + reasoning: body.to_vec(), + ok: true, + ..Default::default() + }); + } + Some(i) => tool_search = i + think_end.len(), + } + } + let Some(rel) = find_substr(&text[tool_search..], SOLAR_TOOL_CALLS.as_bytes()) else { + let (content, reasoning) = + split_reasoning_content(text, text.len(), ChatFormat::SolarOpen2); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + }; + let start = tool_search + rel; + let content_len = trim_tool_separator_ws(text, 0, start); + let mut p = start; + let mut raw_end = None; + let mut calls = Vec::new(); + loop { + if !text[p..].starts_with(SOLAR_TOOL_CALLS.as_bytes()) { + return None; + } + p += SOLAR_TOOL_CALLS.len(); + let Some(nl) = text[p..].iter().position(|&c| c == b'\n') else { + return None; + }; + let name = trim_ascii_span(&text[p..p + nl]); + if name.is_empty() { + return None; + } + let name_s = String::from_utf8_lossy(name).into_owned(); + let order = tool_schema_orders_find(orders, &name_s); + p = p + nl + 1; + let mut args = Vec::new(); + let mut call_closed = false; + while p < text.len() { + while p < text.len() && (text[p] == b'\r' || text[p] == b'\n') { + p += 1; + } + if text[p..].starts_with(SOLAR_TOOL_CALL_END.as_bytes()) { + p += SOLAR_TOOL_CALL_END.len(); + raw_end = Some(p); + call_closed = true; + break; + } + if !text[p..].starts_with(SOLAR_TOOL_ARG_START.as_bytes()) { + return None; + } + p += SOLAR_TOOL_ARG_START.len(); + let Some(vm) = find_substr(&text[p..], SOLAR_TOOL_ARG_VALUE.as_bytes()) else { + return None; + }; + let arg_name = trim_ascii_span(&text[p..p + vm]); + p = p + vm + SOLAR_TOOL_ARG_VALUE.len(); + let Some(ae) = find_substr(&text[p..], SOLAR_TOOL_ARG_END.as_bytes()) else { + return None; + }; + if arg_name.is_empty() { + return None; + } + let arg_name_s = String::from_utf8_lossy(arg_name).into_owned(); + solar_tool_arg_json_add( + &mut args, + arg_name, + &text[p..p + ae], + tool_schema_order_prop_type(order, &arg_name_s), + ); + p = p + ae + SOLAR_TOOL_ARG_END.len(); + } + if !call_closed { + return None; + } + let mut wrapped = vec![b'{']; + wrapped.extend_from_slice(&args); + wrapped.push(b'}'); + calls.push(ToolCall { + name: name_s, + arguments: String::from_utf8_lossy(&wrapped).into_owned(), + ..Default::default() + }); + let next = skip_ws(text, p); + if !text[next..].starts_with(SOLAR_TOOL_CALLS.as_bytes()) { + break; + } + p = next; + } + if calls.is_empty() { + return None; + } + let (content, reasoning) = split_reasoning_content(text, content_len, ChatFormat::SolarOpen2); + Some(ParsedGenerated { + content, + reasoning, + calls, + raw_dsml: String::from_utf8_lossy(&text[start..raw_end.unwrap_or(p)]).into_owned(), + ok: true, + ..Default::default() + }) +} + +fn parse_k2_generated( + text: &[u8], + require_thinking_closed: bool, + orders: &[ToolSchemaOrder], +) -> Option { + let think_end = crate::render::K2_THINK_END.as_bytes(); + let tool_block_start = crate::render::K2_TOOL_CALLS_START.as_bytes(); + let tool_block_end = crate::render::K2_TOOL_CALLS_END.as_bytes(); + let tool_start = crate::render::K2_TOOL_CALL_START.as_bytes(); + let tool_end = crate::render::K2_TOOL_CALL_END.as_bytes(); + let tool_search = if require_thinking_closed { + match find_last_substr(text, think_end) { + Some(i) => i + think_end.len(), + None => { + let (content, reasoning) = + split_reasoning_response(text, ChatFormat::K2Horizon, true); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + } + } + } else { + 0 + }; + let Some(rel) = find_substr(&text[tool_search..], tool_block_start) else { + let (content, reasoning) = + split_reasoning_response(text, ChatFormat::K2Horizon, require_thinking_closed); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + }; + let block_start = tool_search + rel; + let content_len = trim_tool_separator_ws(text, 0, block_start); + let mut p = block_start + tool_block_start.len(); + let mut calls = Vec::new(); + loop { + p = skip_ws(text, p); + if text[p..].starts_with(tool_block_end) { + p += tool_block_end.len(); + break; + } + if !text[p..].starts_with(tool_start) { + return None; + } + p += tool_start.len(); + let rel = find_substr(&text[p..], tool_end)?; + let body = &text[p..p + rel]; + let trimmed = trim_ascii_span(body); + if trimmed.starts_with(b"{") { + calls.push(parse_hermes_tool_call_json(&String::from_utf8_lossy( + trimmed, + ))?); + } else { + let key_start = crate::render::K2_ARG_KEY_START.as_bytes(); + let key_end = crate::render::K2_ARG_KEY_END.as_bytes(); + let value_start = crate::render::K2_ARG_VALUE_START.as_bytes(); + let value_end = crate::render::K2_ARG_VALUE_END.as_bytes(); + let first_key = find_substr(body, key_start).unwrap_or(body.len()); + let name = dsml_unescape_text(trim_ascii_span(&body[..first_key])); + if name.is_empty() { + return None; + } + let name = String::from_utf8_lossy(&name).into_owned(); + let order = tool_schema_orders_find(orders, &name); + let mut q = first_key; + let mut args = Vec::new(); + while q < body.len() { + q = skip_ws(body, q); + if q == body.len() { + break; + } + if !body[q..].starts_with(key_start) { + return None; + } + q += key_start.len(); + let ke = q + find_substr(&body[q..], key_end)?; + let key = dsml_unescape_text(trim_ascii_span(&body[q..ke])); + if key.is_empty() { + return None; + } + q = skip_ws(body, ke + key_end.len()); + if !body[q..].starts_with(value_start) { + return None; + } + q += value_start.len(); + let ve = q + find_substr(&body[q..], value_end)?; + let value = dsml_unescape_text(&body[q..ve]); + let key_name = String::from_utf8_lossy(&key); + solar_tool_arg_json_add( + &mut args, + &key, + &value, + tool_schema_order_prop_type(order, &key_name), + ); + q = ve + value_end.len(); + } + let mut arguments = vec![b'{']; + arguments.extend(args); + arguments.push(b'}'); + calls.push(ToolCall { + name, + arguments: String::from_utf8_lossy(&arguments).into_owned(), + ..Default::default() + }); + } + p += rel + tool_end.len(); + } + if calls.is_empty() { + return None; + } + let (content, reasoning) = split_reasoning_content(text, content_len, ChatFormat::K2Horizon); + Some(ParsedGenerated { + content, + reasoning, + calls, + raw_dsml: String::from_utf8_lossy(&text[block_start..p]).into_owned(), + ok: true, + ..Default::default() + }) +} + +fn parse_qwen_generated( + syntax: ModelSyntax, + text: &[u8], + require_thinking_closed: bool, + orders: &[ToolSchemaOrder], +) -> Option { + // Naive's generation prompt leaves thought unopened. Plain text and + // immediate tools are valid; an emitted opening tag still needs closure. + let require_thinking_closed = require_thinking_closed + && (syntax != ModelSyntax::NaiveN05 || find_substr(text, b"").is_some()); + let mut tool_search = 0usize; + if require_thinking_closed { + match find_last_substr(text, b"") { + None => { + // The template may start a tool call without opening thought. + if syntax != ModelSyntax::Mimo2 + || !text.starts_with(QWEN_TOOL_CALL_START.as_bytes()) + { + let body = text.strip_prefix(b"").unwrap_or(text); + return Some(ParsedGenerated { + content: Vec::new(), + reasoning: body.to_vec(), + ok: true, + ..Default::default() + }); + } + } + Some(i) => tool_search = i + "".len(), + } + } + let Some(rel) = find_substr(&text[tool_search..], QWEN_TOOL_CALL_START.as_bytes()) else { + let (content, reasoning) = split_reasoning_content(text, text.len(), ChatFormat::Qwen4Exp); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + }; + let start = tool_search + rel; + let content_len = trim_tool_separator_ws(text, 0, start); + let mut p = start; + let mut raw_end = None; + let mut calls = Vec::new(); + while text[p..].starts_with(QWEN_TOOL_CALL_START.as_bytes()) { + p += QWEN_TOOL_CALL_START.len(); + p = skip_ws(text, p); + if !text[p..].starts_with(b"')?; + let name = trim_ascii_span(&text[p..p + name_end]); + if name.is_empty() { + return None; + } + let name_s = String::from_utf8_lossy(name).into_owned(); + let order = tool_schema_orders_find(orders, &name_s); + p += name_end + 1; + + let mut args = Vec::new(); + let mut function_closed = false; + while p < text.len() { + p = skip_ws(text, p); + if text[p..].starts_with(b"") { + p += "".len(); + function_closed = true; + break; + } + if !text[p..].starts_with(b"')?; + let arg_name = trim_ascii_span(&text[p..p + arg_name_end]); + if arg_name.is_empty() { + return None; + } + p += arg_name_end + 1; + if syntax != ModelSyntax::NaiveN05 { + if text.get(p) == Some(&b'\r') { + p += 1; + } + if text.get(p) == Some(&b'\n') { + p += 1; + } + } + let arg_end = find_substr(&text[p..], b"")?; + let mut value_end = p + arg_end; + if syntax != ModelSyntax::NaiveN05 { + if value_end > p && text[value_end - 1] == b'\n' { + value_end -= 1; + } + if value_end > p && text[value_end - 1] == b'\r' { + value_end -= 1; + } + } + let arg_name_s = String::from_utf8_lossy(arg_name).into_owned(); + solar_tool_arg_json_add( + &mut args, + arg_name, + &text[p..value_end], + tool_schema_order_prop_type(order, &arg_name_s), + ); + p += arg_end + "".len(); + } + if !function_closed { + return None; + } + p = skip_ws(text, p); + if !text[p..].starts_with(QWEN_TOOL_CALL_END.as_bytes()) { + return None; + } + p += QWEN_TOOL_CALL_END.len(); + raw_end = Some(p); + + let mut wrapped = vec![b'{']; + wrapped.extend_from_slice(&args); + wrapped.push(b'}'); + calls.push(ToolCall { + name: name_s, + arguments: String::from_utf8_lossy(&wrapped).into_owned(), + ..Default::default() + }); + let next = skip_ws(text, p); + if !text[next..].starts_with(QWEN_TOOL_CALL_START.as_bytes()) { + break; + } + p = next; + } + if calls.is_empty() { + return None; + } + let (content, reasoning) = split_reasoning_content(text, content_len, ChatFormat::Qwen4Exp); + Some(ParsedGenerated { + content, + reasoning, + calls, + raw_dsml: String::from_utf8_lossy(&text[start..raw_end.unwrap_or(p)]).into_owned(), + ok: true, + ..Default::default() + }) +} + +/// The request's declared property type, normalised to the JSON Schema +/// spelling the GLM value typing works in. A client declares Rust wire types +/// as readily as JSON Schema ones: open-grok's `search_tool` declares `limit` +/// as `u8` (crates/codegen/xai-grok-tools/src/implementations/ +/// search_tool/types.rs:15), and a spelling the engine does not recognise used +/// to fall through to a JSON string, which the client then rejects -- +/// `invalid type: string "3", expected u8` (Spark session 01a14a82, +/// 2026-10-17, search_tool rejected, the call looped). None means "no +/// declared type the engine knows", and the caller keeps the value as text. +fn glm_declared_type(expected: Option<&str>) -> Option<&'static str> { + Some(match expected?.trim() { + "string" | "str" | "String" | "char" => "string", + "integer" | "u8" | "u16" | "u32" | "u64" | "usize" | "i8" | "i16" | "i32" | "i64" + | "isize" => "integer", + "number" | "f32" | "f64" => "number", + "boolean" | "bool" => "boolean", + "array" => "array", + "object" => "object", + "null" => "null", + _ => return None, + }) +} + +fn glm_arg_type(raw: &[u8], expected: Option<&str>) -> bool { + let Ok(value) = serde_json::from_slice::(raw) else { + return false; + }; + match glm_declared_type(expected) { + Some("number") => value.is_number(), + Some("integer") => value.as_number().is_some_and(|n| n.is_i64() || n.is_u64()), + Some("boolean") => value.is_boolean(), + Some("array") => value.is_array(), + Some("object") => value.is_object(), + Some("null") => value.is_null(), + _ => false, + } +} + +fn parse_glm_generated( + syntax: ModelSyntax, + text: &[u8], + require_thinking_closed: bool, + orders: &[ToolSchemaOrder], +) -> Option { + const ARG_KEY_START: &[u8] = b""; + const ARG_KEY_END: &[u8] = b""; + const ARG_VALUE_START: &[u8] = b""; + const ARG_VALUE_END: &[u8] = b""; + let tool_start = GLM_TOOL_CALL_START.as_bytes(); + let tool_end = GLM_TOOL_CALL_END.as_bytes(); + + let mut tool_search = 0usize; + let mut recovered_unclosed_tool = false; + if require_thinking_closed { + if let Some(end) = find_last_substr(text, b"") { + tool_search = end + "".len(); + } else if let Some(candidate) = find_substr(text, tool_start) { + if find_substr(&text[candidate..], tool_end).is_none() { + let (content, reasoning) = unterminated_reasoning(text); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + } + tool_search = candidate; + recovered_unclosed_tool = true; + } else { + let (content, reasoning) = unterminated_reasoning(text); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + } + } + + let Some(rel) = find_substr(&text[tool_search..], tool_start) else { + let (content, reasoning) = split_reasoning_content(text, text.len(), ChatFormat::DeepSeek); + return Some(ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }); + }; + let start = tool_search + rel; + let raw_start = if start >= 2 && &text[start - 2..start] == b"\n\n" { + start - 2 + } else { + start + }; + let content_len = trim_tool_separator_ws(text, 0, raw_start); + let mut p = start; + let mut calls = Vec::new(); + + loop { + p = skip_ws(text, p); + if !text[p..].starts_with(tool_start) { + break; + } + p += tool_start.len(); + let close = p + find_substr(&text[p..], tool_end)?; + let arg = find_substr(&text[p..close], ARG_KEY_START).map(|at| p + at); + let name_end = arg.unwrap_or(close); + let name = trim_ascii_span(&text[p..name_end]); + if name.is_empty() { + return None; + } + let name = String::from_utf8_lossy(name).into_owned(); + let order = tool_schema_orders_find(orders, &name); + p = name_end; + let mut args = Vec::new(); + + loop { + p = skip_ws(text, p); + if text[p..].starts_with(tool_end) { + p += tool_end.len(); + break; + } + if !text[p..].starts_with(ARG_KEY_START) { + return None; + } + p += ARG_KEY_START.len(); + let key_end = p + find_substr(&text[p..], ARG_KEY_END)?; + if key_end > close { + return None; + } + let raw_key = trim_ascii_span(&text[p..key_end]); + let key = if syntax == ModelSyntax::Glm53 { + raw_key.to_vec() + } else { + dsml_unescape_text(raw_key) + }; + if key.is_empty() { + return None; + } + p = skip_ws(text, key_end + ARG_KEY_END.len()); + if !text[p..].starts_with(ARG_VALUE_START) { + return None; + } + p += ARG_VALUE_START.len(); + let value_end = p + find_substr(&text[p..], ARG_VALUE_END)?; + if value_end > close { + return None; + } + let raw_value = &text[p..value_end]; + if syntax == ModelSyntax::Glm53 { + // The embedded template writes strings verbatim and other + // values as JSON (chat_template.jinja:50), so the text alone is + // ambiguous and the request's declared type decides. schemars + // renders Option as a type list ({"type":["integer","null"]}), + // which the scalar reader cannot see, so the list's head is + // consulted too. With nothing declared the reference's GLM + // rule stands: the value is text (ds4_server.c:6435). Without + // this, a numeric argument went out as a string and the client + // rejected it ('invalid type: string "15", expected usize', + // metasearch, session 01a14aaa, 2026-10-17). + let name = String::from_utf8_lossy(&key); + let expected = tool_schema_order_prop_type(order, &name) + .filter(|t| !t.is_empty()) + .or_else(|| tool_schema_order_prop_type_first(order, &name)); + let is_string = !glm_arg_type(raw_value, expected); + tool_call_json_args_add(&mut args, &key, raw_value, is_string); + } else { + let value = dsml_unescape_text(raw_value); + tool_call_json_args_add(&mut args, &key, &value, true); + } + p = value_end + ARG_VALUE_END.len(); + } + + let mut arguments = vec![b'{']; + arguments.extend(args); + arguments.push(b'}'); + calls.push(ToolCall { + name, + arguments: String::from_utf8_lossy(&arguments).into_owned(), + ..Default::default() + }); + let next = skip_ws(text, p); + if !text[next..].starts_with(tool_start) { + p = next; + break; + } + p = next; + } + if calls.is_empty() { + return None; + } + let (content, reasoning) = if recovered_unclosed_tool { + unterminated_reasoning_before_tool(text, content_len) + } else { + split_reasoning_content(text, content_len, ChatFormat::DeepSeek) + }; + Some(ParsedGenerated { + content, + reasoning, + calls, + raw_dsml: if syntax == ModelSyntax::Glm53 { + String::new() + } else { + String::from_utf8_lossy(&text[raw_start..p]).into_owned() + }, + raw_tool_text: if syntax == ModelSyntax::Glm53 { + String::from_utf8_lossy(&text[raw_start..p]).into_owned() + } else { + String::new() + }, + ok: true, + recovered: recovered_unclosed_tool, + }) +} + +pub fn parse_generated_message( + syntax: ModelSyntax, + text: &[u8], + require_thinking_closed: bool, + format: ChatFormat, + orders: &[ToolSchemaOrder], +) -> ParsedGenerated { + let parsed = match syntax { + ModelSyntax::Motif3 | ModelSyntax::Exaone => { + parse_hermes_generated(text, require_thinking_closed) + } + ModelSyntax::Dots3 => parse_dots3_generated(text, require_thinking_closed), + ModelSyntax::SolarOpen2 => parse_solar_generated(text, require_thinking_closed, orders), + // Step shares this output XML, but renders input through its own Jinja. + ModelSyntax::Qwen4Exp + | ModelSyntax::Step37 + | ModelSyntax::Mimo2 + | ModelSyntax::Qwen35 + | ModelSyntax::NaiveN05 => { + parse_qwen_generated(syntax, text, require_thinking_closed, orders) + } + ModelSyntax::K2Horizon => parse_k2_generated(text, require_thinking_closed, orders), + // Ling emits GLM's tool XML, not Qwen's JSON envelope. + ModelSyntax::Glm53 | ModelSyntax::Ling3Vl => { + parse_glm_generated(syntax, text, require_thinking_closed, orders) + } + // Malformed tool envelopes must not become user-visible raw text. + ModelSyntax::Inkling => return crate::render::inkling::parse(text).unwrap_or_default(), + ModelSyntax::BeamVq => beam::parse(text, orders), + ModelSyntax::IQuestQ1 => Some(iquest::parse(text, require_thinking_closed, orders)), + ModelSyntax::DeepSeek => { + if format == ChatFormat::SolarOpen2 { + parse_solar_generated(text, require_thinking_closed, orders) + } else if format == ChatFormat::Qwen4Exp { + parse_qwen_generated(syntax, text, require_thinking_closed, orders) + } else if format == ChatFormat::K2Horizon { + parse_k2_generated(text, require_thinking_closed, orders) + } else { + parse_dsml_generated(text, require_thinking_closed) + } + } + }; + parsed.unwrap_or(ParsedGenerated { + content: text.to_vec(), + ok: false, + ..Default::default() + }) +} + +pub fn parse_generated_for_model_id( + model_id: i32, + text: &[u8], + require_thinking_closed: bool, + orders: &[ToolSchemaOrder], +) -> ParsedGenerated { + let syntax = syntax_for_model_id(model_id); + let format = crate::generate::chat_format_for_syntax(syntax); + parse_generated_message(syntax, text, require_thinking_closed, format, orders) +} + +fn split_reasoning_response( + text: &[u8], + format: ChatFormat, + require_thinking_closed: bool, +) -> (Vec, Vec) { + let end = think_end(format).as_bytes(); + if require_thinking_closed && find_last_substr(text, end).is_none() { + let start = think_start(format).as_bytes(); + let body = if text.starts_with(start) { + &text[start.len()..] + } else { + text + }; + return (Vec::new(), body.to_vec()); + } + split_reasoning_content(text, text.len(), format) +} + +/// C `parse_generated_message_for_response_model`. No-tools never +/// extracts calls. Parse failure with a seen start becomes raw-text +/// fallback; finish becomes `stop` unless it was `length` or `error`. +pub fn parse_generated_for_response( + syntax: ModelSyntax, + text: &[u8], + has_tools: bool, + saw_tool_start: bool, + require_thinking_closed: bool, + format: ChatFormat, + orders: &[ToolSchemaOrder], + finish: &str, +) -> (ParsedGenerated, &'static str) { + // Channel-framed output always parses its channels. + if matches!(syntax, ModelSyntax::Inkling | ModelSyntax::BeamVq) { + let mut parsed = + parse_generated_message(syntax, text, require_thinking_closed, format, orders); + if !has_tools { + parsed.calls.clear(); + } + if !parsed.ok { + parsed.recovered = has_tools && saw_tool_start && finish != "error"; + return ( + parsed, + if finish == "length" { + "length" + } else { + "error" + }, + ); + } + return (parsed, intern_finish(finish)); + } + if !has_tools { + let (content, reasoning) = if syntax == ModelSyntax::IQuestQ1 { + let (body, reasoning) = iquest::split(text, require_thinking_closed); + (body.to_vec(), reasoning) + } else { + split_reasoning_response(text, format, require_thinking_closed) + }; + return ( + ParsedGenerated { + content, + reasoning, + ok: true, + ..Default::default() + }, + intern_finish(finish), + ); + } + let parsed = parse_generated_message(syntax, text, require_thinking_closed, format, orders); + if parsed.ok { + return (parsed, intern_finish(finish)); + } + let recovered = has_tools && saw_tool_start && finish != "error"; + let out_finish = if recovered { + if finish == "length" { + "length" + } else { + "stop" + } + } else { + intern_finish(finish) + }; + ( + ParsedGenerated { + content: text.to_vec(), + ok: false, + recovered, + ..Default::default() + }, + out_finish, + ) +} + +fn intern_finish(finish: &str) -> &'static str { + match finish { + "tool_calls" => "tool_calls", + "length" => "length", + "error" => "error", + _ => "stop", + } +} + +/// The artifact's quoted tool root: the bare tool name with the residue of the +/// `name="` attribute the reference's own prompt teaches (``, +/// ds4_server.c:2580-2589), which the finish-level parse recovers in +/// dsml_tolerant_tool_root_scan. The predicate is that function's, on +/// purpose: the tracker must recognise exactly the blocks the parse serves, +/// because a tag the parse ignores would leave the client without both the +/// markup and the calls. A tag naming no declared tool is model text. +fn dsml_find_declared_tool_root(s: &[u8], orders: &[ToolSchemaOrder]) -> Option { + let mut best: Option = None; + for o in orders { + for n in [o.name.as_str(), o.wire_name.as_str()] { + if n.is_empty() { + continue; + } + + let mut open = Vec::with_capacity(n.len() + 1); + open.push(b'<'); + open.extend_from_slice(n.as_bytes()); + + let mut from = 0; + while let Some(rel) = find_substr(&s[from..], &open) { + let body_start = from + rel + open.len(); + let Some(gt) = s[body_start..].iter().position(|&c| c == b'>') else { + break; + }; + + // Empty body or the one stray quote; any other body is not + // this rule's shape (dsml_tolerant_tool_root_scan, tools.rs). + let body = &s[body_start..body_start + gt]; + if body.is_empty() || body == b"\"" { + let at = from + rel; + if best.map_or(true, |b| at < b) { + best = Some(at); + } + break; + } + from = body_start; + } + } + } + best +} + +pub fn find_tool_start(s: &[u8], format: ChatFormat, orders: &[ToolSchemaOrder]) -> Option { + match format { + ChatFormat::IQuestQ1 => find_substr(s, crate::render::IQUEST_TOOL_CALL_START.as_bytes()), + ChatFormat::SolarOpen2 => find_substr(s, SOLAR_TOOL_CALLS.as_bytes()), + ChatFormat::Exaone => find_substr(s, b""), + ChatFormat::Qwen4Exp => find_substr(s, QWEN_TOOL_CALL_START.as_bytes()), + ChatFormat::K2Horizon => find_substr(s, crate::render::K2_TOOL_CALLS_START.as_bytes()), + ChatFormat::Inkling => find_substr(s, crate::render::inkling::INVOKE.as_bytes()), + // `<|constrain|>` never occurs in prose; ` to=` may. + ChatFormat::Beam => find_substr(s, beam::TOOL.as_bytes()), + ChatFormat::DeepSeek => { + let cands = [ + DSML_TOOL_CALLS_START.as_bytes(), + DSML_TOOL_CALLS_START_SHORT.as_bytes(), + // The official spaced generation (the trained spelling for the + // <|System|> tokenizer): `<| calls>` / `<| invoke`. + DSML_TOOL_CALLS_START_SPACED.as_bytes(), + DSML_INVOKE_START_SPACED.as_bytes(), + b"", + b"", + b"", + // The V4.1 artifact sometimes drops the wrapper entirely and + // opens straight with an invoke (observed 2026-10-11, + // open-grok session 01a127cf), and the shim records a short + // `<...calls>` wrapper (~/youngai/tool-shim/shim.py:44-48). + // Without these the live stream never enters tool mode, so + // the block streams as visible content while the + // finish-level tolerant parse still serves the calls. + DSML_INVOKE_START.as_bytes(), + DSML_INVOKE_START_SHORT.as_bytes(), + b"", + b"", + b"", + ]; + // The earliest occurrence wins across both families: a quoted + // root and a decorated wrapper can appear in one turn, and the + // block starts where the first of them does. + let marker = cands.iter().filter_map(|p| find_substr(s, p)).min(); + [marker, dsml_find_declared_tool_root(s, orders)] + .into_iter() + .flatten() + .min() + } + } +} + +pub fn find_tool_end(s: &[u8], format: ChatFormat) -> Option { + match format { + ChatFormat::IQuestQ1 => find_substr(s, crate::render::IQUEST_TOOL_CALL_END.as_bytes()), + ChatFormat::SolarOpen2 => find_substr(s, SOLAR_TOOL_CALL_END.as_bytes()), + ChatFormat::Exaone => find_substr(s, b""), + ChatFormat::Qwen4Exp => find_substr(s, QWEN_TOOL_CALL_END.as_bytes()), + ChatFormat::K2Horizon => find_substr(s, crate::render::K2_TOOL_CALLS_END.as_bytes()), + ChatFormat::Inkling => find_substr(s, crate::render::inkling::END.as_bytes()), + ChatFormat::Beam => find_substr(s, beam::CALL.as_bytes()), + ChatFormat::DeepSeek => { + let cands = [ + DSML_TOOL_CALLS_END.as_bytes(), + DSML_TOOL_CALLS_END_SHORT.as_bytes(), + // The official spaced generation: the engine's + // find_any_tool_end walks all four syntaxes + // (server_dsml_parse.c:88-93). Without the spaced close the + // live stream never closed a spaced block, so the loop drew + // one extra token (EOS) after the wrapper -- the engine stops + // at the tag (chat gate, 2026-10-11). + DSML_TOOL_CALLS_END_SPACED.as_bytes(), + b"", + b"", + b"", + ]; + cands.iter().filter_map(|p| find_substr(s, p)).min() + } + } +} + +#[allow(dead_code)] +pub fn observe_tool_markers( + scan: &[u8], + saw_start: &mut bool, + saw_end: &mut bool, + format: ChatFormat, + orders: &[ToolSchemaOrder], +) -> (bool, bool) { + let had_start = *saw_start; + let start = find_tool_start(scan, format, orders); + if start.is_some() { + *saw_start = true; + } + let end_scan = if had_start { Some(0) } else { start }; + let end = end_scan.and_then(|off| find_tool_end(&scan[off..], format)); + if end.is_some() { + *saw_end = true; + } + let entered = *saw_start && !had_start; + let closed = *saw_end && end.is_some(); + // closed should be "became true this observe". Caller tracks old_end. + (entered, closed) +} + +struct ToolIdState { + next: u128, + blocked: HashMap<(u8, u128), usize>, +} + +static TOOL_ID_STATE: OnceLock> = OnceLock::new(); + +fn tool_id_seed() -> u128 { + let mut bytes = [0u8; 16]; + if File::open("/dev/urandom") + .and_then(|mut file| file.read_exact(&mut bytes)) + .is_ok() + { + return u128::from_le_bytes(bytes); + } + let time = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or(0); + time ^ u128::from(std::process::id()) +} + +fn tool_id_kind(prefix: &str) -> u8 { + u8::from(prefix == "toolu_") +} + +#[cfg(any(feature = "native", test))] +fn tool_id_key(id: &str) -> Option<(u8, u128)> { + let (kind, hex) = if let Some(hex) = id.strip_prefix("call_") { + (0, hex) + } else if let Some(hex) = id.strip_prefix("toolu_") { + (1, hex) + } else { + return None; + }; + if hex.len() != 32 { + return None; + } + Some((kind, u128::from_str_radix(hex, 16).ok()?)) +} + +fn mint_tool_id_from(state: &mut ToolIdState, prefix: &str) -> String { + loop { + let value = state.next; + state.next = state.next.wrapping_add(1); + if !state.blocked.contains_key(&(tool_id_kind(prefix), value)) { + return format!("{prefix}{value:032x}"); + } + } +} + +pub(crate) fn mint_tool_id(prefix: &str) -> String { + let state = TOOL_ID_STATE.get_or_init(|| { + Mutex::new(ToolIdState { + next: tool_id_seed(), + blocked: HashMap::new(), + }) + }); + mint_tool_id_from( + &mut state.lock().unwrap_or_else(|error| error.into_inner()), + prefix, + ) +} + +#[cfg(any(feature = "native", test))] +pub(crate) fn reserve_tool_id(id: &str) { + let Some(key) = tool_id_key(id) else { + return; + }; + let state = TOOL_ID_STATE.get_or_init(|| { + Mutex::new(ToolIdState { + next: tool_id_seed(), + blocked: HashMap::new(), + }) + }); + let mut state = state.lock().unwrap_or_else(|error| error.into_inner()); + *state.blocked.entry(key).or_default() += 1; +} + +#[cfg(any(feature = "native", test))] +pub(crate) fn release_tool_id(id: &str) { + let Some(key) = tool_id_key(id) else { + return; + }; + let Some(state) = TOOL_ID_STATE.get() else { + return; + }; + let mut state = state.lock().unwrap_or_else(|error| error.into_inner()); + if let Some(refs) = state.blocked.get_mut(&key) { + if *refs <= 1 { + state.blocked.remove(&key); + } else { + *refs -= 1; + } + } +} + +fn assign_tool_ids_with( + calls: &mut [ToolCall], + prefix: &str, + mut mint: impl FnMut(&str) -> String, +) { + let mut used: HashSet = calls + .iter() + .filter(|call| !call.id.is_empty()) + .map(|call| call.id.clone()) + .collect(); + for tc in calls { + while tc.id.is_empty() { + let id = mint(prefix); + if used.insert(id.clone()) { + tc.id = id; + } + } + } +} + +pub fn assign_tool_ids(calls: &mut [ToolCall], prefix: &str) { + assign_tool_ids_with(calls, prefix, mint_tool_id); +} + +fn stop_list_find_from(stops: &[String], text: &[u8], from: usize) -> Option<(usize, usize)> { + if stops.is_empty() || from > text.len() { + return None; + } + let mut best: Option<(usize, usize)> = None; + for s in stops { + if s.is_empty() { + continue; + } + let needle = s.as_bytes(); + if from + needle.len() > text.len() { + continue; + } + if let Some(rel) = find_substr(&text[from..], needle) { + let pos = from + rel; + if best.map(|(p, _)| pos < p).unwrap_or(true) { + best = Some((pos, needle.len())); + } + } + } + best +} + +fn stop_list_max_len(stops: &[String]) -> usize { + stops.iter().map(|s| s.len()).max().unwrap_or(0) +} + +fn stop_list_stream_safe_len(stops: &[String], text_len: usize) -> usize { + let max = stop_list_max_len(stops); + if max <= 1 || text_len <= max - 1 { + return if max <= 1 { text_len } else { 0 }; + } + text_len - (max - 1) +} + +fn tool_marker_stream_safe_len(text: &[u8], format: ChatFormat) -> usize { + let marks: &[&[u8]] = match format { + ChatFormat::IQuestQ1 => &[crate::render::IQUEST_TOOL_CALL_START.as_bytes()], + ChatFormat::SolarOpen2 => &[SOLAR_TOOL_CALLS.as_bytes()], + ChatFormat::Exaone => &[b""], + ChatFormat::Qwen4Exp => &[QWEN_TOOL_CALL_START.as_bytes()], + ChatFormat::K2Horizon => &[crate::render::K2_TOOL_CALLS_START.as_bytes()], + ChatFormat::Inkling => &[crate::render::inkling::INVOKE.as_bytes()], + ChatFormat::Beam => &[beam::TOOL.as_bytes()], + ChatFormat::DeepSeek => &[ + DSML_TOOL_CALLS_START.as_bytes(), + DSML_TOOL_CALLS_START_SHORT.as_bytes(), + b"", + b"", + ], + }; + if text.is_empty() { + return 0; + } + let mut safe = text.len(); + for m in marks { + if m.len() <= 1 { + continue; + } + let mut k = (m.len() - 1).min(text.len()); + while k >= 1 { + if text[text.len() - k..].starts_with(&m[..k]) { + let pos = text.len() - k; + if pos < safe { + safe = pos; + } + break; + } + k -= 1; + } + } + safe +} + +#[derive(Debug, Clone)] +pub struct SemAccum { + pub text: Vec, + pub track_tools: bool, + cut_tool_syntax: bool, + think_gates: bool, + chat_format: ChatFormat, + /// The request's declared tools: the start scan recognises the artifact's + /// quoted root (``) only for a name some tool declares, the + /// same predicate the finish-level tolerant parse applies + /// (dsml_find_declared_tool_root). Empty for a request with no tools. + tool_orders: Vec, + thinking_inside: bool, + think_tail: Vec, + stop_scan_from: usize, + tool_scan_from: usize, + tool_scan_waiting: bool, + pub saw_tool_start: bool, + pub saw_tool_end: bool, + pub verdict: Option<&'static str>, + pub matched_stop: Option, + pub completion: i32, + pub reasoning_tokens: i32, + pub dsml: DsmlDecodeTracker, + required_tool_prefix_pos: i32, + required_think_end_prefix_pos: i32, +} + +#[derive(Debug, Clone, Default)] +pub struct SemFeed { + pub emit_limit: usize, + pub hit_stop: bool, + pub tool_block_closed: bool, + pub entered_tool_block: bool, + pub tool_syntax_cut: bool, +} + +impl SemAccum { + pub fn init( + kind_chat: bool, + has_tools: bool, + think_enabled: bool, + format: ChatFormat, + prompt: &[u8], + ) -> Self { + let mut a = Self { + text: Vec::new(), + track_tools: kind_chat && has_tools, + cut_tool_syntax: kind_chat && !has_tools && format != ChatFormat::IQuestQ1, + think_gates: think_enabled || matches!(format, ChatFormat::Inkling | ChatFormat::Beam), + chat_format: format, + tool_orders: Vec::new(), + thinking_inside: false, + think_tail: Vec::new(), + stop_scan_from: 0, + tool_scan_from: 0, + tool_scan_waiting: think_enabled, + saw_tool_start: false, + saw_tool_end: false, + verdict: None, + matched_stop: None, + completion: 0, + reasoning_tokens: 0, + dsml: DsmlDecodeTracker::default(), + required_tool_prefix_pos: 0, + required_think_end_prefix_pos: 0, + }; + if !prompt.is_empty() { + a.feed_thinking(prompt); + } else if think_enabled && format != ChatFormat::Inkling { + a.thinking_inside = true; + } + // Harmony opens every reply with its analysis message. + if format == ChatFormat::Beam { + a.thinking_inside = true; + } + a.tool_scan_waiting = a.think_gates && a.thinking_inside; + a + } + + /// Declare the request's tools after `init`, so the start scan can take + /// the artifact's quoted root into tool mode. Separate from `init` so the + /// callers with no parse at hand (and the tapes) keep their signature. + pub fn set_tool_orders(&mut self, orders: &[ToolSchemaOrder]) { + self.tool_orders.clear(); + self.tool_orders.extend_from_slice(orders); + } + + fn feed_thinking(&mut self, piece: &[u8]) { + if self.chat_format == ChatFormat::Beam { + // Analysis closes at `<|end|>` or a bare `<|start|>assistant`; + // a call header's analysis channel (spaced) opens nothing. + let (open, ends) = ( + beam::ANALYSIS.as_bytes(), + [beam::END.as_bytes(), beam::ASSISTANT.as_bytes()], + ); + let keep = beam::ASSISTANT.len().max(open.len()); + for &byte in piece { + if self.think_tail.len() == keep { + self.think_tail.remove(0); + } + self.think_tail.push(byte); + if self.think_tail.ends_with(open) { + self.thinking_inside = true; + } else if ends.iter().any(|end| self.think_tail.ends_with(end)) { + self.thinking_inside = false; + } + } + return; + } + if self.chat_format == ChatFormat::Inkling { + let start = crate::render::inkling::THINK.as_bytes(); + let end = crate::render::inkling::END.as_bytes(); + let keep = start.len().max(end.len()); + for &byte in piece { + if self.think_tail.len() == keep { + self.think_tail.remove(0); + } + self.think_tail.push(byte); + if self.think_tail.ends_with(start) { + self.thinking_inside = true; + } else if self.think_tail.ends_with(end) { + self.thinking_inside = false; + } + } + return; + } + for &c in piece { + if self.think_tail.len() == 16 { + self.think_tail.remove(0); + } + self.think_tail.push(c); + if tail_ends_with(&self.think_tail, b"") + || tail_ends_with(&self.think_tail, SOLAR_THINK_START.as_bytes()) + || tail_ends_with(&self.think_tail, crate::render::K2_THINK_START.as_bytes()) + { + self.thinking_inside = true; + } else if tail_ends_with(&self.think_tail, b"") + || tail_ends_with(&self.think_tail, SOLAR_THINK_END.as_bytes()) + || tail_ends_with(&self.think_tail, crate::render::K2_THINK_END.as_bytes()) + { + self.thinking_inside = false; + } + } + } + + /// The engine's v41_syntax_greedy (server_generate_v41.c): a COPY of the + /// tracker and the thinking state over the emitted text + the ahead pieces; + /// true = the next position is tool-call syntax, so the native takes the raw + /// argmax there. The engine keeps that state in its native v41_gen; the + /// port's text lives here, so the caller supplies the ahead pieces and this + /// rebuilds the copy on a clone (the DSML tracker re-scans from the start, + /// the thinking scanner follows the same windows). The request gate is the + /// engine's own: only a chat request with tools asks at all. + pub fn syntax_greedy(&self, ahead: &[Vec]) -> bool { + let mut probe = self.clone(); + for piece in ahead { + probe.feed_thinking(piece); + probe.dsml.update(&probe.text); + } + !probe.thinking_inside && dsml_decode_state_is_syntax(probe.dsml.decode) + } + + /// A stop token's text (Beam's `<|call|>` closes the last call): parsers + /// and streams see it, usage does not count it, as no stop token is. + pub fn feed_stop(&mut self, piece: &[u8], stops: &[String]) -> SemFeed { + let counts = (self.completion, self.reasoning_tokens); + let feed = self.feed(piece, stops); + (self.completion, self.reasoning_tokens) = counts; + feed + } + + pub fn feed(&mut self, piece: &[u8], stops: &[String]) -> SemFeed { + let mut f = SemFeed::default(); + if self.think_gates && self.thinking_inside { + self.reasoning_tokens += 1; + } + self.feed_thinking(piece); + self.text.extend_from_slice(piece); + self.completion += 1; + if self.track_tools && self.chat_format == ChatFormat::DeepSeek { + self.dsml.update(&self.text); + } + + let hit = stop_list_find_from(stops, &self.text, self.stop_scan_from); + f.emit_limit = if let Some((pos, _)) = hit { + pos + } else { + stop_list_stream_safe_len(stops, self.text.len()) + }; + if f.emit_limit > self.text.len() { + f.emit_limit = self.text.len(); + } + if hit.is_none() { + let max = stop_list_max_len(stops); + if max > 1 { + let hold = max - 1; + self.stop_scan_from = if self.text.len() > hold { + self.text.len() - hold + } else { + 0 + }; + } + } + if self.cut_tool_syntax { + let safe = tool_marker_stream_safe_len(&self.text, self.chat_format); + if f.emit_limit > safe { + f.emit_limit = safe; + } + } + + let mut cut_pos = None; + if self.track_tools || self.cut_tool_syntax { + if self.think_gates && self.thinking_inside { + self.tool_scan_waiting = true; + self.tool_scan_from = self.text.len(); + } else { + if self.tool_scan_waiting { + let marker = think_end(self.chat_format).as_bytes(); + self.tool_scan_from = find_last_substr(&self.text, marker) + .map(|i| i + marker.len()) + .unwrap_or(self.text.len()) + .min(self.text.len()); + self.tool_scan_waiting = false; + } + if self.tool_scan_from > self.text.len() { + self.tool_scan_from = self.text.len(); + } + let scan = &self.text[self.tool_scan_from..]; + let old_start = self.saw_tool_start; + let old_end = self.saw_tool_end; + // The reference reads the tracker for the block's markers + // (ds4_server.c:7755 observe_tool_markers). GLM's argument + // values are free text, so a `` written into one is + // payload; the text scan would close the block there and cut + // the call (ds4_server.c:20608 test_tool_control_text_inside_arguments). + // The opening follows the same tracker: a DSML spelling in GLM + // prose is text, and opening on it would leave a block the GLM + // tracker never closes. + let glm = self.dsml.tool_syntax() == ToolSyntax::Glm; + let start = if !glm { + find_tool_start(scan, self.chat_format, &self.tool_orders) + } else if self.dsml.block_started() { + find_substr(scan, GLM_TOOL_CALL_START.as_bytes()) + } else { + None + }; + if start.is_some() { + self.saw_tool_start = true; + } + let closed = if glm { + self.saw_tool_start && self.dsml.block_closed() + } else { + let end_off = if old_start { Some(0) } else { start }; + end_off + .and_then(|o| find_tool_end(&scan[o..], self.chat_format)) + .is_some() + }; + if closed { + self.saw_tool_end = true; + } + f.entered_tool_block = self.saw_tool_start && !old_start; + f.tool_block_closed = self.saw_tool_end && !old_end; + if f.entered_tool_block && self.cut_tool_syntax { + if let Some(m) = start { + cut_pos = Some(self.tool_scan_from + m); + } + } + let marker_hold = 80; + let hold_from = if self.text.len() > marker_hold { + self.text.len() - marker_hold + } else { + 0 + }; + if hold_from > self.tool_scan_from { + self.tool_scan_from = hold_from; + } + } + } + + if let Some((pos, len)) = hit { + self.matched_stop = + Some(String::from_utf8_lossy(&self.text[pos..pos + len]).into_owned()); + self.text.truncate(pos); + self.verdict = Some("stop"); + f.hit_stop = true; + f.emit_limit = pos; + } else if self.cut_tool_syntax && cut_pos.is_some() && self.verdict.is_none() { + let pos = cut_pos.unwrap(); + self.text.truncate(pos); + self.verdict = Some("stop"); + f.hit_stop = true; + f.tool_syntax_cut = true; + if f.emit_limit > pos { + f.emit_limit = pos; + } + } else if self.track_tools + && self.saw_tool_end + && self.verdict.is_none() + && self.chat_format == ChatFormat::DeepSeek + { + // C only auto-verdicts DSML format, not Solar/EXAONE. + self.verdict = Some("tool_calls"); + } + f + } + + pub fn thinking_inside(&self) -> bool { + self.thinking_inside + } + + pub fn dsml_state(&self) -> DsmlDecodeState { + if self.track_tools && self.chat_format == ChatFormat::DeepSeek { + self.dsml.decode + } else { + DsmlDecodeState::Outside + } + } + + /// C `tracker.model_syntax = ...` (ds4_server.c:7595, :20591): the dialect is + /// chosen from the served family before the first update, because the start + /// search differs per dialect. GLM's `` block is not in the DSML + /// syntax table, so without this its structural bytes were never recognized + /// and every position kept the request's draw instead of the raw argmax. + pub fn set_model_syntax(&mut self, syntax: ModelSyntax) { + self.dsml.set_tool_syntax(match syntax { + ModelSyntax::Glm53 => ToolSyntax::Glm, + _ => ToolSyntax::Dsml, + }); + } + + pub fn sampling_override(&mut self, p: &SamplePolicy<'_>) -> SampleOverride { + let forced = crate::dsml::sampling_override( + self.track_tools, + self.saw_tool_start, + self.thinking_inside, + self.completion, + self.chat_format, + self.dsml.decode, + &mut self.required_tool_prefix_pos, + &mut self.required_think_end_prefix_pos, + p, + ); + // Beam's Harmony headers follow its grammar, not the drifted weights. + if forced == SampleOverride::None && self.chat_format == ChatFormat::Beam { + if let Some(token) = beam::forced_header(&self.text, self.track_tools) { + return SampleOverride::Token(token); + } + } + forced + } +} + +fn tail_ends_with(tail: &[u8], s: &[u8]) -> bool { + tail.len() >= s.len() && tail[tail.len() - s.len()..] == s[..] +} + +#[cfg(test)] +mod tool_id_tests { + use super::{assign_tool_ids_with, mint_tool_id_from, ToolIdState}; + use crate::parse::ToolCall; + use std::collections::{HashMap, VecDeque}; + + #[test] + fn assignment_retries_ids_already_present_in_the_turn() { + let mut calls = vec![ + ToolCall { + id: "call_taken".into(), + ..Default::default() + }, + ToolCall::default(), + ]; + let mut minted = VecDeque::from(["call_taken".to_string(), "call_fresh".to_string()]); + assign_tool_ids_with(&mut calls, "call_", |_| minted.pop_front().unwrap()); + assert_eq!(calls[1].id, "call_fresh"); + } + + #[test] + fn allocator_skips_reserved_checkpoint_ids() { + let mut state = ToolIdState { + next: 7, + blocked: HashMap::from([((0, 7), 1)]), + }; + assert_eq!( + mint_tool_id_from(&mut state, "call_"), + "call_00000000000000000000000000000008" + ); + } +} diff --git a/crates/ds4-server/src/tools/beam.rs b/crates/ds4-server/src/tools/beam.rs new file mode 100644 index 0000000000000000000000000000000000000000..d1c2b8888c63a1c9a349f3ac0bb4873c6a264a00 --- /dev/null +++ b/crates/ds4-server/src/tools/beam.rs @@ -0,0 +1,327 @@ +//! Beam-1.0 Harmony output (tokenizer_config.json `response_template`). +//! +//! The prompt ends at `<|start|>assistant`. Reasoning is an `analysis` +//! message, the answer a `final` one, and each tool call a +//! ` to=NAME<|channel|>analysis <|constrain|>xml<|message|>` message of +//! `\nV\n` parameters joined by +//! newlines. Messages end at `<|end|>` (then `<|start|>assistant` opens the +//! next); `<|return|>` and `<|call|>` are stop tokens, and the decode loop +//! feeds `<|call|>` so the last call keeps its terminator. + +use serde_json::{Map, Value}; + +use crate::parse::{ToolCall, ToolSchemaOrder}; +use crate::stream::ChatFormat; + +use super::{tool_schema_order_prop_type, tool_schema_orders_find, ParsedGenerated}; + +pub(crate) const ASSISTANT: &str = "<|start|>assistant"; +pub(crate) const ANALYSIS: &str = "<|channel|>analysis<|message|>"; +pub(crate) const FINAL: &str = "<|channel|>final<|message|>"; +pub(crate) const TOOL: &str = "<|channel|>analysis <|constrain|>xml<|message|>"; +pub(crate) const RECIPIENT: &str = " to="; +pub(crate) const END: &str = "<|end|>"; +pub(crate) const CALL: &str = "<|call|>"; +pub(crate) const RETURN: &str = "<|return|>"; +// The template's analysis close also accepts a bare `<|start|>assistant` +// and `<|constrain|><|start|>assistant`. +const CONSTRAIN: &str = "<|constrain|>"; +const PARAM: &str = "\n"; +const PARAM_END: &str = "\n"; + +// Header tokens of the release tokenizer (pinned with the release; ds4-core's +// beam_vq test encodes the header pieces against them). +const T_START: i32 = 151669; +const T_ASSISTANT: i32 = 77091; +const T_CHANNEL: i32 = 151672; +const T_MESSAGE: i32 = 151671; +const T_CONSTRAIN: i32 = 151673; +const T_ANALYSIS: i32 = 34484; +const T_FINAL: i32 = 11822; +const T_SPACE: i32 = 220; +const T_XML: i32 = 6455; + +fn rfind(hay: &[u8], needle: &str) -> Option { + hay.windows(needle.len()) + .rposition(|w| w == needle.as_bytes()) +} + +/// The token Harmony's header grammar admits alone after the reply so far. +/// The released VQ weights drift at header positions (the H100 goldens put +/// "final" or `<|channel|>` where the teacher writes "analysis" with a 17 +/// logit margin, and replies open `<|channel|> final`), so headers are forced: +/// a reply opens with analysis, a message after `<|end|>` restarts as +/// `<|start|>assistant`, a second channel is final, and a call's header ends +/// ` <|constrain|>xml<|message|>`. Bodies, the recipient and the choice between +/// an answer and a call (when tools exist) stay sampled. +pub(crate) fn forced_header(text: &[u8], tools: bool) -> Option { + let ends = |s: &str| text.ends_with(s.as_bytes()); + if text.is_empty() { + return Some(T_CHANNEL); + } + if ends(END) { + return Some(T_START); + } + if ends("<|start|>") { + return Some(T_ASSISTANT); + } + if ends(ASSISTANT) { + return (!tools).then_some(T_CHANNEL); + } + // The current message's header follows its `<|start|>assistant`. + let header = &text[rfind(text, ASSISTANT).map_or(0, |at| at + ASSISTANT.len())..]; + let recipient = header.starts_with(RECIPIENT.as_bytes()); + if ends("<|channel|>") { + return Some(if recipient || find(text, ANALYSIS).is_none() { + T_ANALYSIS + } else { + T_FINAL + }); + } + if recipient { + return if ends("<|channel|>analysis") { + Some(T_SPACE) + } else if ends("<|channel|>analysis ") { + Some(T_CONSTRAIN) + } else if ends("<|constrain|>") { + Some(T_XML) + } else if ends("<|constrain|>xml") { + Some(T_MESSAGE) + } else { + None + }; + } + (ends("<|channel|>analysis") || ends("<|channel|>final")).then_some(T_MESSAGE) +} + +/// `<|call|>` is a stop token that also closes the last call: the decode +/// lanes feed its text so parsers and streams see the terminator. +pub(crate) fn stop_text(format: ChatFormat, token: i32, eot: i32) -> Option<&'static [u8]> { + (format == ChatFormat::Beam && token >= 0 && token == eot).then_some(CALL.as_bytes()) +} + +fn find(hay: &[u8], needle: &str) -> Option { + hay.windows(needle.len()) + .position(|w| w == needle.as_bytes()) +} + +/// The earliest of `markers` in `hay`: (offset, marker). +pub(crate) fn earliest(hay: &[u8], markers: &[&'static str]) -> Option<(usize, &'static str)> { + markers + .iter() + .filter_map(|&m| find(hay, m).map(|at| (at, m))) + .min_by_key(|&(at, _)| at) +} + +// A declared string stays text (the official value parser keeps it +// unstripped); anything else is the template's JSON with a text fallback. +fn call_body(name: &str, body: &str, orders: &[ToolSchemaOrder]) -> Option { + let order = tool_schema_orders_find(orders, name); + let mut args = Map::new(); + let mut rest = body; + loop { + rest = rest.trim_start_matches('\n'); + if rest.is_empty() { + break; + } + let tail = rest.strip_prefix(PARAM)?; + let head = tail.find(PARAM_HEAD_END)?; + let key = &tail[..head]; + if key.is_empty() || key.contains('"') { + return None; + } + let value_start = &tail[head + PARAM_HEAD_END.len()..]; + // A missing close tag is legal when the next parameter follows. + let (raw, next) = match value_start.find(PARAM_END) { + Some(end) => (&value_start[..end], &value_start[end + PARAM_END.len()..]), + None => { + let end = value_start.find(&format!("\n{PARAM}"))?; + (&value_start[..end], &value_start[end..]) + } + }; + let value = if tool_schema_order_prop_type(order, key) == Some("string") { + Value::String(raw.to_owned()) + } else { + serde_json::from_str(raw.trim()).unwrap_or_else(|_| Value::String(raw.to_owned())) + }; + args.insert(key.to_owned(), value); + rest = next; + } + Some(ToolCall { + name: name.to_owned(), + arguments: Value::Object(args).to_string(), + ..Default::default() + }) +} + +// ` to=NAME` before a tool header: NAME has no whitespace or `<`. +fn recipient(head: &[u8]) -> Option<&str> { + let name = std::str::from_utf8(head).ok()?.strip_prefix(RECIPIENT)?; + (!name.is_empty() && !name.contains(|c: char| c.is_whitespace() || c == '<')).then_some(name) +} + +pub(crate) fn parse(raw: &[u8], orders: &[ToolSchemaOrder]) -> Option { + let mut out = ParsedGenerated { + ok: true, + ..Default::default() + }; + let (mut pos, mut framed) = (0, false); + while pos < raw.len() { + if raw[pos..].starts_with(ASSISTANT.as_bytes()) { + pos += ASSISTANT.len(); + } + // Unframed output is not Harmony (surface it raw); a budget can end + // inside the next header: publish nothing of it. + let Some((at, marker)) = earliest(&raw[pos..], &[FINAL, ANALYSIS, TOOL]) else { + if !framed && !raw[pos..].starts_with(b"<|") { + return None; + } + break; + }; + framed = true; + let head = &raw[pos..pos + at]; + let name = if marker == TOOL { + Some(recipient(head)?) + } else if head.is_empty() { + None + } else { + return None; + }; + let body_start = pos + at + marker.len(); + let closers: &[&'static str] = match marker { + ANALYSIS => &[END, ASSISTANT, CALL, RETURN], + FINAL => &[END, RETURN, CALL], + _ => &[END, CALL], + }; + let end = earliest(&raw[body_start..], closers).map(|(at, m)| (body_start + at, m)); + let body_end = end.map_or(raw.len(), |(at, _)| at); + let mut body = &raw[body_start..body_end]; + match marker { + ANALYSIS => { + if end.is_some_and(|(_, m)| m == ASSISTANT) { + body = body.strip_suffix(CONSTRAIN.as_bytes()).unwrap_or(body); + } + out.reasoning.extend_from_slice(body); + } + FINAL => out.content.extend_from_slice(body), + _ => { + let (close_at, close) = end?; + let call = call_body(name?, std::str::from_utf8(body).ok()?, orders)?; + out.calls.push(call); + out.raw_tool_text.push_str(ASSISTANT); + out.raw_tool_text + .push_str(std::str::from_utf8(&raw[pos..close_at + close.len()]).ok()?); + } + } + match end { + // A bare `<|start|>assistant` closer opens the next message. + Some((at, ASSISTANT)) => pos = at, + Some((_, CALL | RETURN)) => break, + Some((at, closer)) => pos = at + closer.len(), + None => break, + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reasoning_answer_and_calls() { + let raw = b"<|channel|>analysis<|message|>think<|end|><|start|>assistant<|channel|>final<|message|>hi"; + let p = parse(raw, &[]).unwrap(); + assert_eq!(p.reasoning, b"think"); + assert_eq!(p.content, b"hi"); + assert!(p.calls.is_empty()); + + let raw = "<|channel|>analysis<|message|>plan<|end|><|start|>assistant to=get_weather<|channel|>analysis \ + <|constrain|>xml<|message|>\nSeoul\n\n\ + \n3\n<|end|><|start|>assistant to=time\ + <|channel|>analysis <|constrain|>xml<|message|><|call|>"; + let p = parse(raw.as_bytes(), &[]).unwrap(); + assert_eq!(p.reasoning, b"plan"); + assert_eq!(p.calls.len(), 2); + assert_eq!(p.calls[0].name, "get_weather"); + assert_eq!(p.calls[0].arguments, r#"{"city":"Seoul","days":3}"#); + assert_eq!(p.calls[1].name, "time"); + assert_eq!(p.calls[1].arguments, "{}"); + } + + // Walk the grammar from an empty reply, appending each forced token's text. + fn walk(mut text: String, tools: bool, steps: usize) -> String { + let piece = |t: i32| match t { + T_START => "<|start|>", + T_ASSISTANT => "assistant", + T_CHANNEL => "<|channel|>", + T_MESSAGE => "<|message|>", + T_CONSTRAIN => "<|constrain|>", + T_ANALYSIS => "analysis", + T_FINAL => "final", + T_SPACE => " ", + T_XML => "xml", + _ => unreachable!(), + }; + for _ in 0..steps { + match forced_header(text.as_bytes(), tools) { + Some(t) => text.push_str(piece(t)), + None => break, + } + } + text + } + + #[test] + fn forced_headers() { + assert_eq!(walk(String::new(), false, 8), ANALYSIS); + // After analysis: restart the message, then final without tools. + let answer = walk(format!("{ANALYSIS}plan{END}"), false, 8); + assert_eq!(answer, format!("{ANALYSIS}plan{END}{ASSISTANT}{FINAL}")); + // With tools the model picks ` to=` or `<|channel|>` itself. + assert_eq!( + walk(format!("{ANALYSIS}plan{END}"), true, 8), + format!("{ANALYSIS}plan{END}{ASSISTANT}") + ); + let call = walk( + format!("{ANALYSIS}plan{END}{ASSISTANT} to=f<|channel|>"), + true, + 8, + ); + assert_eq!(call, format!("{ANALYSIS}plan{END}{ASSISTANT} to=f{TOOL}")); + // Bodies stay sampled. + assert_eq!( + forced_header(format!("{ANALYSIS}pl").as_bytes(), false), + None + ); + } + + #[test] + fn truncated_and_malformed() { + // A call cut by the budget has no terminator. + let raw = b"<|channel|>analysis<|message|>x<|end|><|start|>assistant to=f<|channel|>analysis <|constrain|>xml<|message|>\n1"; + assert!(parse(raw, &[]).is_none()); + // A header cut by the budget publishes nothing of it. + let p = parse( + b"<|channel|>analysis<|message|>x<|end|><|start|>assistant<|chan", + &[], + ) + .unwrap(); + assert_eq!(p.reasoning, b"x"); + assert!(p.content.is_empty()); + // Text outside any channel is not Harmony. + assert!(parse(b"hello", &[]).is_none()); + assert!(parse(b"junk<|channel|>final<|message|>hi", &[]).is_none()); + // The template's bare closer of an analysis message. + let p = parse( + b"<|channel|>analysis<|message|>x<|start|>assistant<|channel|>final<|message|>y", + &[], + ) + .unwrap(); + assert_eq!( + (p.reasoning.as_slice(), p.content.as_slice()), + (&b"x"[..], &b"y"[..]) + ); + } +} diff --git a/crates/ds4-server/src/tools/iquest.rs b/crates/ds4-server/src/tools/iquest.rs new file mode 100644 index 0000000000000000000000000000000000000000..da4194619eebc83c56854ea026744d4d95107141 --- /dev/null +++ b/crates/ds4-server/src/tools/iquest.rs @@ -0,0 +1,101 @@ +//! IQuest XML values follow its official parser's schema-aware JSON decoding. +use serde_json::{Map, Value}; + +use crate::parse::{ToolCall, ToolSchemaOrder}; +use crate::render::{IQUEST_TOOL_CALL_END, IQUEST_TOOL_CALL_START}; + +use super::{find_substr, tool_schema_order_prop_type, tool_schema_orders_find, ParsedGenerated}; + +const KEY_START: &str = ""; +const KEY_END: &str = ""; +const VALUE_START: &str = ""; +const VALUE_END: &str = ""; + +fn python_space(c: char) -> bool { + c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c) +} + +pub(crate) fn call_body(body: &[u8], orders: &[ToolSchemaOrder]) -> Option { + let body = std::str::from_utf8(body).ok()?; + let first = body.find(KEY_START).unwrap_or(body.len()); + let name = body[..first].trim_matches(python_space); + if name.is_empty() { + return None; + } + let order = tool_schema_orders_find(orders, name); + let mut tail = &body[first..]; + let mut args = Map::new(); + while !tail.trim_start_matches(python_space).is_empty() { + tail = tail + .trim_start_matches(python_space) + .strip_prefix(KEY_START)?; + let end = tail.find(KEY_END)?; + let key = tail[..end].trim_matches(python_space); + if key.is_empty() { + return None; + } + tail = tail[end + KEY_END.len()..] + .trim_start_matches(python_space) + .strip_prefix(VALUE_START)?; + let end = tail.find(VALUE_END)?; + let raw = &tail[..end]; + let value = if tool_schema_order_prop_type(order, key) == Some("string") { + Value::String(raw.to_owned()) + } else { + let trimmed = raw.trim_matches(python_space); + serde_json::from_str(trimmed).unwrap_or_else(|_| Value::String(trimmed.to_owned())) + }; + args.insert(key.to_owned(), value); + tail = &tail[end + VALUE_END.len()..]; + } + Some(ToolCall { + name: name.to_owned(), + arguments: Value::Object(args).to_string(), + ..Default::default() + }) +} + +pub(super) fn split(text: &[u8], thinking: bool) -> (&[u8], Vec) { + if thinking { + let start = b""; + let raw = find_substr(text, start) + .map(|at| &text[at + start.len()..]) + .unwrap_or(text); + let Some(end) = find_substr(raw, b"") else { + return (&[], raw.to_vec()); + }; + (&raw[end + b"".len()..], raw[..end].to_vec()) + } else { + (text, Vec::new()) + } +} + +pub(super) fn parse(text: &[u8], thinking: bool, orders: &[ToolSchemaOrder]) -> ParsedGenerated { + let (body, reasoning) = split(text, thinking); + let start = IQUEST_TOOL_CALL_START.as_bytes(); + let end = IQUEST_TOOL_CALL_END.as_bytes(); + let mut cursor = 0; + let mut kept = 0; + let mut out = ParsedGenerated { + reasoning, + ok: true, + ..Default::default() + }; + while let Some(rel) = find_substr(&body[cursor..], start) { + let open = cursor + rel; + let first = open + start.len(); + let Some(close) = find_substr(&body[first..], end).map(|at| first + at) else { + break; + }; + cursor = close + end.len(); + if let Some(call) = call_body(&body[first..close], orders) { + out.content.extend_from_slice(&body[kept..open]); + out.raw_tool_text + .push_str(&String::from_utf8_lossy(&body[open..cursor])); + out.calls.push(call); + kept = cursor; + } + } + out.content.extend_from_slice(&body[kept..]); + out +} diff --git a/crates/ds4-server/src/trace.rs b/crates/ds4-server/src/trace.rs new file mode 100644 index 0000000000000000000000000000000000000000..f3df7e5602e6ead98f22dc7ba017e081e872c6b4 --- /dev/null +++ b/crates/ds4-server/src/trace.rs @@ -0,0 +1,273 @@ +//! C `--trace FILE` (ds4_help.c:323, "Write prompt, token, and DSML debug +//! trace"; the server opens the file `w` at ds4_server.c:16010-16019 and logs +//! "tracing session to %s"). One section per request, no KV state read. +//! +//! The reference writes it from four points: the identity and the rendered +//! prompt at `trace_begin` (:11967-12017), the prompt token ids with the +//! engine's own text for each (`trace_write_token`, :11936-11941), the cache +//! diagnostics (:12000-12015), and the parsed message with its tool calls at +//! `trace_finish` (:12044-12107). +//! +//! Named deviations, so the file is read for what it is: +//! - one section, written at the finish, instead of writes at each point: the +//! lines are grouped rather than interleaved with generation; +//! - the raw request body block (:12008-12016) is absent -- this host's parse +//! does not retain the body; +//! - the live-vs-prompt token window (:11920-11950) is absent: the host keeps +//! no per-position live id array. The prompt's own ids are written, taken by +//! tokenizing the rendered prompt (`GenerateEngine::tokenize_rendered_chat`), +//! which is the reference's "prompt" column; +//! - the replay account is the host's cached read/write pair, not the +//! reference's mem/disk/canonical/missing split. + +use std::fs::OpenOptions; +use std::io::Write; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::OnceLock; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::parse::ToolCall; +use crate::route::{Api, ReqKind, ThinkMode}; + +static PATH: OnceLock = OnceLock::new(); +static SEQ: AtomicU64 = AtomicU64::new(0); + +/// Set once at boot from `--trace FILE`; a later call is ignored (one server, +/// one trace file, the reference's `s->trace`). +pub fn set_path(path: PathBuf) { + let _ = PATH.set(path); +} + +pub fn enabled() -> bool { + PATH.get().is_some() +} + +/// One request's section. Borrowed only: the caller writes it after the turn, +/// when the prompt, the account and the generated text all exist. +pub struct TraceRequest<'a> { + pub api: Api, + pub kind: ReqKind, + pub stream: bool, + pub model_id: i32, + pub model: &'a str, + pub tools: usize, + pub think: ThinkMode, + pub prompt: &'a [u8], + /// The prompt's own token ids; empty when the caller did not tokenize. + pub tokens: &'a [i32], + pub cached: i32, + pub prompt_tokens: i32, + pub max_tokens: i32, + pub temperature: f32, + pub top_k: i32, + pub top_p: f32, + pub min_p: f32, + pub seed: u64, + pub finish: &'a str, + pub completion: i32, + pub saw_tool_start: bool, + pub saw_tool_end: bool, + pub elapsed_sec: f64, + pub content: &'a [u8], + pub reasoning: &'a [u8], + pub calls: &'a [ToolCall], +} + +fn api_name(api: Api) -> &'static str { + match api { + Api::Openai => "openai", + Api::Anthropic => "anthropic", + Api::Responses => "responses", + } +} + +fn kind_name(kind: ReqKind) -> &'static str { + match kind { + ReqKind::Chat => "chat", + ReqKind::Completion => "completion", + } +} + +fn think_name(think: ThinkMode) -> &'static str { + match think { + ThinkMode::None => "none", + ThinkMode::Low => "low", + ThinkMode::High => "high", + ThinkMode::Max => "max", + } +} + +/// `trace_write_token` (:11936-11941): the engine's own text for one id, +/// escaped so a control token cannot break the line layout. +fn token_text_escaped(text: &[u8]) -> String { + let mut out = String::with_capacity(text.len()); + for &byte in text { + match byte { + b'\n' => out.push_str("\\n"), + b'\r' => out.push_str("\\r"), + b'\t' => out.push_str("\\t"), + b'\\' => out.push_str("\\\\"), + 0x20..=0x7e => out.push(char::from(byte)), + _ => out.push_str(&format!("\\x{byte:02x}")), + } + } + out +} + +/// Write one section. `text_of` is the engine's token decoder; the caller +/// passes it because the trace module holds no engine. +pub fn write_request(req: &TraceRequest<'_>, text_of: &mut dyn FnMut(i32) -> Vec) { + let Some(path) = PATH.get() else { + return; + }; + let Ok(mut file) = OpenOptions::new().create(true).append(true).open(path) else { + return; + }; + let id = SEQ.fetch_add(1, Ordering::Relaxed) + 1; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); + let mut out = String::new(); + out.push_str(&format!("\n===== request {id} {now:.3} =====\n")); + out.push_str(&format!( + "kind: {}\napi: {}\nmodel: {} (id {})\nstream: {}\ntools: {}\nthink_mode: {}\n\ + prompt_tokens: {}\ncached_tokens: {}\nmax_tokens: {}\ntemperature: {:.3}\n\ + top_k: {}\ntop_p: {:.3}\nmin_p: {:.3}\nseed: {}\n", + kind_name(req.kind), + api_name(req.api), + req.model, + req.model_id, + u8::from(req.stream), + req.tools, + think_name(req.think), + req.prompt_tokens, + req.cached, + req.max_tokens, + req.temperature, + req.top_k, + req.top_p, + req.min_p, + req.seed, + )); + out.push_str("--- rendered prompt ---\n"); + out.push_str(&String::from_utf8_lossy(req.prompt)); + if !out.ends_with('\n') { + out.push('\n'); + } + out.push_str("--- prompt tokens ---\n"); + for (pos, &token) in req.tokens.iter().enumerate() { + out.push_str(&format!( + "{pos:>7} {token:>8} {}\n", + token_text_escaped(&text_of(token)) + )); + } + out.push_str("--- generated text ---\n"); + out.push_str(&String::from_utf8_lossy(req.reasoning)); + out.push_str(&String::from_utf8_lossy(req.content)); + if !out.ends_with('\n') { + out.push('\n'); + } + out.push_str(&format!( + "--- parsed message ---\nfinish: {}\ngenerated_tokens: {}\ndsml_start: {}\n\ + dsml_end: {}\nelapsed_sec: {:.3}\n", + req.finish, + req.completion, + u8::from(req.saw_tool_start), + u8::from(req.saw_tool_end), + req.elapsed_sec, + )); + if !req.reasoning.is_empty() { + out.push_str(&format!( + "\nreasoning:\n{}\n", + String::from_utf8_lossy(req.reasoning) + )); + } + if !req.content.is_empty() { + out.push_str(&format!( + "\ncontent:\n{}\n", + String::from_utf8_lossy(req.content) + )); + } + for (i, call) in req.calls.iter().enumerate() { + out.push_str(&format!( + "\ntool_call[{i}]:\nid: {}\nname: {}\narguments:\n{}\n", + call.id, call.name, call.arguments + )); + } + let _ = file.write_all(out.as_bytes()); + let _ = file.flush(); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// One section carries the identity, the rendered prompt, the prompt token + /// ids with the engine's text for each, and the parsed message with its + /// calls (ds4_server.c:11976-12017 and :12044-12107). + #[test] + fn a_section_carries_the_prompt_tokens_and_parsed_message() { + let path = std::env::temp_dir().join(format!("ds4-trace-{}.txt", std::process::id())); + let _ = std::fs::remove_file(&path); + set_path(path.clone()); + + let prompt = b"<|user|>hi<|assistant|>"; + let tokens = [7i32, 8]; + let calls = [ToolCall { + id: "call_1".into(), + name: "bash".into(), + arguments: r#"{"command":"pwd"}"#.into(), + }]; + let mut text_of = |token: i32| match token { + 7 => b"<|user|>".to_vec(), + _ => b"hi".to_vec(), + }; + write_request( + &TraceRequest { + api: Api::Openai, + kind: ReqKind::Chat, + stream: false, + model_id: 5, + model: "deepseek-v4.1-flash", + tools: 1, + think: ThinkMode::Low, + prompt, + tokens: &tokens, + cached: 0, + prompt_tokens: tokens.len() as i32, + max_tokens: 64, + temperature: 1.0, + top_k: 0, + top_p: 1.0, + min_p: 0.0, + seed: 7, + finish: "tool_calls", + completion: 12, + saw_tool_start: true, + saw_tool_end: true, + elapsed_sec: 1.5, + content: b"", + reasoning: b"thinking", + calls: &calls, + }, + &mut text_of, + ); + + let got = std::fs::read_to_string(&path).expect("the trace file is written"); + assert!(got.contains("kind: chat"), "{got}"); + assert!( + got.contains("--- rendered prompt ---\n<|user|>hi<|assistant|>\n"), + "{got}" + ); + assert!(got.contains(" 0 7 <|user|>"), "{got}"); + assert!(got.contains("finish: tool_calls"), "{got}"); + assert!(got.contains("dsml_start: 1\ndsml_end: 1"), "{got}"); + assert!( + got.contains("tool_call[0]:\nid: call_1\nname: bash"), + "{got}" + ); + let _ = std::fs::remove_file(&path); + } +} diff --git a/crates/ds4-server/src/v41_disk.rs b/crates/ds4-server/src/v41_disk.rs new file mode 100644 index 0000000000000000000000000000000000000000..c4a24bcfee49cae157d87756ced9da7ff210abc5 --- /dev/null +++ b/crates/ds4-server/src/v41_disk.rs @@ -0,0 +1,579 @@ +//! The V4.1 disk half (item 4): the checkpoint payload in the KVC disk +//! store. +//! +//! The store is crates/ds4-kv (the port of ds4_kvstore.c): file format, SHA +//! key, budget, eviction and the artifact identity are its contract, never +//! duplicated here. This module is the V4.1 caller's half: +//! +//! - the store's text block interleaves each token (LE i32) with its image +//! signature (LE u64, zero for text), so its byte-prefix match preserves +//! image identity and order; native restore repeats the comparison +//! (ds4_ds41_kvreuse.inc), so this match only selects; +//! - the trailer carries what the store cannot know: the payload ABI, the +//! packed-KV class, and the mount digest (the amplifier identity the +//! native drops the in-memory checkpoint on, v41_run.rs); +//! - the store's reason carries the checkpoint class: "cold" for a checkpoint +//! taken at the prompt end and "continued" for the turn checkpoint, the +//! frontier the run left (the reference's own names for these two stores, +//! server_kv_cache.c:106-141). The class also rides the payload meta, so +//! the import lands a candidate in that class's own slot; +//! - `store` stages the native payload export (a temp file the native +//! writes) through write_payload_file, which owns the atomic rename and +//! the eviction; `find` answers the payload range the native imports. + +use std::path::Path; + +use ds4_core::{V41KvClass, V41KvDisk, V41KvHit}; +use ds4_kv::{read_trailer, Reason as KvReason, Store as KvStore}; + +/// The quant slot the store header carries. The V4.1 payload is the packed +/// FP4-nibble + E4M3-scale KV (ds41_kvfmt.h), so the class is 4 -- one of +/// the quant classes ds4_kvstore_quant_bits_supported admits. A class +/// marker, not an artifact quant. +const KV_QUANT_BITS: u8 = 4; +const TOKEN_KEY_BYTES: usize = std::mem::size_of::() + std::mem::size_of::(); + +/// The V4.1 trailer (12 bytes; the store appends its identity block after +/// it): [0] payload ABI (the span order and the meta in +/// ds4_ds41_kvreuse.inc; bump on change), [1] packed-KV class (1 = +/// ds41_kvfmt.h 288/72), [4..12] the mount digest LE. +const TRAILER_BYTES: usize = 12; +/// ABI 3 is the payload whose meta carries the checkpoint class (the native +/// v41_kv_class); ABI 2 lacks those four bytes and must not be read as one. +const TRAILER_ABI: u8 = 3; +const TRAILER_CLASS: u8 = 1; + +/// The trailer read cap: ours plus the store's identity block and slack. +const TRAILER_READ_MAX: u64 = 4096; + +pub struct V41Disk<'a> { + store: &'a mut KvStore, + model_id: u8, + ctx: u32, +} + +impl<'a> V41Disk<'a> { + pub fn new(store: &'a mut KvStore, model_id: u8, ctx: u32) -> Self { + Self { + store, + model_id, + ctx, + } + } +} + +/// Interleave each token with its image signature (zero for text). Prefix +/// lookup then rejects changed images while admitting later appended images. +fn tokens_text(tokens: &[i32], sig: &[u64]) -> Option> { + // An empty slice denotes text. A partial image map must never alias it. + if !sig.is_empty() && sig.len() != tokens.len() { + return None; + } + + let mut out = Vec::with_capacity(tokens.len().checked_mul(TOKEN_KEY_BYTES)?); + for (i, token) in tokens.iter().enumerate() { + let image_sig = if sig.is_empty() { 0 } else { sig[i] }; + out.extend_from_slice(&token.to_le_bytes()); + out.extend_from_slice(&image_sig.to_le_bytes()); + } + Some(out) +} + +fn trailer(digest: u64) -> [u8; TRAILER_BYTES] { + let mut out = [0u8; TRAILER_BYTES]; + out[0] = TRAILER_ABI; + out[1] = TRAILER_CLASS; + out[4..12].copy_from_slice(&digest.to_le_bytes()); + out +} + +/// The store reason a class is written under. This is the reference's own +/// vocabulary (server_kv_cache.c:106-141): "cold" for the checkpoint at a +/// prompt boundary, "continued" for a conversation that has grown far enough +/// -- our turn checkpoint, the live frontier the run left. Both are reason +/// family 1, which is the family the automatic exact-replay search admits +/// (crates/ds4-kv format.rs `is_automatic_exact_replay`), and a continued +/// entry whose text is a strict prefix of an incoming one is scored down so +/// it is evicted first (policy.rs:163-181, :206-215). +fn class_reason(class: V41KvClass) -> KvReason { + match class { + V41KvClass::Prompt => KvReason::Cold, + V41KvClass::Turn => KvReason::Continued, + } +} + +/// The V4.1 trailer check: ABI, class, mount digest. The store's own +/// header/identity checks (model, quant class, ctx, artifact identity) have +/// already run in the candidate lookup; this is the mount identity the store +/// cannot know. +fn trailer_ok(raw: &[u8], digest: u64) -> bool { + raw.len() >= TRAILER_BYTES + && raw[0] == TRAILER_ABI + && raw[1] == TRAILER_CLASS + && u64::from_le_bytes(raw[4..12].try_into().unwrap()) == digest +} + +impl V41KvDisk for V41Disk<'_> { + fn find(&mut self, tokens: &[i32], sig: &[u64], digest: u64, have: u32) -> Option { + let text = tokens_text(tokens, sig)?; + let (path, env) = self + .store + .text_prefix_candidate(&text, self.model_id, KV_QUANT_BITS, self.ctx) + .ok()??; + // The store's token count must not exceed this prompt: the native + // import copies that many tokens from the caller's slice. + if env.header.tokens <= have + || env.header.tokens as usize > tokens.len() + || env.text.len() != env.header.tokens as usize * TOKEN_KEY_BYTES + { + return None; + } + let (_, raw) = read_trailer(&path, TRAILER_READ_MAX).ok()?; + if !trailer_ok(&raw, digest) { + return None; + } + Some(V41KvHit { + path, + offset: env.payload_offset, + bytes: env.header.payload_bytes, + tokens: env.header.tokens, + }) + } + + fn store( + &mut self, + tokens: &[i32], + sig: &[u64], + digest: u64, + class: V41KvClass, + save: &mut dyn FnMut(&Path) -> bool, + ) { + if (tokens.len() as i64) < i64::from(self.store.opt.min_tokens) { + return; + } + let Some(text) = tokens_text(tokens, sig) else { + return; + }; + let trailer = trailer(digest); + // An entry under this exact text is current only when ITS trailer + // carries the same mount digest. A different mount leaves a stale + // payload behind, and the store's reuse fast path would rewrite only + // the trailer (keeping the old mount's KV), so a stale entry is + // discarded before the write. A shorter prefix entry is left alone: + // it is a different checkpoint (a different name). + if let Ok(Some((path, env))) = + self.store + .text_prefix_candidate(&text, self.model_id, KV_QUANT_BITS, self.ctx) + { + if env.text == text { + let current = read_trailer(&path, TRAILER_READ_MAX) + .map(|(_, raw)| trailer_ok(&raw, digest)) + .unwrap_or(false); + if current { + return; + } + let _ = self.store.discard(&path); + } + } + let Ok(temp) = self.store.payload_temp() else { + return; + }; + if !save(temp.path()) { + return; + } + // kv_header is the port's header builder (generate.rs; the session + // route stores with the same shape). payload/text bytes are filled + // by write_payload_file, and the reason is this class's own. + let mut header = + crate::generate::kv_header(self.model_id, KV_QUANT_BITS, self.ctx, tokens.len() as u32); + header.reason = class_reason(class); + let _ = self + .store + .write_payload_file(header, &text, temp.path(), &trailer); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ds4_kv::Options; + use std::path::PathBuf; + + fn tmpdir(name: &str) -> PathBuf { + use std::sync::atomic::{AtomicUsize, Ordering}; + static SEQ: AtomicUsize = AtomicUsize::new(0); + let n = SEQ.fetch_add(1, Ordering::Relaxed); + let p = std::env::temp_dir().join(format!("ds4-v41-disk-{n}-{name}")); + let _ = std::fs::remove_dir_all(&p); + std::fs::create_dir_all(&p).unwrap(); + p + } + + fn open_store(dir: &Path, min_tokens: i32) -> KvStore { + let opt = Options { + min_tokens, + ..Options::default() + }; + KvStore::open(dir, 16, false, opt).unwrap() + } + + fn save_zeros(path: &Path, n: usize) -> bool { + std::fs::write(path, vec![0u8; n]).is_ok() + } + + fn entries(dir: &Path) -> Vec { + std::fs::read_dir(dir) + .unwrap() + .flatten() + .map(|e| e.path()) + .filter(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .and_then(ds4_kv::sha_hex_name) + .is_some() + }) + .collect() + } + + #[test] + fn key_interleaves_token_and_sig() { + assert_eq!( + tokens_text(&[1, -2], &[0x0807_0605_0403_0201, 0]).unwrap(), + vec![ + 1, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0xfe, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0 + ] + ); + assert_eq!(tokens_text(&[1, -2], &[]), tokens_text(&[1, -2], &[0; 2])); + } + + #[test] + fn store_then_find_roundtrip() { + let dir = tmpdir("roundtrip"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let tokens: Vec = (100..140).collect(); + let mut save = |p: &Path| save_zeros(p, 128); + disk.store(&tokens, &[], 0xabc, V41KvClass::Prompt, &mut save); + + let hit = disk.find(&tokens, &[], 0xabc, 0).expect("hit"); + assert_eq!(hit.tokens, 40); + assert_eq!(hit.bytes, 128); + assert!(hit.path.exists()); + + // The in-memory checkpoint already at this length leaves nothing to + // gain; a different mount digest refuses. + assert!(disk.find(&tokens, &[], 0xabc, 40).is_none()); + assert!(disk.find(&tokens, &[], 0xdef, 0).is_none()); + + // A longer prompt matches the 40-token prefix; an unrelated one is a + // miss. + let mut longer = tokens.clone(); + longer.push(999); + assert_eq!( + disk.find(&longer, &[], 0xabc, 0).expect("prefix").tokens, + 40 + ); + let other: Vec = (500..540).collect(); + assert!(disk.find(&other, &[], 0xabc, 0).is_none()); + } + + #[test] + fn image_prefix_survives_append() { + let dir = tmpdir("append-image"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let tokens = [100, 900, 900, 101, 900, 900, 102]; + let sig = [0, 11, 12, 0, 21, 22, 0]; + disk.store(&tokens, &sig, 0xabc, V41KvClass::Prompt, &mut |p| { + save_zeros(p, 128) + }); + + let mut longer = tokens.to_vec(); + longer.extend_from_slice(&[900, 900, 103]); + let mut longer_sig = sig.to_vec(); + longer_sig.extend_from_slice(&[31, 32, 0]); + let hit = disk + .find(&longer, &longer_sig, 0xabc, 0) + .expect("an appended image preserves the stored prefix"); + assert_eq!(hit.tokens, tokens.len() as u32); + assert_eq!(hit.bytes, 128); + } + + #[test] + fn changed_images_miss() { + let dir = tmpdir("changed-images"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let tokens = [100, 900, 900, 101, 900, 900, 102]; + let sig = [0, 11, 12, 0, 21, 22, 0]; + disk.store(&tokens, &sig, 0xabc, V41KvClass::Prompt, &mut |p| { + save_zeros(p, 128) + }); + + let replacement = [0, 31, 32, 0, 21, 22, 0]; + let reordered = [0, 21, 22, 0, 11, 12, 0]; + assert!(disk.find(&tokens, &replacement, 0xabc, 0).is_none()); + assert!(disk.find(&tokens, &reordered, 0xabc, 0).is_none()); + assert!(disk.find(&tokens, &[], 0xabc, 0).is_none()); + assert!(disk.find(&tokens, &sig, 0xabc, 0).is_some()); + } + + #[test] + fn image_identity_survives_reopen() { + let dir = tmpdir("reopen-image"); + let tokens = [100, 900, 900, 101]; + let sig = [0, 11, 12, 0]; + let original = { + let mut store = open_store(&dir, 4); + store.bind_identity([7; 32]); + let mut disk = V41Disk::new(&mut store, 7, 4096); + disk.store(&tokens, &sig, 0xabc, V41KvClass::Prompt, &mut |p| { + save_zeros(p, 128) + }); + disk.find(&tokens, &sig, 0xabc, 0).expect("stored").path + }; + + let mut reopened = open_store(&dir, 4); + reopened.bind_identity([7; 32]); + let mut disk = V41Disk::new(&mut reopened, 7, 4096); + let hit = disk.find(&tokens, &sig, 0xabc, 0).expect("reopened"); + assert_eq!(hit.path, original); + assert_eq!(hit.bytes, 128); + assert!(disk.find(&tokens, &[0, 21, 22, 0], 0xabc, 0).is_none()); + assert!(disk.find(&tokens, &sig, 0xdef, 0).is_none()); + drop(reopened); + + let mut other = open_store(&dir, 4); + other.bind_identity([9; 32]); + assert!(V41Disk::new(&mut other, 7, 4096) + .find(&tokens, &sig, 0xabc, 0) + .is_none()); + } + + #[test] + fn legacy_payload_abi_is_rejected() { + let dir = tmpdir("legacy-abi"); + let mut store = open_store(&dir, 4); + let tokens = [100, 101, 102, 103]; + let mut old_trailer = trailer(0xabc); + old_trailer[0] = 1; + let payload = store.payload_temp().unwrap(); + assert!(save_zeros(payload.path(), 128)); + let header = crate::generate::kv_header(7, KV_QUANT_BITS, 4096, tokens.len() as u32); + // Use the current key to prove ABI rejection independently of the + // changed key width: even an otherwise matching legacy payload misses. + store + .write_payload_file( + header, + &tokens_text(&tokens, &[]).unwrap(), + payload.path(), + &old_trailer, + ) + .unwrap(); + let mut disk = V41Disk::new(&mut store, 7, 4096); + assert!(disk.find(&tokens, &[], 0xabc, 0).is_none()); + + disk.store(&tokens, &[], 0xabc, V41KvClass::Prompt, &mut |p| { + save_zeros(p, 256) + }); + assert_eq!( + disk.find(&tokens, &[], 0xabc, 0).expect("new ABI").bytes, + 256 + ); + assert_eq!(entries(&dir).len(), 1); + } + + #[test] + fn malformed_signatures_miss() { + let dir = tmpdir("malformed-find"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let tokens = [100, 101, 102, 103]; + disk.store(&tokens, &[], 0xabc, V41KvClass::Prompt, &mut |p| { + save_zeros(p, 128) + }); + let invalid: [&[u64]; 2] = [&[0; 3], &[0; 5]]; + let hits: Vec<_> = invalid + .into_iter() + .filter(|sig| disk.find(&tokens, sig, 0xabc, 0).is_some()) + .map(<[u64]>::len) + .collect(); + assert!(hits.is_empty(), "invalid signature lengths hit: {hits:?}"); + assert!(disk.find(&tokens, &[], 0xabc, 0).is_some()); + assert!(disk.find(&tokens, &[0; 4], 0xabc, 0).is_some()); + } + + #[test] + fn malformed_signatures_not_saved() { + let dir = tmpdir("malformed-store"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let tokens = [100, 101, 102, 103]; + let invalid: [&[u64]; 2] = [&[0; 3], &[0; 5]]; + let mut exports = 0; + for sig in invalid { + disk.store( + &tokens, + sig, + sig.len() as u64, + V41KvClass::Prompt, + &mut |p| { + exports += 1; + save_zeros(p, 128) + }, + ); + } + assert_eq!(exports, 0, "invalid signatures reached native export"); + assert!(entries(&dir).is_empty()); + } + + #[test] + fn current_entry_is_not_rewritten() { + let dir = tmpdir("current"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let tokens: Vec = (100..140).collect(); + let mut save_small = |p: &Path| save_zeros(p, 128); + disk.store(&tokens, &[], 0xabc, V41KvClass::Prompt, &mut save_small); + // A second store under the same digest must skip the export (a + // rewrite would carry the new size). + let mut save_big = |p: &Path| save_zeros(p, 256); + disk.store(&tokens, &[], 0xabc, V41KvClass::Prompt, &mut save_big); + assert_eq!(entries(&dir).len(), 1); + assert_eq!(disk.find(&tokens, &[], 0xabc, 0).expect("hit").bytes, 128); + } + + #[test] + fn stale_mount_entry_is_replaced() { + let dir = tmpdir("stale"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let tokens: Vec = (100..140).collect(); + let mut save_small = |p: &Path| save_zeros(p, 128); + disk.store(&tokens, &[], 0xabc, V41KvClass::Prompt, &mut save_small); + // The mount changed: the old payload is the old amplifier's KV and + // must not survive under the new trailer. + let mut save_big = |p: &Path| save_zeros(p, 256); + disk.store(&tokens, &[], 0xdef, V41KvClass::Prompt, &mut save_big); + assert_eq!(entries(&dir).len(), 1); + assert!(disk.find(&tokens, &[], 0xabc, 0).is_none()); + assert_eq!(disk.find(&tokens, &[], 0xdef, 0).expect("hit").bytes, 256); + } + + #[test] + fn min_tokens_gate_skips_short_prompts() { + let dir = tmpdir("min-tokens"); + let mut store = open_store(&dir, 64); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let tokens: Vec = (100..140).collect(); + let mut save = |p: &Path| save_zeros(p, 128); + disk.store(&tokens, &[], 0xabc, V41KvClass::Prompt, &mut save); + assert!(entries(&dir).is_empty()); + assert!(disk.find(&tokens, &[], 0xabc, 0).is_none()); + } + + #[test] + fn find_works_with_bound_identity() { + let dir = tmpdir("identity"); + let mut store = open_store(&dir, 4); + store.bind_identity([7; 32]); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let tokens: Vec = (100..140).collect(); + let mut save = |p: &Path| save_zeros(p, 128); + disk.store(&tokens, &[], 0xabc, V41KvClass::Prompt, &mut save); + // The identity block follows our 12-byte trailer; the check must + // read only its own prefix. + assert_eq!(disk.find(&tokens, &[], 0xabc, 0).expect("hit").tokens, 40); + // A store bound to a different identity refuses the entry. + let mut other = open_store(&dir, 4); + other.bind_identity([9; 32]); + let mut other_disk = V41Disk::new(&mut other, 7, 4096); + assert!(other_disk.find(&tokens, &[], 0xabc, 0).is_none()); + } + + #[test] + fn each_class_is_stored_under_its_own_reason() { + let dir = tmpdir("class-reason"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let prompt: Vec = (100..140).collect(); + let mut turn = prompt.clone(); + turn.extend(140..150); + disk.store(&prompt, &[], 0xabc, V41KvClass::Prompt, &mut |p| { + save_zeros(p, 128) + }); + disk.store(&turn, &[], 0xabc, V41KvClass::Turn, &mut |p| { + save_zeros(p, 256) + }); + assert_eq!(entries(&dir).len(), 2); + + // The reason is the reference's own name for each class, and the + // exact-replay search admits both families (crates/ds4-kv + // format.rs is_automatic_exact_replay). + let prompt_text = tokens_text(&prompt, &[]).unwrap(); + let turn_text = tokens_text(&turn, &[]).unwrap(); + let (_, cold) = disk + .store + .text_prefix_candidate(&prompt_text, 7, KV_QUANT_BITS, 4096) + .unwrap() + .expect("prompt entry"); + assert_eq!(cold.header.reason, KvReason::Cold); + let (_, continued) = disk + .store + .text_prefix_candidate(&turn_text, 7, KV_QUANT_BITS, 4096) + .unwrap() + .expect("turn entry"); + assert_eq!(continued.header.reason, KvReason::Continued); + } + + #[test] + fn the_longest_candidate_wins_between_the_classes() { + let dir = tmpdir("class-longest"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let prompt: Vec = (100..140).collect(); + let mut turn = prompt.clone(); + turn.extend(140..150); + disk.store(&prompt, &[], 0xabc, V41KvClass::Prompt, &mut |p| { + save_zeros(p, 128) + }); + disk.store(&turn, &[], 0xabc, V41KvClass::Turn, &mut |p| { + save_zeros(p, 256) + }); + + // The turn entry's key extends the prompt entry's, so a prompt that + // continues the turn gets the longer checkpoint, whichever class it + // was stored as. + let mut next = turn.clone(); + next.extend(150..155); + let hit = disk.find(&next, &[], 0xabc, 0).expect("turn"); + assert_eq!(hit.tokens, 50); + assert_eq!(hit.bytes, 256); + // One that only reaches the prompt end gets the shorter entry. + let hit = disk.find(&prompt, &[], 0xabc, 0).expect("prompt"); + assert_eq!(hit.tokens, 40); + assert_eq!(hit.bytes, 128); + } + + #[test] + fn turn_entry_is_refused_across_a_mount_change() { + let dir = tmpdir("turn-stale"); + let mut store = open_store(&dir, 4); + let mut disk = V41Disk::new(&mut store, 7, 4096); + let mut turn: Vec = (100..140).collect(); + turn.extend(140..150); + disk.store(&turn, &[], 0xabc, V41KvClass::Turn, &mut |p| { + save_zeros(p, 256) + }); + assert!(disk.find(&turn, &[], 0xdef, 0).is_none()); + assert_eq!(disk.find(&turn, &[], 0xabc, 0).expect("hit").bytes, 256); + + // The new mount's KV replaces the old amplifier's payload. + disk.store(&turn, &[], 0xdef, V41KvClass::Turn, &mut |p| { + save_zeros(p, 512) + }); + assert_eq!(entries(&dir).len(), 1); + assert!(disk.find(&turn, &[], 0xabc, 0).is_none()); + assert_eq!(disk.find(&turn, &[], 0xdef, 0).expect("hit").bytes, 512); + } +} diff --git a/crates/ds4-server/src/worker.rs b/crates/ds4-server/src/worker.rs new file mode 100644 index 0000000000000000000000000000000000000000..259b76ecfe758ca2aa6f67994fb7dae40f0c99a6 --- /dev/null +++ b/crates/ds4-server/src/worker.rs @@ -0,0 +1,191 @@ +//! Worker-role launch: assemble path, never HTTP. + +use ds4_core::WeightSlice; +use ds4_dist::{Layers, Role}; + +pub const WORKER_REQUIRES_MODEL: &str = "ds4-server-rs: --role worker requires -m/--model"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ServerLaunch { + Http, + Worker, +} + +impl ServerLaunch { + /// Apply the allocation policy for the selected launch path before quoting. + pub fn configure_serving(self, req: &mut ds4_core::ServingRequest) { + if self == Self::Worker { + // The assembled worker creates one serial session, with no + // HTTP batch slabs or continuous scheduler allocation. + req.max_seqs = ds4_core::MaxSeqs::Off; + req.lane = ds4_core::LaneMode::Serial; + } + } +} + +/// Layer interval this process keeps resident, or `None` for a whole model. +/// +/// Native `ds4_engine_open` slices the model map whenever a distributed role +/// carries `--layers`, and the coordinator always keeps the output head. +/// `A:output` runs to the last block, which the quote spells `u32::MAX`. +pub fn dist_weight_slice(role: Role, layers: &Layers) -> Option { + if role == Role::None || !layers.set { + return None; + } + + Some(WeightSlice { + start: layers.start, + end: if layers.has_output { + u32::MAX + } else { + layers.end + }, + output: layers.has_output || role == Role::Coordinator, + }) +} + +pub fn server_launch(role: Role, has_model: bool) -> Result { + match role { + Role::Worker => { + if !has_model { + return Err(WORKER_REQUIRES_MODEL.to_string()); + } + Ok(ServerLaunch::Worker) + } + Role::None | Role::Coordinator => Ok(ServerLaunch::Http), + } +} + +#[cfg(test)] +mod tests { + use super::{dist_weight_slice, server_launch, ServerLaunch, WORKER_REQUIRES_MODEL}; + use ds4_dist::{Layers, Role}; + + #[test] + fn worker_quote_has_no_http_banks() { + use ds4_core::{ + fill_quote_facts, resolve_plan, serving_caps, EngineFacts, MaxSeqs, ModelFamily, + QuoteHost, ServingRequest, Variant, SHAPE_FLASH, + }; + let caps = serving_caps(ModelFamily::DeepSeek4, Variant::Flash); + let mut req = ServingRequest { + distribution: ds4_core::Distribution::Sliced, + ..ServingRequest::default() + }; + server_launch(Role::Worker, true) + .unwrap() + .configure_serving(&mut req); + let mut facts = EngineFacts::default(); + fill_quote_facts( + &mut facts, + &req, + caps, + Some(SHAPE_FLASH), + QuoteHost { + weights_bytes: 0, + mtp_bytes: 0, + available_bytes: u64::MAX, + native_chunk: None, + vision: false, + }, + ); + assert_eq!(facts.per_bank_bytes, Some(0)); + assert_eq!(req.max_seqs, MaxSeqs::Off); + let plan = resolve_plan(&req, Some(caps), &facts); + assert!(plan.may_listen(), "{:?}", plan.issues); + assert_eq!(plan.effective.max_seqs, 1); + let budget = plan.quote.unwrap().total; + for (available, allowed) in [(budget, true), (budget - 1, false)] { + facts.host_available_bytes = Some(available); + let plan = resolve_plan(&req, Some(caps), &facts); + assert_eq!(plan.may_listen(), allowed, "{:?}", plan.issues); + } + let mut http = ServingRequest::default(); + ServerLaunch::Http.configure_serving(&mut http); + assert_eq!(http.max_seqs, ServingRequest::default().max_seqs); + } + + #[test] + fn worker_quote_prices_its_layer_slice() { + // Given: a middle worker, a tail worker and a coordinator + let middle = Layers { + start: 10, + end: 19, + has_output: false, + set: true, + }; + let tail = Layers { + start: 20, + end: 0, + has_output: true, + set: true, + }; + + // When: the launch derives what each one keeps resident + let mid = dist_weight_slice(Role::Worker, &middle).unwrap(); + let end = dist_weight_slice(Role::Worker, &tail).unwrap(); + let coord = dist_weight_slice(Role::Coordinator, &middle).unwrap(); + + // Then: only the slice is priced, and the head keeps the output + assert_eq!((mid.start, mid.end, mid.output), (10, 19, false)); + assert_eq!((end.start, end.end, end.output), (20, u32::MAX, true)); + assert!(coord.output, "coordinator owns the output head"); + + // And: an undistributed or layer-less launch prices the whole model + assert!(dist_weight_slice(Role::None, &middle).is_none()); + assert!(dist_weight_slice(Role::Worker, &Layers::default()).is_none()); + } + + #[test] + fn worker_without_model_requires_m() { + // Given: --role worker and no -m/--model + // When: decide launch + let err = server_launch(Role::Worker, false).unwrap_err(); + + // Then: C/shadow error token the bin prints + assert_eq!(err, WORKER_REQUIRES_MODEL); + assert!(err.contains("requires -m/--model")); + } + + #[test] + fn worker_role_does_not_start_http() { + // Given: worker role with a model path + // When: decide launch + let kind = server_launch(Role::Worker, true).unwrap(); + + // Then: HTTP accept loop is not selected + assert_eq!(kind, ServerLaunch::Worker); + assert_ne!(kind, ServerLaunch::Http); + assert_eq!( + server_launch(Role::None, false).unwrap(), + ServerLaunch::Http + ); + assert_eq!( + server_launch(Role::Coordinator, true).unwrap(), + ServerLaunch::Http + ); + } + + #[test] + fn worker_assemble_uses_bound_listen_port() { + // Given: worker layers and an ephemeral data listener + let layers = Layers { + start: 20, + end: 20, + has_output: true, + set: true, + }; + let meta = ds4_dist::slice_meta(7, 43, 129_280, 4096, 7168, &layers); + let (_listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap(); + + // When: plan HELLO via the dist assemble helper + let plan = ds4_dist::worker_plan(&meta, 2, u32::from(port), "deepseek4"); + + // Then: HELLO carries the bound nonzero data port + assert_ne!(plan.hello.listen_port, 0); + assert_eq!(plan.hello.listen_port, u32::from(port)); + assert_eq!(plan.hello.layer_start, 20); + assert_eq!(plan.hello.has_output, 1); + assert_eq!(plan.model_name, "deepseek4"); + } +} diff --git a/crates/ds4-server/src/worker_run.rs b/crates/ds4-server/src/worker_run.rs new file mode 100644 index 0000000000000000000000000000000000000000..ea57d5883c0ac0602c7bae7442cf618acf237cc4 --- /dev/null +++ b/crates/ds4-server/src/worker_run.rs @@ -0,0 +1,218 @@ +//! Session-backed assemble + reconnect for `--role worker`. + +use std::cell::RefCell; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::rc::Rc; + +use ds4_core::{LayerPayloadLoad, LayerSliceEval, Model, ModelFamily, Session, Shape}; +use ds4_dist::{ + assemble_worker, copy_chunked, create_temp, open_data_listener, print_worker_listen_banner, + slice_meta, token_hash_prefix, worker_listen_port, worker_plan, Options, SliceExec, SliceMeta, + SnapshotLoad, SnapshotSave, SnapshotStore, TempShard, WorkOutput, WorkRequest, + WorkerListenBanner, LOAD_PREFIX, SAVE_PREFIX, +}; + +fn hidden_values(shape: &Shape) -> u64 { + match shape.family { + ModelFamily::SolarOpen2 + | ModelFamily::ExaoneMoe + | ModelFamily::Qwen4Exp + | ModelFamily::Inkling + | ModelFamily::Step37 + | ModelFamily::Ling3Vl + | ModelFamily::Ling31Vq + | ModelFamily::BeamVq + | ModelFamily::Mimo2 + | ModelFamily::Qwen35 + | ModelFamily::IQuestQ1 + | ModelFamily::NaiveN05 => u64::from(shape.n_embd), + ModelFamily::DeepSeek4 + | ModelFamily::Motif3 + | ModelFamily::Dots3Note + | ModelFamily::Glm53 + | ModelFamily::Glm53Vq => u64::from(shape.n_hc) * u64::from(shape.n_embd), + } +} + +struct SessionSliceExec<'m> { + session: Rc>>, + meta: SliceMeta, +} + +impl SliceExec for SessionSliceExec<'_> { + fn model_id(&self) -> u32 { + self.meta.model_id + } + fn n_layers(&self) -> u32 { + self.meta.n_layers + } + fn vocab(&self) -> u32 { + self.meta.vocab + } + fn ctx_size(&self) -> u32 { + self.meta.ctx_size + } + fn hidden_values(&self) -> u64 { + self.meta.hidden_values + } + fn has_output(&self) -> bool { + self.meta.has_output + } + fn layer_start(&self) -> u32 { + self.meta.layer_start + } + fn layer_end(&self) -> u32 { + self.meta.layer_end + } + + fn eval(&mut self, req: &WorkRequest) -> Result { + if req.reset { + self.session + .borrow_mut() + .layer_slice_reset() + .map_err(|error| error.to_string())?; + } + let n = req.tokens.len(); + let mut output_hc = req + .produce_hidden + .then(|| vec![0.0; (n as u64).saturating_mul(self.meta.hidden_values) as usize]); + let mut logits = req + .produce_logits + .then(|| vec![0.0; n.saturating_mul(self.meta.vocab as usize)]); + self.session + .borrow_mut() + .eval_layer_slice(LayerSliceEval { + tokens: &req.tokens, + pos0: req.pos0, + layer_start: req.layer_start, + layer_end: req.layer_end, + input_hc: (!req.input_hc.is_empty()).then_some(req.input_hc.as_slice()), + output_hc: output_hc.as_deref_mut(), + logits: logits.as_deref_mut(), + }) + .map_err(|error| error.to_string())?; + Ok(WorkOutput { + hidden: output_hc, + logits, + }) + } +} + +struct SessionSnapshotStore<'m> { + session: Rc>>, + layer_start: u32, + layer_end: u32, +} + +impl SnapshotStore for SessionSnapshotStore<'_> { + type SaveReader = TempShard; + + fn save(&mut self, req: SnapshotSave) -> Result<(TempShard, u64), String> { + let live = self.session.borrow(); + if live.host().tokens().len() as u32 != req.token_count { + return Err("worker snapshot token count mismatch".into()); + } + if token_hash_prefix(live.host().tokens()) != req.token_hash { + return Err("worker snapshot token hash mismatch".into()); + } + drop(live); + let mut tmp = create_temp(SAVE_PREFIX) + .map_err(|_| "failed to create worker snapshot temp file".to_string())?; + self.session + .borrow() + .save_layer_payload(tmp.path(), self.layer_start, self.layer_end) + .map_err(|_| "failed to save worker KV shard".to_string())?; + tmp.rewind() + .map_err(|_| "failed to rewind worker KV shard".to_string())?; + let payload_bytes = tmp + .seek(SeekFrom::End(0)) + .map_err(|_| "failed to measure worker KV shard".to_string())?; + tmp.rewind() + .map_err(|_| "failed to rewind worker KV shard".to_string())?; + Ok((tmp, payload_bytes)) + } + + fn load(&mut self, req: SnapshotLoad<'_>, payload: &mut dyn Read) -> Result<(), String> { + let mut tmp = create_temp(LOAD_PREFIX) + .map_err(|_| "failed to restore worker KV shard".to_string())?; + copy_chunked(payload, &mut tmp, req.payload_bytes) + .map_err(|_| "failed to restore worker KV shard".to_string())?; + tmp.flush() + .map_err(|_| "failed to restore worker KV shard".to_string())?; + self.session + .borrow_mut() + .load_layer_payload(LayerPayloadLoad { + path: tmp.path(), + payload_bytes: req.payload_bytes, + tokens: req.tokens, + layer_start: self.layer_start, + layer_end: self.layer_end, + }) + .map_err(|_| "failed to restore worker KV shard".to_string())?; + Ok(()) + } +} + +pub fn run_assembled_worker(model: &Model, ctx: i32, dist: &Options) -> Result { + let session = Rc::new(RefCell::new(model.session(ctx).map_err(|e| e.to_string())?)); + let (listener, bound) = open_data_listener( + dist.listen_host.as_deref(), + worker_listen_port(dist.listen_port), + ) + .map_err(|e| e.to_string())?; + let shape = &model.bind_plan().shape; + let vocab = u32::try_from(model.vocab().n_vocab().max(0)).unwrap_or(0); + let meta = slice_meta( + u32::try_from(model.model_id()).unwrap_or(0), + shape.n_layer, + vocab, + u32::try_from(ctx).unwrap_or(0), + hidden_values(shape), + &dist.layers, + ); + let plan = worker_plan( + &meta, + u32::try_from(model.routed_quant_bits().max(0)).unwrap_or(0), + u32::from(bound), + shape.name, + ); + let exec = SessionSliceExec { + session: Rc::clone(&session), + meta, + }; + let store = SessionSnapshotStore { + session, + layer_start: meta.layer_start, + layer_end: meta.layer_end, + }; + let mut assembled = assemble_worker(exec, store, plan.hello, plan.model_name); + let host = dist + .coordinator_host + .as_deref() + .ok_or("--role worker requires --coordinator HOST PORT")?; + print_worker_listen_banner(&WorkerListenBanner { + layer_start: assembled.hello.layer_start, + has_output: assembled.hello.has_output != 0, + layer_end: assembled.hello.layer_end, + model_id: model.model_id(), + listen_host: dist.listen_host.as_deref(), + listen_port: assembled.hello.listen_port, + coordinator_host: host, + coordinator_port: dist.coordinator_port, + }); + let port = u16::try_from(dist.coordinator_port) + .map_err(|_| "--role worker requires --coordinator HOST PORT")?; + ds4_dist::reconnect_local( + &mut assembled.worker, + ds4_dist::LocalReconnect { + connect: || ds4_dist::connect_endpoint(host, port), + hello: &assembled.hello, + model_name: &assembled.model_name, + sleep: ds4_dist::sleep_reconnect, + should_stop: || false, + listener: Some(&listener), + }, + ) + .map_err(|e| e.to_string())?; + Ok(0) +} diff --git a/crates/ds4-server/tests/admit.rs b/crates/ds4-server/tests/admit.rs new file mode 100644 index 0000000000000000000000000000000000000000..4cf4e9396bd78cd85e9aee46de2a7f74445c49c4 --- /dev/null +++ b/crates/ds4-server/tests/admit.rs @@ -0,0 +1,168 @@ +//! C↔Rust enqueue / job-id / host-metrics fragment. + +use ds4_server::{ + enqueue, mint_job_id, preparse_shed, render_metrics_fragment, AdmitState, EnqVerdict, ReqKind, + RouteMetrics, SHED_CLIENTS, SHED_QUEUE_BYTES, SHED_QUEUE_DEPTH, +}; + +use std::path::PathBuf; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_ADMIT_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/admit_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/admit_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_str(args: &[&str]) -> String { + let out = Command::new(require_oracle()) + .args(args) + .output() + .expect("run admit_c_oracle"); + assert!( + out.status.success(), + "oracle {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() +} + +fn rust_enqueue( + stopping: i32, + queued: i32, + max_queue: i32, + inflight: u64, + max_bytes: u64, + body: u64, +) -> (EnqVerdict, i32, u64) { + let mut s = AdmitState { + stopping: stopping != 0, + queued, + max_queue, + inflight_body_bytes: inflight, + max_queue_bytes: max_bytes, + ..AdmitState::default() + }; + let v = enqueue(&mut s, body); + (v, s.queued, s.inflight_body_bytes) +} + +fn verdict_name(v: EnqVerdict) -> &'static str { + match v { + EnqVerdict::Ok => "ok", + EnqVerdict::Stopping => "stopping", + EnqVerdict::ShedQueueDepth => "shed_queue_depth", + EnqVerdict::ShedQueueBytes => "shed_queue_bytes", + } +} + +#[test] +fn enqueue_verdicts_match_c() { + let rows = [ + (0, 0, 0, 0u64, 0u64, 10u64), + (1, 0, 8, 0, 0, 10), + (0, 8, 8, 0, 0, 10), + (0, 2, 8, 90, 100, 20), + (0, 2, 8, 90, 100, 5), + (0, 0, 1, 0, 0, 1), + ]; + for (st, q, mq, inf, mb, body) in rows { + let (v, nq, ninf) = rust_enqueue(st, q, mq, inf, mb, body); + let c = c_str(&[ + "enqueue", + &st.to_string(), + &q.to_string(), + &mq.to_string(), + &inf.to_string(), + &mb.to_string(), + &body.to_string(), + ]); + let rust = format!("{} queued={} inflight={}\n", verdict_name(v), nq, ninf); + assert_eq!( + rust, c, + "enqueue st={st} q={q} mq={mq} inf={inf} mb={mb} body={body}" + ); + } +} + +#[test] +fn mint_job_id_matches_c() { + assert_eq!( + mint_job_id(ReqKind::Chat, 1), + c_str(&["mint", "0", "1"]).trim() + ); + assert_eq!( + mint_job_id(ReqKind::Completion, 42), + c_str(&["mint", "1", "42"]).trim() + ); + assert_eq!(mint_job_id(ReqKind::Chat, 1), "chatcmpl-1"); + assert_eq!(mint_job_id(ReqKind::Completion, 7), "cmpl-7"); +} + +#[test] +fn preparse_shed_matches_c() { + let mut s = AdmitState { + max_clients: 2, + clients: 3, + ..AdmitState::default() + }; + let rust = preparse_shed(&s, true, true, 10).unwrap(); + let c = c_str(&["preparse", "2", "3", "0", "0", "0", "0", "10", "1", "1"]); + assert_eq!( + format!("shed {} {} {} {}\n", "clients", rust.1, rust.2, rust.3), + c + ); + assert_eq!(rust.0, SHED_CLIENTS); + + s = AdmitState { + max_queue: 1, + queued: 1, + ..AdmitState::default() + }; + let rust = preparse_shed(&s, true, true, 10).unwrap(); + let c = c_str(&["preparse", "0", "0", "1", "1", "0", "0", "10", "1", "1"]); + assert_eq!( + format!("shed {} {} {} {}\n", "queue_depth", rust.1, rust.2, rust.3), + c + ); + assert_eq!(rust.0, SHED_QUEUE_DEPTH); + + s = AdmitState { + max_queue_bytes: 10, + inflight_body_bytes: 8, + ..AdmitState::default() + }; + let rust = preparse_shed(&s, true, true, 8).unwrap(); + let c = c_str(&["preparse", "0", "0", "0", "0", "10", "8", "8", "1", "1"]); + assert_eq!( + format!("shed {} {} {} {}\n", "queue_bytes", rust.1, rust.2, rust.3), + c + ); + assert_eq!(rust.0, SHED_QUEUE_BYTES); + + s = AdmitState::default(); + assert!(preparse_shed(&s, true, true, 100).is_none()); + assert_eq!( + c_str(&["preparse", "0", "0", "0", "0", "0", "0", "100", "1", "1"]), + "ok\n" + ); +} + +#[test] +fn metrics_zero_fragment_matches_c() { + let rust = render_metrics_fragment(&RouteMetrics::default(), &AdmitState::default()); + let c = c_str(&["metrics-zero"]); + assert_eq!(rust, c); +} diff --git a/crates/ds4-server/tests/beam_protocol.rs b/crates/ds4-server/tests/beam_protocol.rs new file mode 100644 index 0000000000000000000000000000000000000000..5aa58046cacd25f246eb6d9aa1a9179bf6e46f6c --- /dev/null +++ b/crates/ds4-server/tests/beam_protocol.rs @@ -0,0 +1,193 @@ +//! Beam-1.0's Harmony output protocol; all tests are model-free. +use ds4_server::generate::chat_format_for_syntax; +use ds4_server::parse::ChatMsg; +use ds4_server::render::{render_chat, tool_start_marker}; +use ds4_server::tools::{parse_generated_for_model_id, parse_generated_for_response}; +use ds4_server::{ + openai_sse_finish_live, openai_sse_stream_update, openai_stream_start, syntax_for_model_id, + SemAccum, StreamReq, ThinkMode, ToolSchemaOrder, Writer, CREATED_TEST, +}; +use serde_json::{json, Value}; + +const BEAM: i32 = 19; +const PROMPT: &[u8] = b"<|start|>user<|message|>hi<|end|><|start|>assistant"; +const ANSWER: &str = + "<|channel|>analysis<|message|>plan<|end|><|start|>assistant<|channel|>final<|message|>Hello"; +const CALL: &str = concat!( + " to=lookup<|channel|>analysis <|constrain|>xml<|message|>", + "\n 123 \n\n", + "\n2\n" +); + +fn orders() -> Vec { + vec![ToolSchemaOrder { + name: "lookup".into(), + prop: vec!["text".into(), "count".into()], + prop_type: vec!["string".into(), "integer".into()], + ..Default::default() + }] +} + +// Two calls closed as the decode lanes leave them: `<|call|>` is fed at the stop. +fn calls() -> String { + format!( + "<|channel|>analysis<|message|>plan<|end|><|start|>assistant{CALL}<|end|><|start|>assistant \ + to=ping<|channel|>analysis <|constrain|>xml<|message|><|call|>" + ) +} + +#[test] +fn beam_model_mapping() { + let syntax = syntax_for_model_id(BEAM); + assert_eq!(syntax as i32, BEAM); + assert_eq!(tool_start_marker(syntax), "<|start|>assistant to"); + let msgs = [ChatMsg { + role: "user".into(), + content: "Hello".into(), + ..Default::default() + }]; + // Beam input is rendered by its official Jinja template only. + assert!(render_chat(syntax, &msgs, "", ThinkMode::Low).is_err()); +} + +#[test] +fn beam_reasoning_and_answer() { + let parsed = parse_generated_for_model_id(BEAM, ANSWER.as_bytes(), true, &[]); + assert!(parsed.ok && parsed.calls.is_empty()); + assert_eq!(parsed.reasoning, b"plan"); + assert_eq!(parsed.content, b"Hello"); + // Unframed output is surfaced as raw text, not dropped. + let plain = parse_generated_for_model_id(BEAM, b"Hello", true, &[]); + assert!(!plain.ok && plain.content == b"Hello"); +} + +#[test] +fn beam_typed_parallel_calls() { + let raw = calls(); + let parsed = parse_generated_for_model_id(BEAM, raw.as_bytes(), true, &orders()); + assert!(parsed.ok); + assert_eq!(parsed.reasoning, b"plan"); + assert_eq!(parsed.calls.len(), 2); + assert_eq!(parsed.calls[0].name, "lookup"); + let args: Value = serde_json::from_str(&parsed.calls[0].arguments).unwrap(); + // A declared string keeps its text unstripped; other values are JSON. + assert_eq!(args, json!({"text": " 123 ", "count": 2})); + assert_eq!(parsed.calls[1].name, "ping"); + assert_eq!(parsed.calls[1].arguments, "{}"); + let syntax = syntax_for_model_id(BEAM); + let format = chat_format_for_syntax(syntax); + let (no_tools, finish) = parse_generated_for_response( + syntax, + raw.as_bytes(), + false, + true, + true, + format, + &orders(), + "stop", + ); + assert!(no_tools.calls.is_empty() && no_tools.reasoning == b"plan"); + assert_eq!(finish, "stop"); +} + +#[test] +fn beam_budget_cut_call_is_not_published() { + let raw = format!("<|channel|>analysis<|message|>plan<|end|><|start|>assistant{CALL}"); + let syntax = syntax_for_model_id(BEAM); + let (parsed, finish) = parse_generated_for_response( + syntax, + raw.as_bytes(), + true, + true, + true, + chat_format_for_syntax(syntax), + &orders(), + "length", + ); + assert!(parsed.calls.is_empty()); + assert_eq!(finish, "length"); +} + +#[test] +fn beam_bytewise_semantic_markers() { + let format = chat_format_for_syntax(syntax_for_model_id(BEAM)); + let raw = calls(); + for width in 1..=raw.len() { + // Every Harmony reply opens with analysis. + let mut acc = SemAccum::init(true, true, true, format, PROMPT); + assert!(acc.thinking_inside(), "width {width}"); + for chunk in raw.as_bytes().chunks(width) { + acc.feed(chunk, &[]); + } + assert!(!acc.thinking_inside(), "width {width}"); + assert!(acc.saw_tool_start && acc.saw_tool_end, "width {width}"); + } +} + +#[test] +fn beam_call_terminator_is_not_counted() { + // `<|call|>` is a stop token: the parser sees it, usage counts it as + // little as `<|return|>`. + let format = chat_format_for_syntax(syntax_for_model_id(BEAM)); + let mut acc = SemAccum::init(true, true, true, format, PROMPT); + let raw = calls(); + acc.feed(raw.strip_suffix("<|call|>").unwrap().as_bytes(), &[]); + acc.feed_stop(b"<|call|>", &[]); + assert_eq!((acc.completion, acc.reasoning_tokens), (1, 1)); + assert!(acc.saw_tool_end); +} + +fn events(bytes: &[u8]) -> Vec { + std::str::from_utf8(bytes) + .unwrap() + .lines() + .filter_map(|line| line.strip_prefix("data: ")) + .filter(|line| *line != "[DONE]") + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +#[test] +fn beam_openai_stream_every_split() { + let syntax = syntax_for_model_id(BEAM); + let r = StreamReq { + syntax, + chat_format: chat_format_for_syntax(syntax), + think_mode: ThinkMode::Low, + ..Default::default() + }; + let raw = ANSWER.as_bytes(); + for width in 1..=raw.len() { + let mut w = Writer::new(CREATED_TEST); + let mut state = openai_stream_start(&r); + let mut current = Vec::new(); + for chunk in raw.chunks(width) { + current.extend(chunk); + assert!(openai_sse_stream_update( + &mut w, &r, "job", &mut state, ¤t, false + )); + } + assert!(openai_sse_finish_live( + &mut w, + &r, + "job", + &mut state, + ¤t, + "stop", + 2, + 5, + &[] + )); + let (mut text, mut thinking) = (String::new(), String::new()); + for e in events(&w.out) { + let delta = &e["choices"][0]["delta"]; + text.push_str(delta["content"].as_str().unwrap_or("")); + thinking.push_str(delta["reasoning_content"].as_str().unwrap_or("")); + } + assert_eq!( + (thinking.as_str(), text.as_str()), + ("plan", "Hello"), + "width {width}" + ); + } +} diff --git a/crates/ds4-server/tests/chat_input.rs b/crates/ds4-server/tests/chat_input.rs new file mode 100644 index 0000000000000000000000000000000000000000..74f105ccc65471b8bf753220f67ffb33c0dce08e --- /dev/null +++ b/crates/ds4-server/tests/chat_input.rs @@ -0,0 +1,458 @@ +use ds4_core::chat_template::{ChatOptions, RenderClock, Template}; +use ds4_core::ChatThinkMode; +use ds4_server::{ + chat_input, parse_anthropic_request, parse_chat_request, parse_responses_request, ParseEnv, + ParsedRequest, ThinkMode, +}; +use serde_json::{json, Value}; + +const INKLING: i32 = 9; +const QWEN: i32 = 6; +const GLM: i32 = 7; +const K2: i32 = 8; +const SOLAR: i32 = 2; +const STEP: i32 = ds4_core::Variant::Step37Flash as i32; +const IQUEST: i32 = ds4_core::Variant::IQuestQ1 as i32; +const PNG: &str = concat!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP8", + "z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" +); +const WAV: &str = "UklGRiYAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQIAAAAAAA=="; + +fn env() -> ParseEnv { + ParseEnv { + default_model: "inkling-small-mq85gb".into(), + default_effort: ThinkMode::High, + ..ParseEnv::default() + } +} + +fn template(model_id: i32) -> Template { + let source = match model_id { + IQUEST => { + include_str!("../../../tests/fixtures/chat-template/models/iquest/chat_template.jinja") + } + STEP => include_str!("../../../tests/fixtures/step37/chat_template.jinja"), + INKLING => { + include_str!("../../../tests/fixtures/chat-template/models/inkling/chat_template.jinja") + } + QWEN => { + include_str!("../../../tests/fixtures/chat-template/models/qwen/chat_template.jinja") + } + GLM => { + include_str!("../../../tests/fixtures/chat-template/models/glm/chat_template.jinja") + } + K2 => { + include_str!("../../../tests/fixtures/chat-template/models/k2/chat_template.jinja") + } + SOLAR => { + include_str!("../../../tests/fixtures/chat-template/models/solar/chat_template.jinja") + } + _ => panic!("unknown fixture model {model_id}"), + }; + Template::compile(source, RenderClock::Fixed(0)).unwrap() +} + +#[test] +fn iquest_keeps_developer_role() { + let chat = parse_chat_request( + &env(), + r#"{"messages":[{"role":"system","content":"System"},{"role":"developer","content":"Developer"},{"role":"user","content":"Hello"}]}"#, + ) + .unwrap(); + let responses = parse_responses_request( + &env(), + r#"{"input":[{"role":"system","content":"System"},{"role":"developer","content":"Developer"},{"role":"user","content":"Hello"}]}"#, + ) + .unwrap(); + for parsed in [chat, responses] { + let rendered = chat_input::render(&template(IQUEST), IQUEST, &parsed).unwrap(); + assert_eq!( + String::from_utf8(rendered).unwrap(), + concat!( + "<|iquest_system|>System<|iquest_end|>", + "<|iquest_developer|>Developer<|iquest_end|>", + "<|iquest_user|>Hello<|iquest_end|>", + "<|iquest_assistant|>", + ) + ); + } +} + +#[test] +fn iquest_tool_history_numbers() { + for number in [ + "2", + "18446744073709551617", + "340282366920938463463374607431768211456", + ] { + let body = format!( + r#"{{"reasoning_effort":"none","messages":[{{"role":"user","content":"Look up"}},{{"role":"assistant","content":"","tool_calls":[{{"id":"call_1","type":"function","function":{{"name":"lookup","arguments":"{{\"count\":{number}}}"}}}}]}},{{"role":"tool","tool_call_id":"call_1","content":"Found"}}]}}"# + ); + let chat = parse_chat_request(&env(), &body).unwrap(); + let body = format!( + r#"{{"reasoning":{{"effort":"none"}},"input":[{{"role":"user","content":"Look up"}},{{"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{{\"count\":{number}}}"}},{{"type":"function_call_output","call_id":"call_1","output":"Found"}}]}}"# + ); + let responses = parse_responses_request(&env(), &body).unwrap(); + let body = format!( + r#"{{"thinking":{{"type":"disabled"}},"messages":[{{"role":"user","content":"Look up"}},{{"role":"assistant","content":[{{"type":"tool_use","id":"call_1","name":"lookup","input":{{"count":{number}}}}}]}},{{"role":"user","content":[{{"type":"tool_result","tool_use_id":"call_1","content":"Found"}}]}}]}}"# + ); + let anthropic = parse_anthropic_request(&env(), &body).unwrap(); + for parsed in [chat, responses, anthropic] { + let rendered = chat_input::render(&template(IQUEST), IQUEST, &parsed).unwrap(); + let rendered = String::from_utf8(rendered).unwrap(); + let expected = format!( + "lookupcount{number}" + ); + assert!(rendered.contains(&expected), "{rendered}"); + assert!(rendered.ends_with("<|iquest_assistant|>")); + } + } +} + +#[test] +fn legacy_developer_uses_system() { + let parsed = parse_chat_request( + &env(), + r#"{"messages":[{"role":"developer","content":"Use Korean"},{"role":"user","content":"Hello"}]}"#, + ) + .unwrap(); + let messages = json!([ + {"role":"system","content":"Use Korean"}, + {"role":"user","content":"Hello"}, + ]); + for model_id in [QWEN, SOLAR, K2, GLM, INKLING, STEP] { + assert_render(model_id, &parsed, messages.clone(), json!([])); + } +} + +fn assert_render(model_id: i32, parsed: &ParsedRequest, messages: Value, tools: Value) { + let template = template(model_id); + let expected = template + .render_chat( + messages.as_array().unwrap(), + tools.as_array().unwrap(), + ChatOptions::new(model_id, ChatThinkMode::High), + ) + .unwrap(); + let actual = chat_input::render(&template, model_id, parsed).unwrap(); + assert_eq!(String::from_utf8(actual).unwrap(), expected); +} + +#[test] +fn k2_plain_assistant() { + let body = r#"{ + "model":"k2-horizon-375b-a23b", + "reasoning_effort":"high", + "messages":[ + {"role":"user","content":"첫 질문"}, + {"role":"assistant","content":"이전 답"}, + {"role":"user","content":"계속해 줘."} + ] + }"#; + let parsed = parse_chat_request(&env(), body).unwrap(); + let canonical = json!([ + {"role":"user","content":"첫 질문"}, + {"role":"assistant","content":"이전 답","reasoning_content":""}, + {"role":"user","content":"계속해 줘."} + ]); + assert_render(K2, &parsed, canonical, json!([])); +} + +#[test] +fn openai_arguments_reasoning() { + let body = r#"{ + "reasoning_effort":"high", + "messages":[ + {"role":"user","content":"서울을 찾아 줘."}, + {"role":"assistant","content":"","reasoning_content":"위치를 확인한다.", + "tool_calls":[{"id":"call_01","type":"function","function":{ + "name":"lookup","arguments":"{\"z\":2,\"a\":\"서울\",\"nested\":{\"labels\":[\"café\",\"漢字\"],\"ok\":true}}" + }}]}, + {"role":"tool","tool_call_id":"call_01","content":"서울: 맑음"} + ] + }"#; + let parsed = parse_chat_request(&env(), body).unwrap(); + let canonical = json!([ + {"role":"user","content":"서울을 찾아 줘."}, + {"role":"assistant","content":"","reasoning_content":"위치를 확인한다.", + "tool_calls":[{"id":"call_01","type":"function","function":{ + "name":"lookup","arguments":{"z":2,"a":"서울","nested":{"labels":["café","漢字"],"ok":true}} + }}]}, + {"role":"tool","tool_call_id":"call_01","content":"서울: 맑음"} + ]); + assert_render(INKLING, &parsed, canonical, json!([])); +} + +#[test] +fn schema_array_key_order() { + let body = r#"{ + "messages":[{"role":"user","content":"도구 목록"}], + "tools":[ + {"type":"function","function":{"name":"zeta","description":"첫째", + "parameters":{"type":"object","properties":{ + "z_first":{"type":"integer"},"a_second":{"type":"string"} + },"required":["a_second"]}}}, + {"type":"function","function":{"name":"alpha","description":"둘째", + "parameters":{"type":"object","properties":{}}}} + ] + }"#; + let parsed = parse_chat_request(&env(), body).unwrap(); + let canonical_tools = json!([ + {"type":"function","function":{"name":"zeta","description":"첫째", + "parameters":{"type":"object","properties":{ + "z_first":{"type":"integer"},"a_second":{"type":"string"} + },"required":["a_second"]}}}, + {"type":"function","function":{"name":"alpha","description":"둘째", + "parameters":{"type":"object","properties":{}}}} + ]); + // These official templates expose schema order; Inkling deliberately sorts it. + for model_id in [QWEN, SOLAR, STEP] { + assert_render( + model_id, + &parsed, + json!([{"role":"user","content":"도구 목록"}]), + canonical_tools.clone(), + ); + } +} + +#[test] +fn anthropic_result_pair() { + let body = r#"{ + "messages":[ + {"role":"user","content":"두 결과를 확인해."}, + {"role":"assistant","content":[ + {"type":"thinking","thinking":"두 도구를 확인한다."}, + {"type":"tool_use","id":"call_a","name":"lookup","input":{"city":"서울","nested":{"n":[1,2]}}}, + {"type":"tool_use","id":"call_b","name":"measure","input":{"unit":"°C"}} + ]}, + {"role":"user","content":[ + {"type":"tool_result","tool_use_id":"call_b","content":"B </tool_result> 서울"}, + {"type":"tool_result","tool_use_id":"call_a","content":"A café"} + ]} + ] + }"#; + let parsed = parse_anthropic_request(&env(), body).unwrap(); + // Escaped and literal closing tags cannot be recovered from the old flat text. + let canonical = json!([ + {"role":"user","content":"두 결과를 확인해."}, + {"role":"assistant","content":"","reasoning_content":"두 도구를 확인한다.", + "tool_calls":[ + {"id":"call_a","type":"function","function":{"name":"lookup","arguments":{"city":"서울","nested":{"n":[1,2]}}}}, + {"id":"call_b","type":"function","function":{"name":"measure","arguments":{"unit":"°C"}}} + ]}, + {"role":"tool","tool_call_id":"call_b","content":"B </tool_result> 서울"}, + {"role":"tool","tool_call_id":"call_a","content":"A café"} + ]); + for model_id in [INKLING, QWEN, SOLAR, STEP] { + assert_render(model_id, &parsed, canonical.clone(), json!([])); + } +} + +#[test] +fn anthropic_result_and_text() { + let body = r#"{ + "messages":[ + {"role":"user","content":"찾아 줘."}, + {"role":"assistant","content":[ + {"type":"tool_use","id":"call_a","name":"lookup","input":{"city":"서울"}} + ]}, + {"role":"user","content":[ + {"type":"tool_result","tool_use_id":"call_a","content":[ + {"type":"text","text":"Result: "},{"type":"text","text":"서울"} + ]}, + {"type":"text","text":"이제 한 문장으로 설명해."} + ]} + ] + }"#; + let parsed = parse_anthropic_request(&env(), body).unwrap(); + let canonical = json!([ + {"role":"user","content":"찾아 줘."}, + {"role":"assistant","content":"","tool_calls":[ + {"id":"call_a","type":"function","function":{"name":"lookup","arguments":{"city":"서울"}}} + ]}, + {"role":"tool","tool_call_id":"call_a","content":"Result: 서울"}, + {"role":"user","content":"이제 한 문장으로 설명해."} + ]); + assert_render(INKLING, &parsed, canonical, json!([])); +} + +#[test] +fn anthropic_system_first() { + let body = r#"{ + "system":"Be concise. 한국어로 답해.", + "messages":[{"role":"user","content":"안녕"}] + }"#; + let parsed = parse_anthropic_request(&env(), body).unwrap(); + let canonical = json!([ + {"role":"system","content":"Be concise. 한국어로 답해."}, + {"role":"user","content":"안녕"} + ]); + assert_render(QWEN, &parsed, canonical, json!([])); +} + +#[test] +fn anthropic_tool_schema() { + let body = r#"{ + "messages":[{"role":"user","content":"도구를 사용해."}], + "tools":[{"name":"lookup","description":"장소 검색","input_schema":{ + "type":"object","properties":{"z":{"type":"integer"},"a":{"type":"string"}},"required":["a"] + }}] + }"#; + let parsed = parse_anthropic_request(&env(), body).unwrap(); + let canonical_tools = json!([ + {"type":"function","function":{"name":"lookup","description":"장소 검색","parameters":{ + "type":"object","properties":{"z":{"type":"integer"},"a":{"type":"string"}},"required":["a"] + }}} + ]); + assert_render( + SOLAR, + &parsed, + json!([{"role":"user","content":"도구를 사용해."}]), + canonical_tools, + ); +} + +#[test] +fn responses_reasoning_tools() { + let body = r#"{ + "instructions":"한국어로 답해.", + "reasoning":{"effort":"high"}, + "input":[ + {"type":"message","role":"user","content":[{"type":"input_text","text":"서울을 찾아 줘."}]}, + {"type":"reasoning","summary":[{"type":"summary_text","text":"위치를 확인한다."}]}, + {"type":"function_call","call_id":"call_01","name":"lookup", + "arguments":"{\"city\":\"서울\",\"nested\":{\"labels\":[\"café\",\"漢字\"],\"ok\":true}}"}, + {"type":"function_call_output","call_id":"call_01","output":"서울: 맑음"} + ], + "tools":[{"type":"function","name":"lookup","description":"장소 검색", + "parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]} + }] + }"#; + let parsed = parse_responses_request(&env(), body).unwrap(); + let canonical = json!([ + {"role":"system","content":"한국어로 답해."}, + {"role":"user","content":"서울을 찾아 줘."}, + {"role":"assistant","content":"","reasoning_content":"위치를 확인한다.","tool_calls":[ + {"id":"call_01","type":"function","function":{"name":"lookup", + "arguments":{"city":"서울","nested":{"labels":["café","漢字"],"ok":true}}}} + ]}, + {"role":"tool","tool_call_id":"call_01","content":"서울: 맑음"} + ]); + let canonical_tools = json!([ + {"type":"function","function":{"name":"lookup","description":"장소 검색", + "parameters":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}} + ]); + for model_id in [INKLING, STEP] { + assert_render( + model_id, + &parsed, + canonical.clone(), + canonical_tools.clone(), + ); + } +} + +#[test] +fn inkling_media_order() { + let body = json!({"messages":[{"role":"user","content":[ + {"type":"text","text":"앞"}, + {"type":"image_url","image_url":format!("data:image/png;base64,{PNG}")}, + {"type":"text","text":"중간"}, + {"type":"input_audio","input_audio":{"format":"wav","data":WAV}}, + {"type":"image_url","image_url":format!("data:image/png;base64,{PNG}")}, + {"type":"text","text":"끝"} + ]}]}); + let parsed = parse_chat_request(&env(), &body.to_string()).unwrap(); + let canonical = json!([{"role":"user","content":[ + {"type":"text","text":"앞"}, + {"type":"image"}, + {"type":"text","text":"중간"}, + {"type":"audio"}, + {"type":"image"}, + {"type":"text","text":"끝"} + ]}]); + assert_render(INKLING, &parsed, canonical, json!([])); +} + +#[test] +fn qwen_images_all_apis() { + let chat = json!({"messages":[{"role":"user","content":[ + {"type":"text","text":"앞"}, + {"type":"image_url","image_url":format!("data:image/png;base64,{PNG}")}, + {"type":"text","text":"뒤"} + ]}]}); + let messages = json!({"messages":[{"role":"user","content":[ + {"type":"text","text":"앞"}, + {"type":"image","source":{"type":"base64","media_type":"image/png","data":PNG}}, + {"type":"text","text":"뒤"} + ]}]}); + let responses = json!({"input":[{"type":"message","role":"user","content":[ + {"type":"input_text","text":"앞"}, + {"type":"input_image","image_url":format!("data:image/png;base64,{PNG}")}, + {"type":"input_text","text":"뒤"} + ]}]}); + let parsed = [ + parse_chat_request(&env(), &chat.to_string()).unwrap(), + parse_anthropic_request(&env(), &messages.to_string()).unwrap(), + parse_responses_request(&env(), &responses.to_string()).unwrap(), + ]; + let canonical = json!([{"role":"user","content":[ + {"type":"text","text":"앞"}, + {"type":"image"}, + {"type":"text","text":"뒤"} + ]}]); + for request in &parsed { + for model_id in [QWEN, STEP] { + assert_render(model_id, request, canonical.clone(), json!([])); + } + } +} + +#[test] +fn glm_image_placeholder() { + let body = json!({ + "model":"glm-5.3-flash", + "reasoning_effort":"high", + "messages":[{"role":"user","content":[ + {"type":"text","text":"앞"}, + {"type":"image_url","image_url":format!("data:image/png;base64,{PNG}")}, + {"type":"text","text":"뒤"} + ]}] + }); + let parsed = parse_chat_request(&env(), &body.to_string()).unwrap(); + let expected_png = b"\x89PNG\r\n\x1a\n\ + \x00\x00\x00\x0dIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90\x77\x53\xde\ + \x00\x00\x00\x0cIDAT\x78\xda\x63\xfc\xcf\xc0\x00\x00\x03\x01\x01\x00\xc9\xfe\x92\xef\ + \x00\x00\x00\x00IEND\xae\x42\x60\x82"; + assert_eq!(parsed.images.len(), 1); + assert_eq!(parsed.images[0].data.as_ref(), expected_png); + + // Native image expansion owns this marker; official Jinja still owns framing. + let canonical = json!([{ + "role":"user", + "content":"앞<|begin_of_image|><|image|><|end_of_image|>뒤" + }]); + assert_render(GLM, &parsed, canonical, json!([])); + assert_eq!(parsed.images[0].data.as_ref(), expected_png); +} + +#[test] +fn invalid_json_arguments() { + for arguments in ["{", "{} trailing", "[broken]"] { + let body = json!({"messages":[ + {"role":"user","content":"도구를 호출해."}, + {"role":"assistant","content":"","tool_calls":[{ + "id":"call_01","type":"function","function":{"name":"lookup","arguments":arguments} + }]} + ]}); + let parsed = parse_chat_request(&env(), &body.to_string()).unwrap(); + for model_id in [INKLING, QWEN, SOLAR, STEP] { + assert!( + chat_input::render(&template(model_id), model_id, &parsed).is_err(), + "model {model_id} accepted invalid arguments: {arguments}" + ); + } + } +} diff --git a/crates/ds4-server/tests/cont.rs b/crates/ds4-server/tests/cont.rs new file mode 100644 index 0000000000000000000000000000000000000000..0fa1329490eae379d75dc1f1df2a745bcfd60d65 --- /dev/null +++ b/crates/ds4-server/tests/cont.rs @@ -0,0 +1,200 @@ +//! C↔Rust Inc 5a/5b/5c continuation registry (tape oracle, no model). + +use ds4_server::{ + dump_script, handle_client_inner, unix_now, Api, ScriptedDecode, ServerConfig, ServerInner, +}; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::PathBuf; +use std::process::Command; +use std::sync::Mutex; +use std::thread; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_CONT_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/cont_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/cont_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_str(script: &str) -> String { + let out = Command::new(require_oracle()) + .arg(script) + .output() + .expect("run cont_c_oracle"); + assert!( + out.status.success(), + "oracle {script} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("oracle utf8") +} + +fn assert_script(name: &str) { + let rust = dump_script(name); + let c = c_str(name); + if rust != c { + panic!("{name} mismatch\n--- rust ---\n{rust}\n--- c ---\n{c}"); + } +} + +#[test] +fn publish_resolve_demote_matches_c() { + assert_script("publish-resolve-demote"); +} + +#[test] +fn supersede_cap_matches_c() { + assert_script("supersede-cap"); +} + +#[test] +fn grace_hold_matches_c() { + assert_script("grace-hold"); +} + +#[test] +fn ttl_matches_c() { + assert_script("ttl"); +} + +#[test] +fn bank_claim_matches_c() { + assert_script("bank-claim"); +} + +#[test] +fn bank_protection_retry_matches_c() { + assert_script("bank-protection"); +} + +fn http_post(addr: std::net::SocketAddr, path: &str, body: &str) -> String { + let mut c = TcpStream::connect(addr).unwrap(); + let req = format!( + "POST {path} HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + c.write_all(req.as_bytes()).unwrap(); + let mut out = Vec::new(); + c.read_to_end(&mut out).unwrap(); + String::from_utf8_lossy(&out).into_owned() +} + +#[test] +fn serial_hold_sheds_unrelated_with_retry_after() { + let mut cfg = ServerConfig::test_cfg(); + cfg.default_tokens = 16; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let now = unix_now() as f64; + let h = thread::spawn(move || { + let (mut s, _) = listener.accept().unwrap(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + { + let mut g = inner.lock().unwrap(); + g.creg + .publish_serial(Api::Anthropic, &["toolu_hold".into()], 4, 70, now); + } + let mut engine = ScriptedDecode::from_pieces(&[b"ok"]); + handle_client_inner(&cfg, &inner, &mut s, Some(&mut engine), None); + }); + let body = r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#; + let s = http_post(addr, "/v1/chat/completions", body); + h.join().unwrap(); + assert!(s.starts_with("HTTP/1.1 503 "), "{s}"); + assert!(s.to_ascii_lowercase().contains("retry-after:"), "{s}"); + assert!(s.contains("live tool continuation"), "{s}"); +} + +#[test] +fn live_only_tool_result_conflicts_when_reference_moved() { + let mut cfg = ServerConfig::test_cfg(); + cfg.default_tokens = 16; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let now = unix_now() as f64; + let h = thread::spawn(move || { + let (mut s, _) = listener.accept().unwrap(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + { + let mut g = inner.lock().unwrap(); + g.creg + .publish_serial(Api::Anthropic, &["toolu_x".into()], 7, 100, now); + } + let mut engine = ScriptedDecode::from_pieces(&[b"ok"]); + handle_client_inner(&cfg, &inner, &mut s, Some(&mut engine), None); + }); + let body = r#"{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_x","content":"ok"}]}],"max_tokens":8}"#; + let s = http_post(addr, "/v1/messages", body); + h.join().unwrap(); + assert!(s.starts_with("HTTP/1.1 409 "), "{s}"); + assert!( + s.contains("Anthropic continuation state is not available"), + "{s}" + ); +} + +#[test] +fn anthropic_tool_turn_publishes_and_holds_next_seat() { + let mut cfg = ServerConfig::test_cfg(); + cfg.default_tokens = 16; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let h = thread::spawn(move || { + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let mut engine = ScriptedDecode::from_pieces(&[concat!( + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"bash\">\n", + "<|DSML|parameter name=\"command\" string=\"true\">ls", + "\n", + "\n", + "" + ) + .as_bytes()]); + for _ in 0..3 { + let (mut s, _) = listener.accept().unwrap(); + handle_client_inner(&cfg, &inner, &mut s, Some(&mut engine), None); + } + let g = inner.lock().unwrap(); + assert_eq!(g.creg.n_live(), 0); + }); + let tools = r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":16,"thinking":{"type":"disabled"},"tools":[{"name":"bash","input_schema":{"type":"object"}}]}"#; + let first = http_post(addr, "/v1/messages", tools); + assert!(first.starts_with("HTTP/1.1 200 "), "{first}"); + assert!(first.contains("\"tool_use\""), "{first}"); + let tool_id = first + .split("\"type\":\"tool_use\",\"id\":\"") + .nth(1) + .unwrap() + .split('"') + .next() + .unwrap(); + assert_eq!(tool_id.len(), 38); + assert!(tool_id.starts_with("toolu_")); + let second = http_post( + addr, + "/v1/chat/completions", + r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#, + ); + assert!(second.starts_with("HTTP/1.1 503 "), "{second}"); + assert!(second.contains("live tool continuation"), "{second}"); + let resumed = http_post( + addr, + "/v1/messages", + &format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"tool_result","tool_use_id":"{tool_id}","content":"ok"}}]}}],"max_tokens":8}}"# + ), + ); + h.join().unwrap(); + assert!(resumed.starts_with("HTTP/1.1 200 "), "{resumed}"); +} diff --git a/crates/ds4-server/tests/dsml.rs b/crates/ds4-server/tests/dsml.rs new file mode 100644 index 0000000000000000000000000000000000000000..075fd77bca691b0be1bce173a89ce0dd9ce7c8bf --- /dev/null +++ b/crates/ds4-server/tests/dsml.rs @@ -0,0 +1,419 @@ +//! C↔Rust DSML decode-state + sampling-override tapes (no model). + +use ds4_server::dsml_dump_script; +use ds4_server::parse::ToolSchemaOrder; +use ds4_server::tools::find_tool_start; +use ds4_server::tools::parse_dsml_tolerant; +use ds4_server::tools::{ + DSML_INVOKE_END_SPACED, DSML_INVOKE_START_SPACED, DSML_PARAM_END_SPACED, + DSML_PARAM_START_SPACED, DSML_TOOL_CALLS_END_SPACED, DSML_TOOL_CALLS_START_SPACED, +}; +use ds4_server::{ + dsml_decode_state_for_text, ChatFormat, DsmlDecodeState, DsmlDecodeTracker, ModelSyntax, + SemAccum, ToolSyntax, +}; +use std::path::PathBuf; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_DSML_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/dsml_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/dsml_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_str(name: &str) -> String { + let out = Command::new(require_oracle()) + .arg(name) + .output() + .expect("run dsml_c_oracle"); + assert!( + out.status.success(), + "oracle {name} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("oracle utf8") +} + +fn assert_script(name: &str) { + let rust = dsml_dump_script(name); + let c = c_str(name); + if rust != c { + panic!("{name} mismatch\n--- rust ---\n{rust}\n--- c ---\n{c}"); + } +} + +#[test] +fn dsml_states_match_c() { + for name in [ + // The GLM vectors are the reference's own (ds4_server.c:20571, :20608). + "glm-boundaries-token", + "glm-boundaries-byte", + "glm-control-text", + "state-prefix", + "state-path-param", + "state-path-closing", + "state-json-struct", + "state-json-string", + "state-done", + ] { + assert_script(name); + } +} + +#[test] +fn sampling_override_matches_c() { + for name in [ + "override-required", + "override-think-cap", + "override-tool-result", + "cap-low-128", + "cap-high-128", + "cap-max-600", + ] { + assert_script(name); + } + let req = dsml_dump_script("override-required"); + assert_eq!(req.trim(), "103 204"); + let think = dsml_dump_script("override-think-cap"); + assert_eq!(think.trim(), "0 305"); + let tr = dsml_dump_script("override-tool-result"); + assert_eq!(tr.trim(), "0 305"); +} + +/// The tool-syntax greedy question (the engine's v41_syntax_greedy + +/// test_syntax_greedy_spaced_dsml, server_generate_v41.c): the tag, the +/// parameter head and the JSON punctuation take the raw argmax; the string +/// value body and the thinking section keep the request's draw. +fn greedy_after(text: &str, thinking_inside: bool) -> bool { + let mut acc = SemAccum::init(true, true, thinking_inside, ChatFormat::DeepSeek, b""); + acc.feed(text.as_bytes(), &[]); + acc.syntax_greedy(&[]) +} + +#[test] +fn syntax_greedy_spaced_dsml() { + let head = format!( + "Listing.\n\n{}{} name=\"bash\">\n", + DSML_TOOL_CALLS_START_SPACED, DSML_INVOKE_START_SPACED + ); + assert!(!greedy_after("Listing.", false)); + assert!(greedy_after(&head, false)); + assert!(!greedy_after(&head, true)); + // The parameter value keeps the request's draw. + assert!(!greedy_after( + &format!("{head}{DSML_PARAM_START_SPACED} name=\"command\" string=\"true\">ls -la"), + false + )); + // The JSON value's structural position (the number / bracket head) is greedy. + assert!(greedy_after( + &format!("{head}{DSML_PARAM_START_SPACED} name=\"n\" string=\"false\">"), + false + )); + // The value closed: back at a tag position, greedy again. + assert!(greedy_after( + &format!( + "{head}{DSML_PARAM_START_SPACED} name=\"command\" string=\"true\">ls{DSML_PARAM_END_SPACED}\n" + ), + false + )); +} + +/// The stream marker scan must see the official spaced close: find_tool_end's +/// DeepSeek candidate list carried the LONG / SHORT / PLAIN wrapper ends but +/// not the SPACED one (found 2026-10-11 on the chat gate against the 221b9e2 +/// oracle: the live stream never closed the block, so the loop drew one extra +/// token -- EOS -- after the wrapper; the engine stops at the tag, its +/// find_any_tool_end walks all four syntaxes, server_dsml_parse.c:88-93). +#[test] +fn stream_closes_on_spaced_wrapper() { + let mut acc = SemAccum::init(true, true, false, ChatFormat::DeepSeek, b""); + let block = format!( + "Checking.\n\n{tcs} name=\"get_weather\">\n{ps} name=\"city\" string=\"true\">Paris{pe}\n{ie}\n{tce}", + tcs = DSML_TOOL_CALLS_START_SPACED, + ps = DSML_PARAM_START_SPACED, + pe = DSML_PARAM_END_SPACED, + ie = DSML_INVOKE_END_SPACED, + tce = DSML_TOOL_CALLS_END_SPACED, + ); + acc.feed(block.as_bytes(), &[]); + assert!( + acc.saw_tool_start, + "the spaced wrapper open must enter tool mode" + ); + assert!( + acc.saw_tool_end, + "the spaced wrapper close must close the block (the engine's find_any_tool_end walks all four syntaxes)" + ); +} + +/// The artifact mixes spellings inside one block (f302be7, 2026-10-11): the +/// spaced block carrying bare parameter tags. The tracker must take the bare +/// open and close as parameter boundaries and reach Outside at the block end; +/// before the aliases it stopped at Structural, holding the block open. +#[test] +fn tracker_accepts_bare_parameter_tags() { + let block = format!( + "{tcs}\n{is} name=\"a\">\nv\n{ie}\n{tce}", + tcs = DSML_TOOL_CALLS_START_SPACED, + is = DSML_INVOKE_START_SPACED, + ie = DSML_INVOKE_END_SPACED, + tce = DSML_TOOL_CALLS_END_SPACED, + ); + assert_eq!( + dsml_decode_state_for_text(block.as_bytes()), + DsmlDecodeState::Outside + ); +} + +// --------------------------------------------------------------------------- +// The live served turn (Spark ds41 deployment, open-grok session 01a14f5d, +// 2026-10-18). Fixtures are the assistant content exactly as the client +// recorded it. +// --------------------------------------------------------------------------- + +fn live_orders() -> Vec { + let mut out = Vec::new(); + for (name, props, types) in [ + ("memory_search", vec!["query"], vec!["string"]), + ( + "read_file", + vec!["target_file", "limit"], + vec!["string", "integer"], + ), + ( + "list_dir", + vec!["target_directory", "description"], + vec!["string", "string"], + ), + ] { + out.push(ToolSchemaOrder { + name: name.to_string(), + wire_name: name.to_string(), + namespace: name.to_string(), + responses_tool_search: false, + prop: props.into_iter().map(str::to_string).collect(), + prop_type: types.iter().copied().map(str::to_string).collect(), + // These fixtures declare a scalar type, so the head of the `type` + // list is the type itself (parse.rs schema_property_type_first). + prop_type_first: types.iter().copied().map(str::to_string).collect(), + }); + } + out +} + +fn live_text(file: &str) -> Vec { + let path = format!( + "{}/tests/fixtures/ds41-dsml/{file}", + env!("CARGO_MANIFEST_DIR") + ); + std::fs::read(&path).unwrap_or_else(|e| panic!("read {path}: {e}")) +} + +fn live_turn(file: &str) -> Vec { + let text = live_text(file); + let parsed = parse_dsml_tolerant(&text, &live_orders(), false) + .unwrap_or_else(|| panic!("{file}: the finished turn recovered no call at all")); + parsed.calls +} + +/// The artifact roots each call on the TOOL NAME and never writes the +/// wrapper or invoke opens; its closes are the decorated DS41 forms, and the +/// root tag carries the residue of the `name="` attribute the reference's own +/// prompt teaches (``). Measured before this case: the tolerant +/// parse recovered nothing from either turn, so the route served the block as +/// assistant text and no tool ran (the client's turn recorded zero tool +/// calls). +#[test] +fn live_tool_root_block_recovers_its_calls() { + let t1 = live_turn("live-01a14f5d-turn1.txt"); + let names: Vec<&str> = t1.iter().map(|c| c.name.as_str()).collect(); + assert_eq!( + names, + ["memory_search", "memory_search", "read_file", "read_file"] + ); + assert!(t1[0] + .arguments + .contains("SolidCraft GUI worker app visible display headless session")); + + let t2 = live_turn("live-01a14f5d-turn2.txt"); + let names: Vec<&str> = t2.iter().map(|c| c.name.as_str()).collect(); + assert_eq!( + names, + ["read_file", "read_file", "read_file", "list_dir"], + "the malformed list_dir (its child close is the wrong tag) must be dropped, not invented" + ); + assert!( + t2[2].arguments.contains("\"limit\": 200"), + "a declared integer child keeps its type: {}", + t2[2].arguments + ); + assert!(t2[3].arguments.contains("/data/3D")); +} + +/// The tracker half of the same defect. The parser now serves the calls, but +/// the live stream still drew the block as assistant content, because the +/// start scan listed the wrapper and invoke opens only and the artifact's root +/// is the bare tool name. A block the scan does not recognise is a block the +/// tracker cannot hold back. +#[test] +fn tracker_enters_tool_mode_on_the_quoted_tool_root() { + let text = live_text("live-01a14f5d-turn1.txt"); + + let at = find_tool_start(&text, ChatFormat::DeepSeek, &live_orders()) + .expect("the artifact's root tag must start a tool block"); + assert_eq!( + &text[at..at + b"".len()], + b"", + "the scan must stop at the root tag, not at some later marker" + ); + + let mut acc = SemAccum::init(true, true, false, ChatFormat::DeepSeek, b""); + acc.set_tool_orders(&live_orders()); + acc.feed(&text, &[]); + assert!( + acc.saw_tool_start, + "the quoted tool root must take the live tracker into tool mode" + ); + assert!( + acc.saw_tool_end, + "the decorated wrapper close must close the block" + ); +} + +/// The reference's GLM tracker vectors, ds4_server.c:20571 +/// (`test_glm_decode_tracker_boundaries` in /data/ds4): the block's fragments, +/// token-sized and byte-by-byte, must land on the states the reference asserts. +/// GLM's block is not in the DSML syntax table, so before the dialect the +/// tracker never left Outside and every structural position kept the request's +/// draw instead of the raw argmax. +#[test] +fn glm_tracker_boundaries_match_the_reference() { + let parts = [ + "prose bashcommand", + "printf ''", + "<", + "/arg_", + "value>", + "descriptiontest", + "", + ]; + let expected = [ + DsmlDecodeState::Structural, + DsmlDecodeState::StringBody, + // A lone '<' stays payload; two bytes of the close marker are structure. + DsmlDecodeState::StringBody, + DsmlDecodeState::Structural, + DsmlDecodeState::Structural, + DsmlDecodeState::StringBody, + DsmlDecodeState::Outside, + ]; + for bytewise in [false, true] { + let mut tracker = DsmlDecodeTracker::default(); + tracker.set_tool_syntax(ToolSyntax::Glm); + let mut raw = Vec::new(); + for (i, part) in parts.iter().enumerate() { + if bytewise { + for byte in part.bytes() { + raw.push(byte); + tracker.update(&raw); + } + } else { + raw.extend_from_slice(part.as_bytes()); + tracker.update(&raw); + } + assert_eq!( + tracker.decode, + expected[i], + "fragment {i} bytewise={bytewise} raw={}", + String::from_utf8_lossy(&raw) + ); + } + } +} + +/// Before the dialect the tracker recognized no GLM block at all: `` +/// is not one of the DSML spellings (dsml.rs `SYNTAXES`), so the whole block read +/// Outside and every structural position kept the request's draw. This test +/// pins the defect the GLM dialect fixes. +#[test] +fn glm_text_under_the_dsml_dialect_is_untracked() { + let raw = b"prose bashcommandls -la"; + let mut tracker = DsmlDecodeTracker::default(); + tracker.set_tool_syntax(ToolSyntax::Dsml); + tracker.update(raw); + assert_eq!(tracker.decode, DsmlDecodeState::Outside); +} + +/// ds4_server.c (test_tool_control_text_inside_arguments): protocol-looking +/// bytes inside an `` body are payload. The block stays open, so the +/// stream does not cut the argument at a `` the model wrote as text. +#[test] +fn glm_control_text_inside_an_argument_is_payload() { + // The reference body (ds4_server.c:20609): control-looking text written + // INTO the argument value. + let body = "literal "; + let raw = + format!("reasonwritecontent{body}"); + let mut acc = SemAccum::init(true, true, false, ChatFormat::DeepSeek, b""); + acc.set_model_syntax(ModelSyntax::Glm53); + acc.feed(raw.as_bytes(), &[]); + assert_eq!(acc.dsml_state(), DsmlDecodeState::StringBody); + assert!(acc.saw_tool_start, "the block opened"); + assert!( + !acc.saw_tool_end, + "an argument body must not close the block (the reference reads the tracker, ds4_server.c:7755)" + ); +} + +/// GLM's block opens only on its own ``. A DSML spelling such as +/// `` in GLM prose is text: if it opened the block, the GLM tracker +/// would never close it, and the turn's EOS would read as an unterminated call +/// (non-streaming retries it, streaming reports an error). +#[test] +fn glm_dsml_spelling_in_prose_does_not_open_a_block() { + let mut acc = SemAccum::init(true, true, false, ChatFormat::DeepSeek, b""); + acc.set_model_syntax(ModelSyntax::Glm53); + acc.feed( + b"DeepSeek wraps calls in blocks; GLM does not.", + &[], + ); + assert!( + !acc.saw_tool_start, + "a DSML spelling is not GLM's opening marker" + ); + assert!(!acc.saw_tool_end); + + acc.feed( + b" bashcommandls", + &[], + ); + assert!(acc.saw_tool_start, "GLM's own marker opens the block"); + assert!(acc.saw_tool_end, "and its tracker closes it"); +} + +/// The sampling consequence of the dialect: the tag head and the argument tags +/// take the raw argmax, the value body keeps the request's draw +/// (ds4_server.c:7575 `dsml_decode_state_is_syntax`). +#[test] +fn glm_structural_positions_take_the_argmax() { + let mut acc = SemAccum::init(true, true, false, ChatFormat::DeepSeek, b""); + acc.set_model_syntax(ModelSyntax::Glm53); + acc.feed(b"prose bashcommand", &[]); + assert!(acc.dsml_state().is_tool(), "the GLM block opens tool mode"); + assert!(acc.syntax_greedy(&[]), "the tag head is structural"); + acc.feed(b"ls -la", &[]); + assert!(!acc.syntax_greedy(&[]), "an argument body keeps the draw"); + acc.feed(b"", &[]); + assert!(!acc.dsml_state().is_tool(), "the block closed"); +} diff --git a/crates/ds4-server/tests/expected_plan.rs b/crates/ds4-server/tests/expected_plan.rs new file mode 100644 index 0000000000000000000000000000000000000000..3fe41379f199a2586aa928356c88ebb81721a9d3 --- /dev/null +++ b/crates/ds4-server/tests/expected_plan.rs @@ -0,0 +1,122 @@ +//! Exercise the production CLI without opening a device or loading weights. +use serde_json::{json, Value}; +use std::{fs, process::Command}; + +#[test] +#[ignore = "needs DS4_EXPECT_PLAN_SERVER, DS4_EXPECT_PLAN_MODEL and memory for the preflight quote"] +fn preflight_checks_policy_without_post_open_quote() { + let server = std::env::var("DS4_EXPECT_PLAN_SERVER") + .expect("set DS4_EXPECT_PLAN_SERVER to the frozen native server binary"); + let model = std::env::var("DS4_EXPECT_PLAN_MODEL") + .expect("set DS4_EXPECT_PLAN_MODEL to a validated local GGUF"); + let command = || { + let mut command = Command::new(&server); + for (name, _) in std::env::vars_os() { + if name.to_string_lossy().starts_with("DS4_") { + command.env_remove(name); + } + } + command.args([ + "--model", + &model, + "--check-config", + "--backend", + "cpu", + "--max-seqs", + "1", + "--ctx", + "512", + "--mtp-mode", + "off", + "--prefix-reuse", + "off", + ]); + command + }; + let baseline = command().output().unwrap(); + assert!( + baseline.status.success(), + "{}", + String::from_utf8_lossy(&baseline.stderr) + ); + let plan: Value = serde_json::from_slice(&baseline.stdout).unwrap(); + let mut expected = json!({ + "family":plan["family"], "backend":plan["requested"]["backend"], + "effective":plan["effective"], "qualified":plan["qualified"], + "controls":plan["controls"], "post_open_quote":{"shared_weights":0} + }); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let guard = + std::env::temp_dir().join(format!("ds4-preflight-{}-{nonce}.json", std::process::id())); + fs::write(&guard, serde_json::to_vec(&expected).unwrap()).unwrap(); + let accepted = command().arg("--expect-plan").arg(&guard).output().unwrap(); + expected["effective"]["ctx"] = json!(513); + fs::write(&guard, serde_json::to_vec(&expected).unwrap()).unwrap(); + let rejected = command().arg("--expect-plan").arg(&guard).output().unwrap(); + fs::remove_file(guard).unwrap(); + assert!( + accepted.status.success(), + "{}", + String::from_utf8_lossy(&accepted.stderr) + ); + assert!(!rejected.status.success()); + assert!(String::from_utf8_lossy(&rejected.stderr).contains("mismatch: effective")); +} + +#[test] +#[ignore = "needs DS4_EXPECT_PLAN_SERVER and DS4_EXPECT_PLAN_MODEL"] +fn preflight_loads_the_eos_policy_vocab() { + // A nondefault EOS policy makes the check load the vocabulary itself; a + // GVQ1 release directory has no single GGUF to open for it. + let server = std::env::var("DS4_EXPECT_PLAN_SERVER").expect("DS4_EXPECT_PLAN_SERVER"); + let model = std::env::var("DS4_EXPECT_PLAN_MODEL").expect("DS4_EXPECT_PLAN_MODEL"); + let mut command = Command::new(&server); + for (name, _) in std::env::vars_os() { + if name.to_string_lossy().starts_with("DS4_") { + command.env_remove(name); + } + } + let output = command + .args(["--model", &model, "--check-config", "--ignore-eos"]) + .args(["--max-seqs", "1", "--ctx", "512", "--mtp-mode", "off"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +#[ignore = "needs DS4_EXPECT_PLAN_SERVER and a GLM VQ DS4_EXPECT_PLAN_MODEL"] +fn glm_vq_context_defaults_to_native() { + // Omitted --ctx follows the family's native maximum; explicit --ctx stays. + let server = std::env::var("DS4_EXPECT_PLAN_SERVER").expect("DS4_EXPECT_PLAN_SERVER"); + let model = std::env::var("DS4_EXPECT_PLAN_MODEL").expect("DS4_EXPECT_PLAN_MODEL"); + let effective_ctx = |extra: &[&str]| { + let mut command = Command::new(&server); + for (name, _) in std::env::vars_os() { + if name.to_string_lossy().starts_with("DS4_") { + command.env_remove(name); + } + } + let output = command + .args(["--model", &model, "--check-config", "--max-seqs", "1"]) + .args(extra) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let plan: Value = serde_json::from_slice(&output.stdout).unwrap(); + plan["effective"]["ctx"].as_u64().unwrap() + }; + assert_eq!(effective_ctx(&[]), 1_048_576); + assert_eq!(effective_ctx(&["--ctx", "16384"]), 16_384); +} diff --git a/crates/ds4-server/tests/fixtures/ds41-dsml/live-01a14f5d-turn1.txt b/crates/ds4-server/tests/fixtures/ds41-dsml/live-01a14f5d-turn1.txt new file mode 100644 index 0000000000000000000000000000000000000000..bbeae6d07a67fee720af900523e5736505e8d3b7 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-dsml/live-01a14f5d-turn1.txt @@ -0,0 +1,16 @@ +I'll start by gathering the necessary knowledge: the modeling skill, the SolidCraft GUI-worker rules, and how to open a visible app on the display. + + +SolidCraft GUI worker app visible display headless session + + +solidcraft palm tree modeling organic shapes + + +/home/user/.opengrok/skills/solidcraft-modeling/SKILL.md + + +/home/user/.opengrok/project-work-rules.md + + +Understood — I'll work from the current sources and skills only, no memory lookup. diff --git a/crates/ds4-server/tests/fixtures/ds41-dsml/live-01a14f5d-turn2.txt b/crates/ds4-server/tests/fixtures/ds41-dsml/live-01a14f5d-turn2.txt new file mode 100644 index 0000000000000000000000000000000000000000..20327c568c57782436c091803470a3ceda6f8f58 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-dsml/live-01a14f5d-turn2.txt @@ -0,0 +1,21 @@ +Understood — I'll work from the current sources and skills only, no memory lookup. + + +/home/user/.opengrok/skills/solidcraft-modeling/SKILL.md + + +/home/user/.opengrok/skills/solidcraft-editing/SKILL.md + + +/home/user/.opengrok/project-work-rules.md +200 + + +/data +The workspace is under /data; I need to locate the SolidCraft application source, docs, and how its GUI worker is launched. + + + +/data/3D + + diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-complex-args.calls.json b/crates/ds4-server/tests/fixtures/ds41-ref/dec-complex-args.calls.json new file mode 100644 index 0000000000000000000000000000000000000000..de6105be6a2279c578e68c75d8308a7d34ff5c31 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-complex-args.calls.json @@ -0,0 +1,14 @@ +{ + "role": "assistant", + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "read_file", + "arguments": "{\"target_file\": \"/tmp/x.txt\", \"limit\": 40, \"force\": true, \"ratio\": 1.5, \"note\": null, \"deps\": [\"a\", \"b\"], \"opts\": {\"k\": 1}}" + } + } + ] +} \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-complex-args.txt b/crates/ds4-server/tests/fixtures/ds41-ref/dec-complex-args.txt new file mode 100644 index 0000000000000000000000000000000000000000..e375abfcc4113cc91b832a75f7d6eed2ee7b2ee6 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-complex-args.txt @@ -0,0 +1,13 @@ + + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/x.txt +<|DSML| parameter name="limit" string="false">40 +<|DSML| parameter name="force" string="false">true +<|DSML| parameter name="ratio" string="false">1.5 +<|DSML| parameter name="note" string="false">null +<|DSML| parameter name="deps" string="false">["a", "b"] +<|DSML| parameter name="opts" string="false">{"k": 1} + +<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-marker-in-value.calls.json b/crates/ds4-server/tests/fixtures/ds41-ref/dec-marker-in-value.calls.json new file mode 100644 index 0000000000000000000000000000000000000000..804d932488fa2a7f4df9ea93536b9e7ba0f07c44 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-marker-in-value.calls.json @@ -0,0 +1,14 @@ +{ + "role": "assistant", + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "read_file", + "arguments": "{\"target_file\": \"quote the <|DSML| calls> tag\"}" + } + } + ] +} \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-marker-in-value.txt b/crates/ds4-server/tests/fixtures/ds41-ref/dec-marker-in-value.txt new file mode 100644 index 0000000000000000000000000000000000000000..f03e9a97c43fa9e6a6ba80381a1166f7e99f544b --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-marker-in-value.txt @@ -0,0 +1,7 @@ + + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">quote the <|DSML| calls> tag + +<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-one-call.calls.json b/crates/ds4-server/tests/fixtures/ds41-ref/dec-one-call.calls.json new file mode 100644 index 0000000000000000000000000000000000000000..35b064414cca5aa60b9fc3e077313c330f9d2eb3 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-one-call.calls.json @@ -0,0 +1,14 @@ +{ + "role": "assistant", + "content": "Reading it now.", + "reasoning_content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "read_file", + "arguments": "{\"target_file\": \"/tmp/x.txt\", \"limit\": 40}" + } + } + ] +} \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-one-call.txt b/crates/ds4-server/tests/fixtures/ds41-ref/dec-one-call.txt new file mode 100644 index 0000000000000000000000000000000000000000..8ac7573bef4595caf1a53801a7a8b63d60eba991 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-one-call.txt @@ -0,0 +1,8 @@ +Reading it now. + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/x.txt +<|DSML| parameter name="limit" string="false">40 + +<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-thinking.calls.json b/crates/ds4-server/tests/fixtures/ds41-ref/dec-thinking.calls.json new file mode 100644 index 0000000000000000000000000000000000000000..fde6ba52516f2697ec77ef564f2e699b0caf0ba9 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-thinking.calls.json @@ -0,0 +1,14 @@ +{ + "role": "assistant", + "content": "Reading it now.", + "reasoning_content": "The user wants /tmp/x.txt read.", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "read_file", + "arguments": "{\"target_file\": \"/tmp/x.txt\"}" + } + } + ] +} \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-thinking.txt b/crates/ds4-server/tests/fixtures/ds41-ref/dec-thinking.txt new file mode 100644 index 0000000000000000000000000000000000000000..02f97b9e12c077a9b145c10077911125cb2ab893 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-thinking.txt @@ -0,0 +1,7 @@ +The user wants /tmp/x.txt read.Reading it now. + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/x.txt + +<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-two-calls.calls.json b/crates/ds4-server/tests/fixtures/ds41-ref/dec-two-calls.calls.json new file mode 100644 index 0000000000000000000000000000000000000000..bd05cb4e1de09349aa5f52c6e67b305c2c1b7724 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-two-calls.calls.json @@ -0,0 +1,21 @@ +{ + "role": "assistant", + "content": "Checking both.", + "reasoning_content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "read_file", + "arguments": "{\"target_file\": \"/tmp/a.txt\"}" + } + }, + { + "type": "function", + "function": { + "name": "read_file", + "arguments": "{\"target_file\": \"/tmp/b.txt\"}" + } + } + ] +} \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-two-calls.txt b/crates/ds4-server/tests/fixtures/ds41-ref/dec-two-calls.txt new file mode 100644 index 0000000000000000000000000000000000000000..6a3da7547113c950222dd98731352692242e79dc --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-two-calls.txt @@ -0,0 +1,10 @@ +Checking both. + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/a.txt + +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/b.txt + +<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-unicode.calls.json b/crates/ds4-server/tests/fixtures/ds41-ref/dec-unicode.calls.json new file mode 100644 index 0000000000000000000000000000000000000000..a72209289784ff4e16ad4dcecbc5084cc5429b0d --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-unicode.calls.json @@ -0,0 +1,14 @@ +{ + "role": "assistant", + "content": "", + "reasoning_content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "read_file", + "arguments": "{\"target_file\": \"/tmp/smörgås.txt\"}" + } + } + ] +} \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/dec-unicode.txt b/crates/ds4-server/tests/fixtures/ds41-ref/dec-unicode.txt new file mode 100644 index 0000000000000000000000000000000000000000..d13cea157b837bb4a356a4b52e547547a34b5703 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/dec-unicode.txt @@ -0,0 +1,7 @@ + + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/smörgås.txt + +<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-args-double.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-args-double.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..c3fec6dc110bda3115d2e2faf912d6e987586d88 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-args-double.prompt.txt @@ -0,0 +1,34 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file /tmp/x.txt<|Assistant|>Calling it now. + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/x.txt + +<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-args-nondict.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-args-nondict.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b7f32b2f92b19478437656b86b00b15cf13b0fe --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-args-nondict.prompt.txt @@ -0,0 +1,34 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file /tmp/x.txt<|Assistant|>Calling it now. + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="arguments" string="true">[1, 2] + +<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-args-plain.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-args-plain.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..b16d0549dc2383c4daaf887759f25805b4d66211 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-args-plain.prompt.txt @@ -0,0 +1,34 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file /tmp/x.txt<|Assistant|>Calling it now. + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="arguments" string="true">not json + +<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-argtypes-chat.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-argtypes-chat.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..c2566996bca49f4c5501bcb41ef5dc98151ee972 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-argtypes-chat.prompt.txt @@ -0,0 +1,42 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}, "limit": {"type": "integer"}, "force": {"type": "boolean"}, "ratio": {"type": "number"}, "note": {"type": "string"}, "deps": {"type": "array", "items": {"type": "string"}}, "opts": {"type": "object"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file and label it.<|Assistant|>Reading it now. + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/x.txt +<|DSML| parameter name="limit" string="false">40 +<|DSML| parameter name="force" string="false">true +<|DSML| parameter name="ratio" string="false">1.5 +<|DSML| parameter name="note" string="false">null +<|DSML| parameter name="deps" string="false">["a", "b"] +<|DSML| parameter name="opts" string="false">{"k": 1} + +<|end▁of▁sentence|><|User|>file body + +Now summarise it.<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-blocks-chat.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-blocks-chat.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..587ad9d26403b52b1bef473ee92f7d1fdb44e4dd --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-blocks-chat.prompt.txt @@ -0,0 +1,3 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant.<|User|>First part. + +Second part.<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-drop-thinking.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-drop-thinking.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..a0c8b7090fa2d604e62b58506ab2e9e953a7f698 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-drop-thinking.prompt.txt @@ -0,0 +1 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant.<|User|>Say hi.<|Assistant|>Hi.<|end▁of▁sentence|><|User|>Say it again.<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-image-chat.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-image-chat.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..4304a681b4bd09e1746f6ab8676a56a95c7d57ff --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-image-chat.prompt.txt @@ -0,0 +1,3 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant.<|User|>What is in this image? + +<|deepseek_image|><|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-midsystem-chat.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-midsystem-chat.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..b2ec34cd32e8d1e41ee78817f6f09f2c882d2b76 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-midsystem-chat.prompt.txt @@ -0,0 +1,28 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file /tmp/x.txt<|System|>Answer in one line.<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-plain-chat.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-plain-chat.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..cb2a9030d55c8bf45396bfc38138844b80385a69 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-plain-chat.prompt.txt @@ -0,0 +1 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant.<|User|>Say hello.<|Assistant|>Hello.<|end▁of▁sentence|><|User|>Say it again.<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-budget.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-budget.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..2c0b1374cf68c48e60ba8043c65765996399458d --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-budget.prompt.txt @@ -0,0 +1,30 @@ +<|begin▁of▁sentence|><|System|>Reasoning Effort: 40 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file /tmp/x.txt<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-drop-thinking.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-drop-thinking.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..42c05af2d04c207758b5aadc3bebbd0c486bea44 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-drop-thinking.prompt.txt @@ -0,0 +1,3 @@ +<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant.<|User|>Say hi.<|Assistant|>Hi.<|end▁of▁sentence|><|User|>Say it again.<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-low.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-low.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac8a96e14ee979f00c0058185e4849897bb7e0b4 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-low.prompt.txt @@ -0,0 +1,30 @@ +<|begin▁of▁sentence|><|System|>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file /tmp/x.txt<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-max.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-max.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac56328d40f4ec8969bbb9e956f49882a06bdb48 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-max.prompt.txt @@ -0,0 +1,30 @@ +<|begin▁of▁sentence|><|System|>Reasoning Effort: 100 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file /tmp/x.txt<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-tail-reasoning.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-tail-reasoning.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..56464cf8e8b900e79f4536ac3eb4857010290706 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-tail-reasoning.prompt.txt @@ -0,0 +1,3 @@ +<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant.<|User|>Say hi.<|Assistant|>TAIL-THOUGHT-MARKERHi.<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-tools.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-tools.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..599117e12e797d930fe803d891539fb34946e8a0 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-tools.prompt.txt @@ -0,0 +1,30 @@ +<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file /tmp/x.txt<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-two-assistants.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-two-assistants.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..04deb85b35d3a2da8d9a8ef2b8b66cd028df86b4 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-think-two-assistants.prompt.txt @@ -0,0 +1,3 @@ +<|begin▁of▁sentence|><|System|>Reasoning Effort: 75 (range 1-100, the higher the value, the more thorough the reasoning) + +You are a helpful assistant.<|User|>Say hi.<|Assistant|>One.<|end▁of▁sentence|>SECOND-THOUGHT-MARKERTwo.<|end▁of▁sentence|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-toolresult-only.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-toolresult-only.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..4969f0c5d1a3942b6fc7cbf1fc18e7b76fbd0722 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-toolresult-only.prompt.txt @@ -0,0 +1,34 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file.<|Assistant|> + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/x.txt + +<|end▁of▁sentence|><|User|>body<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-toolresults-order.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-toolresults-order.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..73770a93925e0a3293ade632b45d6953a0b4800c --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-toolresults-order.prompt.txt @@ -0,0 +1,41 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Check both files.<|Assistant|>Checking. + +<|DSML| calls> +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/a.txt + +<|DSML| invoke name="read_file"> +<|DSML| parameter name="target_file" string="true">/tmp/b.txt + +<|end▁of▁sentence|><|User|>body a + +body b + +Summarise both.<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/ds41-ref/enc-tools-chat.prompt.txt b/crates/ds4-server/tests/fixtures/ds41-ref/enc-tools-chat.prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..6c555772823d10e038073a0ea1d9888dbc67ca98 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/ds41-ref/enc-tools-chat.prompt.txt @@ -0,0 +1,28 @@ +<|begin▁of▁sentence|><|System|>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML| calls>" block like the following: + +<|DSML| calls> +<|DSML| invoke name="$TOOL_NAME"> +<|DSML| parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<|DSML| invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "read_file", "description": "Read a file", "parameters": {"type": "object", "properties": {"target_file": {"type": "string"}, "limit": {"type": "integer"}}, "required": ["target_file"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<|User|>Read the file /tmp/x.txt<|Assistant|> \ No newline at end of file diff --git a/crates/ds4-server/tests/fixtures/iquest-tool-parser.json b/crates/ds4-server/tests/fixtures/iquest-tool-parser.json new file mode 100644 index 0000000000000000000000000000000000000000..ee091003650fe0d82f4605a3e4167973e0487028 --- /dev/null +++ b/crates/ds4-server/tests/fixtures/iquest-tool-parser.json @@ -0,0 +1,422 @@ +{ + "source_url": "https://github.com/IQuestLab/vllm-iquest-q1/blob/1f305c04f6d63b936e118e5aa4a9eb6b20e38a32/src/vllm_iquest_q1/tool_parser.py", + "source_sha256": "4b6de273cf973808a98879ff258a98a68c2cc1415e491ba653aaf03d5c3789d0", + "scope": "Extracted official buffered and streaming parser methods, complete Unicode input streamed one codepoint at a time; no model inference. Native stream finalization additionally preserves unfinished XML as raw text.", + "parameters": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "active": { + "type": "boolean" + }, + "data": { + "type": "object" + } + } + }, + "cases": [ + { + "name": "typed", + "text": "lookuptext 123 \ncount2activetruedata{\"city\":\"서울\",\"items\":[1,null]}", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "text": " 123 \n", + "count": 2, + "active": true, + "data": { + "city": "서울", + "items": [ + 1, + null + ] + } + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "text": " 123 \n", + "count": 2, + "active": true, + "data": { + "city": "서울", + "items": [ + 1, + null + ] + } + } + } + ] + }, + { + "name": "literal-strings", + "text": "lookuptexttrue & \"quoted\" café", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "text": "true & \"quoted\" café" + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "text": "true & \"quoted\" café" + } + } + ] + }, + { + "name": "string-json", + "text": "lookuptext{\"raw\":true}", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "text": "{\"raw\":true}" + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "text": "{\"raw\":true}" + } + } + ] + }, + { + "name": "unknown-properties", + "text": "lookupvalue null list[1,\"漢字\",false]bad 0123 ", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "value": null, + "list": [ + 1, + "漢字", + false + ], + "bad": "0123" + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "value": null, + "list": [ + 1, + "漢字", + false + ], + "bad": "0123" + } + } + ] + }, + { + "name": "float", + "text": "lookupcount1e-05activefalse", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "count": 1e-05, + "active": false + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "count": 1e-05, + "active": false + } + } + ] + }, + { + "name": "arbitrary-integer", + "text": "lookupcount18446744073709551617data{\"negative\":-18446744073709551617}", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "count": 18446744073709551617, + "data": { + "negative": -18446744073709551617 + } + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "count": 18446744073709551617, + "data": { + "negative": -18446744073709551617 + } + } + } + ] + }, + { + "name": "no-arguments", + "text": "  ping\n", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "ping", + "arguments": {} + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "ping", + "arguments": {} + } + ] + }, + { + "name": "empty-string", + "text": "lookuptext", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "text": "" + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "text": "" + } + } + ] + }, + { + "name": "duplicate-key", + "text": "lookupcount1count2", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "count": 2 + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "count": 2 + } + } + ] + }, + { + "name": "parallel-content", + "text": "前lookuptext123中間ping後", + "tools_enabled": true, + "content": "前中間後", + "calls": [ + { + "name": "lookup", + "arguments": { + "text": "123" + } + }, + { + "name": "ping", + "arguments": {} + } + ], + "stream_content_before_final": "前中間後", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "text": "123" + } + }, + { + "index": 1, + "name": "ping", + "arguments": {} + } + ] + }, + { + "name": "malformed", + "text": "lookuptextbroken", + "tools_enabled": true, + "content": "lookuptextbroken", + "calls": [], + "stream_content_before_final": "lookuptextbroken", + "stream_calls_before_final": [] + }, + { + "name": "malformed-then-valid", + "text": "lookuptextbrokenping", + "tools_enabled": true, + "content": "lookuptextbroken", + "calls": [ + { + "name": "ping", + "arguments": {} + } + ], + "stream_content_before_final": "lookuptextbroken", + "stream_calls_before_final": [ + { + "index": 0, + "name": "ping", + "arguments": {} + } + ] + }, + { + "name": "empty-key", + "text": "lookup 1", + "tools_enabled": true, + "content": "lookup 1", + "calls": [], + "stream_content_before_final": "lookup 1", + "stream_calls_before_final": [] + }, + { + "name": "truncated", + "text": "pingafterlookuptext", + "tools_enabled": true, + "content": "afterlookuptext", + "calls": [ + { + "name": "ping", + "arguments": {} + } + ], + "stream_content_before_final": "after", + "stream_calls_before_final": [ + { + "index": 0, + "name": "ping", + "arguments": {} + } + ] + }, + { + "name": "unicode-space", + "text": " lookup\n text   raw  ", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "text": " raw " + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "text": " raw " + } + } + ] + }, + { + "name": "python-control-space", + "text": "\u001clookup\u001d\u001etext\u001f\u001c raw \u001d", + "tools_enabled": true, + "content": "", + "calls": [ + { + "name": "lookup", + "arguments": { + "text": " raw " + } + } + ], + "stream_content_before_final": "", + "stream_calls_before_final": [ + { + "index": 0, + "name": "lookup", + "arguments": { + "text": " raw " + } + } + ] + }, + { + "name": "disabled", + "text": "lookuptext123", + "tools_enabled": false, + "content": "lookuptext123", + "calls": [], + "stream_content_before_final": "lookuptext123", + "stream_calls_before_final": [] + } + ] +} diff --git a/crates/ds4-server/tests/generate.rs b/crates/ds4-server/tests/generate.rs new file mode 100644 index 0000000000000000000000000000000000000000..f5e17da8537a17a9358d121d003f50bd1879762c --- /dev/null +++ b/crates/ds4-server/tests/generate.rs @@ -0,0 +1,2921 @@ +//! Scripted decode + HTTP generate path. No GGUF. + +use ds4_server::parse::{parse_request, ChatMsg, ChatPart, ImageMime, RequestImage, ToolCall}; +use ds4_server::route::WireSurface; +use ds4_server::tools::{ + DSML_INVOKE_END, DSML_INVOKE_END_SPACED, DSML_INVOKE_START, DSML_INVOKE_START_SPACED, + DSML_PARAM_END, DSML_PARAM_END_SPACED, DSML_PARAM_START, DSML_PARAM_START_SPACED, + DSML_TOOL_CALLS_END, DSML_TOOL_CALLS_END_SPACED, DSML_TOOL_CALLS_START, + DSML_TOOL_CALLS_START_SPACED, +}; +use ds4_server::{ + generate_and_write, generation_blocked, handle_client_inner, render_prompt, + stop_list_find_from, think_start, ChatFormat, ContStepper, DecodeIo, GenerateError, ParseEnv, + ParsedRequest, ReqTimings, ScriptedDecode, ScriptedStep, ServerConfig, ServerInner, ThinkMode, + CREATED_TEST, DSML_ASSISTANT, DSML_BOS, DSML_SYSTEM_TOKEN, DSML_THINK_END, DSML_USER, + TAPE_PLAIN, +}; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::PathBuf; +use std::process::Command; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +fn env() -> ParseEnv { + ParseEnv { + default_model: "ds4".into(), + default_tokens: 16, + default_effort: ThinkMode::None, + default_temp: 0.0, + live_ids: Vec::new(), + engine_defaults: false, + } +} + +fn user_req() -> ParsedRequest { + let mut r = parse_request( + WireSurface::OpenaiChat, + &env(), + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":8,"thinking":{"type":"disabled"}}"#, + ) + .unwrap(); + r.think_mode = ThinkMode::None; + r.temperature = 0.0; + r +} + +#[test] +fn ds41_chat_takes_the_c_renderer_with_the_v41_head() { + // The C-rendered DSML families ship no chat template: model 16 is admitted + // beside model 0 (chat_input.rs) and renders through render_prompt with the + // engine's official encoding.py head (server_dsml_render.c:375-383), the + // scripted model standing in for a V4.1 tokenizer that owns the token. + let parsed = parse_request( + WireSurface::OpenaiChat, + &env(), + r#"{"messages":[{"role":"system","content":"S"},{"role":"user","content":"U"}]}"#, + ) + .unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[b"x"]); + engine.model_id = ds4_core::Variant::DeepSeek41Flash as i32; + let rendered = engine.render_request(&parsed).unwrap(); + assert_eq!( + rendered, + format!("{DSML_BOS}{DSML_SYSTEM_TOKEN}S{DSML_USER}U{DSML_ASSISTANT}{DSML_THINK_END}") + .as_bytes() + ); + // The base V4 family keeps its own head: no system token. + engine.model_id = ds4_core::Variant::Flash as i32; + let rendered = engine.render_request(&parsed).unwrap(); + assert_eq!( + rendered, + format!("{DSML_BOS}S{DSML_USER}U{DSML_ASSISTANT}{DSML_THINK_END}").as_bytes() + ); +} + +struct PromptSyncDecode { + template: Option, + inner: ScriptedDecode, + /// What the sync decided, as the real engine would report it. + reuse: ds4_core::ReuseTaken, + miss: ds4_core::ReuseMiss, + cached_tokens: i32, + effective_prompt_pos: i32, + prompt_sync_calls: usize, + sync_calls: usize, + disk_eligible: Vec, + thinking_visible_eligible: Vec, + prompt_sync_elapsed: Option, + remembered: Vec<(Vec, i32)>, + invalidations: usize, + continued_positions: Vec, + fail_continued: bool, + fail_prompt_sync: bool, + events: Vec<&'static str>, + replay_raw: Option, + replay_prompts: Vec>, + rendered: std::cell::RefCell>>, + remembered_tools: Vec<(Vec, String)>, + prefix_width: usize, + prefix_budgets: Vec, + trimmed_positions: Vec, + sample_params: Vec<(f32, i32, f32, f32)>, +} + +impl PromptSyncDecode { + fn new(inner: ScriptedDecode, cached_tokens: i32, effective_prompt_pos: i32) -> Self { + Self { + template: None, + inner, + reuse: ds4_core::ReuseTaken::Cold, + miss: ds4_core::ReuseMiss::None, + cached_tokens, + effective_prompt_pos, + prompt_sync_calls: 0, + sync_calls: 0, + disk_eligible: Vec::new(), + thinking_visible_eligible: Vec::new(), + prompt_sync_elapsed: None, + remembered: Vec::new(), + invalidations: 0, + continued_positions: Vec::new(), + fail_continued: false, + fail_prompt_sync: false, + events: Vec::new(), + replay_raw: None, + replay_prompts: Vec::new(), + rendered: std::cell::RefCell::new(Vec::new()), + remembered_tools: Vec::new(), + prefix_width: 1, + prefix_budgets: Vec::new(), + trimmed_positions: Vec::new(), + sample_params: Vec::new(), + } + } +} + +impl DecodeIo for PromptSyncDecode { + fn template(&self) -> Option<&ds4_core::chat_template::Template> { + self.template.as_ref() + } + fn model_id(&self) -> i32 { + self.inner.model_id() + } + + fn tokenize_text(&self, text: &str) -> Result, GenerateError> { + self.inner.tokenize_text(text) + } + + fn tokenize_rendered_chat(&self, text: &[u8]) -> Result, GenerateError> { + self.rendered.borrow_mut().push(text.to_vec()); + self.inner.tokenize_rendered_chat(text) + } + + fn tokenizes_control_literals(&self) -> bool { + self.inner.tokenizes_control_literals() + } + + fn token_text(&self, token: i32) -> Result, GenerateError> { + self.inner.token_text(token) + } + + fn token_is_stop(&self, token: i32) -> bool { + self.inner.token_is_stop(token) + } + + fn vision_tokens(&self, data: &[u8]) -> Result, GenerateError> { + self.inner.vision_tokens(data) + } + fn sync_vision_prompt( + &mut self, + tokens: &[i32], + images: &[ds4_server::generate::VisionPromptInput], + ) -> Result<(), GenerateError> { + self.inner.sync_vision_prompt(tokens, images) + } + + fn sync(&mut self, tokens: &[i32]) -> Result<(), GenerateError> { + self.sync_calls += 1; + self.inner.sync(tokens) + } + + fn sync_prompt( + &mut self, + _prompt: &[u8], + tokens: &[i32], + disk_eligible: bool, + thinking_visible_eligible: bool, + ) -> Result { + self.events.push("sync"); + self.prompt_sync_calls += 1; + // The real engine reports what its sync decided; the tape carries + // whatever the situation under test set. + + self.disk_eligible.push(disk_eligible); + self.thinking_visible_eligible + .push(thinking_visible_eligible); + if self.fail_prompt_sync { + return Err(GenerateError::Engine("injected prompt sync failure".into())); + } + self.inner.live = tokens.to_vec(); + self.inner.pos = self.effective_prompt_pos; + Ok(self.cached_tokens) + } + + fn last_reuse(&self) -> ds4_core::ReuseTaken { + self.reuse + } + + fn last_miss(&self) -> ds4_core::ReuseMiss { + self.miss + } + + fn prompt_sync_elapsed(&self) -> Option { + self.prompt_sync_elapsed + } + + fn restore_tool_replay(&mut self, messages: &mut [ChatMsg]) { + self.events.push("restore"); + let Some(raw) = &self.replay_raw else { + return; + }; + for message in messages { + if !message.calls.is_empty() { + message.raw_dsml = raw.clone(); + } + } + } + + fn sync_tool_replay_prompt( + &mut self, + prompt: &[u8], + tokens: &[i32], + ) -> Result { + self.events.push("tool-sync"); + self.replay_prompts.push(prompt.to_vec()); + if self.fail_prompt_sync { + return Err(GenerateError::Engine("injected prompt sync failure".into())); + } + self.inner.live = tokens.to_vec(); + self.inner.pos = self.effective_prompt_pos; + Ok(self.cached_tokens) + } + + fn remember_tool_replay(&mut self, calls: &[ToolCall], raw_dsml: &str) { + self.events.push("remember-tool"); + self.remembered_tools.push(( + calls.iter().map(|call| call.id.clone()).collect(), + raw_dsml.to_string(), + )); + } + + fn eval(&mut self, token: i32) -> Result<(), GenerateError> { + self.events.push("eval"); + self.inner.eval(token) + } + + fn eval_greedy(&mut self, first: i32, budget: i32) -> Result, GenerateError> { + self.prefix_budgets.push(budget); + if self.prefix_width == 0 { + return Ok(Vec::new()); + } + self.eval(first)?; + let mut tokens = vec![first]; + let mut rng = 1; + while tokens.len() < self.prefix_width.min(budget as usize) + && !self.token_is_stop(*tokens.last().unwrap()) + { + let token = self.inner.sample(0.0, 0, 1.0, 0.0, &mut rng); + self.inner.eval(token)?; + tokens.push(token); + } + if self.model_id() == ds4_server::ModelSyntax::Glm53 as i32 { + self.inner.live.extend_from_slice(&tokens); + } + Ok(tokens) + } + + fn trim_greedy(&mut self, pos: i32) -> Result<(), GenerateError> { + if self.model_id() != ds4_server::ModelSyntax::Glm53 as i32 { + self.invalidate(); + return Ok(()); + } + assert!( + (1..=self.inner.pos).contains(&pos), + "invalid journal target" + ); + self.trimmed_positions.push(pos); + self.inner.live.truncate(pos as usize); + self.inner.pos = pos; + Ok(()) + } + + fn sample( + &mut self, + temperature: f32, + top_k: i32, + top_p: f32, + min_p: f32, + rng: &mut u64, + ) -> i32 { + self.events.push("sample"); + self.sample_params.push((temperature, top_k, top_p, min_p)); + self.inner.sample(temperature, top_k, top_p, min_p, rng) + } + + fn pos(&self) -> i32 { + self.inner.pos() + } + + fn ctx(&self) -> i32 { + self.inner.ctx() + } + + fn generation(&self) -> u64 { + self.inner.generation() + } + + fn session_tokens(&self) -> Vec { + self.inner.session_tokens() + } + + fn maybe_store_continued(&mut self) -> Result<(), GenerateError> { + self.events.push("continued"); + self.continued_positions.push(self.pos()); + if self.fail_continued { + Err(GenerateError::Engine( + "injected continued save failure".into(), + )) + } else { + Ok(()) + } + } + + fn remember_thinking_visible_checkpoint(&mut self, text: Vec) { + self.remembered.push((text, self.pos())); + } + + fn invalidate(&mut self) { + self.invalidations += 1; + self.inner.live.clear(); + self.inner.pos = 0; + } +} + +#[test] +fn mtp_prefix_limits() { + for (cap, ctx) in [(2, 8192), (8, 3)] { + let mut parsed = user_req(); + parsed.max_tokens = cap; + let mut script = ScriptedDecode::from_pieces(&[b"<|content_text|>Hello", b" world", b"!"]); + script.ctx = ctx; + script.model_id = 9; + let mut engine = PromptSyncDecode::new(script, 0, 1); + engine.prefix_width = 9; + let mut out = Vec::new(); + let result = generate_and_write( + &mut engine, + &parsed, + "mtp-limit", + CREATED_TEST, + false, + cap, + &mut out, + ) + .unwrap(); + assert_eq!(result.timings.decode_tokens, 2); + assert_eq!(result.timings.decode_steps, 1); + assert_eq!(engine.prefix_budgets, [2]); + assert_eq!(engine.pos(), 3); + let text = String::from_utf8(out).unwrap(); + assert!(text.contains("Hello world"), "{text}"); + } +} + +#[test] +fn mtp_prefix_stops() { + for stream in [false, true] { + for stop in [None, Some(" world")] { + let mut parsed = user_req(); + parsed.stream = stream; + if let Some(stop) = stop { + parsed.stops.push(stop.into()); + } + let mut script = + ScriptedDecode::from_pieces(&[b"<|content_text|>Hello", b" world", b"!"]); + script.model_id = 9; + let mut engine = PromptSyncDecode::new(script, 0, 1); + engine.prefix_width = 9; + let mut out = Vec::new(); + let result = generate_and_write( + &mut engine, + &parsed, + "mtp-stop", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap(); + let text = String::from_utf8(out).unwrap(); + assert_eq!(engine.prefix_budgets, [8]); + assert!(text.contains("Hello"), "{text}"); + assert!(text.contains("\"finish_reason\":\"stop\""), "{text}"); + assert_eq!( + result.timings.decode_tokens, + if stop.is_some() { 2 } else { 3 } + ); + assert!(!text.contains(" world") || stop.is_none(), "{text}"); + if stop.is_some() { + assert_eq!(engine.invalidations, 1); + assert_eq!(engine.pos(), 0); + } + } + } +} + +#[test] +fn mtp_prefix_empty() { + let mut engine = PromptSyncDecode::new(ScriptedDecode::from_pieces(&[b"Hello"]), 0, 1); + engine.prefix_width = 0; + let result = generate_and_write( + &mut engine, + &user_req(), + "mtp-empty", + CREATED_TEST, + false, + 8, + &mut Vec::new(), + ); + assert!(matches!(result, Err(GenerateError::Engine(_)))); + assert_eq!(engine.invalidations, 1); +} + +fn glm_tool_prefix(surface: WireSurface, stream: bool, close: usize) { + const WIDTH: usize = 4; + const TAIL: &[u8] = b"UNEMITTED_SPECULATIVE_TAIL"; + const BODY: &[u8] = b"bashcommandecho hi"; + let body = match surface { + WireSurface::Anthropic => { + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":8,"tools":[{"name":"bash","input_schema":{"type":"object","properties":{"command":{"type":"string"}}}}]}"# + } + WireSurface::Responses => { + r#"{"input":"hi","max_output_tokens":8,"tools":[{"type":"function","name":"bash","parameters":{"type":"object","properties":{"command":{"type":"string"}}}}]}"# + } + _ => unreachable!(), + }; + let mut parsed = parse_request(surface, &env(), body).unwrap(); + parsed.stream = stream; + parsed.think_mode = ThinkMode::None; + parsed.temperature = 0.0; + + // The journal has already committed the whole prefix when output sees + // the close tag. Only its un-emitted tail may be discarded. + let mut pieces = vec![TAIL; WIDTH]; + if close == 0 { + pieces[0] = + b"bashcommandecho hi"; + } else { + pieces[0] = BODY; + pieces[1..close].fill(b"\n"); + pieces[close] = b""; + } + let mut tape = ScriptedDecode::from_pieces(&pieces); + tape.model_id = ds4_server::ModelSyntax::Glm53 as i32; + let mut engine = PromptSyncDecode::new(tape, 0, 1); + engine.prefix_width = WIDTH; + let mut out = Vec::new(); + let result = generate_and_write( + &mut engine, + &parsed, + "glm-tool-prefix", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap(); + + let wire = String::from_utf8(out).unwrap(); + assert_eq!(result.finish, "tool_calls", "{wire}"); + assert_eq!(result.tool_ids.len(), 1, "{wire}"); + assert!(wire.contains(&result.tool_ids[0]), "{wire}"); + assert!(!wire.contains(std::str::from_utf8(TAIL).unwrap()), "{wire}"); + assert_eq!(engine.prefix_budgets, [8]); + assert_eq!(engine.inner.idx, WIDTH, "all accepted rows were evaluated"); + assert_eq!(result.timings.decode_tokens, (close + 1) as i32); + assert_eq!(result.frontier, (close + 2) as i32); + assert_eq!(engine.pos(), result.frontier); + assert_eq!(result.generation, 1); + assert_eq!(engine.invalidations, 0); + let expected_history = std::iter::once(1) + .chain(1..=(close + 1) as i32) + .collect::>(); + assert_eq!(engine.inner.live, expected_history); + let expected_trims = if close + 1 < WIDTH { + vec![result.frontier] + } else { + Vec::new() + }; + assert_eq!(engine.trimmed_positions, expected_trims); +} + +macro_rules! glm_tool_prefix_case { + ($name:ident, $surface:ident, $stream:expr, $close:expr) => { + #[test] + fn $name() { + glm_tool_prefix(WireSurface::$surface, $stream, $close); + } + }; +} + +glm_tool_prefix_case!(glm_tool_prefix_an_b_first, Anthropic, false, 0); +glm_tool_prefix_case!(glm_tool_prefix_an_b_middle, Anthropic, false, 1); +glm_tool_prefix_case!(glm_tool_prefix_an_b_final, Anthropic, false, 3); +glm_tool_prefix_case!(glm_tool_prefix_an_s_first, Anthropic, true, 0); +glm_tool_prefix_case!(glm_tool_prefix_an_s_middle, Anthropic, true, 1); +glm_tool_prefix_case!(glm_tool_prefix_an_s_final, Anthropic, true, 3); +glm_tool_prefix_case!(glm_tool_prefix_re_b_first, Responses, false, 0); +glm_tool_prefix_case!(glm_tool_prefix_re_b_middle, Responses, false, 1); +glm_tool_prefix_case!(glm_tool_prefix_re_b_final, Responses, false, 3); +glm_tool_prefix_case!(glm_tool_prefix_re_s_first, Responses, true, 0); +glm_tool_prefix_case!(glm_tool_prefix_re_s_middle, Responses, true, 1); +glm_tool_prefix_case!(glm_tool_prefix_re_s_final, Responses, true, 3); + +#[test] +fn mtp_tool_tail_invalidates() { + let block = concat!( + "<|DSML|tool_calls><|DSML|invoke name=\"bash\">", + "<|DSML|parameter name=\"command\" string=\"true\">ls", + "" + ); + let tape = ScriptedDecode::from_pieces(&[block.as_bytes(), b"UNEMITTED_TAIL"]); + let mut engine = PromptSyncDecode::new(tape, 0, 1); + engine.prefix_width = 2; + let mut out = Vec::new(); + let result = generate_and_write( + &mut engine, + &tools_req(), + "generic-tool-prefix", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap(); + assert_eq!(result.tool_ids.len(), 1); + assert_eq!(result.frontier, 0); + assert_eq!(engine.invalidations, 1); + assert!(engine.trimmed_positions.is_empty()); + assert!(!String::from_utf8(out).unwrap().contains("UNEMITTED_TAIL")); +} + +#[test] +fn mtp_disconnect() { + struct Disconnect; + impl Write for Disconnect { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if data.windows(5).any(|s| s == b"Hello") { + return Err(std::io::ErrorKind::BrokenPipe.into()); + } + Ok(data.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut parsed = user_req(); + parsed.stream = true; + let mut script = ScriptedDecode::from_pieces(&[b"<|content_text|>Hello", b" world"]); + script.model_id = 9; + let mut engine = PromptSyncDecode::new(script, 0, 1); + engine.prefix_width = 9; + let result = generate_and_write( + &mut engine, + &parsed, + "mtp-disconnect", + CREATED_TEST, + false, + 8, + &mut Disconnect, + ); + assert!(matches!(result, Err(GenerateError::Io))); + assert_eq!(engine.prefix_budgets, [8]); + assert_eq!(engine.invalidations, 1); + assert_eq!(engine.pos(), 0); +} + +#[test] +fn mtp_sampling_policy() { + for mode in 0..3 { + let mut parsed = user_req(); + match mode { + 0 => parsed.temperature = 0.7, + 1 => parsed.think_mode = ThinkMode::Low, + _ => { + parsed.has_tool_results = true; + parsed.required_think_end_prefix = vec![7]; + } + } + let mut script = ScriptedDecode::from_pieces(&[b"<|content_text|>Hello"]); + script.model_id = 9; + let mut engine = PromptSyncDecode::new(script, 0, 1); + engine.prefix_width = 9; + generate_and_write( + &mut engine, + &parsed, + "mtp-policy", + CREATED_TEST, + false, + 8, + &mut Vec::new(), + ) + .unwrap(); + assert!(engine.prefix_budgets.is_empty()); + assert!(engine.events.contains(&"eval")); + } +} + +#[test] +fn iquest_thinking_sampling() { + use ds4_core::chat_template::{RenderClock, Template}; + use ds4_server::parse::{DEFAULT_MIN_P, DEFAULT_TEMPERATURE, DEFAULT_TOP_P}; + let iquest = ds4_core::Variant::IQuestQ1 as i32; + let glm = ds4_core::Variant::Glm53FlashVq as i32; + for model_id in [iquest, glm, 0] { + for temperature in [0.0, 0.4] { + let parsed = parse_request( + WireSurface::OpenaiChat, + &env(), + &format!(r#"{{"messages":[{{"role":"user","content":"2 + 2"}}],"reasoning_effort":"high","temperature":{temperature},"top_k":9,"top_p":0.9,"min_p":0.03}}"#), + ) + .unwrap(); + let mut script = ScriptedDecode::from_pieces(&[b"Compute.", b"", b"4"]); + script.model_id = model_id; + let mut engine = PromptSyncDecode::new(script, 0, 1); + if model_id == iquest { + engine.template = Some(Template::compile( + include_str!("../../../tests/fixtures/chat-template/models/iquest/chat_template.jinja"), + RenderClock::Fixed(0), + ).unwrap()); + } + engine.prefix_width = 2; + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "thinking-sample", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + let expected = if model_id == iquest || model_id == glm { + (temperature, 9, 0.9, 0.03) + } else { + (DEFAULT_TEMPERATURE, 0, DEFAULT_TOP_P, DEFAULT_MIN_P) + }; + assert!(!engine.sample_params.is_empty()); + assert!( + engine + .sample_params + .iter() + .all(|¶ms| params == expected), + "model {model_id}, requested {temperature}: {:?}", + engine.sample_params + ); + // Greedy thinking reaches the engine's greedy commit, where the + // embedded MTP drafts run, only on the families that keep the + // request's sampler. + assert_eq!( + !engine.prefix_budgets.is_empty(), + (model_id == iquest || model_id == glm) && temperature == 0.0 + ); + } + } +} + +#[test] +fn stop_list_find_matches_c_order() { + let stops = vec!["STOP".into(), "END".into()]; + assert_eq!( + stop_list_find_from(&stops, b"hello STOP tail END", 0), + Some((6, 4)) + ); +} + +#[test] +fn iquest_structured_tools_allow_dedicated_output_parser() { + let model_id = ds4_core::Variant::IQuestQ1 as i32; + let plain = user_req(); + assert_eq!(generation_blocked(&plain, model_id), None); + let expected = None; + let mut tools = plain.clone(); + tools.has_tools = true; + assert_eq!(generation_blocked(&tools, model_id), expected); + assert_eq!(generation_blocked(&tools, 0), None); + let mut results = plain.clone(); + results.has_tool_results = true; + assert_eq!(generation_blocked(&results, model_id), expected); + let mut history = plain.clone(); + history.messages[0].calls.push(ToolCall::default()); + assert_eq!(generation_blocked(&history, model_id), expected); +} + +#[test] +fn family_generate_allows_tools() { + let parsed = user_req(); + assert_eq!(generation_blocked(&parsed, 3), None); + assert_eq!(generation_blocked(&parsed, 2), None); + let mut tools = parsed.clone(); + tools.has_tools = true; + assert_eq!(generation_blocked(&tools, 0), None); +} + +#[test] +fn glm_serial_image_expands_placeholder_before_sync() { + let mut parsed = user_req(); + parsed.messages[0].parts = vec![ChatPart::Image(0)]; + parsed.images.push(RequestImage { + mime: ImageMime::Png, + data: Arc::from([1u8]), + }); + let mut engine = ScriptedDecode::from_pieces(&[]); + engine.model_id = 7; + engine.prompt_tokens = vec![154830, 154854, 154831]; + let mut out = Vec::new(); + + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-vision", + CREATED_TEST, + false, + 1, + &mut out, + ) + .unwrap(); + + assert_eq!(engine.live.len(), 18); + assert_eq!(engine.live[0], 154830); + assert!(engine.live[1..17].iter().all(|&token| token == 154854)); + assert_eq!(engine.live[17], 154831); +} + +#[test] +fn step_images_expand_complete_spans() { + let mut parsed = user_req(); + parsed.messages[0].parts = vec![ChatPart::Image(0), ChatPart::Image(1)]; + parsed.images = vec![ + RequestImage { + mime: ImageMime::Png, + data: Arc::from([1u8]) + }; + 2 + ]; + let mut engine = ScriptedDecode::from_pieces(&[]); + engine.model_id = 10; + engine.prompt_tokens = vec![17, 128001, 18, 128001, 19]; + let mut engine = PromptSyncDecode::new(engine, 0, 0); + engine.template = Some( + ds4_core::chat_template::Template::compile( + include_str!("../../../tests/fixtures/step37/chat_template.jinja"), + ds4_core::chat_template::RenderClock::Fixed(0), + ) + .unwrap(), + ); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "step-images", + CREATED_TEST, + false, + 1, + &mut out, + ) + .unwrap(); + let span = [vec![128000], vec![128001; 169], vec![128002]].concat(); + assert_eq!( + engine.inner.live, + [vec![17], span.clone(), vec![18], span, vec![19]].concat() + ); + engine.inner.prompt_tokens = vec![128001]; + assert!(generate_and_write( + &mut engine, + &parsed, + "step-missing-image", + CREATED_TEST, + false, + 1, + &mut out + ) + .is_err()); + engine.inner.prompt_tokens = vec![128001; 3]; + assert!(generate_and_write( + &mut engine, + &parsed, + "step-extra-image", + CREATED_TEST, + false, + 1, + &mut out + ) + .is_err()); +} + +#[test] +fn inkling_images_expand_in_order() { + let mut parsed = user_req(); + parsed.messages[0].parts = vec![ + ChatPart::Image(0), + ChatPart::Text("Then".into()), + ChatPart::Image(1), + ]; + parsed.images = vec![ + RequestImage { + mime: ImageMime::Png, + data: Arc::from([1u8]) + }; + 2 + ]; + let mut engine = ScriptedDecode::from_pieces(&[]); + engine.model_id = 9; + engine.prompt_tokens = vec![200000, 200054, 200010, 200054, 200001]; + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "inkling-images", + CREATED_TEST, + false, + 1, + &mut out, + ) + .unwrap(); + assert_eq!( + engine.live, + [ + vec![200000], + vec![200054; 16], + vec![200010], + vec![200054; 16], + vec![200001] + ] + .concat() + ); + engine.prompt_tokens = vec![200054]; + assert!(generate_and_write( + &mut engine, + &parsed, + "inkling-missing", + CREATED_TEST, + false, + 1, + &mut out + ) + .is_err()); + engine.prompt_tokens = vec![200054; 3]; + assert!(generate_and_write( + &mut engine, + &parsed, + "inkling-extra", + CREATED_TEST, + false, + 1, + &mut out + ) + .is_err()); +} + +#[test] +fn inkling_audio_expands_in_order() { + let body = r#"{"messages":[{"role":"user","content":[{"type":"input_audio","input_audio":{"format":"wav","data":"UklGRiYAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQIAAAAAAA=="}}]}]}"#; + let mut parsed = + ds4_server::parse_chat_request(&ds4_server::ParseEnv::default(), body).unwrap(); + let mut engine = ScriptedDecode::from_pieces(&[]); + engine.model_id = 9; + engine.prompt_tokens = vec![200000, 200053, 200043, 200001]; + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "inkling-audio", + CREATED_TEST, + false, + 1, + &mut out, + ) + .unwrap(); + assert_eq!(engine.live, [200000, 200053, 200053, 200043, 200001]); + for tokens in [vec![200000, 200001], vec![200053, 200053]] { + engine.prompt_tokens = tokens; + assert!(generate_and_write( + &mut engine, + &parsed, + "missing-audio", + CREATED_TEST, + false, + 1, + &mut out + ) + .is_err()); + } + parsed.images.push(RequestImage { + mime: ImageMime::Png, + data: Arc::from([1u8]), + }); + parsed.messages[0].parts.push(ChatPart::Image(0)); + engine.prompt_tokens = vec![200053, 10, 200054, 200001]; + generate_and_write( + &mut engine, + &parsed, + "mixed-audio", + CREATED_TEST, + false, + 1, + &mut out, + ) + .unwrap(); + assert_eq!( + engine.live, + [vec![200053; 2], vec![10], vec![200054; 16], vec![200001]].concat() + ); +} + +#[test] +fn continued_store_is_best_effort_and_runs_before_sampling_without_final_catchup() { + let mut parsed = user_req(); + parsed.max_tokens = 2; + parsed.max_tokens_set = true; + let inner = ScriptedDecode::from_pieces(&[b"a", b"b"]); + let mut engine = PromptSyncDecode::new(inner, 0, 1); + engine.fail_continued = true; + let mut out = Vec::new(); + + let outcome = generate_and_write( + &mut engine, + &parsed, + "chatcmpl-continued-order", + CREATED_TEST, + false, + 2, + &mut out, + ) + .unwrap(); + + assert_eq!(outcome.finish, "length"); + assert_eq!(engine.continued_positions, [1, 1, 2]); + assert_eq!( + engine.events, + [ + "sync", + "continued", + "continued", + "sample", + "eval", + "continued", + "sample", + "eval", + ] + ); + assert_eq!(engine.pos(), 3); +} + +#[test] +fn scripted_buffered_openai_has_text_and_stop() { + let parsed = user_req(); + let mut engine = + ScriptedDecode::from_pieces(&TAPE_PLAIN.iter().map(|s| s.as_bytes()).collect::>()); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-1", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + let s = String::from_utf8(out).unwrap(); + assert!(s.starts_with("HTTP/1.1 200 OK"), "{s}"); + assert!(s.contains("Hello world."), "{s}"); + assert!(s.contains("\"finish_reason\":\"stop\""), "{s}"); + assert!(s.contains("\"object\":\"chat.completion\""), "{s}"); + assert!( + s.contains("\"cache_write_tokens\":1"), + "cold serial prompt should count as a KV write: {s}" + ); + assert!( + s.contains("\"timings\":{\"ttft_ms\":"), + "serial path should emit timings: {s}" + ); + assert!(s.contains("\"prefill_tokens\":1"), "{s}"); +} + +#[test] +fn scripted_responses_stream_activates_after_created() { + let parsed = parse_request( + WireSurface::Responses, + &env(), + r#"{"input":"hi","max_output_tokens":8,"stream":true}"#, + ) + .unwrap(); + let mut engine = ScriptedDecode::from_pieces( + &TAPE_PLAIN + .iter() + .map(|piece| piece.as_bytes()) + .collect::>(), + ); + let mut out = Vec::new(); + + generate_and_write( + &mut engine, + &parsed, + "resp-serial-stream", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + + let out = String::from_utf8(out).unwrap(); + assert!(out.contains("\"type\":\"response.created\""), "{out}"); + assert!( + out.contains("\"type\":\"response.output_text.delta\""), + "{out}" + ); + assert!(out.contains("\"type\":\"response.completed\""), "{out}"); +} + +#[test] +fn prompt_sync_reports_buffered_cache_usage_from_effective_pos() { + let parsed = user_req(); + let inner = + ScriptedDecode::from_pieces(&TAPE_PLAIN.iter().map(|s| s.as_bytes()).collect::>()); + let mut engine = PromptSyncDecode::new(inner, 4, 6); + engine.prompt_sync_elapsed = Some(Duration::from_secs(2)); + let mut out = Vec::new(); + + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-cache-buffered", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + + let s = String::from_utf8(out).unwrap(); + assert!(s.contains("\"prompt_tokens\":6"), "{s}"); + assert!( + s.contains("\"cached_tokens\":4,\"cache_write_tokens\":2"), + "cache writes must use effective engine pos minus cache reads: {s}" + ); + assert!(s.contains("\"prefill_tokens\":2"), "{s}"); + assert!(s.contains("\"prefill_cached_tokens\":4"), "{s}"); + assert!(s.contains("\"prefill_tok_s\":1.0"), "{s}"); + assert_eq!(engine.prompt_sync_calls, 1); + assert_eq!(engine.sync_calls, 0); + assert_eq!(engine.disk_eligible, [true]); + assert_eq!(engine.thinking_visible_eligible, [true]); +} + +#[test] +fn prompt_sync_reports_streaming_cache_usage_from_effective_pos() { + let mut parsed = user_req(); + parsed.stream = true; + parsed.stream_include_usage = true; + let inner = + ScriptedDecode::from_pieces(&TAPE_PLAIN.iter().map(|s| s.as_bytes()).collect::>()); + let mut engine = PromptSyncDecode::new(inner, 4, 6); + let mut out = Vec::new(); + + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-cache-stream", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + + let s = String::from_utf8(out).unwrap(); + assert!(s.contains("\"prompt_tokens\":6"), "{s}"); + assert!( + s.contains("\"cached_tokens\":4,\"cache_write_tokens\":2"), + "stream usage must use effective engine pos minus cache reads: {s}" + ); + assert_eq!(engine.prompt_sync_calls, 1); + assert_eq!(engine.sync_calls, 0); + assert_eq!(engine.disk_eligible, [true]); + assert_eq!(engine.thinking_visible_eligible, [true]); +} + +#[test] +fn streaming_prompt_failure_is_an_sse_error_not_a_second_http_response() { + let mut parsed = user_req(); + parsed.stream = true; + let inner = ScriptedDecode::from_pieces(&[b"unused"]); + let mut engine = PromptSyncDecode::new(inner, 0, 1); + engine.fail_prompt_sync = true; + let mut out = Vec::new(); + + let error = generate_and_write( + &mut engine, + &parsed, + "chatcmpl-sync-fail", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap_err(); + + assert!(matches!(error, GenerateError::Streamed(_))); + let wire = String::from_utf8(out).unwrap(); + assert!(wire.starts_with("HTTP/1.1 200 OK\r\n"), "{wire}"); + assert_eq!(wire.matches("HTTP/1.1").count(), 1, "{wire}"); + assert!(wire.contains("event: error\ndata:"), "{wire}"); + assert!(wire.contains("injected prompt sync failure"), "{wire}"); +} + +#[test] +fn prompt_sync_receives_thinking_visible_surface_gate() { + let cases = [ + ( + WireSurface::OpenaiChat, + r#"{"messages":[{"role":"user","content":"hi"}]}"#, + true, + ), + (WireSurface::OpenaiCompletion, r#"{"prompt":"hi"}"#, false), + ( + WireSurface::Anthropic, + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":8}"#, + true, + ), + (WireSurface::Responses, r#"{"input":"hi"}"#, false), + ]; + + for (surface, body, expected) in cases { + let parsed = parse_request(surface, &env(), body).unwrap(); + let inner = ScriptedDecode::from_pieces(&[b"ok"]); + let mut engine = PromptSyncDecode::new(inner, 0, 1); + let mut out = Vec::new(); + + generate_and_write( + &mut engine, + &parsed, + "visible-surface", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + + assert_eq!(engine.thinking_visible_eligible, [expected], "{surface:?}"); + } +} + +#[test] +fn motif3_no_think_remembers_canonical_visible_checkpoint() { + let parsed = user_req(); + let inner = ScriptedDecode { + model_id: 3, + ..ScriptedDecode::from_pieces(&[b" Clear skies. "]) + }; + let mut engine = PromptSyncDecode::new(inner, 0, 1); + let mut out = Vec::new(); + + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-visible", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + + let mut expected = render_prompt(&parsed, 3).unwrap(); + assert!(expected.ends_with(b"")); + expected.truncate(expected.len() - b"".len()); + expected.extend_from_slice(b"Clear skies."); + assert_eq!(engine.remembered, [(expected, engine.pos())]); + assert!(!engine.remembered[0].0.ends_with(b"<|endofturn|>")); + + let mut length = parsed; + length.max_tokens = 1; + length.max_tokens_set = true; + let inner = ScriptedDecode { + model_id: 3, + ..ScriptedDecode::from_pieces(&[b"partial"]) + }; + let mut engine = PromptSyncDecode::new(inner, 0, 1); + let mut out = Vec::new(); + let outcome = generate_and_write( + &mut engine, + &length, + "chatcmpl-visible-length", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + assert_eq!(outcome.finish, "length"); + assert!(engine.remembered.is_empty()); +} + +#[test] +fn serial_thinking_answer_remembers_visible_history_for_all_keyable_formats() { + for (model_id, close) in [ + (0, b"".as_slice()), + (2, b"<|think:end|>".as_slice()), + (4, b"".as_slice()), + (6, b"".as_slice()), + ] { + let mut parsed = user_req(); + parsed.think_mode = ThinkMode::Low; + let pieces = [b"private plan".as_slice(), close, b"answer".as_slice()]; + let inner = ScriptedDecode { + model_id, + ..ScriptedDecode::from_pieces(&pieces) + }; + let mut engine = PromptSyncDecode::new(inner, 0, 1); + let mut out = Vec::new(); + + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-thinking-visible", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + + let checkpoint = &engine + .remembered + .first() + .unwrap_or_else(|| panic!("model {model_id} did not remember visible history")) + .0; + let mut future = parsed.clone(); + future.messages.push(ChatMsg { + role: "assistant".into(), + content: "answer".into(), + ..ChatMsg::default() + }); + future.messages.push(ChatMsg { + role: "user".into(), + content: "next".into(), + ..ChatMsg::default() + }); + let future_prompt = render_prompt(&future, model_id).unwrap(); + assert!( + future_prompt.starts_with(checkpoint), + "model {model_id} visible checkpoint is not a future-prompt prefix" + ); + } +} + +#[test] +fn motif3_no_think_invalidates_user_stop_and_tool_syntax_cut() { + let cases = [ + (vec!["STOP".into()], b"Clear STOP tail".as_slice()), + (Vec::new(), b"Clear ".as_slice()), + ]; + + for (stops, piece) in cases { + let mut parsed = user_req(); + parsed.stops = stops; + let inner = ScriptedDecode { + model_id: 3, + ..ScriptedDecode::from_pieces(&[piece]) + }; + let mut engine = PromptSyncDecode::new(inner, 0, 1); + let mut out = Vec::new(); + + let outcome = generate_and_write( + &mut engine, + &parsed, + "chatcmpl-visible-cut", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + + assert_eq!(outcome.finish, "stop"); + assert_eq!(engine.invalidations, 1); + assert_eq!(engine.pos(), 0); + assert!(engine.remembered.iter().all(|(_, frontier)| *frontier == 0)); + } +} + +/// P1 gate: drive one reuse situation through the HTTP door and read the +/// trace back from `/v1/stats`, the way an operator would. +fn http_reuse_trace( + reuse: ds4_core::ReuseTaken, + miss: ds4_core::ReuseMiss, + cached: i32, +) -> serde_json::Value { + let mut cfg = ServerConfig::test_cfg(); + cfg.model_id = "ds4".into(); + cfg.model_name = "ds4".into(); + cfg.default_tokens = 16; + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + + let mut engine = PromptSyncDecode::new(ScriptedDecode::from_pieces(&[b"ok"]), cached, 1); + engine.reuse = reuse; + engine.miss = miss; + let body = r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#; + let request = format!( + "POST /v1/chat/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let answer = one_shot_inner(&cfg, &inner, Some(&mut engine), request.as_bytes()); + assert!( + answer.starts_with("HTTP/1.1 200 OK"), + "generation failed: {answer}" + ); + + let stats = one_shot_inner(&cfg, &inner, None, b"GET /v1/stats HTTP/1.1\r\n\r\n"); + let start = stats.find("{\"routes\"").expect(&stats); + serde_json::from_str(stats[start..].trim()).expect(&stats[start..]) +} + +fn one_shot_inner( + cfg: &ServerConfig, + inner: &Mutex, + engine: Option<&mut dyn DecodeIo>, + request: &[u8], +) -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let mut out = Vec::new(); + thread::scope(|scope| { + let client = scope.spawn(move || { + let mut c = TcpStream::connect(addr).unwrap(); + c.write_all(request).unwrap(); + let _ = c.shutdown(std::net::Shutdown::Write); + let mut buf = Vec::new(); + c.read_to_end(&mut buf).unwrap(); + buf + }); + let (mut server, _) = listener.accept().unwrap(); + handle_client_inner(cfg, inner, &mut server, engine, None); + drop(server); + out = client.join().unwrap(); + }); + String::from_utf8_lossy(&out).into_owned() +} + +/// The four P1 situations, each read back from the HTTP surface. `fork` +/// only happens on the native bank lane, so it is covered by the +/// admission unit tests instead. +#[test] +fn http_reports_reuse_situations() { + // Same chat, append: reuse at the frontier, suffix prefilled. + let body = http_reuse_trace(ds4_core::ReuseTaken::Exact, ds4_core::ReuseMiss::None, 3); + assert_eq!(body["last_request"]["reuse_kind"], "exact"); + assert!( + body["last_request"].get("reuse_miss").is_none(), + "a request that refused nothing carries no miss member: {body}" + ); + + // Edit or branch: a checkpoint below the prefix, gap replayed. + let body = http_reuse_trace(ds4_core::ReuseTaken::Partial, ds4_core::ReuseMiss::None, 2); + assert_eq!(body["last_request"]["reuse_kind"], "partial"); + + // Restart with a template that re-rendered: refused, and it says why. + let body = http_reuse_trace( + ds4_core::ReuseTaken::Cold, + ds4_core::ReuseMiss::RenderedPrefix, + 0, + ); + assert_eq!(body["last_request"]["reuse_kind"], "cold"); + assert_eq!( + body["last_request"]["reuse_miss"], + "rendered prefix changed" + ); + + // Restart that found its payload. + let body = http_reuse_trace(ds4_core::ReuseTaken::Exact, ds4_core::ReuseMiss::None, 5); + assert_eq!(body["last_request"]["reuse_kind"], "exact"); + assert_eq!(body["last_request"]["effective_lane"], "serial"); +} + +#[test] +fn scripted_http_door_generates() { + let mut cfg = ServerConfig::test_cfg(); + cfg.model_id = "ds4".into(); + cfg.model_name = "ds4".into(); + cfg.default_tokens = 16; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let h = thread::spawn(move || { + let (mut s, _) = listener.accept().unwrap(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let mut engine = ScriptedDecode::from_pieces(&[b"ok"]); + handle_client_inner(&cfg, &inner, &mut s, Some(&mut engine), None); + }); + let mut c = TcpStream::connect(addr).unwrap(); + let body = r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#; + let req = format!( + "POST /v1/chat/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + c.write_all(req.as_bytes()).unwrap(); + let _ = c.shutdown(std::net::Shutdown::Write); + let mut out = Vec::new(); + c.read_to_end(&mut out).unwrap(); + h.join().unwrap(); + let s = String::from_utf8_lossy(&out); + assert!(s.starts_with("HTTP/1.1 200 OK"), "{s}"); + assert!(s.contains("ok"), "{s}"); +} + +#[test] +fn scripted_motif_generates_over_http() { + let cfg = ServerConfig::test_cfg(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let h = thread::spawn(move || { + let (mut s, _) = listener.accept().unwrap(); + let inner = Mutex::new(ServerInner::from_cfg(&cfg)); + let mut engine = ScriptedDecode { + model_id: 3, + ..ScriptedDecode::from_pieces(&[b"ok"]) + }; + handle_client_inner(&cfg, &inner, &mut s, Some(&mut engine), None); + }); + let mut c = TcpStream::connect(addr).unwrap(); + let body = r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#; + let req = format!( + "POST /v1/chat/completions HTTP/1.1\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + c.write_all(req.as_bytes()).unwrap(); + let _ = c.shutdown(std::net::Shutdown::Write); + let mut out = Vec::new(); + c.read_to_end(&mut out).unwrap(); + h.join().unwrap(); + let s = String::from_utf8_lossy(&out); + assert!(s.starts_with("HTTP/1.1 200 OK"), "{s}"); + assert!(s.contains("ok"), "{s}"); +} + +#[test] +fn cont_stepper_buffered_matches_serial_shape() { + let mut parsed = user_req(); + parsed.stream = false; + let (mut st, head) = ContStepper::new( + &parsed, + 0, + "chatcmpl-1", + CREATED_TEST, + false, + 16, + b"prompt".to_vec(), + 1, + 8192, + ); + assert!(head.is_empty(), "buffered request must not stream a head"); + for p in TAPE_PLAIN { + let step = st.feed(p.as_bytes()); + assert!(step.bytes.is_empty()); + assert!(!step.done); + } + let (bytes, outcome) = st.finalize(true, 0, 1, ReqTimings::default(), false); + let s = String::from_utf8_lossy(&bytes); + assert!(s.starts_with("HTTP/1.1 200 OK"), "{s}"); + assert!(s.contains("Hello world."), "{s}"); + assert!(s.contains("\"finish_reason\":\"stop\""), "{s}"); + assert!( + s.contains("\"cached_tokens\":0,\"cache_write_tokens\":1"), + "engine split maps into the client frame: {s}" + ); + assert_eq!(outcome.finish, "stop"); +} + +#[test] +fn cont_stepper_streams_and_stops_on_budget() { + let mut parsed = user_req(); + parsed.stream = true; + parsed.max_tokens = 2; + parsed.max_tokens_set = true; + let (mut st, head) = ContStepper::new( + &parsed, + 0, + "chatcmpl-2", + CREATED_TEST, + false, + 16, + b"prompt".to_vec(), + 1, + 8192, + ); + let h = String::from_utf8_lossy(&head); + assert!(h.contains("text/event-stream"), "{h}"); + assert!(h.contains("chat.completion.chunk"), "{h}"); + let first = st.feed(TAPE_PLAIN[0].as_bytes()); + assert!(!first.done); + let second = st.feed(TAPE_PLAIN[1].as_bytes()); + assert!(second.done, "host budget must stop the sequence"); + let (bytes, outcome) = st.finalize(false, 0, 1, ReqTimings::default(), false); + let s = String::from_utf8_lossy(&bytes); + assert!(s.contains("\"finish_reason\":\"length\""), "{s}"); + assert!(s.contains("data: [DONE]"), "{s}"); + assert_eq!(outcome.finish, "length"); +} + +fn think_req(surface: WireSurface, stream: bool) -> ParsedRequest { + let body = match surface { + WireSurface::Responses => r#"{"input":"hi","max_output_tokens":32}"#, + _ => r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":32}"#, + }; + let mut parsed = parse_request(surface, &env(), body).unwrap(); + parsed.stream = stream; + parsed.think_mode = ThinkMode::Low; + parsed.reasoning_summary_emit = true; + parsed.temperature = 0.0; + parsed.max_tokens = 32; + parsed.max_tokens_set = true; + parsed +} + +fn step_think( + parsed: &ParsedRequest, + piece: &[u8], + stop: bool, + engine_eos: bool, +) -> (String, String) { + let (mut st, head) = ContStepper::new( + parsed, + 0, + "job-think", + CREATED_TEST, + false, + 32, + b"".to_vec(), + 1, + 8192, + ); + let step = st.feed(piece); + if stop { + st.mark_stop(); + } + let (tail, outcome) = st.finalize(engine_eos, 0, 1, ReqTimings::default(), false); + let mut bytes = head; + bytes.extend(step.bytes); + bytes.extend(tail); + (String::from_utf8(bytes).unwrap(), outcome.finish) +} + +#[test] +fn open_think_eos_reports_stop() { + let parsed = think_req(WireSurface::OpenaiChat, true); + let (s, finish) = step_think(&parsed, b"like `", true, true); + assert_eq!(finish, "stop"); + assert!(s.contains("\"reasoning_content\":\"like `\""), "{s}"); + assert!(s.contains("\"finish_reason\":\"stop\""), "{s}"); + assert!(!s.contains("\"finish_reason\":\"length\""), "{s}"); +} + +#[test] +fn open_think_budget_stays_length() { + let mut parsed = think_req(WireSurface::OpenaiChat, true); + parsed.max_tokens = 1; + let (s, finish) = step_think(&parsed, b"like `", false, false); + assert_eq!(finish, "length"); + assert!(s.contains("\"finish_reason\":\"length\""), "{s}"); +} + +#[test] +fn closed_think_eos_reports_stop() { + let parsed = think_req(WireSurface::OpenaiChat, true); + let (s, finish) = step_think(&parsed, b"planAnswer", true, true); + assert_eq!(finish, "stop"); + assert!(s.contains("\"finish_reason\":\"stop\""), "{s}"); + assert!(s.contains("\"reasoning_content\":\"plan\""), "{s}"); + assert!(s.contains("\"content\":\"Answer\""), "{s}"); +} + +#[test] +fn open_think_eos_responses_keeps_completed() { + let parsed = think_req(WireSurface::Responses, true); + let (s, finish) = step_think(&parsed, b"like `", true, true); + assert_eq!(finish, "stop"); + assert!(s.contains("\"type\":\"response.completed\""), "{s}"); + assert!( + s.contains("\"type\":\"reasoning\",\"status\":\"incomplete\""), + "{s}" + ); + assert!(!s.contains("incomplete_details"), "{s}"); + assert!(!s.contains("max_output_tokens"), "{s}"); +} + +#[test] +fn open_think_eos_responses_buffered_item_stays_incomplete() { + let parsed = think_req(WireSurface::Responses, false); + let (s, finish) = step_think(&parsed, b"like `", true, true); + assert_eq!(finish, "stop"); + assert!(s.contains("\"status\":\"completed\""), "{s}"); + assert!( + s.contains("\"type\":\"reasoning\",\"status\":\"incomplete\""), + "{s}" + ); + assert!(!s.contains("incomplete_details"), "{s}"); +} + +#[test] +fn closed_think_eos_responses_item_completed() { + let parsed = think_req(WireSurface::Responses, true); + let (s, finish) = step_think(&parsed, b"planAnswer", true, true); + assert_eq!(finish, "stop"); + assert!(s.contains("\"type\":\"response.completed\""), "{s}"); + assert!( + s.contains("\"type\":\"reasoning\",\"status\":\"completed\""), + "{s}" + ); +} + +#[test] +fn cont_stepper_streams_anthropic_events() { + let parsed = parse_request( + WireSurface::Anthropic, + &env(), + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":8,"stream":true}"#, + ) + .unwrap(); + let (mut stepper, head) = ContStepper::new( + &parsed, + 0, + "msg-cont-anthropic", + CREATED_TEST, + false, + 16, + b"prompt".to_vec(), + 1, + 8192, + ); + + let head = String::from_utf8(head).unwrap(); + assert!(head.contains("event: message_start"), "{head}"); + let mut deltas = Vec::new(); + for piece in TAPE_PLAIN { + deltas.extend(stepper.feed(piece.as_bytes()).bytes); + } + let (tail, outcome) = stepper.finalize(true, 0, 1, ReqTimings::default(), false); + let deltas = String::from_utf8(deltas).unwrap(); + let tail = String::from_utf8(tail).unwrap(); + assert!(deltas.contains("event: content_block_delta"), "{deltas}"); + assert!(tail.contains("event: message_stop"), "{tail}"); + assert_eq!(outcome.finish, "stop"); +} + +#[test] +fn cont_stepper_buffers_anthropic_message() { + let parsed = parse_request( + WireSurface::Anthropic, + &env(), + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":8}"#, + ) + .unwrap(); + let (mut stepper, head) = ContStepper::new( + &parsed, + 0, + "msg-cont-anthropic-buffered", + CREATED_TEST, + false, + 16, + b"prompt".to_vec(), + 1, + 8192, + ); + + assert!(head.is_empty()); + for piece in TAPE_PLAIN { + assert!(stepper.feed(piece.as_bytes()).bytes.is_empty()); + } + let (body, outcome) = stepper.finalize(true, 0, 1, ReqTimings::default(), false); + let body = String::from_utf8(body).unwrap(); + assert!(body.contains("\"type\":\"message\""), "{body}"); + assert!(body.contains("\"text\":\"Hello world.\""), "{body}"); + assert!(body.contains("\"stop_reason\":\"end_turn\""), "{body}"); + assert_eq!(outcome.finish, "stop"); +} + +#[test] +fn cont_stepper_streams_responses_events() { + let parsed = parse_request( + WireSurface::Responses, + &env(), + r#"{"input":"hi","max_output_tokens":8,"stream":true}"#, + ) + .unwrap(); + let (mut stepper, head) = ContStepper::new( + &parsed, + 0, + "resp-cont-stream", + CREATED_TEST, + false, + 16, + b"prompt".to_vec(), + 1, + 8192, + ); + + let head = String::from_utf8(head).unwrap(); + assert!(head.contains("\"type\":\"response.created\""), "{head}"); + let mut deltas = Vec::new(); + for piece in TAPE_PLAIN { + deltas.extend(stepper.feed(piece.as_bytes()).bytes); + } + let (tail, outcome) = stepper.finalize(true, 0, 1, ReqTimings::default(), false); + let deltas = String::from_utf8(deltas).unwrap(); + let tail = String::from_utf8(tail).unwrap(); + assert!( + deltas.contains("\"type\":\"response.output_text.delta\""), + "{deltas}" + ); + assert!(tail.contains("\"type\":\"response.completed\""), "{tail}"); + assert!(tail.contains("resp_"), "{tail}"); + assert_eq!(outcome.finish, "stop"); +} + +#[test] +fn cont_stepper_buffers_responses_object() { + let parsed = parse_request( + WireSurface::Responses, + &env(), + r#"{"input":"hi","max_output_tokens":8}"#, + ) + .unwrap(); + let (mut stepper, head) = ContStepper::new( + &parsed, + 0, + "resp-cont-buffered", + CREATED_TEST, + false, + 16, + b"prompt".to_vec(), + 1, + 8192, + ); + + assert!(head.is_empty()); + for piece in TAPE_PLAIN { + assert!(stepper.feed(piece.as_bytes()).bytes.is_empty()); + } + let (body, outcome) = stepper.finalize(true, 0, 1, ReqTimings::default(), false); + let body = String::from_utf8(body).unwrap(); + assert!(body.contains("\"object\":\"response\""), "{body}"); + assert!(body.contains("\"type\":\"output_text\""), "{body}"); + assert!(body.contains("Hello world."), "{body}"); + assert!(body.contains("resp_"), "{body}"); + assert_eq!(outcome.finish, "stop"); +} + +#[test] +fn responses_counts_tokens_generated_inside_reasoning_on_both_lanes() { + let parsed = parse_request( + WireSurface::Responses, + &env(), + r#"{"input":"hi","max_output_tokens":8,"reasoning":{"effort":"high","summary":"auto"}}"#, + ) + .unwrap(); + let (mut stepper, _) = ContStepper::new( + &parsed, + 0, + "resp-cont-reasoning-usage", + CREATED_TEST, + false, + 16, + b"".to_vec(), + 1, + 8192, + ); + + stepper.feed(b"hidden"); + stepper.feed(b""); + stepper.feed(b"answer"); + let (body, _) = stepper.finalize(true, 0, 1, ReqTimings::default(), false); + let body = String::from_utf8(body).unwrap(); + + assert!(body.contains("\"reasoning_tokens\":2"), "{body}"); + + let mut engine = ScriptedDecode::from_pieces(&[b"hidden", b"", b"answer"]); + let mut body = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "resp-serial-reasoning-usage", + CREATED_TEST, + false, + 16, + &mut body, + ) + .unwrap(); + let body = String::from_utf8(body).unwrap(); + assert!(body.contains("\"reasoning_tokens\":2"), "{body}"); +} + +#[test] +fn cont_stepper_stream_tool_id_matches_outcome() { + let mut parsed = tools_req(); + parsed.stream = true; + let (mut stepper, _) = ContStepper::new( + &parsed, + 0, + "chatcmpl-cont-tool", + CREATED_TEST, + false, + 64, + b"prompt".to_vec(), + 1, + 8192, + ); + let block = concat!( + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"bash\">\n", + "<|DSML|parameter name=\"command\" string=\"true\">ls", + "\n", + "\n", + "" + ); + let streamed = stepper.feed(block.as_bytes()); + assert!(streamed.done); + let (terminal, outcome) = stepper.finalize(false, 0, 1, ReqTimings::default(), false); + assert_eq!(outcome.tool_ids.len(), 1); + let id = &outcome.tool_ids[0]; + assert!(id.starts_with("call_")); + assert_eq!(id.len(), 37); + assert!(String::from_utf8_lossy(&streamed.bytes).contains(id)); + assert!(String::from_utf8_lossy(&terminal).contains("data: [DONE]")); +} + +#[test] +fn bridge_null_oracle_ok() { + let p = if let Ok(v) = std::env::var("DS4_BRIDGE_NULL_ORACLE") { + PathBuf::from(v) + } else { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/bridge_null_oracle") + }; + assert!( + p.exists(), + "build the C oracle first: make tests/parity/bridge_null_oracle (missing {})", + p.display() + ); + let out = Command::new(&p).output().expect("run bridge_null_oracle"); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(out.stdout, b"ok\n"); +} + +#[test] +fn scripted_dsml_tools_emit_tool_calls() { + let mut parsed = parse_request( + WireSurface::OpenaiChat, + &env(), + r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"},"max_tokens":16,"tools":[{"type":"function","function":{"name":"bash","parameters":{"type":"object","properties":{"command":{"type":"string"}}}}}]}"#, + ) + .unwrap(); + parsed.think_mode = ThinkMode::None; + parsed.temperature = 0.0; + assert!(parsed.has_tools); + + let block = concat!( + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"bash\">\n", + "<|DSML|parameter name=\"command\" string=\"true\">ls", + "\n", + "\n", + "" + ); + let mut engine = ScriptedDecode::from_pieces(&[block.as_bytes()]); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-tools", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + let s = String::from_utf8(out).unwrap(); + assert!(s.starts_with("HTTP/1.1 200 OK"), "{s}"); + assert!(s.contains("\"finish_reason\":\"tool_calls\""), "{s}"); + assert!(s.contains("\"tool_calls\":["), "{s}"); + assert!(s.contains("\"name\":\"bash\""), "{s}"); + let id = s + .split("\"tool_calls\":[{\"id\":\"") + .nth(1) + .unwrap() + .split('"') + .next() + .unwrap(); + assert_eq!(id.len(), 37, "{s}"); + assert!(id.starts_with("call_"), "{s}"); + assert!( + id[5..] + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + "{s}" + ); + assert!(s.contains("ls"), "{s}"); +} + +#[test] +fn dsml_inside_closed_thinking_is_not_a_tool_call() { + // The engine's rule (server_dsml_parse.c:291-316): with thinking on, a + // DSML block inside the reasoning section is model text, never a tool + // call -- the tool search starts after the last response. The route's + // tolerant fallback must scope the same way, or the finish serves a + // call the assistant never emitted as its post-thinking action. + let mut parsed = parse_request( + WireSurface::OpenaiChat, + &env(), + r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"enabled"},"max_tokens":64,"tools":[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]}"#, + ) + .unwrap(); + parsed.think_mode = ThinkMode::Low; + parsed.temperature = 0.0; + assert!(parsed.has_tools); + + let think_open = think_start(ChatFormat::DeepSeek); + let text = format!( + "{think_open}the user wants the weather\n{}\n{} name=\"get_weather\">\n\ + {} name=\"city\" string=\"true\">Paris{}\n{}\n{}\n\n\n\nHello.", + DSML_TOOL_CALLS_START, + DSML_INVOKE_START, + DSML_PARAM_START, + DSML_PARAM_END, + DSML_INVOKE_END, + DSML_TOOL_CALLS_END + ); + // ScriptedV41: the route must take generate_terminal_v41, the only + // path with the tolerant fallback (a plain scripted engine never + // reaches it and the test would pass vacuously). + let mut engine = ScriptedV41::new(&[(1, text.as_str()), (99, "")], 1 << 20); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-think-tools", + CREATED_TEST, + false, + 64, + &mut out, + ) + .unwrap(); + let s = String::from_utf8(out).unwrap(); + assert!(s.starts_with("HTTP/1.1 200 OK"), "{s}"); + assert!(!s.contains("\"tool_calls\""), "{s}"); + assert!(s.contains("\"finish_reason\":\"stop\""), "{s}"); +} + +#[test] +fn tool_replay_restores_raw_dsml_before_render_and_uses_scoped_sync() { + let parsed = parse_request( + WireSurface::OpenaiChat, + &env(), + r#"{"messages":[{"role":"user","content":"run"},{"role":"assistant","tool_calls":[{"id":"call_saved","type":"function","function":{"name":"bash","arguments":"{\"a\":1,\"b\":2}"}}]},{"role":"tool","tool_call_id":"call_saved","content":"ok"},{"role":"assistant","content":"finished"},{"role":"user","content":"next"}],"thinking":{"type":"disabled"},"tools":[{"type":"function","function":{"name":"bash","parameters":{"type":"object","properties":{"a":{"type":"integer"},"b":{"type":"integer"}}}}}]}"#, + ) + .unwrap(); + assert!(!parsed.has_tool_results); + let canonical = render_prompt(&parsed, 0).unwrap(); + let raw = concat!( + "\n\n<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"bash\">\n", + "<|DSML|parameter name=\"b\" string=\"false\">2\n", + "<|DSML|parameter name=\"a\" string=\"false\">1\n", + "\n", + "" + ); + assert!(!canonical + .windows(raw.len()) + .any(|window| window == raw.as_bytes())); + let inner = ScriptedDecode::from_pieces(&[b"done"]); + let mut engine = PromptSyncDecode::new(inner, 1, 1); + engine.replay_raw = Some(raw.into()); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-replay", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap(); + + assert_eq!(engine.replay_prompts.len(), 1); + assert!(engine.replay_prompts[0] + .windows(raw.len()) + .any(|window| window == raw.as_bytes())); + let restore = engine + .events + .iter() + .position(|event| *event == "restore") + .unwrap(); + let sync = engine + .events + .iter() + .position(|event| *event == "tool-sync") + .unwrap(); + let sample = engine + .events + .iter() + .position(|event| *event == "sample") + .unwrap(); + assert!(restore < sync && sync < sample); +} + +#[test] +fn tool_producer_remembers_final_wire_ids_with_sampled_dsml() { + let parsed = tools_req(); + let raw = concat!( + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"bash\">\n", + "<|DSML|parameter name=\"command\" string=\"true\">ls", + "\n", + "\n", + "" + ); + let inner = ScriptedDecode::from_pieces(&[raw.as_bytes()]); + let mut engine = PromptSyncDecode::new(inner, 0, 1); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-producer", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + + assert_eq!(engine.remembered_tools.len(), 1); + let (ids, remembered) = &engine.remembered_tools[0]; + assert_eq!(ids.len(), 1); + assert!(ids[0].starts_with("call_")); + assert_eq!(remembered, raw); + let remember = engine + .events + .iter() + .position(|event| *event == "remember-tool") + .unwrap(); + let sample = engine + .events + .iter() + .position(|event| *event == "sample") + .unwrap(); + assert!(sample < remember); +} + +fn tools_req() -> ParsedRequest { + let mut parsed = parse_request( + WireSurface::OpenaiChat, + &env(), + r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"},"max_tokens":16,"tools":[{"type":"function","function":{"name":"bash","parameters":{"type":"object","properties":{"command":{"type":"string"}}}}}]}"#, + ) + .unwrap(); + parsed.think_mode = ThinkMode::None; + parsed.temperature = 0.0; + parsed +} + +fn retrying_tool_decode() -> ScriptedDecode { + let invalid = concat!( + "<|DSML|tool_calls>\n", + "<|DSML|invoke>\n", + "\n", + "" + ); + let valid = concat!( + "<|DSML|tool_calls>\n", + "<|DSML|invoke name=\"bash\">\n", + "<|DSML|parameter name=\"command\" string=\"true\">ls", + "\n", + "\n", + "" + ); + ScriptedDecode { + steps: vec![ + ScriptedStep { + token: 1, + piece: invalid.as_bytes().to_vec(), + stop: false, + }, + ScriptedStep { + token: 3, + piece: valid.as_bytes().to_vec(), + stop: false, + }, + ScriptedStep { + token: 4, + piece: Vec::new(), + stop: true, + }, + ], + suffix_tokens: vec![10, 11], + ..ScriptedDecode::from_pieces(&[b"x"]) + } +} + +#[test] +fn scripted_invalid_dsml_retries_and_emits_tool_calls() { + let parsed = tools_req(); + let mut engine = retrying_tool_decode(); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-retry", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + let s = String::from_utf8(out).unwrap(); + assert!(s.starts_with("HTTP/1.1 200 OK"), "{s}"); + assert!(s.contains("\"finish_reason\":\"tool_calls\""), "{s}"); + assert!(s.contains("\"name\":\"bash\""), "{s}"); + assert!(s.contains("ls"), "{s}"); + assert!( + engine.idx >= 2, + "second decode pass should consume the valid call" + ); +} + +#[test] +fn recovery_suffix_uses_sync_not_prompt_sync() { + let parsed = tools_req(); + let mut engine = PromptSyncDecode::new(retrying_tool_decode(), 0, 1); + let mut out = Vec::new(); + + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-cache-retry", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + + assert_eq!( + engine.prompt_sync_calls, 1, + "only the initial prompt uses the hook" + ); + assert_eq!( + engine.sync_calls, 1, + "the recovery suffix uses ordinary sync" + ); + assert_eq!(engine.disk_eligible, [false]); + assert!( + engine.inner.idx >= 2, + "the retry must run a second decode pass" + ); +} + +#[test] +fn scripted_motif_does_not_retry_invalid_tools() { + let parsed = tools_req(); + let invalid = concat!( + "<|DSML|tool_calls>\n", + "<|DSML|invoke>\n", + "\n", + "" + ); + let mut engine = ScriptedDecode { + model_id: 3, + steps: vec![ + ScriptedStep { + token: 1, + piece: invalid.as_bytes().to_vec(), + stop: false, + }, + ScriptedStep { + token: 2, + piece: Vec::new(), + stop: true, + }, + ScriptedStep { + token: 3, + piece: b"should-not-run".to_vec(), + stop: false, + }, + ], + ..ScriptedDecode::from_pieces(&[b"x"]) + }; + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-motif", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + let s = String::from_utf8(out).unwrap(); + assert!(s.starts_with("HTTP/1.1 200 OK"), "{s}"); + assert!(!s.contains("should-not-run"), "{s}"); + assert!(s.contains("DSML"), "{s}"); + assert_eq!(engine.idx, 1, "Motif must not consume a second decode pass"); +} + +fn inkling_retry_case(invalid: &[u8], closure: &str) { + let parsed = tools_req(); + let valid = br#"bash<|content_invoke_tool_json|>{"name":"bash","args":{"command":"ls"}}<|end_message|>"#; + let mut tape = ScriptedDecode::from_pieces(&[invalid, valid]); + tape.model_id = 9; + tape.steps.insert( + 1, + ScriptedStep { + token: 98, + piece: Vec::new(), + stop: true, + }, + ); + let mut engine = PromptSyncDecode::new(tape, 0, 1); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "inkling-retry", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + let response = String::from_utf8(out).unwrap(); + assert_eq!( + engine.sync_calls, 1, + "exactly one corrective retry: {response}" + ); + let rendered = engine.rendered.borrow(); + let suffix = String::from_utf8(rendered.last().unwrap().clone()).unwrap(); + assert!( + suffix.starts_with(&format!( + "{closure}<|message_tool|><|content_text|>Tool error: invalid Inkling tool call" + )), + "{suffix}" + ); + assert!( + suffix.ends_with("<|end_message|><|message_model|>"), + "{suffix}" + ); + assert!(!suffix.contains("DSML"), "{suffix}"); + assert!( + response.contains("\"finish_reason\":\"tool_calls\""), + "{response}" + ); + assert!(response.contains("\"name\":\"bash\""), "{response}"); + assert!(!response.contains("Tool error:"), "{response}"); +} + +#[test] +fn inkling_unterminated_retry() { + inkling_retry_case( + br#"bash<|content_invoke_tool_json|>{"name":"bash","args":{"command":"ls"}"#, + "<|end_message|><|content_model_end_sampling|>", + ); +} + +#[test] +fn jinja_retry_renders_full_chat() { + use ds4_core::chat_template::{RenderClock, Template}; + let parsed = tools_req(); + let bad = br#"bash<|content_invoke_tool_json|>{"name":"bash","args":[]}<|end_message|>"#; + let good = br#"bash<|content_invoke_tool_json|>{"name":"bash","args":{"command":"ls"}}<|end_message|>"#; + let mut tape = ScriptedDecode::from_pieces(&[bad, good]); + tape.model_id = 9; + tape.steps.insert( + 1, + ScriptedStep { + token: 98, + piece: Vec::new(), + stop: true, + }, + ); + let mut engine = PromptSyncDecode::new(tape, 0, 1); + engine.template = Some( + Template::compile( + include_str!( + "../../../tests/fixtures/chat-template/models/inkling/chat_template.jinja" + ), + RenderClock::Fixed(0), + ) + .unwrap(), + ); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "jinja-retry", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + let renders = engine.rendered.borrow(); + assert_eq!(renders.len(), 2); + let retry = String::from_utf8_lossy(&renders[1]); + assert!(retry.starts_with("<|message_system|>"), "{retry}"); + assert!(retry.contains("Tool error:"), "{retry}"); + assert!( + retry.contains("hi\n\nTool error:"), + "original request retained: {retry}" + ); + assert!( + !retry.contains("\"args\":[]"), + "failed output is not committed: {retry}" + ); + assert_eq!(engine.invalidations, 1); + assert!(String::from_utf8(out) + .unwrap() + .contains("\"finish_reason\":\"tool_calls\"")); +} + +#[test] +fn inkling_malformed_retry() { + inkling_retry_case( + br#"bash<|content_invoke_tool_json|>{"name":"bash","args":[]}<|end_message|>"#, + "<|content_model_end_sampling|>", + ); +} + +#[test] +fn inkling_second_tool_retry() { + inkling_retry_case(br#"bash<|content_invoke_tool_json|>{"name":"bash","args":{"command":"ls"}}<|end_message|><|message_model|>bash<|content_invoke_tool_json|>{"name":"bash","args":{}"#, + "<|end_message|><|content_model_end_sampling|>"); +} + +#[test] +fn inkling_bad_tool_terminal() { + let bad = br#"bash<|content_invoke_tool_json|>{"name":"bash","args":[]}<|end_message|>"#; + for stream in [false, true] { + for cap in [1, 16] { + let mut parsed = tools_req(); + parsed.stream = stream; + parsed.max_tokens = cap; + let mut tape = ScriptedDecode::from_pieces(&[bad, bad]); + tape.model_id = 9; + tape.steps.insert( + 1, + ScriptedStep { + token: 98, + piece: Vec::new(), + stop: true, + }, + ); + let mut engine = PromptSyncDecode::new(tape, 0, 1); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "inkling-terminal", + CREATED_TEST, + false, + 16, + &mut out, + ) + .unwrap(); + let response = String::from_utf8(out).unwrap(); + let finish = if cap == 1 { "length" } else { "error" }; + assert!( + response.contains(&format!("\"finish_reason\":\"{finish}\"")), + "{response}" + ); + assert!(!response.contains("<|"), "{response}"); + assert!(!response.contains("Tool error:"), "{response}"); + assert_eq!(engine.sync_calls, usize::from(!stream && cap != 1)); + } + } +} + +// ---- the V4.1 push route (P5.5) ------------------------------------------- + +/// A scripted V4.1 engine: `is_v41` + the push entry, nothing session-shaped +/// behind it. The script is (token, text) pairs; `stop_id` ends generation +/// the way the native checks it, and every run records what the route handed +/// the engine (prompt tokens, budget). +struct ScriptedV41 { + script: Vec<(i32, Vec)>, + stop_id: i32, + ctx: i32, + runs: std::cell::RefCell, i32)>>, + /// What the route handed `remember_tool_replay`: (wire ids, sampled bytes). + remembered: std::cell::RefCell, String)>>, +} + +impl ScriptedV41 { + fn new(script: &[(i32, &str)], ctx: i32) -> Self { + Self { + script: script + .iter() + .map(|(id, s)| (*id, s.as_bytes().to_vec())) + .collect(), + stop_id: 99, + ctx, + runs: std::cell::RefCell::new(Vec::new()), + remembered: std::cell::RefCell::new(Vec::new()), + } + } +} + +impl DecodeIo for ScriptedV41 { + fn model_id(&self) -> i32 { + 16 // Variant::DeepSeek41Flash + } + + fn is_v41(&self) -> bool { + true + } + + /// The real engine writes the sampled block here (`NativeDecode`, + /// generate.rs:4525, a port of `tool_memory_remember` at + /// ds4_server.c:14524). The fake records what the route handed it, so the + /// write is observable without a model. + fn remember_tool_replay(&mut self, calls: &[ToolCall], raw_dsml: &str) { + self.remembered.borrow_mut().push(( + calls.iter().map(|call| call.id.clone()).collect(), + raw_dsml.to_string(), + )); + } + + fn tokenize_text(&self, text: &str) -> Result, GenerateError> { + Ok(text.bytes().map(i32::from).collect()) + } + + fn tokenize_rendered_chat(&self, text: &[u8]) -> Result, GenerateError> { + Ok(text.iter().map(|b| i32::from(*b)).collect()) + } + + fn token_text(&self, token: i32) -> Result, GenerateError> { + Ok(self + .script + .iter() + .find(|(id, _)| *id == token) + .map(|(_, piece)| piece.clone()) + .unwrap_or_default()) + } + + fn token_is_stop(&self, token: i32) -> bool { + token == self.stop_id + } + + fn eos_id(&self) -> i32 { + self.stop_id + } + + fn v41_generate( + &self, + prompt: &[i32], + n_predict: i32, + _sampling: Option, + _images: Option<&ds4_core::v41_media::V41Images>, + _greedy: &mut dyn FnMut(*const i32, u32) -> bool, + emit: &mut dyn FnMut(i32) -> bool, + progress: &mut dyn FnMut(&str, i32, i32) -> bool, + ) -> Result<(), GenerateError> { + self.runs.borrow_mut().push((prompt.to_vec(), n_predict)); + // One prefill chunk: the route's keepalive path runs (a no-op + // off-stream, the disconnect probe on it). + if !progress("prefill_chunk", prompt.len() as i32, prompt.len() as i32) { + return Ok(()); + } + // The native loop's shape: `while (!stop && produced < n_predict)`. + let mut produced = 0i32; + for (id, _) in &self.script { + if produced >= n_predict { + break; + } + produced += 1; + if !emit(*id) { + break; + } + } + Ok(()) + } + + fn sync(&mut self, _tokens: &[i32]) -> Result<(), GenerateError> { + Ok(()) + } + + fn eval(&mut self, _token: i32) -> Result<(), GenerateError> { + Ok(()) + } + + fn sample(&mut self, _t: f32, _k: i32, _p: f32, _m: f32, _rng: &mut u64) -> i32 { + -1 + } + + fn pos(&self) -> i32 { + 0 + } + + fn ctx(&self) -> i32 { + self.ctx + } + + fn generation(&self) -> u64 { + 0 + } + + fn invalidate(&mut self) {} +} + +fn v41_completion_req(body: &str) -> ParsedRequest { + let mut r = parse_request(WireSurface::OpenaiCompletion, &env(), body).unwrap(); + r.temperature = 0.0; + r +} + +/// The ds41 route builds its own outcome and returns before the shared finish +/// path, so it has to write the sampled block itself, the way +/// `tool_memory_remember` does at ds4_server.c:14524. Before this the route +/// wrote nothing and the next turn's attach read an empty map, which is what +/// kept replay at zero on this family. +#[test] +fn v41_tool_turn_hands_the_sampled_block_to_the_tool_memory() { + // The artifact's own spelling: the spaced wrapper with a plain invocation. + let block = format!( + "{DSML_TOOL_CALLS_START_SPACED}\n{DSML_INVOKE_START_SPACED} name=\"read_file\">\n\ + {DSML_PARAM_START} name=\"target_file\" string=\"true\">/tmp/x.txt{DSML_PARAM_END}\n\ + {DSML_INVOKE_END_SPACED}\n{DSML_TOOL_CALLS_END_SPACED}" + ); + let body = r#"{"model":"t","messages":[{"role":"user","content":"Read the file /tmp/x.txt"}],"tools":[{"type":"function","function":{"name":"read_file","parameters":{"type":"object","properties":{"target_file":{"type":"string"}},"required":["target_file"]}}}],"max_tokens":192,"temperature":0}"#; + let parsed = parse_request(WireSurface::OpenaiChat, &env(), body).unwrap(); + let mut engine = ScriptedV41::new(&[(1, block.as_str()), (99, "")], 1 << 20); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "v41-mem", + CREATED_TEST, + false, + 192, + &mut out, + ) + .unwrap(); + + let remembered = engine.remembered.borrow(); + assert_eq!( + remembered.len(), + 1, + "the ds41 route never wrote the sampled block" + ); + assert_eq!( + remembered[0].1, block, + "the remembered bytes are not the sampled block" + ); + assert_eq!(remembered[0].0.len(), 1, "{:?}", remembered[0].0); + assert!( + remembered[0].0[0].starts_with("call_"), + "the map key must be the id that reached the wire: {:?}", + remembered[0].0 + ); +} + +/// TP-17's signal half (CTR 8.7, NC-07). A turn cut by the output budget +/// inside its tool block must report `length`: the block is unterminated, so +/// there is no executable call to serve, but the truncation itself is what +/// tells the agent loop its output was cut. The reference keeps it by name -- +/// `tool_parse_failure_recovery_finish` (ds4_server.c:6776-6784) returns +/// "length" when the incoming finish is "length" and "stop" otherwise, and +/// `test_incomplete_tool_call_keeps_stop_reason` (:19613) pins both arms. +/// This route pre-clobbered the pre-set `length` (generate.rs:3250) with +/// `error` before the parse-failure mapping could read it, and the wire remap +/// at :3389 then flattened that to `stop`. +#[test] +fn v41_budget_cut_inside_the_block_reports_length() { + let body = r#"{"model":"t","messages":[{"role":"user","content":"Read the file /tmp/x.txt"}],"tools":[{"type":"function","function":{"name":"read_file","parameters":{"type":"object","properties":{"target_file":{"type":"string"}},"required":["target_file"]}}}],"max_tokens":3,"temperature":0}"#; + let parsed = parse_request(WireSurface::OpenaiChat, &env(), body).unwrap(); + assert!(parsed.has_tools); + + // Four pieces against a budget of three. The engine's own loop stops at + // n_predict, so the fourth never feeds and the turn ends with the block + // open and its invocation unfinished -- the cut the budget sweep measures. + let prose = String::from("Checking."); + let open = + format!(" {DSML_TOOL_CALLS_START_SPACED}\n{DSML_INVOKE_START_SPACED} name=\"read_file\">"); + let param = + format!("\n{DSML_PARAM_START_SPACED} name=\"target_file\" string=\"true\">/tmp/x.txt"); + let close = + format!("{DSML_PARAM_END_SPACED}\n{DSML_INVOKE_END_SPACED}\n{DSML_TOOL_CALLS_END_SPACED}"); + let script: Vec<(i32, &str)> = vec![ + (1, prose.as_str()), + (2, open.as_str()), + (3, param.as_str()), + (4, close.as_str()), + ]; + let mut engine = ScriptedV41::new(&script, 1 << 20); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-cut", + CREATED_TEST, + false, + 3, + &mut out, + ) + .unwrap(); + let s = String::from_utf8(out).unwrap(); + + assert!(s.contains("\"finish_reason\":\"length\""), "{s}"); + assert!(!s.contains("\"finish_reason\":\"stop\""), "{s}"); + // No block closed, so no call is executable: the raw text is the content. + assert!(!s.contains("\"tool_calls\""), "{s}"); + assert!(s.contains(DSML_INVOKE_START_SPACED), "{s}"); +} + +/// The other arm of the same reference function: an unterminated block that +/// the model ended itself is a normal assistant stop, and the wire remap that +/// clause 2.3.3 relies on still turns the route's `error` into `stop`. This +/// guards the reclassification above from widening that remap. +#[test] +fn v41_unterminated_block_on_eos_still_reports_stop() { + let body = r#"{"model":"t","messages":[{"role":"user","content":"Read the file /tmp/x.txt"}],"tools":[{"type":"function","function":{"name":"read_file","parameters":{"type":"object","properties":{"target_file":{"type":"string"}},"required":["target_file"]}}}],"max_tokens":64,"temperature":0}"#; + let parsed = parse_request(WireSurface::OpenaiChat, &env(), body).unwrap(); + + let open = format!( + "Checking. {DSML_TOOL_CALLS_START_SPACED}\n{DSML_INVOKE_START_SPACED} name=\"read_file\">" + ); + // Same content as the budget arm, stopped one piece earlier: the model + // ends the turn itself with the parameter still open. + let param = + format!("\n{DSML_PARAM_START_SPACED} name=\"target_file\" string=\"true\">/tmp/x.txt"); + let script: Vec<(i32, &str)> = vec![(1, open.as_str()), (2, param.as_str()), (99, "")]; + let mut engine = ScriptedV41::new(&script, 1 << 20); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "chatcmpl-cut-eos", + CREATED_TEST, + false, + 64, + &mut out, + ) + .unwrap(); + let s = String::from_utf8(out).unwrap(); + + assert!(s.contains("\"finish_reason\":\"stop\""), "{s}"); + assert!(!s.contains("\"finish_reason\":\"length\""), "{s}"); + assert!(!s.contains("\"tool_calls\""), "{s}"); +} + +#[test] +fn v41_completion_pushes_the_script() { + let parsed = v41_completion_req(r#"{"prompt":"hi","max_tokens":8}"#); + let mut engine = ScriptedV41::new(&[(1, "Hello"), (2, " world"), (99, "")], 1 << 20); + let mut out = Vec::new(); + let result = generate_and_write( + &mut engine, + &parsed, + "v41-cmpl", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap(); + let response = String::from_utf8(out).unwrap(); + assert!(response.contains("Hello world"), "{response}"); + assert!( + response.contains("\"finish_reason\":\"stop\""), + "{response}" + ); + let runs = engine.runs.borrow(); + assert_eq!(runs.len(), 1); + // The prompt reached the engine as the tokenized bytes, not the text. + assert_eq!(runs[0].0, vec![i32::from(b'h'), i32::from(b'i')]); + assert_eq!(runs[0].1, 8); + assert_eq!(result.timings.decode_tokens, 2); + assert_eq!(result.frontier, 4); + assert!(!result.speculation_active); +} + +#[test] +fn v41_stream_carries_deltas_and_done() { + let parsed = v41_completion_req(r#"{"prompt":"hi","max_tokens":8,"stream":true}"#); + assert!(parsed.stream); + let mut engine = ScriptedV41::new(&[(1, "Hello"), (2, " world"), (99, "")], 1 << 20); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "v41-stream", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap(); + let sse = String::from_utf8(out).unwrap(); + assert!(sse.contains("data: "), "{sse}"); + assert!(sse.contains("Hello"), "{sse}"); + assert!(sse.contains(" world"), "{sse}"); + assert!(sse.contains("\"finish_reason\":\"stop\""), "{sse}"); + assert!(sse.contains("[DONE]"), "{sse}"); +} + +#[test] +fn v41_budget_stops_at_max_tokens() { + let parsed = v41_completion_req(r#"{"prompt":"hi","max_tokens":2}"#); + let mut engine = ScriptedV41::new(&[(1, "a"), (2, "b"), (3, "c")], 1 << 20); + let mut out = Vec::new(); + let result = generate_and_write( + &mut engine, + &parsed, + "v41-budget", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap(); + let response = String::from_utf8(out).unwrap(); + assert!( + response.contains("\"finish_reason\":\"length\""), + "{response}" + ); + assert!(response.contains("\"ab\""), "{response}"); + assert!(!response.contains("abc"), "{response}"); + assert_eq!(result.timings.decode_tokens, 2); +} + +#[test] +fn v41_budget_clamps_to_the_metadata_context() { + // ctx 5, prompt 2 -> room 3; the request asks for 8. + let parsed = v41_completion_req(r#"{"prompt":"hi","max_tokens":8}"#); + let mut engine = ScriptedV41::new(&[(1, "a")], 5); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "v41-ctx", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap(); + let runs = engine.runs.borrow(); + assert_eq!(runs[0].1, 3); +} + +#[test] +fn v41_stop_string_truncates_and_stops() { + let parsed = v41_completion_req(r#"{"prompt":"hi","max_tokens":8,"stop":" world"}"#); + let mut engine = ScriptedV41::new(&[(1, "Hello"), (2, " world"), (3, "!")], 1 << 20); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "v41-stop", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap(); + let response = String::from_utf8(out).unwrap(); + assert!(response.contains("Hello"), "{response}"); + assert!(!response.contains(" world"), "{response}"); + assert!( + response.contains("\"finish_reason\":\"stop\""), + "{response}" + ); +} + +#[test] +fn v41_empty_prompt_is_refused_before_the_engine() { + let parsed = v41_completion_req(r#"{"prompt":""}"#); + let mut engine = ScriptedV41::new(&[(1, "a")], 1 << 20); + let mut out = Vec::new(); + let error = generate_and_write( + &mut engine, + &parsed, + "v41-empty", + CREATED_TEST, + false, + 8, + &mut out, + ) + .unwrap_err(); + assert!(error.to_string().contains("empty prompt"), "{error}"); + assert!(engine.runs.borrow().is_empty()); +} + +/// The tracker half of P1.6, measured on a served turn: the recorded artifact +/// turn (open-grok session 01a14f5d, the fixtures beside `tests/dsml.rs`) +/// roots its calls on the bare tool name with the stray quote, so the start +/// scan found no marker and the whole block reached the client as content. +/// Here the turn is scripted in three pieces -- prose, block, prose -- and the +/// streamed body must carry the calls while carrying no markup at all. +#[test] +fn v41_live_tool_root_turn_streams_calls_without_markup() { + let text = std::fs::read( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ds41-dsml/live-01a14f5d-turn1.txt"), + ) + .unwrap(); + + // The request declares the three tools the turn calls; the start scan only + // takes a root tag some tool declares. + let body = r#"{"model":"t","stream":true,"messages":[{"role":"user","content":"open solidcraft"}], + "tools":[ + {"type":"function","function":{"name":"memory_search","parameters":{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}}}, + {"type":"function","function":{"name":"read_file","parameters":{"type":"object","properties":{"target_file":{"type":"string"}},"required":["target_file"]}}}, + {"type":"function","function":{"name":"list_dir","parameters":{"type":"object","properties":{"target_directory":{"type":"string"}},"required":["target_directory"]}}} + ],"max_tokens":512,"temperature":0}"#; + let mut parsed = parse_request(WireSurface::OpenaiChat, &env(), body).unwrap(); + parsed.think_mode = ThinkMode::None; + parsed.temperature = 0.0; + + // Split at the fixture's own boundaries -- the root tag and the wrapper + // close -- without asking the scan under test, so the arm that turns the + // scan off still feeds the writer the same three pieces. + let root: &[u8] = b""; + let at = text + .windows(root.len()) + .position(|w| w == root) + .expect("the recorded turn keeps its quoted tool root"); + let close = DSML_TOOL_CALLS_END_SPACED.as_bytes(); + let end = at + + text[at..] + .windows(close.len()) + .position(|w| w == close) + .expect("the recorded turn keeps its wrapper close") + + close.len(); + + let pieces: Vec> = vec![ + text[..at].to_vec(), + text[at..end].to_vec(), + text[end..].to_vec(), + ]; + let script: Vec<(i32, String)> = pieces + .iter() + .enumerate() + .map(|(i, p)| (i as i32 + 1, String::from_utf8_lossy(p).into_owned())) + .chain(std::iter::once((99, String::new()))) + .collect(); + + let mut engine = ScriptedV41::new( + &script + .iter() + .map(|(i, s)| (*i, s.as_str())) + .collect::>(), + 1 << 20, + ); + let mut out = Vec::new(); + generate_and_write( + &mut engine, + &parsed, + "v41-live-root", + CREATED_TEST, + true, + 512, + &mut out, + ) + .unwrap(); + let sse = String::from_utf8(out).unwrap(); + + assert!( + sse.contains("\"tool_calls\""), + "the turn's calls must reach the client: {sse}" + ); + assert!( + sse.contains("memory_search") && sse.contains("read_file"), + "the sampled calls must be named on the wire: {sse}" + ); + for marker in [ + DSML_TOOL_CALLS_START_SPACED, + DSML_TOOL_CALLS_END_SPACED, + DSML_INVOKE_START_SPACED, + DSML_INVOKE_END_SPACED, + " ChatMsg { + ChatMsg { + role: role.into(), + content: content.into(), + ..ChatMsg::default() + } +} + +/// C `make_bash_order()` (ds4_server.c:17070): the `bash` order the GLM vectors +/// use to fix argument order. +fn bash_order() -> Vec { + vec![ToolSchemaOrder { + name: "bash".into(), + prop: vec!["command".into()], + ..Default::default() + }] +} + +/// One assistant turn carrying a single tool call, empty content. +fn call(name: &str, arguments: &str) -> ChatMsg { + ChatMsg { + role: "assistant".into(), + calls: vec![ToolCall { + id: "call1".into(), + name: name.into(), + arguments: arguments.into(), + }], + ..ChatMsg::default() + } +} + +#[test] +fn render_glm53_chat_prompt_text_matches_c() { + // C `test_render_glm_chat_prompt_text` (ds4_server.c:18960). The `hidden` + // schema is dropped (defer_loading) and `strict` stripped, so only `bash` + // survives; the expected string is the C `expected` literal verbatim. + let schemas = concat!( + r#"{"name":"hidden","defer_loading":true,"parameters":{}}"#, + "\n", + r#"{"name":"bash","parameters":{"type":"object","properties":{"command":{}}},"strict":true}"#, + ); + let msgs = [msg("system", "You are terse."), msg("user", "Hello")]; + let prompt = render_chat_choice( + ModelSyntax::Glm53, + &msgs, + schemas, + &bash_order(), + ThinkMode::High, + ToolChoice::Auto, + ) + .unwrap(); + assert_eq!( + prompt, + concat!( + "[gMASK]<|system|>Reasoning Effort: High<|system|>\n", + "# Tools\n\n", + "You may call one or more functions to assist with the user query.\n\n", + "You are provided with function signatures within XML tags:\n", + "\n\n", + "{\"name\": \"bash\", \"parameters\": {\"type\":\"object\",\"properties\":{\"command\":{}}}}\n\n\n", + "\n\n", + "For each function call, output the function name and arguments within the following XML format:\n", + "{function-name}{arg-key-1}", + "{arg-value-1}{arg-key-2}", + "{arg-value-2}...", + "<|system|>You are terse.<|user|>Hello<|assistant|>\x3cthink\x3e", + ) + .as_bytes() + ); +} + +#[test] +fn render_glm53_drops_old_reasoning_without_tools() { + // C `test_render_glm_drops_old_reasoning_without_tools` (ds4_server.c:19000): + // an assistant turn before the last user turn loses its reasoning when the + // history carries no tool context. + let mut assistant = msg("assistant", "first answer"); + assistant.reasoning = "old hidden reasoning".into(); + let msgs = [msg("user", "first"), assistant, msg("user", "second")]; + + let prompt = + String::from_utf8(render_chat(ModelSyntax::Glm53, &msgs, "", ThinkMode::High).unwrap()) + .unwrap(); + + assert!(!prompt.contains("old hidden reasoning")); + assert!(prompt.contains("<|assistant|>\x3cthink\x3e\x3c/think\x3efirst answer")); + assert!(prompt.contains("<|user|>second<|assistant|>\x3cthink\x3e")); +} + +#[test] +fn render_glm53_preserves_reasoning_with_tools() { + // C `test_render_glm_preserves_reasoning_with_tools` (ds4_server.c:19027). + // A tool message in the history keeps the assistant reasoning, and the two + // tool calls render inline with no newline between them. + let mut assistant = call("bash", r#"{"command":"pwd"}"#); + assistant.reasoning = "tool reasoning".into(); + assistant.calls.push(ToolCall { + id: "call2".into(), + name: "bash".into(), + arguments: r#"{"command":"ls"}"#.into(), + }); + let msgs = [msg("user", "first"), assistant, msg("tool", "/tmp")]; + + let prompt = render_chat_choice( + ModelSyntax::Glm53, + &msgs, + "", + &bash_order(), + ThinkMode::High, + ToolChoice::Auto, + ) + .unwrap(); + + assert_eq!( + prompt, + concat!( + "[gMASK]<|system|>Reasoning Effort: High", + "<|user|>first<|assistant|>\x3cthink\x3etool reasoning\x3c/think\x3e", + "bashcommandpwd", + "bashcommand", + "ls", + "<|observation|>/tmp", + "<|assistant|>\x3cthink\x3e", + ) + .as_bytes() + ); +} + +#[test] +fn glm53_tool_calls_render_matches_template() { + // C `test_glm_tool_calls_render_matches_template` (ds4_server.c:19070): the + // reconstructed block must byte-match the sampled live block. GLM emits it + // inline (C append_glm_tool_calls_text, ds4_server.c:2983-2985: "no newline + // before or after"), and a re-render from the raw splice must equal the + // reconstructed render. + let tools = "[{\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\ + \"parameters\":{\"type\":\"object\",\"properties\":\ + {\"city\":{\"type\":\"string\"}},\"required\":[\"city\"]}}}]"; + let raw_block = + "get_weathercityParis"; + let msgs = [ + msg("user", "What is the weather in Paris?"), + call("get_weather", r#"{"city":"Paris"}"#), + msg("tool", "18C, light rain."), + ]; + + let canonical = + String::from_utf8(render_chat(ModelSyntax::Glm53, &msgs, tools, ThinkMode::High).unwrap()) + .unwrap(); + assert!(canonical.contains(raw_block)); + assert!(canonical.contains("\x3c/think\x3e")); + assert!(canonical.contains("<|observation|>")); + assert!(!canonical.contains("\x3c/think\x3e\n")); + assert!(!canonical.contains("\n")); + + // Raw leg. C splices the sampled block from `calls.raw_tool_text` + // (ds4_server.c:19104). + let mut spliced = msgs.clone(); + let sampled = raw_block.replace("Paris", "Paris"); + spliced[1].raw_tool_text = sampled.clone(); + let raw = render_chat(ModelSyntax::Glm53, &spliced, tools, ThinkMode::High).unwrap(); + assert_eq!(raw, canonical.replace(raw_block, &sampled).as_bytes()); +} + +#[test] +fn render_glm53_groups_tool_results() { + // C `test_render_glm_groups_tool_results` (ds4_server.c:19116). + // Part A: consecutive tool-role messages collapse under one observation, + // each wrapped in its own . + let msgs = [ + msg("user", "inspect both"), + msg("tool", "first result"), + msg("tool", "second result"), + ]; + let prompt = + String::from_utf8(render_chat(ModelSyntax::Glm53, &msgs, "", ThinkMode::High).unwrap()) + .unwrap(); + assert_eq!(prompt.matches("<|observation|>").count(), 1); + assert!(prompt.contains(concat!( + "<|observation|>first result", + "second result", + "<|assistant|>\x3cthink\x3e", + ))); + + // Part B: a user message carrying multiple tool-call ids holds + // blocks that normalize into one observation, never + // rendering as user text. + let wrapped = ChatMsg { + role: "user".into(), + content: "onetwo".into(), + tool_call_ids: vec!["call_1".into(), "call_2".into()], + ..ChatMsg::default() + }; + let prompt = String::from_utf8( + render_chat(ModelSyntax::Glm53, &[wrapped], "", ThinkMode::High).unwrap(), + ) + .unwrap(); + assert!(prompt.contains(concat!( + "<|observation|>one", + "two", + "<|assistant|>\x3cthink\x3e", + ))); + assert!(!prompt.contains("<|user|>")); +} + +#[test] +fn render_glm53_live_tool_tail() { + // C `render_glm_live_tool_tail` (ds4_server.c:3792), through the port's + // render_live_tool_tail: the trailing tool results form one observation + // with one per result, ending on the assistant think head. + let msgs = [ + msg("user", "inspect both"), + msg("tool", "first result"), + msg("tool", "second result"), + ]; + let tail = + render_live_tool_tail(ModelSyntax::Glm53, Api::Responses, &msgs, ThinkMode::High).unwrap(); + assert_eq!( + tail, + concat!( + "<|observation|>first result", + "second result", + "<|assistant|>\x3cthink\x3e", + ) + .as_bytes() + ); +} + +#[test] +fn glm53_parse_tool_call_message_matches_c() { + // C `test_parse_glm_tool_call_message` (ds4_server.c:19262). With no + // declared schema the reference's GLM rule stands: every argument is text + // (ds4_server.c:6435), so `timeout` is the string "10". + let generated = concat!( + "\x3cthink\x3eneed bash\x3c/think\x3eOK\n\n", + "bash", + "commandecho hi", + "timeout10", + "", + ); + + let parsed = parse_generated_message( + ModelSyntax::Glm53, + generated.as_bytes(), + true, + ChatFormat::DeepSeek, + &[], + ); + + assert!(parsed.ok); + assert_eq!(parsed.reasoning, b"need bash"); + assert_eq!(parsed.content, b"OK"); + assert_eq!(parsed.calls.len(), 1); + assert_eq!(parsed.calls[0].name, "bash"); + assert!(parsed.calls[0] + .arguments + .contains("\"command\": \"echo hi\"")); + assert!(parsed.calls[0].arguments.contains("\"timeout\": \"10\"")); + // C keeps the sampled block on `calls.raw_tool_text` (ds4_server.c:19262). + assert!(parsed.raw_tool_text.starts_with("\n\nbash")); +} diff --git a/crates/ds4-server/tests/glm_output.rs b/crates/ds4-server/tests/glm_output.rs new file mode 100644 index 0000000000000000000000000000000000000000..a88f2ef6fcd3235ecdc2ccbe7148b4a8618df74b --- /dev/null +++ b/crates/ds4-server/tests/glm_output.rs @@ -0,0 +1,155 @@ +use ds4_core::chat_template::{ChatOptions, RenderClock, Template}; +use ds4_core::ChatThinkMode; +use ds4_server::{ + openai_sse_finish_live, openai_sse_stream_update, openai_stream_start, parse_chat_request, + parse_generated_for_response, parse_generated_message, ChatFormat, ModelSyntax, ParseEnv, + StreamReq, ThinkMode, ToolSchemaOrder, Writer, +}; +use serde_json::{json, Value}; + +const TEMPLATE: &str = + include_str!("../../../tests/fixtures/chat-template/models/glm-uncensored/chat_template.jinja"); + +fn source_call(arguments: Value) -> Vec { + let template = Template::compile(TEMPLATE, RenderClock::Fixed(0)).unwrap(); + let messages = [ + json!({"role":"user","content":"Use inspect."}), + json!({"role":"assistant","content":"", "tool_calls":[{ + "type":"function", "function":{"name":"inspect","arguments":arguments} + }]}), + ]; + let rendered = template + .render_chat(&messages, &[], ChatOptions::new(7, ChatThinkMode::None)) + .unwrap(); + let start = rendered.find("").unwrap(); + let end = start + rendered[start..].find("").unwrap() + "".len(); + rendered.as_bytes()[start..end].to_vec() +} + +fn orders(properties: Value) -> Vec { + let env = ParseEnv { + default_model: "glm-5.3-flash".into(), + default_tokens: 2048, + default_effort: ds4_server::ThinkMode::None, + default_temp: ds4_server::default_temperature(), + live_ids: Vec::new(), + engine_defaults: false, + }; + let request = json!({"messages":[{"role":"user","content":"Use inspect."}], + "tools":[{"type":"function","function":{"name":"inspect", + "parameters":{"type":"object","properties":properties}}}]}); + parse_chat_request(&env, &request.to_string()) + .unwrap() + .tool_orders +} + +fn parse_call(text: &[u8], orders: &[ToolSchemaOrder]) -> Value { + let (parsed, finish) = parse_generated_for_response( + ModelSyntax::Glm53, + text, + true, + true, + false, + ChatFormat::DeepSeek, + orders, + "tool_calls", + ); + assert!(parsed.ok); + assert_eq!(finish, "tool_calls"); + assert_eq!(parsed.calls.len(), 1); + serde_json::from_str(&parsed.calls[0].arguments).unwrap() +} + +#[test] +fn glm_source_types_round_trip() { + let arguments = json!({ + "text":" & < true [1] {\"x\":2} ", + "number":1.25, "integer":7, "flag":true, + "list":[1,"&"], "map":{"x":2}, "empty":null, + }); + let schemas = orders(json!({ + "text":{"type":"string"}, "number":{"type":"number"}, + "integer":{"type":"integer"}, "flag":{"type":"boolean"}, + "list":{"type":"array"}, "map":{"type":"object"}, "empty":{"type":"null"}, + })); + assert_eq!( + parse_call(&source_call(arguments.clone()), &schemas), + arguments + ); +} + +#[test] +fn glm_ambiguous_values_stay_raw() { + let arguments = json!({ + "unknown":"[1]", "union":"true", "mismatch":"[1]", "escaped":"&", + }); + let schemas = orders(json!({ + "union":{"type":["string","boolean"]}, "mismatch":{"type":"integer"}, + "escaped":{"type":"string"}, + })); + assert_eq!( + parse_call(&source_call(arguments.clone()), &schemas), + arguments + ); + assert_eq!(parse_call(&source_call(arguments.clone()), &[]), arguments); +} + +#[test] +fn glm_stream_keeps_source_types() { + let arguments = json!({"text":"&", "number":1.25}); + let raw = source_call(arguments.clone()); + let req = StreamReq { + syntax: ModelSyntax::Glm53, + chat_format: ChatFormat::DeepSeek, + has_tools: true, + think_mode: ThinkMode::None, + tool_orders: orders(json!({"text":{"type":"string"}, "number":{"type":"number"}})), + ..StreamReq::default() + }; + let mut writer = Writer::new(0); + let mut stream = openai_stream_start(&req); + for end in 1..=raw.len() { + assert!(openai_sse_stream_update( + &mut writer, + &req, + "glm", + &mut stream, + &raw[..end], + false, + )); + } + let parsed = + parse_generated_message(req.syntax, &raw, false, req.chat_format, &req.tool_orders); + assert!(openai_sse_finish_live( + &mut writer, + &req, + "glm", + &mut stream, + &raw, + "tool_calls", + 1, + 1, + &parsed.calls, + )); + let tape = String::from_utf8(writer.out).unwrap(); + let mut emitted = String::new(); + for data in tape.lines().filter_map(|line| line.strip_prefix("data: ")) { + if data == "[DONE]" { + continue; + } + let frame: Value = serde_json::from_str(data).unwrap(); + let delta = &frame["choices"][0]["delta"]; + assert!( + delta.get("content").is_none(), + "raw tool XML leaked: {frame}" + ); + if let Some(calls) = delta["tool_calls"].as_array() { + for call in calls { + if let Some(part) = call["function"]["arguments"].as_str() { + emitted.push_str(part); + } + } + } + } + assert_eq!(serde_json::from_str::(&emitted).unwrap(), arguments); +} diff --git a/crates/ds4-server/tests/http_door.rs b/crates/ds4-server/tests/http_door.rs new file mode 100644 index 0000000000000000000000000000000000000000..1531ac364a72413e8994cd142d195302fcf3c460 --- /dev/null +++ b/crates/ds4-server/tests/http_door.rs @@ -0,0 +1,486 @@ +//! C↔Rust HTTP door: escape, envelopes, format refusal, model-id, /v1/models. + +use ds4_server::{ + anthropic_error_body, anthropic_error_type, handle_client, header_accepts_json, header_chunked, + header_end, http_head, json_escape, json_models_array_dup, model_alias_known, + model_id_from_gguf_path, model_one_json, models_list_json, openai_error_body, + openai_error_type, output_format_type_supported, parse_output_config_effort, + parse_output_format_value, parse_responses_text_value, read_http_request, retry_after_header, + wire_http_error_bytes, Json, ServerConfig, WireSurface, +}; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::PathBuf; +use std::process::Command; +use std::thread; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_SERVER_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/server_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/server_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_out(args: &[&str]) -> Vec { + let out = Command::new(require_oracle()) + .args(args) + .output() + .expect("run server_c_oracle"); + assert!( + out.status.success(), + "oracle {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); + out.stdout +} + +fn c_str(args: &[&str]) -> String { + String::from_utf8(c_out(args)).unwrap() +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[test] +fn json_escape_matches_c() { + let rows = [ + "", + "plain", + "quote\"slash\\", + "line\nret\rtab\t", + "ctrl\u{0001}bell\u{0007}", + "한글", + ]; + for s in rows { + assert_eq!(json_escape(s), c_str(&["escape", s]), "escape {s:?}"); + } +} + +#[test] +fn error_envelopes_match_c() { + let codes = [400, 404, 409, 429, 500, 503]; + for code in codes { + assert_eq!( + openai_error_type(code), + c_str(&["openai-type", &code.to_string()]).trim() + ); + assert_eq!( + anthropic_error_type(code), + c_str(&["anth-type", &code.to_string()]).trim() + ); + let msg = "boom"; + assert_eq!( + openai_error_body(code, msg), + c_str(&["openai-error", &code.to_string(), msg]) + ); + assert_eq!( + anthropic_error_body(code, msg), + c_str(&["anth-error", &code.to_string(), msg]) + ); + } + assert_eq!( + openai_error_body(400, "say \"hi\"\n"), + c_str(&["openai-error", "400", "say \"hi\"\n"]) + ); +} + +#[test] +fn http_head_and_wire_error_match_c() { + let rust = http_head(404, Some("application/json"), None, false, 12); + let c = c_str(&["http-head", "404", "application/json", "0", "-", "12"]); + assert_eq!(rust, c); + + let rust = http_head(204, None, None, true, 0); + let c = c_str(&["http-head", "204", "-", "1", "-", "0"]); + assert_eq!(rust, c); + assert!(rust.contains("Access-Control-Allow-Origin: *\r\n")); + assert!(rust.contains("Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n")); + assert!(!rust.contains(" Access-Control")); + + let extra = retry_after_header(5); + let rust = http_head(429, Some("application/json"), Some(&extra), false, 3); + let c = c_str(&[ + "http-head", + "429", + "application/json", + "0", + "Retry-After: 5\r\n", + "3", + ]); + assert_eq!(rust, c); + + let rust = wire_http_error_bytes( + WireSurface::OpenaiChat, + 400, + "bad HTTP request", + false, + None, + ); + let c = c_out(&["wire-http", "0", "400", "0", "-1", "bad HTTP request"]); + assert_eq!(rust, c); + + let rust = wire_http_error_bytes(WireSurface::Anthropic, 503, "boom", true, Some(10)); + let c = c_out(&["wire-http", "2", "503", "1", "10", "boom"]); + assert_eq!(rust, c); + let s = String::from_utf8_lossy(&rust); + assert!(s.contains("\"type\":\"overloaded_error\"")); + assert!(s.contains("Retry-After: 10\r\n")); +} + +fn fmt_rust(field: &str, json: &str) -> Result<(), String> { + let mut p = Json::new(json); + parse_output_format_value(&mut p, field) +} + +fn normalize_c_schema_refusal(error: &str) -> String { + error.replacen( + "is not implemented: structured output is unsupported", + "is not supported: structured output is unsupported", + 1, + ) +} + +fn fmt_c(field: &str, json: &str) -> Result<(), String> { + let s = c_str(&["format-value", field, json]); + if s.starts_with("OK\n") { + Ok(()) + } else { + Err(normalize_c_schema_refusal( + s.strip_prefix("ERROR\n") + .unwrap_or(&s) + .trim_end_matches('\n'), + )) + } +} + +#[test] +fn schema_format_refusal_matches_c() { + let field = "response_format"; + assert!(output_format_type_supported(field, "text").is_ok()); + assert!(c_str(&["format-type", field, "text"]).starts_with("OK")); + + let refuse = output_format_type_supported(field, "json_object").unwrap_err(); + assert_eq!(refuse, fmt_c(field, "\"json_object\"").unwrap_err()); + assert!(refuse.contains("is not supported")); + assert_eq!( + output_format_type_supported(field, "xml").unwrap_err(), + c_str(&["format-type", field, "xml"]) + .strip_prefix("ERROR\n") + .unwrap() + .trim_end_matches('\n') + ); + + let rows = [ + ("null", true), + ("{}", true), + ("{\"type\":\"text\"}", true), + ("{\"type\":\"json_object\"}", false), + ( + "{\"json_schema\":{\"schema\":{\"type\":\"object\"}},\"type\":\"json_schema\"}", + false, + ), + ("{\"type\":\"xml\"}", false), + ("\"json_object\"", false), + ("\"text\"", true), + ]; + for (json, ok) in rows { + assert_eq!(fmt_rust(field, json).is_ok(), ok, "rust {json}"); + assert_eq!(fmt_c(field, json).is_ok(), ok, "c {json}"); + if !ok { + assert_eq!( + fmt_rust(field, json).unwrap_err(), + fmt_c(field, json).unwrap_err(), + "{json}" + ); + } + } + + let mut p = + Json::new("{\"format\":{\"type\":\"json_schema\",\"name\":\"x\"},\"verbosity\":\"low\"}"); + let rust = parse_responses_text_value(&mut p).unwrap_err(); + let c = c_str(&[ + "text-value", + "{\"format\":{\"type\":\"json_schema\",\"name\":\"x\"},\"verbosity\":\"low\"}", + ]); + assert!(c.starts_with("ERROR")); + assert_eq!( + rust, + normalize_c_schema_refusal(c.strip_prefix("ERROR\n").unwrap().trim_end_matches('\n')) + ); + assert!(rust.contains("text.format type 'json_schema' is not supported")); + + let mut p = Json::new("{\"effort\":\"high\",\"format\":{\"type\":\"json_object\"}}"); + let rust = parse_output_config_effort(&mut p).unwrap_err(); + let c = c_str(&[ + "output-config", + "{\"effort\":\"high\",\"format\":{\"type\":\"json_object\"}}", + ]); + assert_eq!( + rust, + normalize_c_schema_refusal(c.strip_prefix("ERROR\n").unwrap().trim_end_matches('\n')) + ); + + let mut p = Json::new("{\"effort\":\"banana\",\"format\":{\"type\":\"text\"}}"); + assert!(parse_output_config_effort(&mut p).is_err()); + assert!(c_str(&[ + "output-config", + "{\"effort\":\"banana\",\"format\":{\"type\":\"text\"}}" + ]) + .starts_with("ERROR")); +} + +#[test] +fn model_id_from_gguf_path_matches_c() { + let paths = [ + "foo.gguf", + "/tmp/foo.gguf", + "/home/x/gguf/ds4flash.gguf", + "/home/x/models/Motif-3-Mixed-Quant-GGUF/Motif-3-MQ87-88-FIT.gguf", + "/home/x/models/Solar-Open2-250B-Mixed-Quant-GGUF/Solar-Open2-250B-MXQ-v1-00001-of-00011.gguf", + "K-EXAONE-236B-A23B-MXQ-v1-00001-of-00003.gguf", + "/tmp/", + "", + ]; + for path in paths { + let rust = model_id_from_gguf_path(path); + let c = c_str(&["model-id", path]); + match rust { + Some(id) => assert_eq!(format!("{id}\n"), c, "{path}"), + None => assert_eq!(c, "NULL\n", "{path}"), + } + } + assert!(model_alias_known("motif-3")); + assert!(model_alias_known("Motif-3-Mixed-Quant-GGUF")); + assert!(!model_alias_known("custom-id")); +} + +#[test] +fn models_json_matches_c() { + let id = "Motif-3-Mixed-Quant-GGUF"; + let rust = models_list_json(id, id, 196608, 1024, None); + let c = c_str(&["models-list", id, id, "196608", "1024"]); + assert_eq!(rust, c); + assert!(rust.contains("\"owned_by\":\"ds4.c\"")); + assert!(rust.contains("\"created\":1767225600")); + assert!(rust.ends_with('\n')); + + let rust = model_one_json(id, id, 8192, 393216); + let c = c_str(&["model-one", id, id, "8192", "393216"]); + assert_eq!(rust, c); + + let catalog = "{\"object\":\"list\",\"models\":[{\"id\":\"x\",\"context_window\":8192}]}"; + let rust = json_models_array_dup(catalog).unwrap(); + let c = c_str(&["models-array", catalog]); + assert_eq!(rust, c); + let rust_list = models_list_json("ds4", "ds4", 8192, 128, Some(&rust)); + let c_list = c_str(&["models-list", "ds4", "ds4", "8192", "128", &rust]); + assert_eq!(rust_list, c_list); +} + +#[test] +fn header_helpers_match_c() { + let headers = [ + b"GET /v1/models HTTP/1.1\r\nHost: x\r\n\r\n".as_slice(), + b"POST /v1/chat/completions HTTP/1.1\nContent-Length: 12abc\n\n".as_slice(), + b"POST /x HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n".as_slice(), + b"GET /x HTTP/1.1\r\nAccept: text/html, application/json\r\n\r\n".as_slice(), + b"GET /x HTTP/1.1\nAccept: text/plain\n\n".as_slice(), + b"POST /x HTTP/1.1\r\nContent-Length: -1\r\n\r\n".as_slice(), + ]; + for h in headers { + let hx = hex(h); + assert_eq!( + header_end(h).map(|n| n as i64).unwrap_or(-1), + c_str(&["header-end", &hx]).trim().parse::().unwrap(), + "{:?}", + String::from_utf8_lossy(h) + ); + assert_eq!( + ds4_server::http::content_length(h), + c_str(&["content-length", &hx]) + .trim() + .parse::() + .unwrap() + ); + assert_eq!( + header_chunked(h) as i32, + c_str(&["header-chunked", &hx]) + .trim() + .parse::() + .unwrap() + ); + assert_eq!( + header_accepts_json(h) as i32, + c_str(&["accept-json", &hx]).trim().parse::().unwrap() + ); + } +} + +#[test] +fn read_http_content_length_and_chunked() { + let raw = b"POST /v1/chat/completions HTTP/1.1\r\nContent-Length: 5\r\n\r\nabcdeEXTRA"; + let req = read_http_request(&mut &raw[..], true).unwrap(); + assert_eq!(req.method, "POST"); + assert_eq!(req.path, "/v1/chat/completions"); + assert_eq!(req.body, b"abcde"); + + let raw = b"GET /v1/models?foo=1 HTTP/1.1\r\n\r\n"; + let req = read_http_request(&mut &raw[..], true).unwrap(); + assert_eq!(req.path, "/v1/models"); + + let raw = + b"POST /v1/messages HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n"; + let req = read_http_request(&mut &raw[..], true).unwrap(); + assert_eq!(req.body, b"hello"); + + let raw = b"POST /v1/messages HTTP/1.1\r\nTransfer-Encoding: chunked\r\nContent-Length: 99\r\n\r\n5\r\nhello\r\n0\r\n\r\n"; + let req = read_http_request(&mut &raw[..], true).unwrap(); + assert_eq!(req.body, b"hello"); + + let raw = + b"POST /v1/messages HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n"; + assert!(read_http_request(&mut &raw[..], false).is_some()); + let req = read_http_request(&mut &raw[..], false).unwrap(); + assert!(req.body.is_empty()); + + let raw = b"POST /x HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\nG\r\nxx\r\n0\r\n\r\n"; + assert!(read_http_request(&mut &raw[..], true).is_none()); +} + +#[test] +fn chunked_trailers_share_the_header_size_limit() { + let mut raw = b"POST /x HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n".to_vec(); + for _ in 0..9 { + raw.extend_from_slice(b"x:"); + raw.extend(std::iter::repeat_n(b'a', 8_180)); + raw.extend_from_slice(b"\r\n"); + } + raw.extend_from_slice(b"\r\n"); + + assert!(read_http_request(&mut &raw[..], true).is_none()); +} + +fn one_shot(cfg: &ServerConfig, req: &[u8]) -> Vec { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let cfg = cfg.clone(); + let h = thread::spawn(move || { + let (mut s, _) = listener.accept().unwrap(); + handle_client(&cfg, &mut s); + }); + let mut c = TcpStream::connect(addr).unwrap(); + c.write_all(req).unwrap(); + let _ = c.shutdown(std::net::Shutdown::Write); + let mut out = Vec::new(); + c.read_to_end(&mut out).unwrap(); + h.join().unwrap(); + out +} + +#[test] +fn tcp_models_options_unknown_bad_http() { + let mut cfg = ServerConfig::default(); + cfg.model_id = "Motif-3-Mixed-Quant-GGUF".into(); + cfg.model_name = "Motif-3-Mixed-Quant-GGUF".into(); + cfg.ctx = 196608; + cfg.default_tokens = 1024; + cfg.cors = true; + cfg.serving_plan = Some(ds4_core::resolve_plan( + &ds4_core::ServingRequest::default(), + Some(ds4_core::serving_caps( + ds4_core::ModelFamily::Qwen4Exp, + ds4_core::Variant::Qwen38FlashNext, + )), + &ds4_core::EngineFacts::default(), + )); + + let out = one_shot(&cfg, b"GET /v1/models HTTP/1.1\r\n\r\n"); + let s = String::from_utf8_lossy(&out); + assert!(s.starts_with("HTTP/1.1 200 OK\r\n")); + assert!(s.contains("Access-Control-Allow-Origin: *")); + let body = models_list_json( + &cfg.model_id, + &cfg.model_name, + cfg.ctx, + cfg.default_tokens, + None, + ); + assert!(s.ends_with(&body)); + + let out = one_shot(&cfg, b"GET /v1/models/motif-3 HTTP/1.1\r\n\r\n"); + let s = String::from_utf8_lossy(&out); + assert!(s.starts_with("HTTP/1.1 200 OK")); + assert!(s.contains(&model_one_json( + "motif-3", + &cfg.model_name, + cfg.ctx, + cfg.default_tokens + ))); + + let out = one_shot(&cfg, b"OPTIONS /v1/chat/completions HTTP/1.1\r\n\r\n"); + let rust_head = http_head(204, None, None, true, 0); + assert_eq!(out, rust_head.into_bytes()); + + let out = one_shot(&cfg, b"GET /metrics HTTP/1.1\r\n\r\n"); + let s = String::from_utf8_lossy(&out); + assert!(s.starts_with("HTTP/1.1 200 OK")); + assert!(s.contains("ds4_uptime_seconds ")); + assert!(s.contains("ds4_route_requests_total{surface=\"openai_chat\",lane=\"serial\"} 0")); + assert!(s.contains("ds4_requests_shed_total{reason=\"clients\"} 0")); + assert!(s.contains("ds4_memory_census_supported 0")); + assert!(s.contains("ds4_memory_substrate_outstanding_bytes 0")); + + let out = one_shot(&cfg, b"GET /v1/stats HTTP/1.1\r\n\r\n"); + let s = String::from_utf8_lossy(&out); + assert!(s.starts_with("HTTP/1.1 200 OK")); + assert!(s.contains("\"routes\":{")); + assert!(s.contains("\"sheds\":{")); + assert!(s.contains("\"memory\":{\"census_supported\":false")); + assert!(s.contains("\"governor\":{\"shadow\":true")); + assert!(s.contains("\"serving\":{")); + assert!(s.contains("\"requested\":{")); + assert!(s.contains("\"effective\":{")); + assert!(s.contains("\"qualified\":{")); + + let out = one_shot( + &cfg, + b"POST /v1/chat/completions HTTP/1.1\r\nContent-Length: 45\r\n\r\n{\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}]}", + ); + assert_eq!( + out, + wire_http_error_bytes( + WireSurface::OpenaiChat, + 503, + "server shutting down", + true, + None, + ) + ); + + let out = one_shot( + &cfg, + b"POST /v1/chat/completions HTTP/1.1\r\nContent-Length: 2\r\n\r\n{}", + ); + assert_eq!( + out, + wire_http_error_bytes(WireSurface::OpenaiChat, 400, "missing messages", true, None) + ); + + let out = one_shot(&cfg, b"NOTHTTP"); + assert_eq!( + out, + wire_http_error_bytes(WireSurface::OpenaiChat, 400, "bad HTTP request", true, None) + ); +} diff --git a/crates/ds4-server/tests/inkling.rs b/crates/ds4-server/tests/inkling.rs new file mode 100644 index 0000000000000000000000000000000000000000..e5bb49e18c216374c7d449e948a155f05f71439f --- /dev/null +++ b/crates/ds4-server/tests/inkling.rs @@ -0,0 +1,375 @@ +use ds4_server::{render_chat, syntax_for_model_id, ChatMsg, ChatPart, ThinkMode, ToolCall}; +use serde_json::Value; + +fn stream_events(api: ds4_server::Api, think: ThinkMode, raw: &[u8], width: usize) -> Vec { + use ds4_server::stream::*; + let req = StreamReq { + api, + think_mode: think, + chat_format: ChatFormat::Inkling, + reasoning_summary_emit: true, + has_tools: true, + ..StreamReq::default() + }; + let mut writer = Writer::new(1); + let calls = ds4_server::parse_generated_for_model_id(9, raw, true, &[]).calls; + let finish = if calls.is_empty() { + "stop" + } else { + "tool_calls" + }; + match api { + ds4_server::Api::Openai => { + let mut state = openai_stream_start(&req); + for end in (width..raw.len()) + .step_by(width) + .chain(std::iter::once(raw.len())) + { + assert!(openai_sse_stream_update( + &mut writer, + &req, + "test", + &mut state, + &raw[..end], + false + )); + } + assert!(openai_sse_finish_live( + &mut writer, + &req, + "test", + &mut state, + raw, + finish, + 1, + 10, + &calls + )); + } + ds4_server::Api::Anthropic => { + let mut state = anthropic_sse_start_live(&mut writer, &req, "test", 1); + for end in (width..raw.len()) + .step_by(width) + .chain(std::iter::once(raw.len())) + { + assert!(anthropic_sse_stream_update( + &mut writer, + &req, + "test", + &mut state, + &raw[..end], + false + )); + } + assert!(anthropic_sse_finish_live( + &mut writer, + &req, + "test", + &mut state, + raw, + finish, + None, + 10, + &calls + )); + } + ds4_server::Api::Responses => { + let mut state = responses_stream_init(&req, "resp_test", "rs_test", "msg_test"); + responses_sse_created(&mut writer, &req, &mut state, 1); + for end in (width..raw.len()) + .step_by(width) + .chain(std::iter::once(raw.len())) + { + assert!(responses_sse_stream_update( + &mut writer, + &req, + &mut state, + &raw[..end], + false + )); + } + assert!(responses_sse_finish_live( + &mut writer, + &req, + &mut state, + raw, + finish, + 1, + 10, + 2, + 1, + &calls + )); + } + } + String::from_utf8(writer.out) + .unwrap() + .lines() + .filter_map(|line| line.strip_prefix("data: ")) + .filter(|line| *line != "[DONE]") + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +#[test] +fn streamed_channels() { + use ds4_server::Api; + let raw = "<|content_thinking|>Plan🙂<|end_message|><|message_model|><|content_text|>답<|end_message|><|message_model|><|content_text|>변<|end_message|><|content_model_end_sampling|>".as_bytes(); + for api in [Api::Openai, Api::Anthropic, Api::Responses] { + for think in [ThinkMode::None, ThinkMode::High] { + for width in [1, 7, raw.len()] { + let events = stream_events(api, think, raw, width); + let (mut text, mut reasoning) = (String::new(), String::new()); + for event in &events { + match api { + Api::Openai => { + text.push_str( + event["choices"][0]["delta"]["content"] + .as_str() + .unwrap_or(""), + ); + reasoning.push_str( + event["choices"][0]["delta"]["reasoning_content"] + .as_str() + .unwrap_or(""), + ); + } + Api::Anthropic if event["type"] == "content_block_delta" => { + text.push_str(event["delta"]["text"].as_str().unwrap_or("")); + reasoning.push_str(event["delta"]["thinking"].as_str().unwrap_or("")); + } + Api::Responses if event["type"] == "response.output_text.delta" => { + text.push_str(event["delta"].as_str().unwrap()) + } + Api::Responses + if event["type"] == "response.reasoning_summary_text.delta" => + { + reasoning.push_str(event["delta"].as_str().unwrap()) + } + _ => {} + } + } + assert_eq!(text, "답변", "{api:?} {think:?} width={width}"); + assert_eq!(reasoning, "Plan🙂", "{api:?} {think:?} width={width}"); + assert!(!serde_json::to_string(&events).unwrap().contains("<|")); + if api == Api::Responses { + let output = events.last().unwrap()["response"]["output"] + .as_array() + .unwrap(); + assert_eq!(output[0]["summary"][0]["text"], "Plan🙂"); + assert_eq!(output[1]["content"][0]["text"], "답변"); + } + } + } + } +} + +#[test] +fn inkling_thinking_tool_gate() { + let mut state = ds4_server::SemAccum::init( + true, + true, + false, + ds4_server::ChatFormat::Inkling, + b"<|message_model|>", + ); + state.feed(b"<|content_thinking|>", &[]); + assert!(state.thinking_inside()); + state.feed(b"Mention <|content_invoke_tool_json|> literally.", &[]); + assert!(!state.saw_tool_start); + state.feed(b"<|end_message|>", &[]); + assert!(!state.thinking_inside()); + state.feed(b"<|message_model|>weather<|content_invoke_tool_json|>", &[]); + assert!(state.saw_tool_start); +} + +#[test] +fn generated_channels() { + let cases: &[(&[u8], &[u8], &[u8])] = &[ + (b"<|content_text|>4<|end_message|>", b"4", b""), + (b"<|content_thinking|>Plan<|end_message|><|message_model|><|content_text|>Answer<|end_message|>", b"Answer", b"Plan"), + (b"<|content_thinking|><|end_message|><|message_model|><|content_text|>4", b"4", b""), + (b"<|content_thinking|>Not finished", b"", b"Not finished"), + ]; + for &(raw, content, reasoning) in cases { + let parsed = ds4_server::parse_generated_for_model_id(9, raw, true, &[]); + assert!(parsed.ok); + assert_eq!(parsed.content, content); + assert_eq!(parsed.reasoning, reasoning); + assert!(parsed.calls.is_empty()); + } +} + +#[test] +fn generated_tool_envelope() { + let raw = br#"weather<|content_invoke_tool_json|>{"name":"weather","args":{"city":"Seoul"}}<|end_message|>"#; + let parsed = ds4_server::parse_generated_for_model_id(9, raw, false, &[]); + assert!(parsed.ok); + assert!(parsed.content.is_empty()); + assert_eq!(parsed.calls.len(), 1); + assert_eq!(parsed.calls[0].name, "weather"); + assert_eq!(parsed.calls[0].arguments, r#"{"city":"Seoul"}"#); + for bad in [ + br#"other<|content_invoke_tool_json|>{"name":"weather","args":{}}<|end_message|>"#.as_slice(), + br#"weather<|content_invoke_tool_json|>{"name":"weather","args":[]}<|end_message|>"#, + br#"weather<|content_invoke_tool_json|>{"name":"weather","args":{}} trailing<|end_message|>"#, + ] { + let parsed = ds4_server::parse_generated_for_model_id(9, bad, false, &[]); + assert!(!parsed.ok); + assert!(parsed.calls.is_empty()); + } +} + +#[test] +fn source_chat_vectors() { + let fixture: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/inkling/server-vectors.json" + )) + .unwrap(); + for row in fixture["vectors"].as_array().unwrap() { + let messages: Vec<_> = row["messages"] + .as_array() + .unwrap() + .iter() + .map(|m| { + let mut msg = ChatMsg { + role: m["role"].as_str().unwrap().into(), + name: m["name"].as_str().unwrap_or_default().into(), + content: m["content"].as_str().unwrap_or_default().into(), + reasoning: m["reasoning_content"].as_str().unwrap_or_default().into(), + tool_call_id: m["tool_call_id"].as_str().unwrap_or_default().into(), + ..ChatMsg::default() + }; + if let Some(calls) = m["tool_calls"].as_array() { + msg.calls = calls + .iter() + .map(|call| ToolCall { + id: call["id"].as_str().unwrap().into(), + name: call["function"]["name"].as_str().unwrap().into(), + arguments: call["function"]["arguments"].to_string(), + }) + .collect(); + } + if let Some(parts) = m["content"].as_array() { + msg.parts = parts + .iter() + .map(|part| match part["type"].as_str().unwrap() { + "text" => ChatPart::Text(part["text"].as_str().unwrap().into()), + "image" => ChatPart::Image(0), + "audio" | "input_audio" => ChatPart::Audio(0), + _ => unreachable!(), + }) + .collect(); + } + msg + }) + .collect(); + let think = match row["effort"].as_str().unwrap() { + "none" => ThinkMode::None, + "low" => ThinkMode::Low, + "high" => ThinkMode::High, + "max" => ThinkMode::Max, + _ => unreachable!(), + }; + let tools = if row["tools"].as_array().unwrap().is_empty() { + String::new() + } else { + row["tools"] + .as_array() + .unwrap() + .iter() + .map(Value::to_string) + .collect::>() + .join("\n") + }; + let actual = render_chat(syntax_for_model_id(9), &messages, &tools, think).unwrap(); + assert_eq!( + String::from_utf8(actual).unwrap(), + row["rendered"].as_str().unwrap(), + "{} / {}", + row["name"], + row["effort"] + ); + } +} + +#[test] +fn streamed_unclosed_text() { + let events = stream_events( + ds4_server::Api::Anthropic, + ThinkMode::None, + b"<|content_text|>unfinished", + 1, + ); + let starts = events + .iter() + .filter(|e| e["type"] == "content_block_start") + .count(); + let stops = events + .iter() + .filter(|e| e["type"] == "content_block_stop") + .count(); + assert_eq!(starts, 1); + assert_eq!(starts, stops); +} + +#[test] +fn wire_tool_declarations() { + use ds4_server::route::WireSurface; + let cases = [ + ( + WireSurface::OpenaiChat, + serde_json::json!({"messages": [{"role": "user", "content": "Hi"}], "tools": [ + {"type": "function", "function": {"name": "one", "parameters": {"type": "object"}}}, + {"type": "function", "function": {"name": "two", "description": "Second", "parameters": {"type": "object"}}} + ]}), + ), + ( + WireSurface::Anthropic, + serde_json::json!({"messages": [{"role": "user", "content": "Hi"}], "max_tokens": 16, "tools": [ + {"name": "one", "input_schema": {"type": "object"}}, + {"name": "two", "description": "Second", "input_schema": {"type": "object"}} + ]}), + ), + ( + WireSurface::Responses, + serde_json::json!({"input": "Hi", "tools": [ + {"type": "function", "name": "one", "parameters": {"type": "object"}}, + {"type": "function", "name": "two", "description": "Second", "parameters": {"type": "object"}} + ]}), + ), + ]; + let expected = r#"<|message_system|>tool_declare<|content_xml|>[{"description":"","name":"one","parameters":{"type":"object"},"type":"function"},{"description":"Second","name":"two","parameters":{"type":"object"},"type":"function"}]<|end_message|>"#; + for (surface, body) in cases { + let request = + ds4_server::parse_request(surface, &ds4_server::ParseEnv::default(), &body.to_string()) + .unwrap(); + let actual = ds4_server::render_prompt(&request, 9).unwrap(); + assert!(String::from_utf8(actual).unwrap().starts_with(expected)); + } +} + +#[test] +fn responses_tool_deltas() { + let raw = br#"weather<|content_invoke_tool_json|>{"name":"weather","args":{"city":"Seoul"}}<|end_message|>"#; + let events = stream_events(ds4_server::Api::Responses, ThinkMode::None, raw, 1); + let mut arguments = String::new(); + for event in events { + match event["type"].as_str().unwrap() { + "response.output_item.added" if event["item"]["type"] == "function_call" => { + arguments = event["item"]["arguments"].as_str().unwrap().into(); + } + "response.function_call_arguments.delta" => { + arguments.push_str(event["delta"].as_str().unwrap()); + } + "response.function_call_arguments.done" => { + assert_eq!(arguments, event["arguments"].as_str().unwrap()); + assert_eq!( + serde_json::from_str::(&arguments).unwrap(), + serde_json::json!({"city": "Seoul"}) + ); + } + _ => {} + } + } +} diff --git a/crates/ds4-server/tests/iquest_output.rs b/crates/ds4-server/tests/iquest_output.rs new file mode 100644 index 0000000000000000000000000000000000000000..96bbc479a2f0f23ba253f9b4a59c9f25b8ea73c8 --- /dev/null +++ b/crates/ds4-server/tests/iquest_output.rs @@ -0,0 +1,650 @@ +//! IQuest's source XML output protocol; all tests are model-free. +use ds4_server::generate::chat_format_for_syntax; +use ds4_server::tools::{parse_generated_for_model_id, parse_generated_for_response}; +use ds4_server::{ + anthropic_sse_finish_live, anthropic_sse_start_live, anthropic_sse_stream_update, + openai_sse_finish_live, openai_sse_stream_update, openai_stream_start, responses_sse_created, + responses_sse_finish_live, responses_sse_stream_update, responses_stream_init, + syntax_for_model_id, Api, SemAccum, StreamReq, ThinkMode, ToolSchemaOrder, Writer, + CREATED_TEST, +}; +use serde_json::{json, Value}; + +const IQUEST: i32 = 15; +const PROMPT: &[u8] = b"<|iquest_assistant|>"; +const CALL: &str = concat!( + "lookup", + "text 123 ", + "count2", + "activetrue", + "data{\"city\":\"서울\",\"items\":[1,null]}", + "" +); + +fn orders() -> Vec { + vec![ToolSchemaOrder { + name: "lookup".into(), + prop: vec![ + "text".into(), + "count".into(), + "active".into(), + "data".into(), + ], + prop_type: vec![ + "string".into(), + "integer".into(), + "boolean".into(), + "object".into(), + ], + ..Default::default() + }] +} + +#[test] +fn iquest_model_mapping() { + assert_eq!(syntax_for_model_id(IQUEST) as i32, IQUEST); +} + +#[test] +fn iquest_source_types_and_surrounding_text() { + let raw = + format!("planBefore.{CALL}Between.pingAfter."); + let parsed = parse_generated_for_model_id(IQUEST, raw.as_bytes(), true, &orders()); + assert!(parsed.ok); + assert_eq!(parsed.reasoning, b"plan"); + assert_eq!(parsed.content, b"Before.Between.After."); + assert_eq!(parsed.calls.len(), 2); + assert_eq!(parsed.calls[0].name, "lookup"); + let args: Value = serde_json::from_str(&parsed.calls[0].arguments).unwrap(); + assert_eq!( + args, + json!({"text":" 123 ","count":2,"active":true,"data":{"city":"서울","items":[1,null]}}) + ); + assert_eq!(parsed.calls[1].name, "ping"); + assert_eq!(parsed.calls[1].arguments, "{}"); +} + +#[test] +fn iquest_malformed_and_disabled_tools_stay_text() { + let broken = b"runxbroken"; + let parsed = parse_generated_for_model_id(IQUEST, broken, false, &[]); + assert!(parsed.ok && parsed.calls.is_empty()); + assert_eq!(parsed.content, broken); + let syntax = syntax_for_model_id(IQUEST); + let format = chat_format_for_syntax(syntax); + let (parsed, _) = parse_generated_for_response( + syntax, + CALL.as_bytes(), + false, + false, + false, + format, + &orders(), + "stop", + ); + assert!(parsed.calls.is_empty()); + assert_eq!(parsed.content, CALL.as_bytes()); +} + +#[test] +fn iquest_bytewise_semantic_markers() { + let raw = format!("plan{CALL}"); + let format = chat_format_for_syntax(syntax_for_model_id(IQUEST)); + for width in 1..=raw.len() { + let mut acc = SemAccum::init(true, true, true, format, PROMPT); + for chunk in raw.as_bytes().chunks(width) { + acc.feed(chunk, &[]); + } + assert!(!acc.thinking_inside(), "width {width}"); + assert!(acc.saw_tool_start && acc.saw_tool_end, "width {width}"); + // Closing one call must leave generation open for subsequent calls/text. + assert_eq!(acc.verdict, None, "width {width}"); + } +} + +fn events(bytes: &[u8]) -> Vec { + std::str::from_utf8(bytes) + .unwrap() + .lines() + .filter_map(|line| line.strip_prefix("data: ")) + .filter(|line| *line != "[DONE]") + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +fn request() -> StreamReq { + let syntax = syntax_for_model_id(IQUEST); + StreamReq { + syntax, + chat_format: chat_format_for_syntax(syntax), + has_tools: true, + think_mode: ThinkMode::None, + tool_orders: orders(), + ..Default::default() + } +} + +#[test] +fn iquest_openai_stream_every_split() { + let raw = format!("before{CALL}betweenpingafter"); + let parsed = parse_generated_for_model_id(IQUEST, raw.as_bytes(), false, &orders()); + let r = request(); + for width in 1..=raw.len() { + let mut w = Writer::new(CREATED_TEST); + let mut state = openai_stream_start(&r); + let mut current = Vec::new(); + for chunk in raw.as_bytes().chunks(width) { + current.extend(chunk); + assert!(openai_sse_stream_update( + &mut w, &r, "job", &mut state, ¤t, false + )); + } + assert!(openai_sse_finish_live( + &mut w, + &r, + "job", + &mut state, + ¤t, + "tool_calls", + 2, + 5, + &parsed.calls + )); + let mut text = String::new(); + let mut calls: std::collections::BTreeMap = + Default::default(); + for e in events(&w.out) { + let delta = &e["choices"][0]["delta"]; + text.push_str(delta["content"].as_str().unwrap_or("")); + for call in delta["tool_calls"].as_array().into_iter().flatten() { + let entry = calls.entry(call["index"].as_i64().unwrap()).or_default(); + if let Some(id) = call["id"].as_str() { + entry.0.push_str(id); + } + if let Some(name) = call["function"]["name"].as_str() { + entry.1.push_str(name); + } + if let Some(args) = call["function"]["arguments"].as_str() { + entry.2.push_str(args); + } + } + } + assert_eq!(text, "beforebetweenafter", "width {width}"); + assert_eq!(calls.len(), 2, "width {width}"); + assert_eq!(calls[&0].1, "lookup"); + assert_eq!(calls[&1].1, "ping"); + assert_ne!(calls[&0].0, calls[&1].0); + assert_eq!( + serde_json::from_str::(&calls[&0].2).unwrap(), + serde_json::from_str::(&parsed.calls[0].arguments).unwrap() + ); + assert_eq!(calls[&1].2, "{}"); + } +} + +#[test] +fn iquest_anthropic_and_responses_stream_splits() { + let raw = format!("before{CALL}betweenpingafter"); + let parsed = parse_generated_for_model_id(IQUEST, raw.as_bytes(), false, &orders()); + for width in [1, 3, 23, raw.len()] { + let mut r = request(); + r.api = Api::Anthropic; + let mut w = Writer::new(CREATED_TEST); + let mut st = anthropic_sse_start_live(&mut w, &r, "job", 2); + let mut current = Vec::new(); + for chunk in raw.as_bytes().chunks(width) { + current.extend(chunk); + assert!(anthropic_sse_stream_update( + &mut w, &r, "job", &mut st, ¤t, false + )); + } + assert!(anthropic_sse_finish_live( + &mut w, + &r, + "job", + &mut st, + ¤t, + "tool_calls", + None, + 5, + &parsed.calls + )); + let records = events(&w.out); + let text: String = records + .iter() + .filter_map(|e| e["delta"]["text"].as_str()) + .collect(); + let names: Vec<_> = records + .iter() + .filter_map(|e| e["content_block"]["name"].as_str()) + .collect(); + assert_eq!(text, "beforebetweenafter"); + assert_eq!(names, ["lookup", "ping"]); + assert_eq!( + records + .iter() + .filter(|e| e["type"] == "content_block_start") + .count(), + records + .iter() + .filter(|e| e["type"] == "content_block_stop") + .count() + ); + r.api = Api::Responses; + let mut w = Writer::new(CREATED_TEST); + let mut st = responses_stream_init(&r, "resp", "reason", "message"); + responses_sse_created(&mut w, &r, &mut st, CREATED_TEST); + let mut current = Vec::new(); + for chunk in raw.as_bytes().chunks(width) { + current.extend(chunk); + assert!(responses_sse_stream_update( + &mut w, &r, &mut st, ¤t, false + )); + } + assert!(responses_sse_finish_live( + &mut w, + &r, + &mut st, + ¤t, + "tool_calls", + 2, + 5, + 0, + CREATED_TEST, + &parsed.calls + )); + let records = events(&w.out); + let text: String = records + .iter() + .filter(|e| e["type"] == "response.output_text.delta") + .filter_map(|e| e["delta"].as_str()) + .collect(); + assert_eq!(text, "beforebetweenafter"); + let output = records.last().unwrap()["response"]["output"] + .as_array() + .unwrap(); + assert_eq!(output[0]["content"][0]["text"], "beforebetweenafter"); + assert_eq!(output[1]["name"], "lookup"); + assert_eq!(output[2]["name"], "ping"); + assert!(!std::str::from_utf8(&w.out) + .unwrap() + .contains("iquest_tool_call")); + } +} + +#[test] +fn iquest_stream_malformed_and_disabled() { + let raw = "runxbroken"; + for enabled in [true, false] { + let mut r = request(); + r.has_tools = enabled; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + for end in 1..=raw.len() { + assert!(openai_sse_stream_update( + &mut w, + &r, + "job", + &mut st, + &raw.as_bytes()[..end], + false + )); + } + assert!(openai_sse_finish_live( + &mut w, + &r, + "job", + &mut st, + raw.as_bytes(), + "stop", + 2, + 5, + &[] + )); + let text: String = events(&w.out) + .iter() + .filter_map(|e| e["choices"][0]["delta"]["content"].as_str()) + .collect(); + assert_eq!(text, raw); + let mut acc = SemAccum::init(true, enabled, false, r.chat_format, b""); + for byte in raw.bytes() { + acc.feed(&[byte], &[]); + } + assert_eq!(acc.verdict, None); + } +} + +#[test] +fn iquest_official_parser_vectors() { + let vectors: Value = + serde_json::from_str(include_str!("fixtures/iquest-tool-parser.json")).unwrap(); + for case in vectors["cases"].as_array().unwrap() { + let raw = case["text"].as_str().unwrap().as_bytes(); + let enabled = case["tools_enabled"].as_bool().unwrap(); + let r = request(); + let (parsed, _) = parse_generated_for_response( + r.syntax, + raw, + enabled, + false, + false, + r.chat_format, + &orders(), + "stop", + ); + assert!(parsed.ok, "{}", case["name"]); + assert_eq!( + std::str::from_utf8(&parsed.content).unwrap(), + case["content"].as_str().unwrap(), + "{}", + case["name"] + ); + let calls: Vec = parsed.calls.iter().map(|call| json!({ + "name":call.name,"arguments":serde_json::from_str::(&call.arguments).unwrap() + })).collect(); + assert_eq!(Value::Array(calls), case["calls"], "{}", case["name"]); + let mut r = r; + r.has_tools = enabled; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + for end in 1..=raw.len() { + assert!(openai_sse_stream_update( + &mut w, + &r, + "job", + &mut st, + &raw[..end], + false + )); + } + let records = events(&w.out); + let text: String = records + .iter() + .filter_map(|e| e["choices"][0]["delta"]["content"].as_str()) + .collect(); + assert_eq!( + text, case["stream_content_before_final"], + "{} streamed before final", + case["name"] + ); + let mut calls: std::collections::BTreeMap = Default::default(); + for call in records + .iter() + .filter_map(|e| e["choices"][0]["delta"]["tool_calls"].as_array()) + .flatten() + { + let entry = calls.entry(call["index"].as_i64().unwrap()).or_default(); + entry + .0 + .push_str(call["function"]["name"].as_str().unwrap_or("")); + entry + .1 + .push_str(call["function"]["arguments"].as_str().unwrap_or("")); + } + let calls: Vec = calls.into_iter().map(|(index,(name,args))| + json!({"index":index,"name":name,"arguments":serde_json::from_str::(&args).unwrap()})).collect(); + assert_eq!( + Value::Array(calls), + case["stream_calls_before_final"], + "{} streamed calls", + case["name"] + ); + assert!(openai_sse_finish_live( + &mut w, + &r, + "job", + &mut st, + raw, + "stop", + 1, + 1, + &parsed.calls + )); + let text: String = events(&w.out) + .iter() + .filter_map(|e| e["choices"][0]["delta"]["content"].as_str()) + .collect(); + assert_eq!(text, case["content"], "{} streamed", case["name"]); + } +} + +#[test] +fn iquest_reasoning_boundaries() { + let hidden = format!("private{CALL}"); + let visible = format!("visible{CALL}"); + for prefix in ["", ""] { + let raw = format!("{prefix}{hidden}{visible}"); + let parsed = parse_generated_for_model_id(IQUEST, raw.as_bytes(), true, &orders()); + assert_eq!(parsed.reasoning, hidden.as_bytes()); + assert_eq!(parsed.content, b"visible"); + assert_eq!(parsed.calls.len(), 1); + let mut r = request(); + r.think_mode = ThinkMode::High; + for width in [1, 3, 17, raw.len()] { + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + let mut current = Vec::new(); + for chunk in raw.as_bytes().chunks(width) { + current.extend(chunk); + assert!(openai_sse_stream_update( + &mut w, &r, "job", &mut st, ¤t, false + )); + } + assert!(openai_sse_finish_live( + &mut w, + &r, + "job", + &mut st, + ¤t, + "tool_calls", + 1, + 1, + &parsed.calls + )); + let records = events(&w.out); + let reasoning: String = records + .iter() + .filter_map(|e| e["choices"][0]["delta"]["reasoning_content"].as_str()) + .collect(); + let text: String = records + .iter() + .filter_map(|e| e["choices"][0]["delta"]["content"].as_str()) + .collect(); + assert_eq!(reasoning, hidden); + assert_eq!(text, "visible"); + let calls = records + .iter() + .filter_map(|e| e["choices"][0]["delta"]["tool_calls"].as_array()) + .flatten() + .filter(|call| call["function"]["name"].is_string()) + .count(); + assert_eq!(calls, 1); + } + let unfinished = format!("{prefix}{hidden}"); + let parsed = parse_generated_for_model_id(IQUEST, unfinished.as_bytes(), true, &orders()); + assert_eq!(parsed.reasoning, hidden.as_bytes()); + assert!(parsed.content.is_empty() && parsed.calls.is_empty()); + } + let raw = format!("discardedprivate{visible}"); + let parsed = parse_generated_for_model_id(IQUEST, raw.as_bytes(), true, &orders()); + assert_eq!(parsed.reasoning, b"private"); + let (parsed, _) = parse_generated_for_response( + syntax_for_model_id(IQUEST), + raw.as_bytes(), + false, + false, + true, + chat_format_for_syntax(syntax_for_model_id(IQUEST)), + &orders(), + "stop", + ); + assert_eq!(parsed.reasoning, b"private"); + assert_eq!(parsed.content, visible.as_bytes()); + assert!(parsed.calls.is_empty()); +} + +#[test] +fn iquest_arbitrary_integer_arguments() { + for number in ["18446744073709551617", "-18446744073709551617"] { + let raw = format!("lookupcount{number}"); + let parsed = parse_generated_for_model_id(IQUEST, raw.as_bytes(), false, &orders()); + let expected = format!("{{\"count\":{number}}}"); + assert_eq!(parsed.calls[0].arguments, expected); + let r = request(); + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + assert!(openai_sse_stream_update( + &mut w, + &r, + "job", + &mut st, + raw.as_bytes(), + false + )); + assert!(openai_sse_finish_live( + &mut w, + &r, + "job", + &mut st, + raw.as_bytes(), + "tool_calls", + 1, + 1, + &parsed.calls + )); + let args: String = events(&w.out) + .iter() + .filter_map(|e| e["choices"][0]["delta"]["tool_calls"].as_array()) + .flatten() + .filter_map(|call| call["function"]["arguments"].as_str()) + .collect(); + assert_eq!(args, expected); + } +} + +#[test] +fn iquest_malformed_think_prelude_contract() { + let raw = b"discardedprivatevisible"; + let parsed = parse_generated_for_model_id(IQUEST, raw, true, &orders()); + assert_eq!(parsed.reasoning, b"private"); + assert_eq!(parsed.content, b"visible"); + for width in [1, 2, raw.len()] { + let mut r = request(); + r.think_mode = ThinkMode::High; + r.has_tools = false; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + let mut current = Vec::new(); + for chunk in raw.chunks(width) { + current.extend(chunk); + assert!(openai_sse_stream_update( + &mut w, &r, "job", &mut st, ¤t, false + )); + } + assert!(openai_sse_finish_live( + &mut w, + &r, + "job", + &mut st, + raw, + "stop", + 1, + 1, + &[] + )); + let records = events(&w.out); + let reasoning: String = records + .iter() + .filter_map(|e| e["choices"][0]["delta"]["reasoning_content"].as_str()) + .collect(); + let text: String = records + .iter() + .filter_map(|e| e["choices"][0]["delta"]["content"].as_str()) + .collect(); + // Already emitted malformed prelude bytes cannot be retracted. + assert_eq!(reasoning, "discardedprivate"); + assert_eq!(text, "visible"); + r.api = Api::Anthropic; + let mut w = Writer::new(CREATED_TEST); + let mut st = anthropic_sse_start_live(&mut w, &r, "job", 1); + let mut current = Vec::new(); + for chunk in raw.chunks(width) { + current.extend(chunk); + assert!(anthropic_sse_stream_update( + &mut w, &r, "job", &mut st, ¤t, false + )); + } + assert!(anthropic_sse_finish_live( + &mut w, + &r, + "job", + &mut st, + raw, + "stop", + None, + 1, + &[] + )); + let records = events(&w.out); + let reasoning: String = records + .iter() + .filter_map(|e| e["delta"]["thinking"].as_str()) + .collect(); + let text: String = records + .iter() + .filter_map(|e| e["delta"]["text"].as_str()) + .collect(); + assert_eq!(reasoning, "discardedprivate"); + assert_eq!(text, "visible"); + r.api = Api::Responses; + r.reasoning_summary_emit = true; + let mut w = Writer::new(CREATED_TEST); + let mut st = responses_stream_init(&r, "resp", "reason", "message"); + responses_sse_created(&mut w, &r, &mut st, CREATED_TEST); + let mut current = Vec::new(); + for chunk in raw.chunks(width) { + current.extend(chunk); + assert!(responses_sse_stream_update( + &mut w, &r, &mut st, ¤t, false + )); + } + assert!(responses_sse_finish_live( + &mut w, + &r, + &mut st, + raw, + "stop", + 1, + 1, + 1, + CREATED_TEST, + &[] + )); + let records = events(&w.out); + let output = records.last().unwrap()["response"]["output"] + .as_array() + .unwrap(); + assert_eq!(output[0]["summary"][0]["text"], "discardedprivate"); + assert_eq!(output[1]["content"][0]["text"], "visible"); + } +} + +#[test] +fn iquest_implicit_reasoning_is_incremental() { + let mut r = request(); + r.think_mode = ThinkMode::High; + r.has_tools = false; + let raw = b"ordinary native reasoning continues"; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + assert!(openai_sse_stream_update( + &mut w, &r, "job", &mut st, raw, false + )); + let reasoning: String = events(&w.out) + .iter() + .filter_map(|e| e["choices"][0]["delta"]["reasoning_content"].as_str()) + .collect(); + assert!(!reasoning.is_empty()); + assert!(std::str::from_utf8(raw).unwrap().starts_with(&reasoning)); + assert!(!st.checked_think_prefix || st.mode == ds4_server::stream::OpenaiMode::Thinking); +} diff --git a/crates/ds4-server/tests/metrics.rs b/crates/ds4-server/tests/metrics.rs new file mode 100644 index 0000000000000000000000000000000000000000..3c7c5abd69d2629599ca882d9a0b0506d069f4e3 --- /dev/null +++ b/crates/ds4-server/tests/metrics.rs @@ -0,0 +1,186 @@ +//! C↔Rust `/metrics` + `/v1/stats` memgov porcelain. + +use ds4_server::{ + dump_memgov_names, render_memgov_metrics, render_metrics, render_stats_memgov_json, AdmitState, + MemgovSnap, ReconcileSnap, RouteMetrics, RuntimeMetrics, +}; + +use std::path::PathBuf; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_MEMGOV_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/memgov_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/memgov_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_str(args: &[&str]) -> String { + let out = Command::new(require_oracle()) + .args(args) + .output() + .expect("run memgov_c_oracle"); + assert!( + out.status.success(), + "memgov_c_oracle {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("oracle utf8") +} + +fn supported_runtime() -> RuntimeMetrics { + let mut rt = RuntimeMetrics::default(); + rt.uptime_seconds = 12; + rt.requests_started = 3; + rt.requests_completed = 2; + rt.requests_failed = 1; + rt.requests_inflight = 1; + rt.requests_serial = 1; + rt.out_backlog_bytes = 4096; + rt.cont_admit_rejects = 1; + rt.cont_credit_ext_granted = 2; + rt.cont_credit_ext_refused = 3; + rt.have_batch_ctx = true; + rt.cont_commit_observed = 64; + rt.cont_commit_packed = 48; + rt.cont_admit_band_x1024 = 3; + rt.serial_idle_reaps = 4; + rt.cont_batch_failures = 5; + rt.graph_fit_refusals = 6; + rt.rejected[0][0] = 7; + rt.reclaim_banks[1] = 8; + rt.reclaim_bytes[2] = 9; + rt.admits_cold = 1; + rt.admits_warm = 2; + rt.admits_fork = 3; + rt.admits_partial_fork = 4; + rt.admits_partial_truncate = 5; + rt.tokens_prefilled_computed = 10; + rt.tokens_prefilled_cached = 11; + rt.tokens_decoded = 12; + rt.decode_steps = 13; + rt.spec_drafts = 4; + rt.spec_hits = 2; + rt.spec_quench = 1; + rt.decode_tok_s = 1.5; + rt.prefill_tok_s = 2.25; + rt.tok_per_step = 0.5; + rt.banks_live = 2; + rt.banks_total = 4; + rt.warm_records = 3; + rt.kv_pages_resident = 5; + rt.cont_ontoken_ns = 100; + rt.cont_ontoken_tokens = 10; + rt.batch_genmu_wait_ns = 200; + rt.batch_genmu_waits = 2; + rt.creg_published = 1; + rt.creg_resolved = 2; + rt.creg_missed = 3; + rt.creg_demoted = 4; + rt.creg_records_live = 1; + rt.artifact_source = 2; + rt.derived_artifacts = 3; + rt.derived_artifact_bytes = 4096; + rt.memgov.census_supported = true; + rt.memgov.cells[11][0].requested = 200; + rt.memgov.cells[11][0].committed = 256; + rt.memgov.census_faults = 1; + rt.memgov.torn_fallbacks = 2; + rt.memgov.census_epoch = 7; + rt.memgov.substrate_outstanding = 1024; + rt.memgov.obs_status = 0; + rt.memgov.obs_source = 1; + rt.memgov.obs_free = 100; + rt.memgov.obs_total = 200; + rt.memgov.obs_cuda_free = 90; + rt.memgov.obs_meminfo = 80; + rt.memgov.memobs_calls = [1, 2, 3]; + rt.memgov.memobs_errors = 4; + rt.memgov.own_trim_calls = 5; + rt.memgov.own_trim_recovered = 6; + rt.memgov.reconcile = Some(ReconcileSnap { + residual: -10, + onetime: 20, + }); + rt.memgov.reconcile_flagged = 1; + rt.memgov.gov_epoch = 3; + rt.memgov.leases[0].intent = 10; + rt.memgov.leases[0].resident = 20; + rt.memgov.leases[0].reservation = 30; + rt.memgov.decisions[0][0][0] = 1; + rt.memgov.deficit[3] = 99; + rt.memgov.faults = 8; + rt.memgov.residency_supported = true; + rt.memgov.residency_units[0][0] = 5; + rt.memgov.residency_failures[0][0] = 1; + rt +} + +#[test] +fn memgov_names_match_c() { + assert_eq!(dump_memgov_names(), c_str(&["names"])); +} + +#[test] +fn metrics_full_zero_matches_c() { + let rust = render_metrics( + &RouteMetrics::default(), + &AdmitState::default(), + &RuntimeMetrics::default(), + ); + assert_eq!(rust, c_str(&["full-zero"])); +} + +#[test] +fn memgov_unsupported_matches_c() { + assert_eq!( + render_memgov_metrics(&MemgovSnap::default()), + c_str(&["unsupported"]) + ); +} + +#[test] +fn metrics_supported_matches_c() { + let rust = render_metrics( + &RouteMetrics::default(), + &AdmitState::default(), + &supported_runtime(), + ); + assert_eq!(rust, c_str(&["supported"])); +} + +#[test] +fn mixed_artifact_source_matches_c() { + let mut rt = supported_runtime(); + rt.artifact_source = 3; + let rust = render_metrics(&RouteMetrics::default(), &AdmitState::default(), &rt); + assert!(rust.contains("source=\"mixed\"")); + assert_eq!(rust, c_str(&["mixed"])); +} + +#[test] +fn stats_memgov_zero_matches_c() { + assert_eq!( + render_stats_memgov_json(&MemgovSnap::default()), + c_str(&["stats-zero"]) + ); +} + +#[test] +fn stats_memgov_supported_matches_c() { + assert_eq!( + render_stats_memgov_json(&supported_runtime().memgov), + c_str(&["stats-supported"]) + ); +} diff --git a/crates/ds4-server/tests/naive_protocol.rs b/crates/ds4-server/tests/naive_protocol.rs new file mode 100644 index 0000000000000000000000000000000000000000..abe261ea4d6d254207dff7d29dbb6d3f2b74101e --- /dev/null +++ b/crates/ds4-server/tests/naive_protocol.rs @@ -0,0 +1,80 @@ +use ds4_server::generate::chat_format_for_syntax; +use ds4_server::parse::ChatMsg; +use ds4_server::render::{render_chat, syntax_for_model_id, tool_start_marker}; +use ds4_server::route::ThinkMode; +use ds4_server::tools::parse_generated_for_model_id; +use ds4_server::{SemAccum, ToolSchemaOrder}; + +const MODEL_ID: i32 = 13; + +#[test] +fn naive_requires_official_input() { + let msgs = [ChatMsg { + role: "user".into(), + content: "Hello".into(), + ..Default::default() + }]; + assert!(render_chat(syntax_for_model_id(MODEL_ID), &msgs, "", ThinkMode::None).is_err()); + assert_eq!( + tool_start_marker(syntax_for_model_id(MODEL_ID)), + "" + ); +} + +#[test] +fn naive_splits_thought_and_xml() { + let text = concat!( + "검토\n\n", + "\nnotes.txt\n\n" + ); + let parsed = parse_generated_for_model_id(MODEL_ID, text.as_bytes(), true, &[]); + assert!(parsed.ok); + assert_eq!(parsed.reasoning, "검토".as_bytes()); + assert_eq!(parsed.calls.len(), 1); + assert_eq!(parsed.calls[0].name, "read"); + assert_eq!( + serde_json::from_str::(&parsed.calls[0].arguments).unwrap(), + serde_json::json!({"path": "notes.txt"}) + ); +} + +#[test] +fn naive_preserves_param_lines() { + let text = "\n\n\n notes.txt \n\n\n"; + let orders = [ToolSchemaOrder { + name: "read".into(), + prop: vec!["path".into()], + prop_type: vec!["string".into()], + ..Default::default() + }]; + let parsed = parse_generated_for_model_id(MODEL_ID, text.as_bytes(), true, &orders); + assert!(parsed.ok && parsed.calls.len() == 1); + assert_eq!( + serde_json::from_str::(&parsed.calls[0].arguments).unwrap(), + serde_json::json!({"path": "\n notes.txt \n"}) + ); +} + +#[test] +fn naive_stream_handles_splits() { + let text = "검토\n\n\nnotes.txt\n\n".as_bytes(); + let format = chat_format_for_syntax(syntax_for_model_id(MODEL_ID)); + for width in 1..=text.len() { + let mut acc = SemAccum::init(true, true, true, format, b"<|im_start|>assistant\n"); + assert!(!acc.thinking_inside()); + for chunk in text.chunks(width) { + acc.feed(chunk, &[]); + } + assert!(!acc.thinking_inside(), "width {width}"); + assert!(acc.saw_tool_start && acc.saw_tool_end, "width {width}"); + assert_eq!(acc.verdict, None); + } +} + +#[test] +fn naive_plain_and_open_thought() { + let plain = parse_generated_for_model_id(MODEL_ID, b"Hello", true, &[]); + assert!(plain.ok && plain.content == b"Hello" && plain.reasoning.is_empty()); + let thought = parse_generated_for_model_id(MODEL_ID, b"unfinished", true, &[]); + assert!(thought.ok && thought.content.is_empty() && thought.reasoning == b"unfinished"); +} diff --git a/crates/ds4-server/tests/parse.rs b/crates/ds4-server/tests/parse.rs new file mode 100644 index 0000000000000000000000000000000000000000..e6c63cb81f65a8efb59eec942a474686df904967 --- /dev/null +++ b/crates/ds4-server/tests/parse.rs @@ -0,0 +1,1371 @@ +//! C↔Rust four-surface JSON parsers (tokenize/render stay C). + +use ds4_server::render::render_dsml_chat_choice; +use ds4_server::render::DSML_IMAGE; +use ds4_server::{ + generation_blocked, parse_anthropic_request, parse_chat_request, parse_completion_request, + parse_request, parse_responses_request, ChatPart, ImageMime, ParseEnv, ParsedRequest, + ToolChoice, WireSurface, NEED_IMAGE, +}; + +use std::path::PathBuf; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_PARSE_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/parse_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/parse_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_str(args: &[&str]) -> String { + let out = Command::new(require_oracle()) + .args(args) + .output() + .expect("run parse_c_oracle"); + assert!( + out.status.success(), + "oracle {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).unwrap() +} + +fn env() -> ParseEnv { + ParseEnv { + default_model: "deepseek-v4-flash".into(), + default_tokens: 393216, + default_effort: ds4_server::ThinkMode::Low, + default_temp: ds4_server::default_temperature(), + live_ids: Vec::new(), + engine_defaults: false, + } +} + +fn rust_parse(surface: &str, body: &str) -> Result { + let e = env(); + match surface { + "chat" => parse_chat_request(&e, body), + "completion" => parse_completion_request(&e, body), + "anthropic" => parse_anthropic_request(&e, body), + "responses" => parse_responses_request(&e, body), + _ => panic!("surface {surface}"), + } +} + +fn dump(r: &ParsedRequest) -> String { + let mut s = format!( + "OK\nkind={} api={} model={} from_req={}\nmax_tokens={} max_set={} top_k={} seed={}\ntemp={:.6} top_p={:.6} min_p={:.6}\nstream={} usage={} echo={} think={} tools={} tool_results={} choice={}\nstops={} summary={} needs={} nmsg={}\n", + r.kind as i32, + r.api as i32, + r.model, + r.model_from_request as i32, + r.max_tokens, + r.max_tokens_set as i32, + r.top_k, + r.seed, + r.temperature, + r.top_p, + r.min_p, + r.stream as i32, + r.stream_include_usage as i32, + r.return_token_ids as i32, + r.think_mode as i32, + r.has_tools as i32, + r.has_tool_results as i32, + r.tool_choice as i32, + r.stops.len(), + r.reasoning_summary_emit as i32, + r.needs, + r.messages.len() + ); + for (i, m) in r.messages.iter().take(8).enumerate() { + s.push_str(&format!( + "msg{i}_role={} msg{i}_ncalls={}\n", + m.role, + m.calls.len() + )); + } + s +} + +fn both(surface: &str, body: &str) -> (Result, String) { + (rust_parse(surface, body), c_str(&[surface, body])) +} + +fn normalize_c_schema_refusal(error: &str) -> String { + error.replacen( + "is not implemented: structured output is unsupported", + "is not supported: structured output is unsupported", + 1, + ) +} + +fn assert_err_eq(surface: &str, body: &str, expect: &str) { + let (rust, c) = both(surface, body); + let rust_e = rust.unwrap_err(); + assert!(c.starts_with("ERROR\n"), "{surface} {body}: {c}"); + let c_e = normalize_c_schema_refusal(c.strip_prefix("ERROR\n").unwrap().trim_end_matches('\n')); + assert_eq!(rust_e, c_e, "{surface} {body}"); + assert!( + rust_e.contains(expect) || rust_e == expect, + "{surface} {body}: rust={rust_e:?} expect {expect:?}" + ); +} + +fn assert_ok_eq(surface: &str, body: &str) -> ParsedRequest { + let (rust, c) = both(surface, body); + let rust = rust.unwrap_or_else(|e| panic!("rust {surface} {body}: {e}")); + assert!(c.starts_with("OK\n"), "{surface} {body}: {c}"); + assert_eq!(dump(&rust), c, "{surface} {body}"); + rust +} + +#[test] +fn c_unit_error_cases_match() { + assert_err_eq("chat", r#"{"max_tokens":-1}"#, "max_tokens must be >= 0"); + assert_err_eq( + "chat", + r#"{"max_completion_tokens":-2}"#, + "max_completion_tokens must be >= 0", + ); + assert_err_eq( + "anthropic", + r#"{"max_tokens":-1}"#, + "max_tokens must be >= 0", + ); + assert_err_eq( + "responses", + r#"{"max_output_tokens":-3}"#, + "max_output_tokens must be >= 0", + ); + assert_err_eq( + "completion", + r#"{"max_tokens":-1}"#, + "max_tokens must be >= 0", + ); + + assert_err_eq( + "chat", + r#"{"response_format":{"type":"text"}}"#, + "missing messages", + ); + assert_err_eq("chat", r#"{"response_format":null}"#, "missing messages"); + assert_err_eq("chat", r#"{"response_format":{}}"#, "missing messages"); + assert_err_eq( + "chat", + r#"{"response_format":{"type":"json_object"}}"#, + "response_format type 'json_object' is not supported", + ); + assert_err_eq( + "chat", + r#"{"response_format":{"json_schema":{"schema":{"type":"object"}},"type":"json_schema"}}"#, + "'json_schema' is not supported", + ); + assert_err_eq( + "chat", + r#"{"response_format":{"type":"xml"}}"#, + "response_format type 'xml' is not supported", + ); + assert_err_eq( + "chat", + r#"{"response_format":"json_object"}"#, + "'json_object' is not supported", + ); + assert_err_eq( + "completion", + r#"{"response_format":{"type":"text"}}"#, + "missing prompt", + ); + assert_err_eq( + "completion", + r#"{"response_format":{"type":"json_object"}}"#, + "response_format type 'json_object' is not supported", + ); + assert_err_eq( + "responses", + r#"{"text":{"format":{"type":"text"},"verbosity":"low"}}"#, + "missing input", + ); + assert_err_eq( + "responses", + r#"{"text":{"format":{"type":"json_schema","name":"x"}}}"#, + "text.format type 'json_schema' is not supported", + ); + assert_err_eq("anthropic", r#"{"output_format":null}"#, "missing messages"); + assert_err_eq( + "anthropic", + r#"{"output_format":{"type":"json_schema","schema":{}}}"#, + "output_format type 'json_schema' is not supported", + ); + assert_err_eq( + "anthropic", + r#"{"output_config":{"effort":"high","format":{"type":"json_object"}}}"#, + "output_config.format type 'json_object' is not supported", + ); + + assert_err_eq( + "responses", + r#"{"previous_response_id":"resp_123"}"#, + "previous_response_id is not supported; replay full input instead", + ); + assert_err_eq( + "responses", + r#"{"conversation":"conv_1"}"#, + "conversation is not supported; replay full input instead", + ); +} + +/// The accepted spelling of `response_format` is not a render input. The +/// encoder oracle owns that rendering — it appends the schema to the SYSTEM +/// region as "## Response Format:\n\nYou MUST strictly adhere..." when a +/// message dict carries the field (encoding.py:64-66, :628-629) — and neither +/// reference server reads the field at all. This server refuses the schema +/// forms by name (format.rs:9-15, the v0.6.3 refusal) and accepts "text"; the +/// case pins that the accepted form leaves every render input untouched, so a +/// later change cannot begin trusting a schema unnoticed. +#[test] +fn accepted_response_format_is_not_a_render_input() { + let plain = rust_parse( + "chat", + r#"{"messages":[{"role":"user","content":"two words"}]}"#, + ) + .unwrap(); + let text = rust_parse( + "chat", + r#"{"messages":[{"role":"user","content":"two words"}],"response_format":{"type":"text"}}"#, + ) + .unwrap(); + + assert_eq!(dump(&plain), dump(&text)); + assert_eq!(plain.tool_schemas, text.tool_schemas); + assert!(plain.tool_schemas.is_empty()); + assert_eq!(plain.messages.len(), text.messages.len()); + assert_eq!(plain.messages[0].content, text.messages[0].content); + assert_eq!(plain.messages[0].parts, text.messages[0].parts); +} + +/// A chat tool's name reaches the prompt verbatim, and a tool-level `namespace` +/// key is not folded into it. The encoder oracle does fold — `_tool_name_for_encoding` +/// (encoding.py:116) joins an explicit namespace or a "::"-qualified name into +/// "namespace::name", and :88-97 prefixes the namespace description onto the +/// function's — but that is the encoder's own rule for the message dicts handed +/// to it. Neither reference server reads a tool-level namespace on the chat +/// surface: namespaces exist there for `type:"namespace"` items on the +/// Responses surface (server_parse_tools.c:9-120), which this port mirrors +/// (parse.rs:2147). The case pins both forms this server does see. +#[test] +fn chat_tool_names_pass_through_verbatim() { + // The qualified name the oracle would produce is taken as given. + let qualified = r#"{"messages":[{"role":"user","content":"hi"}],"tools":[{"type":"function","function":{"name":"ns::foo","parameters":{"type":"object","properties":{}}}}]}"#; + let parsed = rust_parse("chat", qualified).unwrap(); + assert_eq!( + parsed.tool_schemas, + r#"{"name":"ns::foo","parameters":{"type":"object","properties":{}}}"# + ); + + // The tool-level key is not part of the schema, and the function object is + // carried byte-for-byte. + let folded = r#"{"messages":[{"role":"user","content":"hi"}],"tools":[{"type":"function","namespace":"ns","function":{"name":"foo","description":"d","parameters":{"type":"object","properties":{}}}}]}"#; + let parsed = rust_parse("chat", folded).unwrap(); + assert_eq!( + parsed.tool_schemas, + r#"{"name":"foo","description":"d","parameters":{"type":"object","properties":{}}}"# + ); +} + +#[test] +fn responses_namespace_tools_keep_both_spellings() { + // The Responses surface is the one place a namespace exists + // (server_parse_tools.c:9-120, mirrored by + // `append_responses_namespace_tool_schemas`, parse.rs:2336). A + // `type:"namespace"` item holds its functions, the prompt advertises the + // namespace concatenated with each function name + // (`responses_namespace_function_schema`, parse.rs:2282), and the order + // keeps both spellings so the answer can be emitted under the name the + // client asked for (`responses_tool_wire_name`, stream.rs:234) plus the + // namespace field (stream.rs:287). The chat surface has no namespace at + // all: `chat_tool_names_pass_through_verbatim` above pins that half. + let body = r#"{"input":[{"type":"message","role":"user","content":[{"type":"input_text","text":"hi"}]}],"tools":[{"type":"namespace","name":"mcp__files","tools":[{"type":"function","name":"read","description":"Read a file","parameters":{"type":"object","properties":{"path":{"type":"string"}}}}]}]}"#; + let parsed = rust_parse("responses", body).unwrap(); + + assert_eq!( + parsed.tool_schemas, + r#"{"name":"mcp__filesread","description":"Read a file","parameters":{"type":"object","properties":{"path":{"type":"string"}}}}"# + ); + assert_eq!(parsed.tool_orders.len(), 1); + assert_eq!(parsed.tool_orders[0].name, "mcp__filesread"); + assert_eq!(parsed.tool_orders[0].wire_name, "read"); + assert_eq!(parsed.tool_orders[0].namespace, "mcp__files"); + + // The prompt the model reads carries the same qualified name. + let prompt = render_dsml_chat_choice( + &parsed.messages, + &parsed.tool_schemas, + parsed.think_mode, + parsed.tool_choice, + ); + let prompt = String::from_utf8(prompt.unwrap()).unwrap(); + assert!(prompt.contains("mcp__filesread"), "{prompt}"); +} + +#[test] +fn missing_required_fields_and_invalid_json() { + assert_err_eq("chat", "{}", "missing messages"); + assert_err_eq("chat", "[]", "invalid JSON request"); + assert_err_eq("chat", "not-json", "invalid JSON request"); + assert_err_eq("completion", "{}", "missing prompt"); + assert_err_eq("anthropic", "{}", "missing messages"); + assert_err_eq("responses", "{}", "missing input"); + assert_err_eq( + "chat", + r#"{"messages":[{"role":"user","content":"hi"}]"#, + "invalid JSON request", + ); +} + +#[test] +fn tool_choice_errors_match_c() { + let s = c_str(&["tool-choice-openai", r#""unsupported""#]); + assert!(s.starts_with("ERROR\n")); + assert!(s.contains("not supported")); + assert_err_eq( + "chat", + r#"{"messages":[],"tool_choice":"unsupported"}"#, + "tool_choice=unsupported not supported", + ); + + assert_err_eq( + "anthropic", + r#"{"messages":[],"tool_choice":{"type":"tool","name":"lookup"}}"#, + "forced tool_choice not supported", + ); + assert_err_eq( + "chat", + r#"{"messages":[],"tool_choice":"required"}"#, + "tool_choice=required requires at least one tool", + ); +} + +/// The named-function tool choice: the OpenAI chat reference defines +/// `{"type":"function","function":{"name":"my_function"}}` as "forces the model +/// to call that tool", and the port serves it by narrowing the request's tool +/// set to that function and requiring the call (OpenAI's own allowed_tools +/// mechanism for the same intent). The production C engine refuses the form, so +/// the divergence is pinned below rather than left to a later parity pass to +/// find by accident. +#[test] +fn named_tool_choice_narrows_to_the_forced_function() { + let body = r#"{"messages":[{"role":"user","content":"hi"}],"tools":[{"type":"function","function":{"name":"other_function","parameters":{"type":"object"}}},{"type":"function","function":{"name":"my_function","parameters":{"type":"object","properties":{"a":{"type":"string"}},"required":["a"]}}}],"tool_choice":{"type":"function","function":{"name":"my_function"}}}"#; + let r = parse_chat_request(&env(), body).unwrap(); + assert_eq!(r.tool_choice, ToolChoice::Required); + assert_eq!(r.forced_tool.as_deref(), Some("my_function")); + assert!(r.has_tools); + assert_eq!(r.tool_orders.len(), 1, "{:?}", r.tool_orders); + assert_eq!(r.tool_orders[0].name, "my_function"); + assert!(r.tool_schemas.contains("my_function"), "{}", r.tool_schemas); + assert!( + !r.tool_schemas.contains("other_function"), + "{}", + r.tool_schemas + ); + + // Responses carries the same OpenAI value and takes the same route. + let resp = parse_responses_request( + &env(), + r#"{"input":"hi","tools":[{"type":"function","function":{"name":"other_function","parameters":{"type":"object"}}},{"type":"function","function":{"name":"my_function","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"my_function"}}}"#, + ) + .unwrap(); + assert_eq!(resp.tool_choice, ToolChoice::Required); + assert_eq!(resp.tool_orders.len(), 1); + assert!(resp.tool_schemas.contains("my_function")); + assert!(!resp.tool_schemas.contains("other_function")); + + // The Responses wire form names the function at the top level, and a + // pretty-printed tool list spans many lines: both narrow the same way. + let resp = parse_responses_request( + &env(), + "{\"input\":\"hi\",\"tools\":[\n {\n \"type\": \"function\",\n \"name\": \"other_function\",\n \"parameters\": {\"type\": \"object\"}\n },\n {\n \"type\": \"function\",\n \"name\": \"my_function\",\n \"parameters\": {\"type\": \"object\"}\n }\n],\"tool_choice\":{\"type\":\"function\",\"name\":\"my_function\"}}", + ) + .unwrap(); + assert_eq!(resp.tool_choice, ToolChoice::Required); + assert_eq!(resp.tool_orders.len(), 1, "{:?}", resp.tool_orders); + assert!(resp.has_tools); + assert!(resp.tool_schemas.contains("my_function")); + assert!(!resp.tool_schemas.contains("other_function")); + let pretty = "{\"messages\":[{\"role\":\"user\",\"content\":\"hi\"}],\"tools\":[\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"my_function\",\n \"parameters\": {\"type\": \"object\"}\n }\n }\n],\"tool_choice\":{\"type\":\"function\",\"function\":{\"name\":\"my_function\"}}}"; + let pr = parse_chat_request(&env(), pretty).unwrap(); + assert!( + pr.has_tools && pr.tool_schemas.contains("my_function"), + "{}", + pr.tool_schemas + ); + + // The renderer is handed exactly what the parse narrowed (generate.rs:1331), + // so the served prompt offers the forced function alone and requires the + // call. + let prompt = render_dsml_chat_choice(&r.messages, &r.tool_schemas, r.think_mode, r.tool_choice); + let prompt = String::from_utf8(prompt.unwrap()).unwrap(); + assert!(prompt.contains("my_function"), "{prompt}"); + assert!(!prompt.contains("other_function"), "{prompt}"); + assert!(prompt.contains("### Required Tool Use"), "{prompt}"); + + // The C engine's refusal of the same body, recorded here. + let c = c_str(&["chat", body]); + assert!(c.starts_with("ERROR\n"), "{c}"); + assert!(c.contains("forced tool_choice not supported"), "{c}"); +} + +#[test] +fn named_tool_choice_rejects_a_name_outside_the_tools() { + assert_eq!( + parse_chat_request( + &env(), + r#"{"messages":[],"tools":[{"type":"function","function":{"name":"other_function","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"my_function"}}}"#, + ) + .unwrap_err(), + "tool_choice function \"my_function\" is not among the request tools" + ); + assert_eq!( + parse_chat_request( + &env(), + r#"{"messages":[],"tool_choice":{"type":"function","function":{"name":"my_function"}}}"#, + ) + .unwrap_err(), + "tool_choice function \"my_function\" is not among the request tools" + ); +} + +#[test] +fn unsupported_tool_choice_objects_stay_refused() { + let cases = [ + ( + r#"{"messages":[],"tool_choice":{"type":"allowed_tools","allowed_tools":{"mode":"auto","tools":[]}}}"#, + "tool_choice type=allowed_tools not supported", + ), + ( + r#"{"messages":[],"tool_choice":{"type":"custom","custom":{"name":"my_function"}}}"#, + "tool_choice type=custom not supported", + ), + ( + r#"{"messages":[],"tool_choice":{"type":"function"}}"#, + "invalid tool_choice", + ), + ( + r#"{"messages":[],"tool_choice":{"type":"function","function":{}}}"#, + "invalid tool_choice", + ), + ( + r#"{"messages":[],"tool_choice":{"function":{"name":"lookup"}}}"#, + "invalid tool_choice", + ), + (r#"{"messages":[],"tool_choice":{}}"#, "invalid tool_choice"), + ]; + for (body, expect) in cases { + assert_eq!( + parse_chat_request(&env(), body).unwrap_err(), + expect, + "{body}" + ); + } +} + +#[test] +fn tool_controls_are_not_silently_ignored() { + let requests = [ + ( + "chat", + r#"{"messages":[],"tools":[{"type":"function","function":{"name":"lookup","parameters":{"type":"object"}}}],"tool_choice":"none"}"#, + ), + ( + "anthropic", + r#"{"messages":[],"tools":[{"name":"lookup","input_schema":{"type":"object"}}],"tool_choice":{"type":"none"}}"#, + ), + ( + "responses", + r#"{"input":[],"tools":[{"type":"function","name":"lookup","parameters":{"type":"object"}}],"tool_choice":"none"}"#, + ), + ]; + for (surface, body) in requests { + let request = rust_parse(surface, body).unwrap(); + assert!(!request.has_tools, "{surface}"); + assert!(request.tool_schemas.is_empty(), "{surface}"); + assert!(request.tool_orders.is_empty(), "{surface}"); + } + + for surface in ["chat", "responses"] { + let input = if surface == "chat" { + r#"{"messages":[],"parallel_tool_calls":false}"# + } else { + r#"{"input":[],"parallel_tool_calls":false}"# + }; + assert_eq!( + rust_parse(surface, input).unwrap_err(), + "parallel_tool_calls=false is not supported" + ); + } + assert!(rust_parse("chat", r#"{"messages":[],"parallel_tool_calls":true}"#).is_ok()); +} + +#[test] +fn simple_success_dumps_match_c() { + let r = assert_ok_eq("chat", r#"{"messages":[{"role":"user","content":"hi"}]}"#); + assert_eq!(r.think_mode as i32, 1); + assert_eq!(r.needs, 6); + + let r = assert_ok_eq( + "chat", + r#"{"messages":[{"role":"user","content":"hi"}],"stream":true,"temperature":0.0,"thinking":{"type":"disabled"}}"#, + ); + assert!(r.stream); + assert_eq!(r.temperature, 0.0); + assert_eq!(r.think_mode as i32, 0); + + assert_ok_eq("completion", r#"{"prompt":"hi","stream":true}"#); + assert_ok_eq( + "anthropic", + r#"{"messages":[{"role":"user","content":"hi"}],"max_tokens":0}"#, + ); + assert_ok_eq("responses", r#"{"input":"hi"}"#); + assert_ok_eq("responses", r#"{"input":[]}"#); + assert_ok_eq("chat", r#"{"messages":[]}"#); + + let r = assert_ok_eq( + "chat", + r#"{"messages":[{"role":"user","content":"hi"}],"model":"deepseek-chat"}"#, + ); + assert_eq!(r.think_mode as i32, 0); + + let r = assert_ok_eq( + "chat", + r#"{"messages":[{"role":"user","content":"hi"}],"model":"deepseek-reasoner"}"#, + ); + assert_eq!(r.think_mode as i32, 1); + + let r = assert_ok_eq( + "anthropic", + r#"{"system":"be brief","messages":[{"role":"user","content":"hi"}]}"#, + ); + assert_eq!(r.messages.len(), 2); + assert_eq!(r.messages[1].role, "system"); + + let r = assert_ok_eq( + "responses", + r#"{"instructions":"sys","input":[{"type":"message","role":"user","content":"hi"}]}"#, + ); + assert_eq!(r.messages[0].role, "system"); + assert_eq!(r.messages[1].role, "user"); + + let r = assert_ok_eq( + "chat", + r#"{"messages":[{"role":"user","content":"hi"}],"tools":[{"type":"function","function":{"name":"lookup","parameters":{"type":"object"}}}],"tool_choice":"auto"}"#, + ); + assert!(r.has_tools); + assert_eq!(r.tool_choice, ToolChoice::Auto); + assert_eq!(r.tool_orders.len(), 1); + assert_eq!(r.tool_orders[0].name, "lookup"); +} + +#[test] +fn parse_request_dispatcher_and_needs() { + let e = env(); + let r = parse_request(WireSurface::OpenaiChat, &e, r#"{"messages":[]}"#).unwrap(); + assert_eq!(r.kind as i32, 0); + let r = parse_request(WireSurface::OpenaiCompletion, &e, r#"{"prompt":""}"#).unwrap(); + assert_eq!(r.kind as i32, 1); +} + +#[test] +fn qwen_image_inputs_normalize_across_all_three_surfaces() { + const PNG: &str = concat!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP8", + "z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + ); + let bodies = [ + format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"text","text":"before"}},{{"type":"image_url","image_url":{{"url":"data:image/png;base64,{PNG}"}}}},{{"type":"text","text":"after"}}]}}]}}"# + ), + format!( + r#"{{"input":[{{"type":"message","role":"user","content":[{{"type":"input_text","text":"before"}},{{"type":"input_image","image_url":"data:image/png;base64,{PNG}"}},{{"type":"input_text","text":"after"}}]}}]}}"# + ), + format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"text","text":"before"}},{{"type":"image","source":{{"type":"base64","media_type":"image/png","data":"{PNG}"}}}},{{"type":"text","text":"after"}}]}}]}}"# + ), + ]; + let parsed = [ + rust_parse("chat", &bodies[0]).unwrap(), + rust_parse("responses", &bodies[1]).unwrap(), + rust_parse("anthropic", &bodies[2]).unwrap(), + ]; + for request in &parsed { + assert_eq!(request.images.len(), 1); + assert_eq!(request.images[0].mime, ImageMime::Png); + assert_eq!(request.messages[0].content, "beforeafter"); + assert_eq!( + request.messages[0].parts, + [ + ChatPart::Text("before".into()), + ChatPart::Image(0), + ChatPart::Text("after".into()), + ] + ); + assert_ne!(request.needs & NEED_IMAGE, 0); + assert_eq!( + generation_blocked(request, 0), + Some("image input is supported only by Qwen4Exp, GLM-5.3, Inkling, Step, Ling or MiMo") + ); + assert_eq!( + generation_blocked(request, 6), + Some("image input requires continuous runtime") + ); + assert_eq!(generation_blocked(request, 7), None); + assert_eq!(generation_blocked(request, 10), None); + assert_eq!(generation_blocked(request, 11), None); + assert_eq!(generation_blocked(request, 12), None); + } + assert_eq!(parsed[0].images[0].data, parsed[1].images[0].data); + assert_eq!(parsed[0].images[0].data, parsed[2].images[0].data); +} + +#[test] +fn image_input_rejects_remote_urls_wrong_magic_and_non_user_roles() { + let remote = r#"{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.com/x.png"}}]}]}"#; + assert_eq!( + rust_parse("chat", remote).unwrap_err(), + "image_url must be a base64 data URI" + ); + + const PNG: &str = concat!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP8", + "z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + ); + let wrong_magic = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"image_url","image_url":"data:image/jpeg;base64,{PNG}"}}]}}]}}"# + ); + assert_eq!( + rust_parse("chat", &wrong_magic).unwrap_err(), + "image bytes do not match declared media type" + ); + let assistant = format!( + r#"{{"messages":[{{"role":"assistant","content":[{{"type":"image_url","image_url":"data:image/png;base64,{PNG}"}}]}}]}}"# + ); + // An assistant image stays a parse error. A tool/function image does not + // (the MCP loop; see openai_tool_message_carries_the_image_placeholder). + assert_eq!( + rust_parse("chat", &assistant).unwrap_err(), + "images are allowed only in user messages or tool results" + ); +} + +/// The oracle's path and URL forms are refused on every surface. Its +/// `parse_tagged_text` / `_extract_image` (encoding.py:273-320) normalize a bare +/// path or an http URL into an image record and leave the fetch to whatever +/// takes it; this server is base64-only by contract, and it fails closed rather +/// than dropping the pixels silently, so each form is a named request error. +#[test] +fn image_refusals_cover_the_oracle_path_and_url_forms() { + // The oracle's path form, and the same path as a plain + // string instead of the object spelling. + for body in [ + r#"{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"/tmp/photo.png"}}]}]}"#, + r#"{"messages":[{"role":"user","content":[{"type":"image_url","image_url":"/tmp/photo.png"}]}]}"#, + ] { + assert_eq!( + rust_parse("chat", body).unwrap_err(), + "image_url must be a base64 data URI" + ); + } + + // Responses input_image with a URL; the data-URI form is pinned by + // qwen_image_inputs_normalize_across_all_three_surfaces. + let responses = r#"{"input":[{"type":"message","role":"user","content":[{"type":"input_image","image_url":"https://example.com/x.png"}]}]}"#; + assert_eq!( + rust_parse("responses", responses).unwrap_err(), + "image_url must be a base64 data URI" + ); + + // Anthropic's url source; its tool_result spelling of the same check is + // pinned by anthropic_tool_result_content_blocks_carry_images. + let anthropic = r#"{"messages":[{"role":"user","content":[{"type":"image","source":{"type":"url","url":"https://example.com/a.png"}}]}]}"#; + assert_eq!( + rust_parse("anthropic", anthropic).unwrap_err(), + "Anthropic image source must use base64 data" + ); +} + +#[test] +fn the_image_path_tag_form_is_text_and_not_markup() { + // The oracle's `parse_tagged_text` rewrites an `path` text + // form into `image_url` blocks and raises on an unbalanced tag + // (encoding.py:273-297, pattern at :40). This carriage serves no path form + // at all -- pixels arrive base64 only (AGENTS.md, "Pixels arrive + // base64-only") -- and it never reads the tag as markup, so the client's + // text reaches the model byte for byte, as ordinary text. Measured + // 2026-10-20: no image is registered, the content keeps the tag, and the + // rendered prompt carries it verbatim. + for text in [ + "look at /tmp/photo.png please", + "unbalanced with no closer", + ] { + let body = format!(r#"{{"messages":[{{"role":"user","content":"{text}"}}]}}"#); + let parsed = rust_parse("chat", &body).unwrap(); + + assert!(parsed.images.is_empty(), "{text}: an image was registered"); + assert_eq!( + parsed.needs & NEED_IMAGE, + 0, + "{text}: an image span is owed" + ); + assert_eq!(parsed.messages[0].content, text, "{text}: content moved"); + + let prompt = render_dsml_chat_choice( + &parsed.messages, + &parsed.tool_schemas, + parsed.think_mode, + parsed.tool_choice, + ); + let prompt = String::from_utf8(prompt.unwrap()).unwrap(); + assert!(prompt.contains(text), "{text}: the prompt rewrote the text"); + } +} + +#[test] +fn anthropic_tool_result_content_blocks_carry_images() { + // The engine's test_anthropic_tool_result_mcp_blocks + // (tests/server_tests_replay.c:65): the real MCP loop replays an assistant + // tool_use plus a user tool_result whose content is a block array, and a + // screenshot-style MCP tool returns pixels inside it. + const PNG: &str = concat!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP8", + "z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + ); + // The replaying assistant turn plus the tool_result block under test. + let mcp_replay = |id: &str, block: &str| { + format!( + r#"{{"messages":[{{"role":"assistant","content":[{{"type":"tool_use","id":"{id}","name":"screenshot","input":{{}}}}]}},{{"role":"user","content":[{block}]}}]}}"# + ) + }; + + // Legacy string content renders byte-identically, including is_error:false. + let string_form = rust_parse( + "anthropic", + &mcp_replay( + "toolu_01", + r#"{"type":"tool_result","tool_use_id":"toolu_01","is_error":false,"content":"ok"}"#, + ), + ) + .unwrap(); + assert_eq!( + string_form.messages[1].content, + "ok" + ); + assert_eq!( + string_form.messages[1].parts, + [ChatPart::ToolResult { + id: "toolu_01".into(), + content: "ok".into(), + }] + ); + assert_eq!(string_form.messages[1].tool_call_ids, ["toolu_01"]); + + // Block array: text + image + text in client order. The image registers + // like a user-message image; the placeholder sits where the block sits in + // the content the DSML head renders, while the ToolResult part keeps the + // plain text the other heads read. + let mixed = rust_parse( + "anthropic", + &mcp_replay( + "toolu_02", + &format!( + r#"{{"type":"tool_result","tool_use_id":"toolu_02","content":[{{"type":"text","text":"shot:"}},{{"type":"image","source":{{"type":"base64","media_type":"image/png","data":"{PNG}"}}}},{{"type":"text","text":"done"}}]}}"# + ), + ), + ) + .unwrap(); + assert_eq!(mixed.images.len(), 1); + assert_eq!(mixed.images[0].mime, ImageMime::Png); + assert_eq!( + mixed.messages[1].content, + format!("shot:{DSML_IMAGE}done") + ); + assert_eq!( + mixed.messages[1].parts, + [ + ChatPart::Image(0), + ChatPart::ToolResult { + id: "toolu_02".into(), + content: "shot:done".into(), + }, + ] + ); + assert_eq!(mixed.messages[1].tool_call_ids, ["toolu_02"]); + assert_ne!(mixed.needs & NEED_IMAGE, 0); + // The V4.1 head (variant 16) admits it. A head that refuses images keeps + // its own message; a head that admits them but cannot place one inside a + // tool result (it renders the plain ToolResult part) is refused by name. + assert_eq!(generation_blocked(&mixed, 16), None); + assert_eq!( + generation_blocked(&mixed, 0), + Some("image input is supported only by Qwen4Exp, GLM-5.3, Inkling, Step, Ling or MiMo") + ); + assert_eq!( + generation_blocked(&mixed, 7), + Some("images inside a tool result are supported only by DeepSeek V4.1 Flash") + ); + + // is_error:true surfaces as an explicit marker inside the envelope; a + // text-only tool_result never needs pixels and renders as before. + let errored = rust_parse( + "anthropic", + &mcp_replay( + "toolu_03", + r#"{"type":"tool_result","tool_use_id":"toolu_03","content":"exit status 1","is_error":true}"#, + ), + ) + .unwrap(); + assert_eq!( + errored.messages[1].content, + "[tool_error] exit status 1" + ); + let text_only = rust_parse( + "anthropic", + &mcp_replay( + "toolu_04", + r#"{"type":"tool_result","tool_use_id":"toolu_04","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}"#, + ), + ) + .unwrap(); + assert_eq!( + text_only.messages[1].content, + "ab" + ); + assert!(text_only.images.is_empty()); + + // Fail closed: an image the server cannot deliver rejects the request + // instead of silently vanishing. + let url_source = mcp_replay( + "toolu_05", + r#"{"type":"tool_result","tool_use_id":"toolu_05","content":[{"type":"image","source":{"type":"url","url":"https://example.com/a.png"}}]}"#, + ); + assert_eq!( + rust_parse("anthropic", &url_source).unwrap_err(), + "Anthropic image source must use base64 data" + ); + let wrong_magic = mcp_replay( + "toolu_06", + &format!( + r#"{{"type":"tool_result","tool_use_id":"toolu_06","content":[{{"type":"image","source":{{"type":"base64","media_type":"image/jpeg","data":"{PNG}"}}}}]}}"# + ), + ); + assert_eq!( + rust_parse("anthropic", &wrong_magic).unwrap_err(), + "image bytes do not match declared media type" + ); +} + +#[test] +fn inkling_audio_chat_input() { + const WAV: &str = "UklGRiYAAABXQVZFZm10IBAAAAABAAEAgD4AAAB9AAACABAAZGF0YQIAAAAAAA=="; + let block = + serde_json::json!({"type":"input_audio", "input_audio":{"format":"wav", "data":WAV}}); + let body = serde_json::json!({"messages":[{"role":"user", "content":[{"type":"text","text":"Transcribe."},block]}]}); + let result = rust_parse("chat", &body.to_string()); + assert!(result.is_ok(), "{result:?}"); + let parsed = result.unwrap(); + assert_eq!(parsed.audios.len(), 1); + assert_eq!( + parsed.messages[0].parts, + [ChatPart::Text("Transcribe.".into()), ChatPart::Audio(0)] + ); + assert_ne!(parsed.needs & ds4_server::route::NEED_AUDIO, 0); + let route = ds4_server::route_decide( + parsed.needs, + WireSurface::OpenaiChat, + &ds4_server::RouteEnv { + have_cont: true, + coalesce: true, + prompt_len: 128, + seq_cap: 1024, + cont_anthropic: true, + cont_responses: true, + cont_tools_anthropic: true, + cont_tools_responses: true, + }, + ); + assert_eq!(route.lane, ds4_server::LANE_SERIAL); + assert_eq!(generation_blocked(&parsed, 9), None); + assert_eq!( + generation_blocked(&parsed, 6), + Some("audio input requires Inkling or MiMo") + ); + let prompt = ds4_server::render_prompt(&parsed, 9).unwrap(); + assert!(String::from_utf8(prompt) + .unwrap() + .contains("<|content_audio_input|><|unused_200053|><|audio_end|>")); + for bad in [ + body.to_string().replace("\"wav\"", "\"mp3\""), + body.to_string().replace("\"user\"", "\"assistant\""), + body.to_string().replace(WAV, "!!!!"), + body.to_string().replace(WAV, "AAAA"), + serde_json::json!({"messages":[{"role":"user","content":vec![block;5]}]}).to_string(), + ] { + assert!(rust_parse("chat", &bad).is_err(), "{bad}"); + } +} + +#[test] +fn mimo_video_stays_on_the_serial_lane() { + const MP4: &str = "AAAAAGZ0eXAAAAAAAAAAAA=="; + let body = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"video_url","video_url":{{"url":"data:video/mp4;base64,{MP4}"}}}}]}}]}}"# + ); + let parsed = rust_parse("chat", &body).unwrap(); + assert_eq!(parsed.videos.len(), 1); + assert_eq!(parsed.messages[0].parts, [ChatPart::Video(0)]); + assert_ne!(parsed.needs & ds4_server::route::NEED_AUDIO, 0); + let route = ds4_server::route_decide( + parsed.needs, + WireSurface::OpenaiChat, + &ds4_server::RouteEnv { + have_cont: true, + coalesce: true, + prompt_len: 128, + seq_cap: 1024, + cont_anthropic: true, + cont_responses: true, + cont_tools_anthropic: true, + cont_tools_responses: true, + }, + ); + assert_eq!(route.lane, ds4_server::LANE_SERIAL); + assert_eq!( + generation_blocked(&parsed, 6), + Some("video input requires MiMo") + ); + assert_eq!(generation_blocked(&parsed, 12), None); + assert_eq!( + rust_parse("chat", &body.replace("video/mp4", "video/webm")).unwrap_err(), + "video input currently requires video/mp4" + ); + assert_eq!( + rust_parse("chat", &body.replace("\"user\"", "\"assistant\"")).unwrap_err(), + "video is allowed only in user messages" + ); +} + +#[test] +fn audio_image_budget_preflight() { + // Two 10 MiB audio payloads exhaust the shared budget. A following + // image must be refused before decoding or inspecting its bytes. + let size = 10 * 1024 * 1024; + let mut data = "UklGRgAAAABXQVZF".to_string(); + data.push_str(&"AAAA".repeat((size - 12) / 3)); + data.push_str("AA=="); + let block = + serde_json::json!({"type":"input_audio", "input_audio":{"format":"wav", "data":data}}); + let body = serde_json::json!({"messages":[{"role":"user", "content":[block.clone(),block,{"type":"image_url","image_url":"data:image/png;base64,AAAA"}]}]}); + assert_eq!( + rust_parse("chat", &body.to_string()).unwrap_err(), + "media exceeds 20 MiB request limit" + ); +} + +/// Unit H: the engine's request-omitted parse defaults and effort-name +/// collapse for its own model set (ParseEnv::engine_defaults, set by the +/// server bin only on the V4.1 route). References: server_parse_chat.c:16-17 +/// (thinking OFF, effort HIGH), server_msgs.c:194-215 (everything non-zero +/// below max collapses to HIGH), ds4.h:54-56 (temp 1.0 / top_p 1.0 / min_p +/// 0.0), server_parse_chat.c:91-127 (penalties + DRY). +#[test] +fn engine_defaults_think_and_sampling() { + use ds4_server::ThinkMode; + let eng = ParseEnv { + engine_defaults: true, + ..env() + }; + let chat = |body: &str| parse_chat_request(&eng, body).unwrap(); + + // Nothing sent: thinking OFF. + assert_eq!( + chat(r#"{"messages":[{"role":"user","content":"hi"}]}"#).think_mode, + ThinkMode::None + ); + // The engine's collapse: low / medium / minimal / xhigh / high -> HIGH. + for name in ["low", "medium", "minimal", "xhigh", "high"] { + let body = format!( + r#"{{"messages":[{{"role":"user","content":"hi"}}],"reasoning_effort":"{name}"}}"# + ); + assert_eq!(chat(&body).think_mode, ThinkMode::High, "{name}"); + } + assert_eq!( + chat(r#"{"messages":[{"role":"user","content":"hi"}],"reasoning_effort":"max"}"#) + .think_mode, + ThinkMode::Max + ); + assert_eq!( + chat(r#"{"messages":[{"role":"user","content":"hi"}],"reasoning_effort":"none"}"#) + .think_mode, + ThinkMode::None + ); + // A bare thinking flag: the engine's HIGH effort default. + assert_eq!( + chat(r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"enabled"}}"#) + .think_mode, + ThinkMode::High + ); + // The reasoner alias alone asks for thinking (server_msgs.c:304-306). + assert_eq!( + chat(r#"{"messages":[{"role":"user","content":"hi"}],"model":"deepseek-reasoner"}"#) + .think_mode, + ThinkMode::High + ); + assert_eq!( + chat(r#"{"messages":[{"role":"user","content":"hi"}],"thinking":{"type":"disabled"}}"#) + .think_mode, + ThinkMode::None + ); + // min_p defaults to the engine's 0.0 on this scope; the port's own default + // (0.05) stays for the other families. + assert_eq!( + chat(r#"{"messages":[{"role":"user","content":"hi"}]}"#).min_p, + 0.0 + ); + assert_eq!( + parse_chat_request(&env(), r#"{"messages":[{"role":"user","content":"hi"}]}"#) + .unwrap() + .min_p, + 0.05 + ); + // The penalties and DRY land on the request. + let r = chat( + r#"{"messages":[{"role":"user","content":"hi"}],"frequency_penalty":0.5,"presence_penalty":-0.2,"dry_multiplier":0.8}"#, + ); + assert_eq!(r.frequency_penalty, 0.5); + assert_eq!(r.presence_penalty, -0.2); + assert!(r.dry_set); + assert_eq!(r.dry_multiplier, 0.8); +} + +/// O-01: the effort a request asked for survives the engine's collapse, so the +/// V4.1 head can render the requested budget (encoding.py:443-473) while the +/// engine keeps the mode it decided. The name mapping is low 50 / high 75 / +/// max 100, and the oracle's numeric form renders as the number itself. +#[test] +fn requested_effort_budget_survives_the_engine_collapse() { + use ds4_server::{RequestedEffort, ThinkMode}; + let eng = ParseEnv { + engine_defaults: true, + ..env() + }; + let chat = |body: &str| parse_chat_request(&eng, body).unwrap(); + let ask = |effort: &str| { + chat(&format!( + r#"{{"messages":[{{"role":"user","content":"hi"}}],"reasoning_effort":{effort}}}"# + )) + }; + + // The engine collapses low into HIGH (server_msgs.c:194-215); the head still + // takes the low mapping's 50. + let low = ask(r#""low""#); + assert_eq!(low.think_mode, ThinkMode::High); + assert_eq!(low.effort, RequestedEffort::named(ThinkMode::Low)); + assert_eq!(low.effort.head_budget(), Some(50)); + + // A bare ask keeps the engine's HIGH default, 75 in the head. + let bare = chat(r#"{"messages":[{"role":"user","content":"hi"}]}"#); + assert_eq!(bare.effort, RequestedEffort::named(ThinkMode::High)); + assert_eq!(bare.effort.head_budget(), Some(75)); + + assert_eq!(ask(r#""max""#).effort.head_budget(), Some(100)); + assert_eq!(ask(r#""none""#).effort.head_budget(), None); + + // The oracle's numeric form: taken as the budget, engine mode collapsed. + let numeric = ask("40"); + assert_eq!(numeric.think_mode, ThinkMode::High); + assert_eq!( + numeric.effort, + RequestedEffort { + mode: ThinkMode::High, + budget: Some(40) + } + ); + assert_eq!(numeric.effort.head_budget(), Some(40)); + + // Out of the oracle's range and its non-integers are refused, as the oracle + // refuses them (encoding.py:464-469). + for bad in ["0", "101", "50.5", "5e1"] { + assert!( + parse_chat_request( + &eng, + &format!( + r#"{{"messages":[{{"role":"user","content":"hi"}}],"reasoning_effort":{bad}}}"# + ) + ) + .is_err(), + "{bad} must be refused" + ); + } +} + +#[test] +fn openai_tool_message_carries_the_image_placeholder() { + // The real MCP loop on the Chat Completions surface: an assistant turn + // calls a screenshot tool, and the tool-role message's content is a block + // array whose image block is the pixels. The image registers like a + // user-message image, and the placeholder lands in that message's content + // in client block order, because that content is what the DSML render + // writes inside its envelope. + const PNG: &str = concat!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP8", + "z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + ); + let body = format!( + r#"{{"messages":[{{"role":"assistant","content":"","tool_calls":[{{"id":"c1","type":"function","function":{{"name":"shot","arguments":"{{}}"}}}}]}},{{"role":"tool","tool_call_id":"c1","content":[{{"type":"text","text":"shot:"}},{{"type":"image_url","image_url":{{"url":"data:image/png;base64,{PNG}"}}}},{{"type":"text","text":"done"}}]}}]}}"# + ); + let parsed = rust_parse("chat", &body).unwrap(); + assert_eq!(parsed.images.len(), 1); + assert_eq!(parsed.images[0].mime, ImageMime::Png); + assert_eq!(parsed.messages[1].role, "tool"); + assert_eq!(parsed.messages[1].content, format!("shot:{DSML_IMAGE}done")); + assert_eq!( + parsed.messages[1].parts, + [ + ChatPart::Text("shot:".into()), + ChatPart::Image(0), + ChatPart::Text("done".into()), + ] + ); + assert_ne!(parsed.needs & NEED_IMAGE, 0); + // The V4.1 head admits it; a head that admits images but cannot place one + // inside a tool result is refused by name rather than leaking the token. + assert_eq!(generation_blocked(&parsed, 16), None); + assert_eq!( + generation_blocked(&parsed, 7), + Some("images inside a tool result are supported only by DeepSeek V4.1 Flash") + ); +} + +mod v41_media { + use super::*; + use serde_json::{json, Value}; + + const PNG: &str = concat!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP8", + "z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + ); + + fn image(surface: &str, data: &str) -> Value { + match surface { + "chat" => { + json!({"type":"image_url","image_url":format!("data:image/png;base64,{data}")}) + } + "responses" => { + json!({"type":"input_image","image_url":format!("data:image/png;base64,{data}")}) + } + "anthropic" => { + json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":data}}) + } + _ => unreachable!(), + } + } + + fn request(surface: &str, blocks: Vec) -> Value { + let messages = json!([{"role":"user","content":blocks}]); + if surface == "responses" { + json!({"input":messages}) + } else { + json!({"messages":messages}) + } + } + + fn parse(surface: &str, body: &Value) -> Result { + let mut env = env(); + env.engine_defaults = true; + let body = body.to_string(); + match surface { + "chat" => parse_chat_request(&env, &body), + "anthropic" => parse_anthropic_request(&env, &body), + "responses" => parse_responses_request(&env, &body), + _ => unreachable!(), + } + } + + #[test] + fn mixed_tool_image_order() { + let nested = json!({"type":"tool_result","tool_use_id":"c1","content":[ + {"type":"text","text":"nested:"},image("anthropic", PNG), + {"type":"text","text":":end"} + ]}); + let outer = image("anthropic", PNG); + for (blocks, want) in [ + ( + vec![outer.clone(), nested.clone()], + format!("{DSML_IMAGE}nested:{DSML_IMAGE}:end"), + ), + ( + vec![nested, outer], + format!("nested:{DSML_IMAGE}:end{DSML_IMAGE}"), + ), + ] { + let body = json!({"messages":[ + {"role":"assistant","content":[{"type":"tool_use","id":"c1","name":"shot","input":{}}]}, + {"role":"user","content":blocks} + ]}); + let parsed = parse("anthropic", &body).unwrap(); + let rendered = + String::from_utf8(ds4_server::render_prompt(&parsed, 16).unwrap()).unwrap(); + assert_eq!(parsed.images.len(), 2); + assert!(rendered.contains(&want), "{rendered}"); + assert_eq!(rendered.matches(DSML_IMAGE).count(), 2); + assert_eq!( + parsed.messages[1] + .parts + .iter() + .filter_map(|part| match part { + ChatPart::Image(index) => Some(*index), + _ => None, + }) + .collect::>(), + [0, 1] + ); + } + } + + #[test] + fn responses_tool_images() { + for (call, output) in [ + ("function_call", "function_call_output"), + ("custom_tool_call", "custom_tool_call_output"), + ] { + let body = json!({"reasoning":{"effort":"none"},"input":[ + {"type":call,"call_id":"c1","name":"shot","arguments":"{}"}, + {"type":output,"call_id":"c1","output":[ + {"type":"input_text","text":"before:"},image("responses", PNG), + {"type":"input_text","text":":after"} + ]} + ]}); + let parsed = parse("responses", &body).unwrap(); + assert_eq!(parsed.images.len(), 1); + assert_eq!( + parsed.messages[1].content, + format!("before:{DSML_IMAGE}:after") + ); + assert_eq!(generation_blocked(&parsed, 16), None); + assert!(generation_blocked(&parsed, 7) + .unwrap() + .contains("tool result")); + let rendered = + String::from_utf8(ds4_server::render_prompt(&parsed, 16).unwrap()).unwrap(); + assert!(rendered.contains(&format!( + "before:{DSML_IMAGE}:after" + ))); + } + } + + #[test] + fn multi_image_admission() { + for surface in ["chat", "anthropic", "responses"] { + for count in [5, 8] { + let body = request(surface, vec![image(surface, PNG); count]); + let parsed = parse(surface, &body).unwrap(); + assert_eq!(parsed.images.len(), count, "{surface}"); + assert!(rust_parse(surface, &body.to_string()).is_err(), "{surface}"); + } + } + } + + #[test] + fn mixed_media_stays_bounded() { + let audio = + json!({"type":"input_audio","input_audio":{"format":"wav","data":"UklGRgAAAABXQVZF"}}); + for first in [true, false] { + let mut blocks = vec![image("chat", PNG); 4]; + blocks.insert(if first { 0 } else { blocks.len() }, audio.clone()); + assert!(parse("chat", &request("chat", blocks)).is_err()); + } + } + + #[test] + fn image_bytes_stay_bounded() { + let limit = 10 * 1024 * 1024; + let mut data = "iVBORw0KGgoA".to_string(); + data.push_str(&"AAAA".repeat((limit - 9) / 3)); + data.push_str("AA=="); + let too_large = "AAAA".repeat(limit / 3 + 1); + for surface in ["chat", "anthropic", "responses"] { + let body = request(surface, vec![image(surface, &too_large)]); + assert_eq!( + parse(surface, &body).unwrap_err(), + "media exceeds 10 MiB decoded limit" + ); + let body = request( + surface, + vec![ + image(surface, &data), + image(surface, &data), + image(surface, PNG), + ], + ); + assert_eq!( + parse(surface, &body).unwrap_err(), + "media exceeds 20 MiB request limit" + ); + } + } +} + +/// CTR clause 2.3.7, the server's half: the client echoes `reasoning_content` on +/// every assistant message, empty included, and the server must not require the +/// key. Both shapes parse, and the reasoning the message carried is what the +/// render sees -- an omitted key is not an error and an empty value stays empty +/// (`types.rs` 526-538; `conversation.rs` 3691-3740). +#[test] +fn assistant_reasoning_is_optional_and_may_be_empty() { + let eng = ParseEnv { + engine_defaults: true, + ..env() + }; + let empty = parse_chat_request( + &eng, + r#"{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"ok","reasoning_content":""},{"role":"user","content":"again"}]}"#, + ) + .unwrap(); + assert_eq!(empty.messages[1].role, "assistant"); + assert_eq!(empty.messages[1].content, "ok"); + assert_eq!(empty.messages[1].reasoning, ""); + + let omitted = parse_chat_request( + &eng, + r#"{"messages":[{"role":"user","content":"hi"},{"role":"assistant","content":"ok"},{"role":"user","content":"again"}]}"#, + ) + .unwrap(); + assert_eq!(omitted.messages[1].content, "ok"); + assert_eq!(omitted.messages[1].reasoning, ""); +} + +/// A numeric reasoning_effort is the V4.1 head's budget; any other family +/// would read it as HIGH, so it is refused off the V4.1 route. +#[test] +fn numeric_effort_is_v41_only() { + let body = r#"{"messages":[{"role":"user","content":"hi"}],"reasoning_effort":5}"#; + let v41 = ParseEnv { + engine_defaults: true, + ..env() + }; + assert_eq!( + parse_chat_request(&v41, body).unwrap().effort.budget, + Some(5) + ); + assert!(parse_chat_request(&env(), body).is_err()); +} diff --git a/crates/ds4-server/tests/ref_constants.rs b/crates/ds4-server/tests/ref_constants.rs new file mode 100644 index 0000000000000000000000000000000000000000..b91100b3e6d465784a1f33d90e7f541fba07cdd1 --- /dev/null +++ b/crates/ds4-server/tests/ref_constants.rs @@ -0,0 +1,248 @@ +//! The constants leg: every literal the two sides must agree on, read out of +//! the vendored oracle's own source instead of copied by hand. +//! +//! `docs/ds41-dsml-function-map.md` section 3 carries one row per literal of +//! `tests/ds41_ref/oracle/encoding.py`. A row that says "pass" because a case +//! happens to contain the literal is weaker than it reads: the case pins the +//! bytes of that one prompt, not the constant. This test closes that gap for +//! the literals that have a counterpart in this crate, by extracting the value +//! the oracle assigns and comparing it with ours -- so a literal that moves on +//! either side fails here with both spellings, and a hand-copied constant can +//! no longer drift from the file it claims to mirror. +//! +//! The extracted literals are the oracle's: the same file +//! `tests/ds41_ref/ref_drive.py` pins by sha256, so a swapped oracle is caught +//! by the reference legs first and by this test second. + +use ds4_server::render::{ + DSML_ASSISTANT, DSML_BOS, DSML_EOS, DSML_IMAGE, DSML_SYSTEM_TOKEN, DSML_THINK_END, + DSML_THINK_START, DSML_TOOL_CALLS, DSML_USER, V41_EFFORT_TEMPLATE, V41_THINK_HIGH_PREFIX, + V41_THINK_MAX_PREFIX, +}; +use ds4_server::tools::{ + DSML_INVOKE_END_SPACED, DSML_INVOKE_START_SPACED, DSML_PARAM_END_SPACED, + DSML_PARAM_START_SPACED, DSML_TOOL_CALLS_END_SPACED, DSML_TOOL_CALLS_START_SPACED, +}; + +/// The oracle's bytes, compiled in. Reading the same file the driver pins is +/// the point: the test compares against the authority, not against a copy. +const ORACLE: &str = include_str!("../../../tests/ds41_ref/oracle/encoding.py"); + +/// The source text of the statement that declares `name`. +/// +/// A python declaration ends at the end of the line that closes the last open +/// bracket, unless the line continues with `+` (the oracle composes +/// `assistant_msg_template` that way). Bracket depth is counted outside +/// string literals, because the effort template carries a `(range 1-100)` +/// parenthesis inside its own text. +fn declaration(name: &str) -> String { + let mut at = None; + let mut offset = 0; + for line in ORACLE.split_inclusive('\n') { + let trimmed = line.trim_start(); + if !trimmed.starts_with('#') { + if let Some(rest) = trimmed.strip_prefix(name) { + if rest.starts_with(':') || rest.starts_with(' ') { + at = Some(offset + (line.len() - trimmed.len())); + break; + } + } + } + offset += line.len(); + } + let src = &ORACLE[at.unwrap_or_else(|| panic!("the oracle no longer declares {name}"))..]; + + let mut depth = 0i32; + let mut in_str = false; + let mut escaped = false; + let mut end = src.len(); + for (i, c) in src.char_indices() { + if escaped { + escaped = false; + continue; + } + if in_str { + match c { + '\\' => escaped = true, + '"' => in_str = false, + _ => {} + } + continue; + } + match c { + '"' => in_str = true, + '(' | '{' | '[' => depth += 1, + ')' | '}' | ']' => depth -= 1, + '\n' if depth <= 0 => { + let line = src[..i].rsplit('\n').next().unwrap_or(""); + let open = line.trim_end().ends_with('+') || line.trim_end().ends_with('\\'); + if !open { + end = i; + break; + } + } + _ => {} + } + } + src[..end].to_string() +} + +/// `name`'s value, with adjacent string literals concatenated as python does +/// and the escapes decoded. A declaration that carries no string literal is a +/// test bug, not an oracle change, so it panics. +fn literal(name: &str) -> String { + let text = declaration(name); + let mut out = String::new(); + let mut chars = text.chars(); + let mut seen = false; + while let Some(c) = chars.next() { + if c != '"' { + continue; + } + seen = true; + while let Some(c) = chars.next() { + match c { + '\\' => match chars.next() { + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('r') => out.push('\r'), + Some('"') => out.push('"'), + Some('\\') => out.push('\\'), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => break, + }, + '"' => break, + other => out.push(other), + } + } + } + assert!(seen, "{name} no longer holds a string literal"); + out +} + +/// One integer member of an oracle dict, e.g. `REASONING_EFFORT_MAPPINGS["low"]`. +fn mapping_int(name: &str, key: &str) -> i64 { + let text = declaration(name); + let needle = format!("\"{key}\":"); + let rest = text + .split_once(&needle) + .unwrap_or_else(|| panic!("{name} no longer maps {key}")) + .1; + let digits: String = rest + .trim_start() + .chars() + .take_while(|c| c.is_ascii_digit() || *c == '-') + .collect(); + digits + .parse() + .unwrap_or_else(|e| panic!("{name}[{key}] is not an integer: {e}")) +} + +/// The five special tokens our renderer writes (encoding.py:29-39, :426). +#[test] +fn special_tokens_match_the_oracle() { + assert_eq!(DSML_BOS, literal("bos_token"), "bos_token"); + assert_eq!(DSML_EOS, literal("eos_token"), "eos_token"); + assert_eq!( + DSML_THINK_START, + literal("thinking_start_token"), + "thinking_start_token" + ); + assert_eq!( + DSML_THINK_END, + literal("thinking_end_token"), + "thinking_end_token" + ); + assert_eq!(DSML_USER, literal("USER_SP_TOKEN"), "USER_SP_TOKEN"); + assert_eq!( + DSML_ASSISTANT, + literal("ASSISTANT_SP_TOKEN"), + "ASSISTANT_SP_TOKEN" + ); + assert_eq!( + DSML_SYSTEM_TOKEN, + literal("SYSTEM_SP_TOKEN"), + "SYSTEM_SP_TOKEN" + ); + assert_eq!( + DSML_IMAGE, + literal("IMAGE_PLACEHOLDER"), + "IMAGE_PLACEHOLDER" + ); +} + +/// The DSML tags, composed from the oracle's own fragments +/// (encoding.py:33, :428-435) so the separators are the oracle's and not +/// ours. +/// +/// `DSML_TOOL_CALLS` is not in this set: it is the artifact's older unspaced +/// root (`|DSML|tool_calls`, no space before the name), a spelling the oracle +/// does not define and the map's section 4 lists beside the short one. What +/// matters about it here is that the renderer's block tag and the parser's +/// block tag are the same string: a renderer writing a tag the parser does not +/// recognize would lose the call. +#[test] +fn tool_tags_match_the_oracle() { + let dsml = literal("dsml_token"); + let block = literal("tool_calls_block_name"); + let call = literal("tool_call_tag_name"); + let param = literal("tool_parameter_tag_name"); + + assert_eq!(DSML_TOOL_CALLS_START_SPACED, format!("<{dsml}{block}>")); + assert_eq!(DSML_TOOL_CALLS_END_SPACED, format!("")); + assert_eq!(DSML_INVOKE_START_SPACED, format!("<{dsml}{call}")); + assert_eq!(DSML_INVOKE_END_SPACED, format!("")); + assert_eq!(DSML_PARAM_START_SPACED, format!("<{dsml}{param}")); + assert_eq!(DSML_PARAM_END_SPACED, format!("")); + + assert_eq!(DSML_TOOL_CALLS, ds4_server::tools::DSML_TOOL_CALLS_START); +} + +/// The effort line and its budget table (encoding.py:443-453). The 75 and 100 +/// constants are the template with the mapping's number substituted, which is +/// what `render_reasoning_effort` does before it formats. +#[test] +fn effort_line_and_budgets_match_the_oracle() { + let template = literal("REASONING_EFFORT_TEMPLATE"); + assert_eq!(V41_EFFORT_TEMPLATE, template, "REASONING_EFFORT_TEMPLATE"); + + let with = |budget: i64| template.replace("{budget}", &budget.to_string()); + assert_eq!( + V41_THINK_HIGH_PREFIX, + with(mapping_int("REASONING_EFFORT_MAPPINGS", "high")), + "high" + ); + assert_eq!( + V41_THINK_MAX_PREFIX, + with(mapping_int("REASONING_EFFORT_MAPPINGS", "max")), + "max" + ); + assert_eq!( + literal("DEFAULT_REASONING_EFFORT"), + "high", + "default effort" + ); +} + +/// The two templates that define a served turn's tail: the tool-result +/// envelope (encoding.py:68) and the assistant composition (encoding.py:60). +/// The assistant template is `{reasoning}{content}{tool_calls}` plus the EOS +/// token, so its own body is what the renderer must write before the EOS. +#[test] +fn turn_templates_match_the_oracle() { + assert_eq!( + literal("tool_output_template"), + "{content}" + ); + + let assistant = literal("assistant_msg_template"); + assert_eq!(assistant, "{reasoning}{content}{tool_calls}"); + assert!( + declaration("assistant_msg_template").contains("+ eos_token"), + "the oracle's assistant template no longer appends the EOS token" + ); + assert_eq!(DSML_EOS, literal("eos_token")); +} diff --git a/crates/ds4-server/tests/ref_decode.rs b/crates/ds4-server/tests/ref_decode.rs new file mode 100644 index 0000000000000000000000000000000000000000..e7d3a7c94494fc09a9dd94352a0945914051f3b0 --- /dev/null +++ b/crates/ds4-server/tests/ref_decode.rs @@ -0,0 +1,191 @@ +//! The reference leg, decode side: our DSML parse against the official +//! DeepSeek-V4.1 encoding implementation (the oracle). +//! +//! The oracle is vendored at `tests/ds41_ref/oracle/encoding.py` (sha256 +//! 13231c592e958fb39e2af6001dae69d90e7a36b265702b9b390236fb387599e2), pinned +//! by that digest in `tests/ds41_ref/ref_drive.py`. Both +//! halves of every case under `tests/fixtures/ds41-ref/` are the oracle's own +//! output: `.txt` is the completion the oracle's templates rendered, and +//! `.calls.json` is the oracle parsing that same text with +//! `parse_message_from_completion_text` (encoding.py:946-990). Regenerate with +//! `bash tests/ds41_ref/regen_fixtures.sh`; the driver refuses any other +//! oracle revision. +//! +//! What is compared: the content and reasoning split byte for byte, and each +//! call's name and arguments as JSON values. The arguments *string* is not +//! compared byte for byte on purpose -- our shape minifies non-string children +//! exactly as the C implementation this port descends from does +//! (`tool_call_json_args_add`, quoted there at ds4_server.c:5971-5982, over +//! json_minify_raw_value :411), while the oracle +//! keeps the parameter body verbatim; both decode to the same value, and the +//! client echoes whichever string it received (requirement 2.3.5). The +//! minified shape is pinned by our own test below, so the difference is +//! recorded rather than silent. + +use ds4_server::{parse_generated_message, ChatFormat, ModelSyntax, ToolSchemaOrder}; +use serde_json::Value; + +/// The oracle's parsed message, as its fixture records it. +struct Oracle { + content: String, + reasoning: String, + calls: Vec<(String, String)>, +} + +fn fixture(name: &str) -> Vec { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ds41-ref") + .join(name); + std::fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +fn oracle(name: &str) -> Oracle { + let raw = fixture(&format!("{name}.calls.json")); + let v: Value = serde_json::from_slice(&raw).expect("oracle fixture is JSON"); + let calls = v["tool_calls"] + .as_array() + .expect("tool_calls array") + .iter() + .map(|c| { + let f = &c["function"]; + ( + f["name"].as_str().expect("name").to_string(), + f["arguments"].as_str().expect("arguments").to_string(), + ) + }) + .collect(); + Oracle { + content: v["content"].as_str().unwrap_or_default().to_string(), + reasoning: v["reasoning_content"] + .as_str() + .unwrap_or_default() + .to_string(), + calls, + } +} + +/// Parse one completion as the ds41 route parses it and check both sides agree. +fn check(name: &str, thinking: bool) { + let text = fixture(&format!("{name}.txt")); + let want = oracle(name); + + let parsed = parse_generated_message( + ModelSyntax::DeepSeek, + &text, + thinking, + ChatFormat::DeepSeek, + &[] as &[ToolSchemaOrder], + ); + + assert!( + parsed.ok, + "{name}: our parse rejected a completion the oracle accepts" + ); + assert_eq!( + String::from_utf8_lossy(&parsed.content), + want.content, + "{name}: content differs" + ); + assert_eq!( + String::from_utf8_lossy(&parsed.reasoning), + want.reasoning, + "{name}: reasoning differs" + ); + + let got: Vec<(String, String)> = parsed + .calls + .iter() + .map(|c| (c.name.clone(), c.arguments.clone())) + .collect(); + + assert_eq!(got.len(), want.calls.len(), "{name}: call count differs"); + for (i, ((gn, ga), (wn, wa))) in got.iter().zip(&want.calls).enumerate() { + assert_eq!(gn, wn, "{name}: call {i} name differs"); + let gv: Value = serde_json::from_str(ga) + .unwrap_or_else(|e| panic!("{name}: our call {i} arguments are not JSON: {e} ({ga})")); + let wv: Value = serde_json::from_str(wa).unwrap_or_else(|e| { + panic!("{name}: oracle call {i} arguments are not JSON: {e} ({wa})") + }); + assert_eq!(gv, wv, "{name}: call {i} arguments differ as values"); + } +} + +/// One call, one string and one integer child: the canonical shape. +#[test] +fn canonical_single_call_matches_official_parser() { + check("dec-one-call", false); +} + +/// Two invokes in one block (the shape TP-03 exercises on the wire). +#[test] +fn two_invokes_match_official_parser() { + check("dec-two-calls", false); +} + +/// Every JSON child kind the type attribute distinguishes, including a nested +/// array and object. +#[test] +fn complex_argument_kinds_match_official_parser() { + check("dec-complex-args", false); +} + +/// A non-ASCII string child: the oracle's json.dumps is ensure_ascii False, so +/// the value round-trips raw and both sides must agree on the character. +#[test] +fn unicode_argument_matches_official_parser() { + check("dec-unicode", false); +} + +/// A block marker inside a parameter value is data, never structure +/// (inventory G9; the reference rejects markers inside values at +/// ds4_server.c:5890 find_tool_structural_text): one call, whose body carries +/// the wrapper's own text. +#[test] +fn marker_inside_a_value_is_not_structure() { + check("dec-marker-in-value", false); +} + +/// Thinking mode: reasoning before the close, prose after it, then the block. +#[test] +fn thinking_split_matches_official_parser() { + check("dec-thinking", true); +} + +/// Our arguments string for a composite child is the minified form, as the +/// reference server builds it (ds4_server.c:5971-5982). The oracle keeps the +/// parameter body verbatim; the values are equal, the bytes are not, and this +/// test pins our side of that difference so a future edit cannot change it +/// unnoticed. +#[test] +fn composite_arguments_are_minified_as_the_reference_does() { + let text = fixture("dec-complex-args.txt"); + let parsed = parse_generated_message( + ModelSyntax::DeepSeek, + &text, + false, + ChatFormat::DeepSeek, + &[] as &[ToolSchemaOrder], + ); + assert_eq!(parsed.calls.len(), 1); + + let ours = &parsed.calls[0].arguments; + let oracle_args = oracle("dec-complex-args").calls[0].1.clone(); + + // Members are joined with ", " (the reference's tool_call_json_args_add), + // while a composite *value* is minified inside. + assert!( + ours.contains(r#""deps": ["a","b"]"#), + "our composite child is not minified: {ours}" + ); + assert!( + oracle_args.contains(r#""deps": ["a", "b"]"#), + "the oracle's child is no longer verbatim: {oracle_args}" + ); + + let gv: Value = serde_json::from_str(ours).expect("our arguments are JSON"); + let wv: Value = serde_json::from_str(&oracle_args).expect("oracle arguments are JSON"); + assert_eq!( + gv, wv, + "the minified and verbatim forms must carry one value" + ); +} diff --git a/crates/ds4-server/tests/ref_encoding.rs b/crates/ds4-server/tests/ref_encoding.rs new file mode 100644 index 0000000000000000000000000000000000000000..9ef0e25587fca89951bd00775ff4fff653a87918 --- /dev/null +++ b/crates/ds4-server/tests/ref_encoding.rs @@ -0,0 +1,501 @@ +//! The reference leg, encode side: our DS41 prompt bytes against the official +//! DeepSeek-V4.1 encoding implementation (the oracle). +//! +//! The oracle is vendored at `tests/ds41_ref/oracle/encoding.py` (sha256 +//! 13231c592e958fb39e2af6001dae69d90e7a36b265702b9b390236fb387599e2), an +//! operator-supplied file that `tests/ds41_ref/ref_drive.py` pins by that +//! digest; the external tree it was first read from is no longer consulted. +//! Each +//! fixture beside this test is the oracle's own output for the case of the +//! same name under `tests/ds41_ref/cases/`, regenerated with +//! `bash tests/ds41_ref/regen_fixtures.sh`; driving the oracle is what +//! `tests/ds41_ref/ref_drive.py` does, and it refuses any other revision. +//! The fixtures therefore pin bytes we did not author. +//! +//! Coverage is traced in `docs/ds41-dsml-reference-matrix.md`: every CTR +//! requirement and every inventory gap names the case that exercises it, and +//! a case that does not yet hold is marked OPEN there rather than dropped. + +use ds4_server::{ + render_dsml_chat_choice_ex, ChatMsg, ChatPart, DsmlHead, ThinkMode, ToolCall, ToolChoice, +}; + +/// The head our ds41 route passes: the engine's V4.1 encoding.py head with the +/// tokenizer's system token (render.rs `dsml_head`). +const V41: DsmlHead = DsmlHead::V41 { + system_token: true, + effort: None, +}; + +fn msg(role: &str, content: &str) -> ChatMsg { + ChatMsg { + role: role.into(), + content: content.into(), + ..Default::default() + } +} + +fn assistant(content: &str, calls: Vec) -> ChatMsg { + ChatMsg { + role: "assistant".into(), + content: content.into(), + calls, + ..Default::default() + } +} + +fn call(id: &str, name: &str, arguments: &str) -> ToolCall { + ToolCall { + id: id.into(), + name: name.into(), + arguments: arguments.into(), + } +} + +/// A `tool` role message as the chat parse builds it: the body plus the id of +/// the call it answers (`add_tool_call_id`, parse.rs:885). The id is what the +/// renderer orders a run of results by, so a case that omits it cannot +/// exercise the order rule. +fn tool_result(id: &str, body: &str) -> ChatMsg { + ChatMsg { + role: "tool".into(), + content: body.into(), + tool_call_id: id.into(), + tool_call_ids: vec![id.into()], + ..Default::default() + } +} + +fn fixture(name: &str) -> Vec { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ds41-ref") + .join(name); + std::fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())) +} + +/// Report the first differing byte, with both sides escaped, so a mismatch is +/// readable without dumping two long prompts. +fn diff_report(got: &[u8], want: &[u8]) -> String { + let at = got + .iter() + .zip(want) + .position(|(a, b)| a != b) + .unwrap_or_else(|| got.len().min(want.len())); + let lo = at.saturating_sub(60); + format!( + "first difference at byte {at} (got {} bytes, want {} bytes)\n got ...{}\n want ...{}", + got.len(), + want.len(), + String::from_utf8_lossy(&got[lo..(at + 60).min(got.len())]), + String::from_utf8_lossy(&want[lo..(at + 60).min(want.len())]), + ) +} + +/// Render one case and compare with the oracle's fixture of the same name. +fn check(name: &str, msgs: &[ChatMsg], schemas: &str, mode: ThinkMode) { + check_head(name, msgs, schemas, mode, V41); +} + +/// The same, with the effort budget the request carried (O-01's numeric half). +fn check_with_budget( + name: &str, + msgs: &[ChatMsg], + schemas: &str, + mode: ThinkMode, + budget: Option, +) { + check_head(name, msgs, schemas, mode, V41.with_effort(budget)); +} + +fn check_head(name: &str, msgs: &[ChatMsg], schemas: &str, mode: ThinkMode, head: DsmlHead) { + let got = render_dsml_chat_choice_ex(msgs, schemas, mode, ToolChoice::Auto, head) + .unwrap_or_else(|e| panic!("{name}: render refused: {}", e.0)); + let want = fixture(&format!("{name}.prompt.txt")); + + assert_eq!( + String::from_utf8_lossy(&got), + String::from_utf8_lossy(&want), + "{name}: {}", + diff_report(&got, &want) + ); +} + +/// A case's tool, as the route receives it: one client JSON value per line +/// (render.rs:475-520 reformats each to json.dumps shape, which is what the +/// oracle emits). Each constant mirrors the `tools` array of the case that +/// uses it -- a schema that declares a property its case does not is a test +/// bug, not a carriage difference. +const READ_FILE_SCHEMA: &str = concat!( + r#"{"name":"read_file","description":"Read a file","parameters":{"type":"object","properties":{"target_file":{"type":"string"},"limit":{"type":"integer"}},"required":["target_file"]}}"# +); +const READ_FILE_SCHEMA_MIN: &str = concat!( + r#"{"name":"read_file","description":"Read a file","parameters":{"type":"object","properties":{"target_file":{"type":"string"}},"required":["target_file"]}}"# +); + +#[test] +fn tools_prompt_matches_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file /tmp/x.txt"), + ]; + check("enc-tools-chat", &msgs, READ_FILE_SCHEMA, ThinkMode::None); +} + +#[test] +fn plain_chat_prompt_matches_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Say hello."), + assistant("Hello.", Vec::new()), + msg("user", "Say it again."), + ]; + check("enc-plain-chat", &msgs, "", ThinkMode::None); +} + +/// The type attribute: `string="true"` for a string child, `string="false"` +/// with the value in json.dumps shape for every other kind +/// (encoding.py encode_arguments_to_dsml:524-556). The echoed arguments are +/// the client's raw JSON string, as requirement 2.3.5 replays them. +#[test] +fn typed_parameters_match_official_encoding() { + let args = r#"{"target_file": "/tmp/x.txt", "limit": 40, "force": true, "ratio": 1.5, "note": null, "deps": ["a", "b"], "opts": {"k": 1}}"#; + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file and label it."), + assistant("Reading it now.", vec![call("call_1", "read_file", args)]), + msg("tool", "file body"), + msg("user", "Now summarise it."), + ]; + let schemas = concat!( + r#"{"name":"read_file","description":"Read a file","parameters":{"type":"object","properties":{"target_file":{"type":"string"},"limit":{"type":"integer"},"force":{"type":"boolean"},"ratio":{"type":"number"},"note":{"type":"string"},"deps":{"type":"array","items":{"type":"string"}},"opts":{"type":"object"}},"required":["target_file"]}}"# + ); + check("enc-argtypes-chat", &msgs, schemas, ThinkMode::None); +} + +/// A replayed assistant's `arguments` is a string on the wire, and the oracle +/// tolerates two shapes beside the object JSON: it decodes a string twice +/// before giving up, and a value that is still not a dict becomes one +/// parameter named `arguments` carrying the value as it stood +/// (encoding.py encode_arguments_to_dsml:524-556). +/// +/// The three arms measured 2026-10-20. A string that decodes to an object on +/// the second pass renders that object's parameters; a JSON array renders one +/// `arguments` parameter with `string="false"` and the json.dumps spelling of +/// the value; a string that is not JSON at all renders one `arguments` +/// parameter with `string="true"` and the text verbatim. +#[test] +fn double_encoded_arguments_match_official_encoding() { + // The wire string is a JSON object wrapped in one more JSON string, so the + // first decode yields a string and the second the object. + let args = r#""{\"target_file\": \"/tmp/x.txt\"}""#; + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file /tmp/x.txt"), + assistant("Calling it now.", vec![call("call_1", "read_file", args)]), + ]; + check( + "enc-args-double", + &msgs, + READ_FILE_SCHEMA_MIN, + ThinkMode::None, + ); +} + +#[test] +fn non_dict_arguments_match_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file /tmp/x.txt"), + assistant( + "Calling it now.", + vec![call("call_1", "read_file", "[1, 2]")], + ), + ]; + check( + "enc-args-nondict", + &msgs, + READ_FILE_SCHEMA_MIN, + ThinkMode::None, + ); +} + +#[test] +fn unparsable_arguments_match_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file /tmp/x.txt"), + assistant( + "Calling it now.", + vec![call("call_1", "read_file", "not json")], + ), + ]; + check( + "enc-args-plain", + &msgs, + READ_FILE_SCHEMA_MIN, + ThinkMode::None, + ); +} + +/// A user message whose content is an array of blocks: a text block beside an +/// image block, and a text-only pair. +/// +/// The oracle joins the pieces the client sent as separate blocks with a blank +/// line -- `_process_image_blocks` turns an image into a placeholder text block +/// and the user branch writes `"\n\n".join(parts)` (encoding.py:323-353, +/// :633-658) -- while this port concatenates them, at parse time for the text +/// pieces (`append_text_part`, parse.rs:587) and at render time for the +/// placeholder (`append_dsml_user_content`, render.rs:1376). +/// +/// Measured 2026-10-20 on both cases: the two prompts carry the same pieces in +/// the same order and differ by that one separator, which is what these cases +/// pin. Recorded as an open row in docs/ds41-dsml-function-map.md row +/// `process_image_messages`; the fix is a render/parse join gated to the V4.1 +/// head, and it is the operator's call because it moves every image-bearing +/// prompt. +fn check_block_join(name: &str, msgs: &[ChatMsg]) { + let got = render_dsml_chat_choice_ex(msgs, "", ThinkMode::None, ToolChoice::Auto, V41) + .unwrap_or_else(|e| panic!("{name}: render refused: {}", e.0)); + let got = String::from_utf8(got).expect("our prompt is UTF-8"); + let want = String::from_utf8(fixture(&format!("{name}.prompt.txt"))).expect("fixture is UTF-8"); + + assert_eq!( + want.matches("\n\n").count(), + 1, + "{name}: the fixture must hold exactly the one separator this test removes" + ); + assert_eq!( + got, + want.replace("\n\n", ""), + "{name}: the two prompts differ beyond the block separator" + ); +} + +/// A text block and an image block in one user message. +#[test] +fn image_block_beside_text_is_pinned() { + let mut user = msg("user", "What is in this image?"); + user.parts = vec![ + ChatPart::Text("What is in this image?".into()), + ChatPart::Image(0), + ]; + let msgs = [msg("system", "You are a helpful assistant."), user]; + check_block_join("enc-image-chat", &msgs); +} + +/// Two text blocks in one user message, with no image anywhere. +#[test] +fn text_blocks_are_pinned() { + let mut user = msg("user", "First part.Second part."); + user.parts = vec![ + ChatPart::Text("First part.".into()), + ChatPart::Text("Second part.".into()), + ]; + let msgs = [msg("system", "You are a helpful assistant."), user]; + check_block_join("enc-blocks-chat", &msgs); +} + +#[test] +fn thinking_tools_prompt_matches_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file /tmp/x.txt"), + ]; + check( + "enc-think-tools", + &msgs, + READ_FILE_SCHEMA_MIN, + ThinkMode::High, + ); +} + +/// The numeric reasoning-effort header: 75 for the default, 100 for max +/// (encoding.py:444-472). Only thinking mode renders it, and only at index 0. +#[test] +fn thinking_max_effort_matches_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file /tmp/x.txt"), + ]; + check("enc-think-max", &msgs, READ_FILE_SCHEMA_MIN, ThinkMode::Max); +} + +/// CLOSED (matrix row O-01): the oracle renders "Reasoning Effort: 50" for the +/// low effort (encoding.py:449-451). Measured before the fix: low rendered no +/// line at all (the requested effort never reached the renderer on the ds41 +/// route, where the engine collapses low into high) and the fixture was 1372 +/// bytes against our 1311. The route now carries the requested effort beside +/// the collapsed mode (`ParsedRequest::effort`, route.rs `RequestedEffort`) and +/// the V4.1 head writes the low mapping's 50 (render.rs `v41_effort_prefix`). +#[test] +fn thinking_low_effort_matches_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file /tmp/x.txt"), + ]; + check("enc-think-low", &msgs, READ_FILE_SCHEMA_MIN, ThinkMode::Low); +} + +/// CLOSED (matrix row O-01, the numeric half): the oracle renders an integer +/// budget in [1,100] as itself (encoding.py:456-473). Measured before the fix: +/// a numeric `reasoning_effort` was refused at the parse, which read a string +/// name and nothing else. The request now carries the number as its own budget +/// and the head writes it verbatim; the engine keeps its collapsed mode. +#[test] +fn thinking_numeric_effort_matches_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file /tmp/x.txt"), + ]; + check_with_budget( + "enc-think-budget", + &msgs, + READ_FILE_SCHEMA_MIN, + ThinkMode::High, + Some(40), + ); +} + +/// A mid-conversation system message: the oracle writes the system token +/// again and lets the message introduce its own turn (encoding.py:621-627, +/// :733-739), and counts that index as user-like for the assistant header +/// that follows (:562-582). Measured before the fix: our renderer hoisted +/// every system message into the head, so "Answer in one line." landed before +/// the tools preamble instead of after the user turn. +#[test] +fn mid_conversation_system_matches_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file /tmp/x.txt"), + msg("system", "Answer in one line."), + ]; + check( + "enc-midsystem-chat", + &msgs, + READ_FILE_SCHEMA_MIN, + ThinkMode::None, + ); +} + +/// Tool results are ordered by the preceding assistant's call order, not by +/// arrival (encoding.py:233-266 sort_tool_results_by_call_order). Measured +/// before the fix: the oracle's block carried body a then body b, ours body b +/// then body a, every other byte of the 1767 matching. +#[test] +fn tool_results_in_call_order_match_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Check both files."), + assistant( + "Checking.", + vec![ + call("call_a", "read_file", r#"{"target_file": "/tmp/a.txt"}"#), + call("call_b", "read_file", r#"{"target_file": "/tmp/b.txt"}"#), + ], + ), + tool_result("call_b", "body b"), + tool_result("call_a", "body a"), + msg("user", "Summarise both."), + ]; + check( + "enc-toolresults-order", + &msgs, + READ_FILE_SCHEMA_MIN, + ThinkMode::None, + ); +} + +/// A historical assistant's `reasoning_content` never reaches the prompt in +/// chat mode, where neither side renders it. This case pins that it does not +/// leak; the thinking-mode case below is the one that decides the drop rule. +#[test] +fn historical_reasoning_not_rendered_in_chat_matches_official_encoding() { + let mut history = assistant("Hi.", Vec::new()); + history.reasoning = "HISTORICAL-THOUGHT-MARKER".into(); + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Say hi."), + history, + msg("user", "Say it again."), + ]; + check("enc-drop-thinking", &msgs, "", ThinkMode::None); +} + +/// A user-like message carrying only tool results does not start a turn +/// (encoding.py `find_last_user_index`:562-582). The client of record sends +/// tool results as their own messages, so a tool turn with no following user +/// message ends on exactly this shape. +#[test] +fn tool_result_only_tail_matches_official_encoding() { + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Read the file."), + assistant( + "", + vec![call( + "call_1", + "read_file", + r#"{"target_file": "/tmp/x.txt"}"#, + )], + ), + tool_result("call_1", "body"), + ]; + check( + "enc-toolresult-only", + &msgs, + READ_FILE_SCHEMA_MIN, + ThinkMode::None, + ); +} + +/// The drop rule itself, measured in the mode that renders reasoning: with no +/// tools anywhere, `_drop_thinking_messages` (encoding.py:749-767) strips the +/// historical assistant's `reasoning_content`, so the `` its +/// predecessor's transition token wrote is all the thinking that message has. +/// Same conversation as the chat-mode case above, so the two verdicts differ +/// only by the mode. +#[test] +fn historical_reasoning_is_dropped_matches_official_encoding() { + let mut history = assistant("Hi.", Vec::new()); + history.reasoning = "HISTORICAL-THOUGHT-MARKER".into(); + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Say hi."), + history, + msg("user", "Say it again."), + ]; + check("enc-think-drop-thinking", &msgs, "", ThinkMode::High); +} + +/// An assistant at or after the last user index keeps its reasoning: the +/// oracle writes `{reasoning}` there (encoding.py:678-685, the +/// `index > last_user_idx` arm of the assistant branch). +#[test] +fn tail_reasoning_is_kept_matches_official_encoding() { + let mut tail = assistant("Hi.", Vec::new()); + tail.reasoning = "TAIL-THOUGHT-MARKER".into(); + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Say hi."), + tail, + ]; + check("enc-think-tail-reasoning", &msgs, "", ThinkMode::High); +} + +/// Two assistant messages in a row share one `` header, so the +/// second has no transition token to open its thinking. The oracle still +/// writes that message's reasoning and its closer inline, because +/// `thinking_part` belongs to the message body and not to the transition token +/// (encoding.py render_message:678-712). +#[test] +fn consecutive_assistant_reasoning_matches_official_encoding() { + let mut second = assistant("Two.", Vec::new()); + second.reasoning = "SECOND-THOUGHT-MARKER".into(); + let msgs = [ + msg("system", "You are a helpful assistant."), + msg("user", "Say hi."), + assistant("One.", Vec::new()), + second, + ]; + check("enc-think-two-assistants", &msgs, "", ThinkMode::High); +} diff --git a/crates/ds4-server/tests/render.rs b/crates/ds4-server/tests/render.rs new file mode 100644 index 0000000000000000000000000000000000000000..d6549e7143935ba9999bc5c88e61569543798c59 --- /dev/null +++ b/crates/ds4-server/tests/render.rs @@ -0,0 +1,1110 @@ +//! C↔Rust DSML no-tools render. + +use ds4_server::render::DSML_IMAGE; +use ds4_server::tools::{ + DSML_INVOKE_END_SPACED, DSML_INVOKE_START_SPACED, DSML_PARAM_END_SPACED, + DSML_PARAM_START_SPACED, DSML_TOOL_CALLS_END_SPACED, DSML_TOOL_CALLS_START_SPACED, +}; +use ds4_server::{ + render_chat, render_chat_choice, render_dsml_chat, render_dsml_chat_choice, + render_dsml_chat_choice_ex, render_live_tool_tail, render_motif3_chat_ex, Api, ChatMsg, + ChatPart, DsmlHead, ModelSyntax, ThinkMode, ToolCall, ToolChoice, ToolSchemaOrder, + DSML_ASSISTANT, DSML_BOS, DSML_EOS, DSML_SYSTEM_TOKEN, DSML_THINK_END, DSML_THINK_START, + DSML_USER, THINK_HIGH_PREFIX, THINK_MAX_PREFIX, V41_THINK_HIGH_PREFIX, +}; + +use std::path::PathBuf; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_RENDER_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/render_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/render_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_out(args: &[&str]) -> Vec { + let out = Command::new(require_oracle()) + .args(args) + .output() + .expect("run render_c_oracle"); + assert!( + out.status.success(), + "oracle {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); + out.stdout +} + +fn msg(role: &str, content: &str) -> ChatMsg { + ChatMsg { + role: role.into(), + content: content.into(), + ..ChatMsg::default() + } +} + +fn msg_reason(role: &str, content: &str, reasoning: &str) -> ChatMsg { + ChatMsg { + role: role.into(), + content: content.into(), + reasoning: reasoning.into(), + ..ChatMsg::default() + } +} + +fn rust_user(think: ThinkMode, content: &str) -> Vec { + render_dsml_chat(&[msg("user", content)], "", think).unwrap() +} + +#[test] +fn prefixes_match_c_literals() { + assert!(THINK_HIGH_PREFIX.starts_with("Reasoning Effort: Absolute maximum")); + assert!(THINK_MAX_PREFIX.starts_with("Reasoning Effort: Beyond maximum")); +} + +#[test] +fn qwen_live_tool_tail_appends_only_eos_result_and_next_assistant() { + let mut result = msg("tool", "/tmp"); + result.tool_call_id = "call_1".into(); + assert_eq!( + render_live_tool_tail(ModelSyntax::Qwen4Exp, Api::Responses, &[result], ThinkMode::None) + .unwrap(), + b"<|im_end|>\n<|im_start|>user\n\n/tmp\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" + ); +} + +#[test] +fn dsml_anthropic_live_tool_tail_does_not_replay_the_assistant_call() { + let mut result = msg("user", "ok"); + result.tool_call_id = "toolu_1".into(); + assert_eq!( + render_live_tool_tail( + ModelSyntax::DeepSeek, + Api::Anthropic, + &[result], + ThinkMode::None + ) + .unwrap(), + "<|end▁of▁sentence|><|User|>ok<|Assistant|>" + .as_bytes() + ); +} + +#[test] +fn dsml_v41_head_matches_official_encoding() { + // The engine's golden byte test `test_render_matches_official_v41_encoding` + // (tests/server_tests_render_v41.c): the official encoding.py render of + // four conversations, plus the V4-tokenizer fallback. A V4.1 tokenizer + // owns the system token; the head writes it once before the effort prefix. + let v41 = DsmlHead::V41 { + system_token: true, + effort: None, + }; + let with_sys = [msg("system", "S"), msg("user", "U")]; + let only_user = [msg("user", "U")]; + let render = |msgs: &[ChatMsg], mode| { + render_dsml_chat_choice_ex(msgs, "", mode, ToolChoice::Auto, v41).unwrap() + }; + // The message list ends in a user turn, so the render closes with the + // pending assistant head (the think-end tag when plain, the think-start tag + // when thinking). + let sys_tail = format!("S{DSML_USER}U{DSML_ASSISTANT}"); + let user_tail = format!("{DSML_USER}U{DSML_ASSISTANT}"); + assert_eq!( + render(&with_sys, ThinkMode::None), + format!("{DSML_BOS}{DSML_SYSTEM_TOKEN}{sys_tail}{DSML_THINK_END}").as_bytes() + ); + // A user-only chat with no effort prefix writes no head token at all. + assert_eq!( + render(&only_user, ThinkMode::None), + format!("{DSML_BOS}{user_tail}{DSML_THINK_END}").as_bytes() + ); + assert_eq!( + render(&with_sys, ThinkMode::High), + format!("{DSML_BOS}{DSML_SYSTEM_TOKEN}{V41_THINK_HIGH_PREFIX}{sys_tail}{DSML_THINK_START}") + .as_bytes() + ); + assert_eq!( + render(&only_user, ThinkMode::High), + format!( + "{DSML_BOS}{DSML_SYSTEM_TOKEN}{V41_THINK_HIGH_PREFIX}{user_tail}{DSML_THINK_START}" + ) + .as_bytes() + ); + // The V4 tokenizer has no system token: BOS then the system body (V4 head). + assert_eq!( + render_dsml_chat_choice_ex( + &with_sys, + "", + ThinkMode::None, + ToolChoice::Auto, + DsmlHead::V4 + ) + .unwrap(), + format!("{DSML_BOS}{sys_tail}{DSML_THINK_END}").as_bytes() + ); +} + +#[test] +fn dsml_live_tool_tail_has_no_head() { + // The engine's tail writes EOS + messages only (server_live_prep.c:21-69; + // the port's own C: ds4_server.c:4700-4748): no effort prefix and no + // system token; the head is already in the live KV. + let mut result = msg("tool", "ok"); + result.tool_call_id = "call_1".into(); + let tail = render_live_tool_tail( + ModelSyntax::DeepSeek, + Api::Responses, + &[result], + ThinkMode::High, + ) + .unwrap(); + assert!(!tail.starts_with(V41_THINK_HIGH_PREFIX.as_bytes())); + let expected = format!( + "{DSML_EOS}{DSML_USER}ok{DSML_ASSISTANT}{DSML_THINK_START}" + ); + assert_eq!(tail, expected.as_bytes()); +} +#[test] +fn user_none_and_low_match_c() { + for (name, mode) in [("none", ThinkMode::None), ("low", ThinkMode::Low)] { + assert_eq!( + rust_user(mode, "Hello"), + c_out(&["user", name, "Hello"]), + "user {name}" + ); + } +} + +#[test] +fn user_high_max_match_c() { + for (name, mode) in [("high", ThinkMode::High), ("max", ThinkMode::Max)] { + assert_eq!( + rust_user(mode, "Why?"), + c_out(&["user", name, "Why?"]), + "user {name}" + ); + } +} + +#[test] +fn system_user_match_c() { + let rust = render_dsml_chat( + &[msg("system", "sys"), msg("user", "ask")], + "", + ThinkMode::None, + ) + .unwrap(); + assert_eq!(rust, c_out(&["system-user", "none", "sys", "ask"])); +} + +#[test] +fn developer_is_system() { + let rust = render_dsml_chat( + &[msg("developer", "dev"), msg("user", "hi")], + "", + ThinkMode::None, + ) + .unwrap(); + assert_eq!(rust, c_out(&["developer"])); +} + +#[test] +fn history_none_and_low_match_c() { + let msgs = [msg("user", "u1"), msg("assistant", "a1"), msg("user", "u2")]; + assert_eq!( + render_dsml_chat(&msgs, "", ThinkMode::None).unwrap(), + c_out(&["history", "none", "u1", "a1", "u2"]) + ); + assert_eq!( + render_dsml_chat(&msgs, "", ThinkMode::Low).unwrap(), + c_out(&["history", "low", "u1", "a1", "u2"]) + ); +} + +#[test] +fn history_with_reasoning_match_c() { + let msgs = [ + msg("user", "u1"), + msg_reason("assistant", "a1", "plan"), + msg("user", "u2"), + ]; + assert_eq!( + render_dsml_chat(&msgs, "", ThinkMode::Low).unwrap(), + c_out(&["think-hist", "low", "u1", "plan", "a1", "u2"]) + ); +} + +#[test] +fn tool_result_and_escape_match_c() { + let rust = + render_dsml_chat(&[msg("user", "q"), msg("tool", "ok")], "", ThinkMode::None).unwrap(); + assert_eq!(rust, c_out(&["tool-result", "none", "q", "ok"])); + let rust = render_dsml_chat( + &[msg("user", "q"), msg("tool", "xy")], + "", + ThinkMode::None, + ) + .unwrap(); + assert_eq!(rust, c_out(&["tool-escape"])); +} + +fn weather_schema() -> &'static str { + r#"{"name":"get_weather","description":"Weather","parameters":{"type":"object","properties":{"city":{"type":"string"},"unit":{"type":"string"}}}}"# +} + +fn call(name: &str, arguments: &str) -> ChatMsg { + ChatMsg { + role: "assistant".into(), + calls: vec![ToolCall { + id: "call1".into(), + name: name.into(), + arguments: arguments.into(), + }], + ..ChatMsg::default() + } +} + +#[test] +fn dsml_tools_and_required_match_c() { + let schema = weather_schema(); + let msgs = [msg("user", "hi")]; + assert_eq!( + render_dsml_chat(&msgs, schema, ThinkMode::None).unwrap(), + c_out(&["dsml-tools", "none", schema, "hi"]) + ); + assert_eq!( + render_dsml_chat_choice(&msgs, schema, ThinkMode::None, ToolChoice::Required).unwrap(), + c_out(&["dsml-tools-req", "none", schema, "hi"]) + ); +} + +#[test] +fn dsml_invoke_match_c() { + let args = r#"{"city":"Seoul","n":2}"#; + let msgs = [msg("user", "q"), call("get_weather", args)]; + assert_eq!( + render_dsml_chat(&msgs, "", ThinkMode::None).unwrap(), + c_out(&["dsml-invoke", "none", "get_weather", args]) + ); + let bad = "not-json"; + let msgs = [msg("user", "q"), call("get_weather", bad)]; + assert_eq!( + render_dsml_chat(&msgs, "", ThinkMode::None).unwrap(), + c_out(&["dsml-invoke", "none", "get_weather", bad]) + ); +} + +#[test] +fn family_tools_match_c() { + let schema = weather_schema(); + let msgs = [msg("user", "hi")]; + for (syntax, fam) in [ + (ModelSyntax::Motif3, "motif"), + (ModelSyntax::Exaone, "exaone"), + (ModelSyntax::Dots3, "dots3"), + (ModelSyntax::SolarOpen2, "solar"), + ] { + assert_eq!( + render_chat(syntax, &msgs, schema, ThinkMode::None).unwrap(), + c_out(&["fam-tools", fam, "none", schema, "hi"]), + "{fam} tools" + ); + } +} + +#[test] +fn family_invoke_match_c() { + let args = r#"{"city":"Seoul","n":2}"#; + let msgs = [msg("user", "q"), call("get_weather", args)]; + for (syntax, fam) in [ + (ModelSyntax::Motif3, "motif"), + (ModelSyntax::Exaone, "exaone"), + (ModelSyntax::Dots3, "dots3"), + (ModelSyntax::SolarOpen2, "solar"), + ] { + assert_eq!( + render_chat(syntax, &msgs, "", ThinkMode::None).unwrap(), + c_out(&["fam-invoke", fam, "none", "get_weather", args]), + "{fam} invoke" + ); + } +} + +#[test] +fn motif_tools_order_match_c() { + let schema = weather_schema(); + let orders = [ToolSchemaOrder { + name: "get_weather".into(), + prop: vec!["city".into(), "unit".into()], + ..ToolSchemaOrder::default() + }]; + assert_eq!( + render_motif3_chat_ex(&[msg("user", "hi")], schema, &orders, ThinkMode::None).unwrap(), + c_out(&["motif-tools-order", schema, "hi"]) + ); +} + +#[test] +fn solar_arg_order_uses_schema() { + let args = r#"{"unit":"c","city":"Seoul"}"#; + let orders = [ToolSchemaOrder { + name: "get_weather".into(), + prop: vec!["city".into(), "unit".into()], + ..ToolSchemaOrder::default() + }]; + let msgs = [msg("user", "q"), call("get_weather", args)]; + let ordered = render_chat_choice( + ModelSyntax::SolarOpen2, + &msgs, + "", + &orders, + ThinkMode::None, + ToolChoice::Auto, + ) + .unwrap(); + let unordered = render_chat(ModelSyntax::SolarOpen2, &msgs, "", ThinkMode::None).unwrap(); + let s = String::from_utf8(ordered.clone()).unwrap(); + let city = s.find("city").expect("city"); + let unit = s.find("unit").expect("unit"); + assert!(city < unit, "{s}"); + assert_ne!(ordered, unordered); +} + +fn fam_user(syntax: ModelSyntax, fam: &str, think: &str, mode: ThinkMode, content: &str) { + assert_eq!( + render_chat(syntax, &[msg("user", content)], "", mode).unwrap(), + c_out(&["fam-user", fam, think, content]), + "{fam} user {think}" + ); +} + +#[test] +fn family_user_none_and_low_match_c() { + for (syntax, fam) in [ + (ModelSyntax::Motif3, "motif"), + (ModelSyntax::Exaone, "exaone"), + (ModelSyntax::Dots3, "dots3"), + (ModelSyntax::SolarOpen2, "solar"), + ] { + fam_user(syntax, fam, "none", ThinkMode::None, "Hello"); + fam_user(syntax, fam, "low", ThinkMode::Low, "Hello"); + } +} + +#[test] +fn family_system_user_match_c() { + let msgs = [msg("system", "sys"), msg("user", "ask")]; + for (syntax, fam) in [ + (ModelSyntax::Motif3, "motif"), + (ModelSyntax::Exaone, "exaone"), + (ModelSyntax::Dots3, "dots3"), + (ModelSyntax::SolarOpen2, "solar"), + ] { + assert_eq!( + render_chat(syntax, &msgs, "", ThinkMode::None).unwrap(), + c_out(&["fam-system-user", fam, "none", "sys", "ask"]), + "{fam} system-user" + ); + } +} + +#[test] +fn family_history_match_c() { + let msgs = [msg("user", "u1"), msg("assistant", "a1"), msg("user", "u2")]; + for (syntax, fam) in [ + (ModelSyntax::Motif3, "motif"), + (ModelSyntax::Exaone, "exaone"), + (ModelSyntax::Dots3, "dots3"), + (ModelSyntax::SolarOpen2, "solar"), + ] { + assert_eq!( + render_chat(syntax, &msgs, "", ThinkMode::None).unwrap(), + c_out(&["fam-history", fam, "none", "u1", "a1", "u2"]), + "{fam} history" + ); + assert_eq!( + render_chat(syntax, &msgs, "", ThinkMode::Low).unwrap(), + c_out(&["fam-history", fam, "low", "u1", "a1", "u2"]), + "{fam} history low" + ); + } +} + +#[test] +fn family_think_history_match_c() { + let msgs = [ + msg("user", "u1"), + msg_reason("assistant", "a1", "plan"), + msg("user", "u2"), + ]; + for (syntax, fam) in [ + (ModelSyntax::Motif3, "motif"), + (ModelSyntax::Exaone, "exaone"), + (ModelSyntax::Dots3, "dots3"), + (ModelSyntax::SolarOpen2, "solar"), + ] { + assert_eq!( + render_chat(syntax, &msgs, "", ThinkMode::Low).unwrap(), + c_out(&["fam-think-hist", fam, "low", "u1", "plan", "a1", "u2"]), + "{fam} think-hist" + ); + } +} + +#[test] +fn family_tool_result_match_c() { + let mut tool = msg("tool", "ok"); + tool.tool_call_id = "call1".into(); + let msgs = [msg("user", "q"), tool]; + for (syntax, fam) in [ + (ModelSyntax::Motif3, "motif"), + (ModelSyntax::Exaone, "exaone"), + (ModelSyntax::Dots3, "dots3"), + (ModelSyntax::SolarOpen2, "solar"), + ] { + assert_eq!( + render_chat(syntax, &msgs, "", ThinkMode::None).unwrap(), + c_out(&["fam-tool", fam, "none", "q", "ok"]), + "{fam} tool" + ); + } +} + +#[test] +fn dots3_embedded_think_match_c() { + let msgs = [ + msg("user", "q"), + msg("assistant", "\nplan\n\n\nAnswer"), + ]; + assert_eq!( + render_chat(ModelSyntax::Dots3, &msgs, "", ThinkMode::Low).unwrap(), + c_out(&["dots3-embed"]) + ); +} + +#[test] +fn syntax_for_model_id_matches_c() { + assert_eq!(ds4_server::syntax_for_model_id(0), ModelSyntax::DeepSeek); + assert_eq!(ds4_server::syntax_for_model_id(2), ModelSyntax::SolarOpen2); + assert_eq!(ds4_server::syntax_for_model_id(3), ModelSyntax::Motif3); + assert_eq!(ds4_server::syntax_for_model_id(4), ModelSyntax::Exaone); + assert_eq!(ds4_server::syntax_for_model_id(5), ModelSyntax::Dots3); + assert_eq!(ds4_server::syntax_for_model_id(6), ModelSyntax::Qwen4Exp); + assert_eq!(ds4_server::syntax_for_model_id(7), ModelSyntax::Glm53); + assert_eq!(ds4_server::syntax_for_model_id(8), ModelSyntax::K2Horizon); +} + +#[test] +fn glm_vq_uses_glm_syntax() { + // GLM VQ keeps GLM-5.3's chat template, tool markers and output parser. + assert_eq!( + ds4_server::syntax_for_model_id(ds4_core::Variant::Glm53FlashVq as i32), + ModelSyntax::Glm53 + ); +} + +#[test] +fn ling31_uses_bailing_syntax() { + // Ling-3.1 keeps Bailing V3's chat template, tool markers and parser. + let id = ds4_core::Variant::Ling31FlashVq as i32; + assert_eq!(ds4_server::syntax_for_model_id(id), ModelSyntax::Ling3Vl); + let raw = b"read itread\npath\na.rs\n"; + let parsed = ds4_server::parse_generated_for_model_id(id, raw, true, &[]); + assert_eq!(parsed.reasoning, b"read it"); + assert_eq!(parsed.calls.len(), 1); + assert_eq!(parsed.calls[0].name, "read"); +} + +#[test] +fn k2_horizon_chat_uses_ifm_not_exaone() { + let prompt = render_chat( + ModelSyntax::K2Horizon, + &[msg("user", "hello")], + "", + ThinkMode::None, + ) + .unwrap(); + assert_eq!( + prompt, + concat!( + "<|ifm|begin_of_text|>", + "<|ifm|im_start|>user\nhello<|ifm|im_end|>", + "<|ifm|im_start|>assistant\n\n\n", + ) + .as_bytes() + ); + let s = String::from_utf8(prompt).unwrap(); + assert!(!s.contains("<|user|>")); + assert!(!s.contains("<|assistant|>")); + assert!(!s.contains("<|User|>")); + assert!(!s.contains("")); +} + +#[test] +fn k2_horizon_json_tools_and_xml_replay_use_official_ifm_blocks() { + let mut assistant = call("get_weather", r#"{"city":"Seoul"}"#); + assistant.reasoning = "check weather".into(); + let prompt = render_chat_choice( + ModelSyntax::K2Horizon, + &[ + msg("system", "Be precise."), + msg("user", "Weather?"), + assistant, + msg("tool", "sunny"), + ], + weather_schema(), + &[], + ThinkMode::High, + ToolChoice::Auto, + ) + .unwrap(); + let s = String::from_utf8(prompt).unwrap(); + assert!(s.starts_with("<|ifm|begin_of_text|><|ifm|im_start|>system\n# Tools")); + assert!(s.contains("\n{\"name\":\"get_weather\"")); + assert!(s.contains("followed by paired and tags")); + assert!(s.contains(concat!( + "\n", + "get_weather\n", + "city\n", + "Seoul\n", + "\n", + "", + ))); + assert!(s.contains("<|ifm|im_start|>tool\nsunny<|ifm|im_end|>")); + assert!(s.ends_with("<|ifm|im_start|>assistant\n\n")); +} + +#[test] +fn k2_horizon_live_tool_tail_does_not_repeat_bos() { + let mut result = msg("tool", "sunny"); + result.tool_call_id = "call_1".into(); + let tail = render_live_tool_tail( + ModelSyntax::K2Horizon, + Api::Responses, + &[result], + ThinkMode::None, + ) + .unwrap(); + assert_eq!( + tail, + concat!( + "<|ifm|im_end|>", + "<|ifm|im_start|>tool\nsunny<|ifm|im_end|>", + "<|ifm|im_start|>assistant\n\n\n", + ) + .as_bytes() + ); +} + +#[test] +fn glm53_chat_and_image_protocol_match_official_template() { + let mut user = msg("user", "beforeafter"); + user.parts = vec![ + ChatPart::Text("before".into()), + ChatPart::Image(0), + ChatPart::Text("after".into()), + ]; + assert_eq!( + render_chat( + ModelSyntax::Glm53, + &[msg("system", "Be precise."), user], + "", + ThinkMode::High, + ) + .unwrap(), + b"[gMASK]<|system|>Reasoning Effort: High<|system|>Be precise.<|user|>before<|begin_of_image|><|image|><|end_of_image|>after<|assistant|>" + ); +} + +/// The artifact's own template honours the two named levels and reads +/// everything else as max (chat_template.jinja:2-3, `low`/`high` in a set, +/// else 'max'), so the low level must reach the model as Low. The port wrote +/// "Reasoning Effort: Max" for it. +#[test] +fn glm53_low_effort_renders_low() { + assert_eq!( + render_chat(ModelSyntax::Glm53, &[msg("user", "hi")], "", ThinkMode::Low).unwrap(), + b"[gMASK]<|system|>Reasoning Effort: Low<|user|>hi<|assistant|>" + ); +} + +#[test] +fn glm53_tools_filter_deferred_schema_and_preserve_argument_order() { + let schemas = concat!( + r#"{"name":"hidden","defer_loading":true,"parameters":{}}"#, + "\n", + r#"{"name":"bash","parameters":{"type":"object"},"strict":true}"#, + ); + let mut assistant = call("bash", r#"{"timeout":10,"command":"pwd"}"#); + assistant.reasoning = "need shell".into(); + let mut result = msg("tool", "okdone"); + result.tool_call_id = "call_1".into(); + let orders = [ToolSchemaOrder { + name: "bash".into(), + prop: vec!["command".into(), "timeout".into()], + ..Default::default() + }]; + let prompt = render_chat_choice( + ModelSyntax::Glm53, + &[msg("user", "run"), assistant, result], + schemas, + &orders, + ThinkMode::High, + ToolChoice::Auto, + ) + .unwrap(); + let prompt = String::from_utf8(prompt).unwrap(); + assert!(!prompt.contains("hidden")); + assert!(!prompt.contains("strict")); + assert!(prompt.contains(r#"{"name": "bash", "parameters": {"type":"object"}}"#)); + assert!(prompt.contains(concat!( + "<|assistant|>need shell", + "bashcommandpwd", + "timeout10", + "<|observation|>ok</tool_response>done", + "<|assistant|>" + ))); +} + +#[test] +fn qwen4exp_native_chat_protocol_matches_c() { + let msgs = [msg("system", "Be precise."), msg("user", "Weather?")]; + assert_eq!( + render_chat(ModelSyntax::Qwen4Exp, &msgs, "", ThinkMode::High).unwrap(), + concat!( + "<|im_start|>system\n", + "Reasoning effort is set to xhigh. Please think carefully through the task, ", + "validate key assumptions, consider plausible alternatives, and prioritize ", + "correctness, consistency, and clarity in the final answer.\n\n", + "Be precise.<|im_end|>\n", + "<|im_start|>user\nWeather?<|im_end|>\n", + "<|im_start|>assistant\n\n", + ) + .as_bytes() + ); + assert_eq!( + render_chat(ModelSyntax::Qwen4Exp, &msgs, "", ThinkMode::None).unwrap(), + concat!( + "<|im_start|>system\nBe precise.<|im_end|>\n", + "<|im_start|>user\nWeather?<|im_end|>\n", + "<|im_start|>assistant\n\n\n\n\n", + ) + .as_bytes() + ); +} + +#[test] +fn qwen4exp_renders_image_parts_in_source_order() { + let mut user = msg("user", "beforeafter"); + user.parts = vec![ + ChatPart::Text("before".into()), + ChatPart::Image(0), + ChatPart::Text("after".into()), + ]; + let prompt = render_chat(ModelSyntax::Qwen4Exp, &[user], "", ThinkMode::None).unwrap(); + assert!(String::from_utf8(prompt) + .unwrap() + .contains("before<|vision_start|><|image_pad|><|vision_end|>after")); +} + +#[test] +fn qwen4exp_tool_replay_uses_native_order_and_escape() { + let orders = [ToolSchemaOrder { + name: "weather".into(), + prop: vec!["city".into(), "days".into()], + ..Default::default() + }]; + let mut assistant = call("weather", r#"{"days":2,"city":"Seoul"}"#); + assistant.content = "Checking".into(); + let mut result = msg("tool", "sunnytoday"); + result.tool_call_id = "call1".into(); + let prompt = render_chat_choice( + ModelSyntax::Qwen4Exp, + &[msg("user", "Weather?"), assistant, result], + "", + &orders, + ThinkMode::None, + ToolChoice::Auto, + ) + .unwrap(); + let prompt = String::from_utf8(prompt).unwrap(); + assert!(prompt.contains( + "Checking\n\n\n\n\nSeoul\n\n\n2\n\n\n" + )); + assert!(prompt.contains( + "<|im_start|>user\n\nsunny</tool_response>today\n<|im_end|>" + )); +} + +/// The engine's SPACED_TOOLS_HEAD (tests/server_tests_render_v41.c, 4294136): +/// the official encoding.py TOOLS_TEMPLATE verbatim. Built from the crate's +/// tag constants, whose bytes are the engine's (od-verified); the think tags +/// use the crate's DSML_THINK_START/DSML_THINK_END spellings. +fn spaced_tools_head() -> String { + let mut s = String::new(); + s.push_str("## Tools\n\n"); + s.push_str("You have access to a set of tools to help answer the user's question. "); + s.push_str("You can invoke tools by writing a \""); + s.push_str(DSML_TOOL_CALLS_START_SPACED); + s.push_str("\" block like the following:\n\n"); + s.push_str(DSML_TOOL_CALLS_START_SPACED); + s.push_str("\n"); + s.push_str(DSML_INVOKE_START_SPACED); + s.push_str(" name=\"$TOOL_NAME\">\n"); + s.push_str(DSML_PARAM_START_SPACED); + s.push_str(" name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE"); + s.push_str(DSML_PARAM_END_SPACED); + s.push_str("\n...\n"); + s.push_str(DSML_INVOKE_END_SPACED); + s.push_str("\n"); + s.push_str(DSML_INVOKE_START_SPACED); + s.push_str(" name=\"$TOOL_NAME2\">\n...\n"); + s.push_str(DSML_INVOKE_END_SPACED); + s.push_str("\n"); + s.push_str(DSML_TOOL_CALLS_END_SPACED); + s.push_str("\n\n"); + s.push_str("String parameters should be specified as is and set `string=\"true\"`. "); + s.push_str("For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\n"); + s.push_str("If thinking_mode is enabled (triggered by "); + s.push_str(DSML_THINK_START); + s.push_str("), you MUST output your complete reasoning inside "); + s.push_str(DSML_THINK_START); + s.push_str("..."); + s.push_str(DSML_THINK_END); + s.push_str(" BEFORE any tool calls or final response.\n\n"); + s.push_str("Otherwise, output directly after "); + s.push_str(DSML_THINK_END); + s.push_str(" with tool calls or final response.\n\n"); + s.push_str("### Available Tool Schemas\n\n"); + s +} + +const SPACED_TOOLS_TAIL: &str = + "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n"; + +fn assert_spaced_eq(got: Vec, want: &str) { + if got == want.as_bytes() { + return; + } + let at = got + .iter() + .zip(want.as_bytes()) + .position(|(a, b)| a != b) + .unwrap_or(got.len().min(want.len())); + panic!( + "spaced render mismatch at byte {at}\n want: {:?}\n got: {:?}", + &want.as_bytes() + [at.min(want.len())..at.min(want.len()) + 120.min(want.len() - at.min(want.len()))], + &got[at.min(got.len())..at.min(got.len()) + 120.min(got.len() - at.min(got.len()))] + ); +} + +#[test] +fn dsml_v41_spaced_tools_match_official_encoding() { + // The engine's golden byte test (tests/server_tests_render_v41.c, 4294136): + // case1 is the official repo's encoding/tests/test_input_1.json, byte-identical + // to its own test_output_1.txt; case2 adds parallel calls, a zero-parameter + // call, json.dumps-shaped values, "<" and "&" inside a value, two adjacent + // tool results merged into one <|User|>, and a schema carrying a \u escape. A + // byte mismatch here is a bug in the spaced template. + let v41 = DsmlHead::V41 { + system_token: true, + effort: None, + }; + + // case1: the official get_weather flow, THINK_HIGH. + let c1 = [ + msg("system", "You are a helpful assistant."), + msg("user", "What's the weather like in Beijing?"), + ChatMsg { + role: "assistant".into(), + reasoning: "The user wants the weather in Beijing. I should call get_weather.".into(), + calls: vec![ToolCall { + id: "call1".into(), + name: "get_weather".into(), + arguments: r#"{"location": "Beijing", "unit": "celsius"}"#.into(), + }], + ..ChatMsg::default() + }, + msg( + "tool", + r#"{"temperature": 22, "condition": "sunny", "humidity": 45}"#, + ), + msg_reason( + "assistant", + "The weather in Beijing is currently sunny with a temperature of 22\u{b0}C and 45% humidity.", + "Got the weather data. Let me format a nice response.", + ), + ]; + let schemas1 = concat!( + r#"{"name":"get_weather","description":"Get the weather for a specific location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city name"},"unit":{"type":"string","enum":["celsius","fahrenheit"],"description":"Temperature unit"}},"required":["location"]}}"#, + "\n", + r#"{"name":"search","description":"Search the web for information","parameters":{"type":"object","properties":{"query":{"type":"string","description":"Search query"},"num_results":{"type":"integer","description":"Number of results to return"}},"required":["query"]}}"#, + ); + let mut want1 = String::new(); + want1.push_str(DSML_BOS); + want1.push_str(DSML_SYSTEM_TOKEN); + want1.push_str(V41_THINK_HIGH_PREFIX); + want1.push_str("You are a helpful assistant.\n\n"); + want1.push_str(&spaced_tools_head()); + want1.push_str("{\"name\": \"get_weather\", \"description\": \"Get the weather for a specific location\", \"parameters\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The city name\"}, \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"], \"description\": \"Temperature unit\"}}, \"required\": [\"location\"]}}\n"); + want1.push_str("{\"name\": \"search\", \"description\": \"Search the web for information\", \"parameters\": {\"type\": \"object\", \"properties\": {\"query\": {\"type\": \"string\", \"description\": \"Search query\"}, \"num_results\": {\"type\": \"integer\", \"description\": \"Number of results to return\"}}, \"required\": [\"query\"]}}"); + want1.push_str(SPACED_TOOLS_TAIL); + want1.push_str(DSML_USER); + want1.push_str("What's the weather like in Beijing?"); + want1.push_str(DSML_ASSISTANT); + want1.push_str(DSML_THINK_START); + want1.push_str("The user wants the weather in Beijing. I should call get_weather."); + want1.push_str(DSML_THINK_END); + want1.push_str("\n\n"); + want1.push_str(DSML_TOOL_CALLS_START_SPACED); + want1.push_str("\n"); + want1.push_str(DSML_INVOKE_START_SPACED); + want1.push_str(" name=\"get_weather\">\n"); + want1.push_str(DSML_PARAM_START_SPACED); + want1.push_str(" name=\"location\" string=\"true\">Beijing"); + want1.push_str(DSML_PARAM_END_SPACED); + want1.push_str("\n"); + want1.push_str(DSML_PARAM_START_SPACED); + want1.push_str(" name=\"unit\" string=\"true\">celsius"); + want1.push_str(DSML_PARAM_END_SPACED); + want1.push_str("\n"); + want1.push_str(DSML_INVOKE_END_SPACED); + want1.push_str("\n"); + want1.push_str(DSML_TOOL_CALLS_END_SPACED); + want1.push_str(DSML_EOS); + want1.push_str(DSML_USER); + want1.push_str( + "{\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}", + ); + want1.push_str(DSML_ASSISTANT); + want1.push_str(DSML_THINK_START); + want1.push_str("Got the weather data. Let me format a nice response."); + want1.push_str(DSML_THINK_END); + want1.push_str( + "The weather in Beijing is currently sunny with a temperature of 22\u{b0}C and 45% humidity.", + ); + want1.push_str(DSML_EOS); + assert_spaced_eq( + render_dsml_chat_choice_ex(&c1, schemas1, ThinkMode::High, ToolChoice::Auto, v41).unwrap(), + &want1, + ); + + // case2: parallel calls, a zero-parameter call, merged tool results, THINK_NONE. + let c2 = [ + msg("system", "S"), + msg("user", "U"), + ChatMsg { + role: "assistant".into(), + calls: vec![ + ToolCall { + id: "call1".into(), + name: "get_weather".into(), + arguments: r#"{"location":"a\n"); + want2.push_str(DSML_PARAM_START_SPACED); + want2.push_str(" name=\"location\" string=\"true\">a3"); + want2.push_str(DSML_PARAM_END_SPACED); + want2.push_str("\n"); + want2.push_str(DSML_PARAM_START_SPACED); + want2.push_str(" name=\"opts\" string=\"false\">{\"a\": [1, 2], \"b\": true}"); + want2.push_str(DSML_PARAM_END_SPACED); + want2.push_str("\n"); + want2.push_str(DSML_INVOKE_END_SPACED); + want2.push_str("\n"); + want2.push_str(DSML_INVOKE_START_SPACED); + want2.push_str(" name=\"now\">\n"); + want2.push_str("\n"); + want2.push_str(DSML_INVOKE_END_SPACED); + want2.push_str("\n"); + want2.push_str(DSML_TOOL_CALLS_END_SPACED); + want2.push_str(DSML_EOS); + want2.push_str(DSML_USER); + want2.push_str("R1"); + want2.push_str("\n\n"); + want2.push_str("R2"); + want2.push_str("\n\n"); + want2.push_str("U2"); + want2.push_str(DSML_ASSISTANT); + want2.push_str(DSML_THINK_END); + assert_spaced_eq( + render_dsml_chat_choice_ex(&c2, schemas2, ThinkMode::None, ToolChoice::Auto, v41).unwrap(), + &want2, + ); +} + +#[test] +fn dsml_v41_places_tool_result_images_inside_the_envelope() { + // A tool_result whose content carried an image reaches the renderer with + // the envelope and the placeholder already written into the content in the + // client's block order (parse.rs json_tool_result_content; the engine's + // text-level render, server_parse_tools.c:330). The user body must write + // that content -- running the parts loop instead would drop the envelope. + let v41 = DsmlHead::V41 { + system_token: true, + effort: None, + }; + let tool_result = ChatMsg { + role: "user".into(), + content: format!("shot:{DSML_IMAGE}done"), + parts: vec![ + ChatPart::Image(0), + ChatPart::ToolResult { + id: "toolu_02".into(), + content: "shot:done".into(), + }, + ], + tool_call_ids: vec!["toolu_02".into()], + ..ChatMsg::default() + }; + let out = + render_dsml_chat_choice_ex(&[tool_result], "", ThinkMode::None, ToolChoice::Auto, v41) + .unwrap(); + let text = String::from_utf8(out).unwrap(); + assert!(text.contains(&format!("shot:{DSML_IMAGE}done"))); + assert_eq!(text.matches(DSML_IMAGE).count(), 1); + + // A plain user image still renders from the parts, placeholder where the + // part sits (the content carries no placeholder for this shape). + let plain = ChatMsg { + role: "user".into(), + content: "beforeafter".into(), + parts: vec![ + ChatPart::Text("before".into()), + ChatPart::Image(0), + ChatPart::Text("after".into()), + ], + ..ChatMsg::default() + }; + let out = + render_dsml_chat_choice_ex(&[plain], "", ThinkMode::None, ToolChoice::Auto, v41).unwrap(); + let text = String::from_utf8(out).unwrap(); + assert!(text.contains(&format!("before{DSML_IMAGE}after"))); +} + +#[test] +fn dsml_v41_writes_a_tool_message_image_inside_its_envelope() { + // The Chat Completions MCP loop: the image arrives in a tool-role message + // whose content is a block array, so parse.rs writes the placeholder into + // that content in client block order and this render wraps the content in + // its envelope. The envelope text is the only place the + // V4.1 head can put the placeholder for a tool result (the engine's + // render, server_dsml_render.c:409-414), so the image must reach the + // prompt here or it is silently dropped. + let v41 = DsmlHead::V41 { + system_token: true, + effort: None, + }; + let tool = ChatMsg { + role: "tool".into(), + content: format!("shot:{DSML_IMAGE}done"), + parts: vec![ + ChatPart::Text("shot:".into()), + ChatPart::Image(0), + ChatPart::Text("done".into()), + ], + tool_call_id: "c1".into(), + ..ChatMsg::default() + }; + let out = + render_dsml_chat_choice_ex(&[tool], "", ThinkMode::None, ToolChoice::Auto, v41).unwrap(); + let text = String::from_utf8(out).unwrap(); + assert!(text.contains(&format!("shot:{DSML_IMAGE}done"))); + assert_eq!(text.matches(DSML_IMAGE).count(), 1); +} + +/// The reference's own test (ds4_server.c:20386 +/// `test_responses_visible_suffix_matches_client_replay`): a final answer's +/// reasoning stays out of the remembered visible prefix even when summaries +/// were requested, while a tool-call turn carries its summary before the think +/// close and the call block after it. +#[test] +fn responses_visible_suffix_matches_client_replay() { + use ds4_server::responses_visible_assistant_suffix; + + let none: Vec = Vec::new(); + let suffix = responses_visible_assistant_suffix( + ModelSyntax::DeepSeek, + true, + ThinkMode::High, + true, + b"5", + b"hidden summary", + &none, + &[], + ); + let text = String::from_utf8(suffix).unwrap(); + assert!(!text.contains("hidden summary"), "{text}"); + assert!(text.contains("5"), "{text}"); + assert!(text.ends_with(DSML_EOS), "{text}"); + + let calls = vec![ToolCall { + id: "call_live".into(), + name: "bash".into(), + arguments: r#"{"command":"pwd"}"#.into(), + }]; + let suffix = responses_visible_assistant_suffix( + ModelSyntax::DeepSeek, + true, + ThinkMode::High, + true, + b"", + b"tool summary", + &calls, + &[], + ); + let text = String::from_utf8(suffix).unwrap(); + assert!(text.contains("tool summary"), "{text}"); + assert!(text.contains(DSML_TOOL_CALLS_START_SPACED), "{text}"); + assert!(text.ends_with(DSML_EOS), "{text}"); +} diff --git a/crates/ds4-server/tests/retry.rs b/crates/ds4-server/tests/retry.rs new file mode 100644 index 0000000000000000000000000000000000000000..851e7bfdb46390daa95691a2559da0cfef9e2d97 --- /dev/null +++ b/crates/ds4-server/tests/retry.rs @@ -0,0 +1,254 @@ +//! C↔Rust corrective-retry tapes (suffix / repair / decide). No model. + +use ds4_server::{ + retry::build_invalid_tool_error_suffix, retry_dump_script, ChatFormat, ModelSyntax, ThinkMode, +}; +use std::path::PathBuf; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_RETRY_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/retry_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/retry_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_str(name: &str) -> String { + let out = Command::new(require_oracle()) + .arg(name) + .output() + .expect("run retry_c_oracle"); + assert!( + out.status.success(), + "oracle {name} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("oracle utf8") +} + +fn assert_script(name: &str) { + let rust = retry_dump_script(name); + let c = c_str(name); + if rust != c { + panic!("{name} mismatch\n--- rust ---\n{rust}\n--- c ---\n{c}"); + } +} + +#[test] +fn terminal_finish_keeps_cause() { + assert_eq!(ds4_server::retry::terminal_finish("stop"), "stop"); + assert_eq!( + ds4_server::retry::terminal_finish("tool_calls"), + "tool_calls" + ); + assert_eq!(ds4_server::retry::terminal_finish("length"), "length"); + assert_eq!(ds4_server::retry::terminal_finish("error"), "error"); + assert_eq!(ds4_server::retry::terminal_finish("other"), "stop"); +} + +#[test] +fn suffix_and_system_match_c() { + for name in [ + "dsml-think", + "dsml-nothink", + "solar-think", + "system-dsml", + "system-solar", + ] { + assert_script(name); + } + let dsml = retry_dump_script("dsml-think"); + assert!(dsml.starts_with("<|end▁of▁sentence|><|User|>")); + assert!(dsml.contains("Tool error: invalid DSML tool call: missing invoke name")); + assert!(dsml.contains("System prompt reminder:\n## Tools\nschema\n\nSystem rule")); + assert!(dsml.contains("<|Assistant|>")); + assert!(!dsml.contains("<|User|>Hi")); + + let solar = retry_dump_script("solar-think"); + assert!(!solar.contains("DSML")); + assert!(solar.contains("invalid Solar tool call")); + assert!(solar.contains("System prompt reminder:\n## System Prompt\n\nStay precise.")); +} + +#[test] +fn repair_and_decide_match_c() { + for name in [ + "repair-dsml", + "repair-dsml-param-open", + "repair-dsml-invoke-open", + "repair-dsml-spaced", + "repair-solar", + "repair-dsml-none", + "decide-unterminated-stop", + "decide-unterminated-length", + "decide-parse-retry", + "decide-parse-motif", + ] { + assert_script(name); + } + assert_eq!(retry_dump_script("repair-dsml-none"), "NONE"); + assert_eq!( + retry_dump_script("decide-unterminated-stop").trim(), + "retry-unterminated" + ); + assert_eq!( + retry_dump_script("decide-unterminated-length").trim(), + "none" + ); + assert_eq!(retry_dump_script("decide-parse-retry").trim(), "true"); + assert_eq!(retry_dump_script("decide-parse-motif").trim(), "false"); +} + +/// The reference's repair self-test, ported case by case +/// (ds4_server.c:19322 `test_dsml_repair_produces_parseable_calls` and the +/// balance cases around it). Only a missing wrapper is repairable: an +/// unfinished invocation can still be missing arguments and an unfinished value +/// can be a truncated command, so closing either "would invent an executable +/// action" (:6697-6700). A repair that closes them was the deviation G8 +/// recorded. +#[test] +fn dsml_repair_matches_the_reference_balance_contract() { + use ds4_server::retry::try_repair_dsml; + use ds4_server::tools::{ + DSML_INVOKE_END, DSML_INVOKE_END_SHORT, DSML_INVOKE_START, DSML_INVOKE_START_SHORT, + DSML_PARAM_END, DSML_PARAM_END_SHORT, DSML_PARAM_START, DSML_PARAM_START_SHORT, + DSML_TOOL_CALLS_END, DSML_TOOL_CALLS_END_SHORT, DSML_TOOL_CALLS_START, + DSML_TOOL_CALLS_START_SHORT, + }; + let repair = + |text: String| try_repair_dsml(text.as_bytes()).map(|v| String::from_utf8(v).unwrap()); + let parse = |text: &str| { + ds4_server::parse_generated_message( + ModelSyntax::DeepSeek, + text.as_bytes(), + false, + ChatFormat::DeepSeek, + &[], + ) + }; + + // The wrapper close alone is missing: repaired, and the call parses. + let full = format!( + "thinking done\n\n{DSML_TOOL_CALLS_START}\n{DSML_INVOKE_START} name=\"bash\">\n\ + {DSML_PARAM_START} name=\"command\" string=\"true\">ls -la{DSML_PARAM_END}\n\ + {DSML_INVOKE_END}\n" + ); + let repaired = repair(full.clone()).expect("a missing wrapper is the repairable case"); + assert_eq!(repaired, format!("{full}{DSML_TOOL_CALLS_END}")); + let got = parse(&repaired); + assert_eq!(got.calls.len(), 1); + assert_eq!(got.calls[0].name, "bash"); + assert_eq!(got.calls[0].arguments, r#"{"command": "ls -la"}"#); + + // An unfinished invocation: refused. + assert!(repair(format!( + "\n\n{DSML_TOOL_CALLS_START}\n{DSML_INVOKE_START} name=\"edit\">\n\ + {DSML_PARAM_START} name=\"path\" string=\"true\">/tmp/test.c{DSML_PARAM_END}\n" + )) + .is_none()); + + // An unfinished value: refused, even though the prefix looks plausible. + assert!(repair(format!( + "\n\n{DSML_TOOL_CALLS_START}\n{DSML_INVOKE_START} name=\"bash\">\n\ + {DSML_PARAM_START} name=\"command\" string=\"true\">echo hello" + )) + .is_none()); + + // The older short spelling repairs the same way. + let short = format!( + "\n\n{DSML_TOOL_CALLS_START_SHORT}\n{DSML_INVOKE_START_SHORT} name=\"write_file\">\n\ + {DSML_PARAM_START_SHORT} name=\"path\" string=\"true\">/tmp/out.txt{DSML_PARAM_END_SHORT}\n\ + {DSML_PARAM_START_SHORT} name=\"content\" string=\"true\">hello world{DSML_PARAM_END_SHORT}\n\ + {DSML_INVOKE_END_SHORT}\n" + ); + let repaired = repair(short.clone()).expect("the short spelling repairs"); + assert_eq!(repaired, format!("{short}{DSML_TOOL_CALLS_END_SHORT}")); + let got = parse(&repaired); + assert_eq!(got.calls.len(), 1); + assert_eq!(got.calls[0].name, "write_file"); + assert_eq!( + got.calls[0].arguments, + r#"{"path": "/tmp/out.txt", "content": "hello world"}"# + ); + + // Plain XML: the same again. + let plain = "\n\n\n\n\ + pwd\n\n" + .to_string(); + let repaired = repair(plain).expect("the plain spelling repairs"); + let got = parse(&repaired); + assert_eq!(got.calls.len(), 1); + assert_eq!(got.calls[0].name, "execute_command"); + assert_eq!(got.calls[0].arguments, r#"{"command": "pwd"}"#); + + // Balanced text, no DSML, and a balanced block without an invocation: no + // repair in any of them. + assert!(repair(full.clone() + DSML_TOOL_CALLS_END).is_none()); + assert!(repair("just plain text, no tools".into()).is_none()); + let balanced_no_invoke = format!( + "Let me analyze this.\n\n{DSML_TOOL_CALLS_START}The write tool truncates this too.\ + {DSML_TOOL_CALLS_END}" + ); + assert!(repair(balanced_no_invoke).is_none()); + let balanced_short_no_invoke = format!( + "thinking...\n\n{DSML_TOOL_CALLS_START_SHORT}some content here{DSML_TOOL_CALLS_END_SHORT}" + ); + assert!(repair(balanced_short_no_invoke).is_none()); + let balanced_xml_no_invoke = + "Let me think.\n\nI need a tool but I don't know which." + .to_string(); + assert!(repair(balanced_xml_no_invoke).is_none()); + + // DSML quoted inside thinking is not executable, so the last + // scopes the scan and nothing follows it to repair. + let thinking_quote = format!( + "The protocol uses {DSML_TOOL_CALLS_START}some text{DSML_TOOL_CALLS_END}, \ + but this is only a quote.\nFinal answer." + ); + assert!(repair(thinking_quote).is_none()); + + // Extra closing tags are not a truncation pattern. + let orphan = + format!("done\n\n{DSML_TOOL_CALLS_START}{DSML_TOOL_CALLS_END}{DSML_TOOL_CALLS_END}"); + assert!(repair(orphan).is_none()); + + // Quoted DSML before the last , real DSML after it: repaired. + let after_think = format!( + "{DSML_TOOL_CALLS_START}quoted, not executable{DSML_TOOL_CALLS_END}\n\n\ + {DSML_TOOL_CALLS_START}\n{DSML_INVOKE_START} name=\"bash\">\n\ + {DSML_PARAM_START} name=\"command\" string=\"true\">date{DSML_PARAM_END}\n{DSML_INVOKE_END}\n" + ); + let repaired = repair(after_think).expect("real DSML after thinking repairs"); + let got = parse(&repaired); + assert_eq!(got.calls.len(), 1); + assert_eq!(got.calls[0].name, "bash"); + assert_eq!(got.calls[0].arguments, r#"{"command": "date"}"#); +} + +#[test] +fn qwen_invalid_tool_retry_uses_native_protocol() { + let suffix = build_invalid_tool_error_suffix( + ChatFormat::Qwen4Exp, + ThinkMode::None, + false, + b"<|im_start|>system\nStay precise.<|im_end|>\n", + "missing function", + ); + let suffix = String::from_utf8(suffix).unwrap(); + assert!(suffix.starts_with( + "<|im_end|>\n<|im_start|>user\n\nTool error: invalid Qwen tool call: missing function" + )); + assert!(suffix.contains("System prompt reminder:\nStay precise.")); + assert!(suffix + .ends_with("\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\n")); +} diff --git a/crates/ds4-server/tests/route_table.rs b/crates/ds4-server/tests/route_table.rs new file mode 100644 index 0000000000000000000000000000000000000000..6f512e9eac389f92f028c65e5a8ed95e17d79b0e --- /dev/null +++ b/crates/ds4-server/tests/route_table.rs @@ -0,0 +1,769 @@ +//! C reason table + C↔Rust `route_decide` / `compute_needs` bytes. + +use ds4_server::{ + compute_needs, route_decide, Api, NeedInput, ReqKind, RouteEnv, WireSurface, LANE_CONTINUOUS, + LANE_NONE, LANE_SERIAL, LANE_STATIC, NEED_BANK_FRONTIER, NEED_CONTINUATION_PUBLISH, + NEED_CORRECTIVE_RECOVERY, NEED_DURABLE_RESPONSE, NEED_IMAGE, NEED_LIVE_FRONTIER, + NEED_PER_ROW_SAMPLING, NEED_PREFILL_ONLY, NEED_STOP_SCAN, NEED_STREAMING, NEED_THINKING, + NEED_TOKEN_IDS, NEED_TOOL_SCAN, REASON_COALESCE_OFF, REASON_CONT, REASON_CONT_BANK, + REASON_CONT_UNAVAILABLE, REASON_NEED_CONTINUATION_PUBLISH, REASON_NEED_DURABLE, + REASON_NEED_LIVE_FRONTIER, REASON_NEED_PREFILL_ONLY, REASON_STATIC_NO_CONT, + REASON_STATIC_PROMPT_BOUNDS, REASON_SURFACE, REASON_TOKEN_IDS_PROJECTION, + REASON_TOOLS_COMPLETION, +}; +use std::path::PathBuf; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_ROUTE_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/route_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/route_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_decide(needs: u32, surf: WireSurface, env: &RouteEnv) -> (u8, u8) { + let out = Command::new(require_oracle()) + .args([ + "decide", + &needs.to_string(), + &(surf as i32).to_string(), + &(env.coalesce as i32).to_string(), + &(env.have_cont as i32).to_string(), + &(env.cont_anthropic as i32).to_string(), + &(env.cont_responses as i32).to_string(), + &(env.cont_tools_anthropic as i32).to_string(), + &(env.cont_tools_responses as i32).to_string(), + &env.seq_cap.to_string(), + &env.prompt_len.to_string(), + ]) + .output() + .expect("run route_c_oracle"); + assert!( + out.status.success(), + "oracle failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let s = String::from_utf8_lossy(&out.stdout); + let mut lane = 0u8; + let mut reason = 0u8; + for part in s.split_whitespace() { + if let Some(v) = part.strip_prefix("lane=") { + lane = v.parse().unwrap(); + } + if let Some(v) = part.strip_prefix("reason=") { + reason = v.parse().unwrap(); + } + } + (lane, reason) +} + +fn assert_row(needs: u32, surf: WireSurface, env: &RouteEnv, lane: u8, reason: u8) { + let d = route_decide(needs, surf, env); + assert_eq!( + (d.lane, d.reason), + (lane, reason), + "rust needs={needs:#x} surf={surf:?} env={env:?}" + ); + let c = c_decide(needs, surf, env); + assert_eq!( + c, + (lane, reason), + "c oracle needs={needs:#x} surf={surf:?} env={env:?}" + ); +} + +fn on() -> RouteEnv { + RouteEnv { + coalesce: true, + have_cont: true, + cont_anthropic: true, + cont_responses: true, + cont_tools_anthropic: true, + cont_tools_responses: true, + seq_cap: 4096, + prompt_len: 16, + } +} + +fn no_ctx() -> RouteEnv { + let mut e = on(); + e.have_cont = false; + e.seq_cap = 0; + e +} + +fn off() -> RouteEnv { + let mut e = on(); + e.coalesce = false; + e +} + +fn big() -> RouteEnv { + let mut e = on(); + e.prompt_len = 4097; + e +} + +fn empty() -> RouteEnv { + let mut e = on(); + e.prompt_len = 0; + e +} + +#[test] +fn reason_table_matches_c() { + let on = on(); + let no_ctx = no_ctx(); + let off = off(); + let big = big(); + let empty = empty(); + let mut anth_off = on; + anth_off.cont_anthropic = false; + let mut resp_off = on; + resp_off.cont_responses = false; + + assert_row( + 0, + WireSurface::OpenaiChat, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + NEED_STREAMING, + WireSurface::OpenaiChat, + &no_ctx, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + assert_row( + NEED_IMAGE, + WireSurface::OpenaiChat, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + NEED_IMAGE, + WireSurface::OpenaiChat, + &no_ctx, + LANE_NONE, + REASON_CONT_UNAVAILABLE, + ); + assert_row( + 0, + WireSurface::OpenaiChat, + &no_ctx, + LANE_STATIC, + REASON_STATIC_NO_CONT, + ); + assert_row( + 0, + WireSurface::OpenaiChat, + &big, + LANE_STATIC, + REASON_STATIC_PROMPT_BOUNDS, + ); + assert_row( + 0, + WireSurface::OpenaiChat, + &empty, + LANE_STATIC, + REASON_STATIC_PROMPT_BOUNDS, + ); + assert_row( + 0, + WireSurface::OpenaiChat, + &off, + LANE_SERIAL, + REASON_COALESCE_OFF, + ); + + assert_row( + NEED_TOKEN_IDS | NEED_STREAMING, + WireSurface::OpenaiChat, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + NEED_TOKEN_IDS, + WireSurface::OpenaiChat, + &on, + LANE_SERIAL, + REASON_TOKEN_IDS_PROJECTION, + ); + assert_row( + NEED_TOKEN_IDS | NEED_STREAMING, + WireSurface::OpenaiCompletion, + &on, + LANE_SERIAL, + REASON_TOKEN_IDS_PROJECTION, + ); + assert_row( + NEED_TOOL_SCAN, + WireSurface::OpenaiChat, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + NEED_TOOL_SCAN, + WireSurface::OpenaiCompletion, + &on, + LANE_SERIAL, + REASON_TOOLS_COMPLETION, + ); + + assert_row(0, WireSurface::Anthropic, &on, LANE_CONTINUOUS, REASON_CONT); + assert_row( + NEED_THINKING | NEED_STOP_SCAN | NEED_PER_ROW_SAMPLING, + WireSurface::Anthropic, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + 0, + WireSurface::Anthropic, + &anth_off, + LANE_SERIAL, + REASON_SURFACE, + ); + assert_row(0, WireSurface::Responses, &on, LANE_CONTINUOUS, REASON_CONT); + assert_row( + NEED_THINKING | NEED_PER_ROW_SAMPLING, + WireSurface::Responses, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + 0, + WireSurface::Responses, + &resp_off, + LANE_SERIAL, + REASON_SURFACE, + ); + assert_row( + 0, + WireSurface::Anthropic, + &resp_off, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + 0, + WireSurface::Responses, + &anth_off, + LANE_CONTINUOUS, + REASON_CONT, + ); + + assert_row( + NEED_STREAMING, + WireSurface::Anthropic, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + NEED_STREAMING | NEED_THINKING | NEED_STOP_SCAN, + WireSurface::Anthropic, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + NEED_STREAMING, + WireSurface::Anthropic, + &anth_off, + LANE_SERIAL, + REASON_SURFACE, + ); + assert_row( + NEED_STREAMING, + WireSurface::Anthropic, + &no_ctx, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + assert_row( + NEED_STREAMING, + WireSurface::Anthropic, + &big, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + assert_row( + NEED_STREAMING, + WireSurface::Responses, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + NEED_STREAMING | NEED_THINKING, + WireSurface::Responses, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + NEED_STREAMING, + WireSurface::Responses, + &resp_off, + LANE_SERIAL, + REASON_SURFACE, + ); + assert_row( + NEED_STREAMING, + WireSurface::Responses, + &no_ctx, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + assert_row( + NEED_STREAMING, + WireSurface::Responses, + &big, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + + assert_row( + 0, + WireSurface::Anthropic, + &big, + LANE_STATIC, + REASON_STATIC_PROMPT_BOUNDS, + ); + assert_row( + 0, + WireSurface::Anthropic, + &no_ctx, + LANE_STATIC, + REASON_STATIC_NO_CONT, + ); + assert_row( + 0, + WireSurface::Responses, + &big, + LANE_STATIC, + REASON_STATIC_PROMPT_BOUNDS, + ); + assert_row( + NEED_STOP_SCAN, + WireSurface::Anthropic, + &no_ctx, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + assert_row( + NEED_THINKING, + WireSurface::Responses, + &big, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + let mut anth_off_noctx = no_ctx; + anth_off_noctx.cont_anthropic = false; + assert_row( + 0, + WireSurface::Anthropic, + &anth_off_noctx, + LANE_SERIAL, + REASON_SURFACE, + ); + let mut resp_off_noctx = no_ctx; + resp_off_noctx.cont_responses = false; + assert_row( + 0, + WireSurface::Responses, + &resp_off_noctx, + LANE_SERIAL, + REASON_SURFACE, + ); + + let stream_tools = NEED_STREAMING | NEED_TOOL_SCAN | NEED_CONTINUATION_PUBLISH; + assert_row( + stream_tools, + WireSurface::Anthropic, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + stream_tools | NEED_THINKING | NEED_PER_ROW_SAMPLING | NEED_STOP_SCAN, + WireSurface::Anthropic, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + assert_row( + stream_tools, + WireSurface::Responses, + &on, + LANE_CONTINUOUS, + REASON_CONT, + ); + let mut tools_off_a = on; + tools_off_a.cont_tools_anthropic = false; + assert_row( + stream_tools, + WireSurface::Anthropic, + &tools_off_a, + LANE_SERIAL, + REASON_NEED_CONTINUATION_PUBLISH, + ); + assert_row( + stream_tools, + WireSurface::Responses, + &tools_off_a, + LANE_CONTINUOUS, + REASON_CONT, + ); + let mut tools_off_r = on; + tools_off_r.cont_tools_responses = false; + assert_row( + stream_tools, + WireSurface::Responses, + &tools_off_r, + LANE_SERIAL, + REASON_NEED_CONTINUATION_PUBLISH, + ); + assert_row( + stream_tools, + WireSurface::Anthropic, + &anth_off, + LANE_SERIAL, + REASON_NEED_CONTINUATION_PUBLISH, + ); + assert_row( + stream_tools, + WireSurface::Anthropic, + &no_ctx, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + assert_row( + stream_tools, + WireSurface::Anthropic, + &big, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + assert_row( + stream_tools, + WireSurface::Anthropic, + &off, + LANE_SERIAL, + REASON_COALESCE_OFF, + ); + + assert_row( + NEED_BANK_FRONTIER, + WireSurface::Anthropic, + &on, + LANE_CONTINUOUS, + REASON_CONT_BANK, + ); + assert_row( + NEED_BANK_FRONTIER | stream_tools, + WireSurface::Responses, + &on, + LANE_CONTINUOUS, + REASON_CONT_BANK, + ); + assert_row( + NEED_BANK_FRONTIER | NEED_TOOL_SCAN | NEED_CONTINUATION_PUBLISH | NEED_CORRECTIVE_RECOVERY, + WireSurface::Anthropic, + &on, + LANE_SERIAL, + REASON_NEED_CONTINUATION_PUBLISH, + ); + assert_row( + NEED_BANK_FRONTIER, + WireSurface::Anthropic, + &tools_off_a, + LANE_SERIAL, + REASON_NEED_LIVE_FRONTIER, + ); + assert_row( + NEED_BANK_FRONTIER, + WireSurface::Anthropic, + &anth_off, + LANE_SERIAL, + REASON_NEED_LIVE_FRONTIER, + ); + assert_row( + NEED_BANK_FRONTIER, + WireSurface::Anthropic, + &no_ctx, + LANE_SERIAL, + REASON_CONT_UNAVAILABLE, + ); + assert_row( + NEED_BANK_FRONTIER, + WireSurface::OpenaiChat, + &on, + LANE_SERIAL, + REASON_NEED_LIVE_FRONTIER, + ); + + assert_row( + NEED_TOOL_SCAN | NEED_CONTINUATION_PUBLISH | NEED_CORRECTIVE_RECOVERY, + WireSurface::Anthropic, + &on, + LANE_SERIAL, + REASON_NEED_CONTINUATION_PUBLISH, + ); + assert_row( + NEED_LIVE_FRONTIER | NEED_TOOL_SCAN, + WireSurface::Responses, + &on, + LANE_SERIAL, + REASON_NEED_LIVE_FRONTIER, + ); + assert_row( + NEED_PREFILL_ONLY, + WireSurface::Anthropic, + &on, + LANE_SERIAL, + REASON_NEED_PREFILL_ONLY, + ); + assert_row( + NEED_DURABLE_RESPONSE, + WireSurface::Responses, + &on, + LANE_NONE, + REASON_NEED_DURABLE, + ); +} + +fn c_needs(r: &NeedInput) -> u32 { + let out = Command::new(require_oracle()) + .args([ + "needs", + &(r.api as i32).to_string(), + &(r.stream as i32).to_string(), + &r.temperature.to_string(), + &(r.think as i32).to_string(), + &r.stop_count.to_string(), + &(r.has_tools as i32).to_string(), + &(r.return_token_ids as i32).to_string(), + &(r.has_images as i32).to_string(), + &(r.responses_requires_live_tool_state as i32).to_string(), + &(r.responses_requires_live_reasoning as i32).to_string(), + &(r.anthropic_requires_live_tool_state as i32).to_string(), + &(r.live_state_bank_owned as i32).to_string(), + &(r.max_tokens_set as i32).to_string(), + &r.max_tokens.to_string(), + ]) + .output() + .expect("run route_c_oracle needs"); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().parse().unwrap() +} + +fn blank() -> NeedInput { + NeedInput { + api: Api::Openai, + kind: ReqKind::Chat, + stream: false, + temperature: 0.0, + think: false, + stop_count: 0, + has_tools: false, + return_token_ids: false, + responses_requires_live_tool_state: false, + responses_requires_live_reasoning: false, + anthropic_requires_live_tool_state: false, + live_state_bank_owned: false, + max_tokens_set: false, + max_tokens: 128, + has_images: false, + } +} + +#[test] +fn compute_needs_matches_c() { + let cases = [ + blank(), + NeedInput { + stream: true, + temperature: 0.7, + think: true, + stop_count: 1, + has_tools: true, + ..blank() + }, + NeedInput { + api: Api::Anthropic, + has_tools: true, + stream: false, + ..blank() + }, + NeedInput { + api: Api::Anthropic, + has_tools: true, + stream: true, + ..blank() + }, + NeedInput { + api: Api::Anthropic, + max_tokens_set: true, + max_tokens: 0, + ..blank() + }, + NeedInput { + api: Api::Responses, + max_tokens_set: true, + max_tokens: 0, + ..blank() + }, + NeedInput { + api: Api::Anthropic, + anthropic_requires_live_tool_state: true, + live_state_bank_owned: true, + ..blank() + }, + NeedInput { + api: Api::Responses, + responses_requires_live_tool_state: true, + live_state_bank_owned: true, + ..blank() + }, + NeedInput { + api: Api::Responses, + responses_requires_live_reasoning: true, + live_state_bank_owned: true, + ..blank() + }, + NeedInput { + has_images: true, + ..blank() + }, + ]; + for r in cases { + let rust = compute_needs(&r); + let c = c_needs(&r); + assert_eq!(rust, c, "needs mismatch rust={rust:#x} c={c:#x}"); + } + let anth_tools = NeedInput { + api: Api::Anthropic, + has_tools: true, + stream: false, + ..blank() + }; + assert_eq!( + compute_needs(&anth_tools), + NEED_TOOL_SCAN | NEED_CONTINUATION_PUBLISH | NEED_CORRECTIVE_RECOVERY + ); + let anth_zero = NeedInput { + api: Api::Anthropic, + max_tokens_set: true, + max_tokens: 0, + ..blank() + }; + assert_eq!(compute_needs(&anth_zero), NEED_PREFILL_ONLY); + let resp_zero = NeedInput { + api: Api::Responses, + max_tokens_set: true, + max_tokens: 0, + ..blank() + }; + assert_eq!(compute_needs(&resp_zero), 0); + let bank = NeedInput { + api: Api::Anthropic, + anthropic_requires_live_tool_state: true, + live_state_bank_owned: true, + ..blank() + }; + assert_eq!(compute_needs(&bank), NEED_BANK_FRONTIER); + let reason = NeedInput { + api: Api::Responses, + responses_requires_live_reasoning: true, + live_state_bank_owned: true, + ..blank() + }; + assert_eq!(compute_needs(&reason), NEED_LIVE_FRONTIER); +} + +#[test] +fn decode_budget_three_states_match_c() { + fn c(set: bool, tokens: i32, def: i32) -> i32 { + let out = Command::new(require_oracle()) + .args([ + "budget", + &(set as i32).to_string(), + &tokens.to_string(), + &def.to_string(), + ]) + .output() + .unwrap(); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).trim().parse().unwrap() + } + let def = 393216; + assert_eq!(ds4_server::decode_budget(false, def, def), def); + assert_eq!(c(false, def, def), def); + assert_eq!(ds4_server::decode_budget(true, 4096, def), 4096); + assert_eq!(c(true, 4096, def), 4096); + assert_eq!(ds4_server::decode_budget(true, 0, def), 0); + assert_eq!(c(true, 0, def), 0); + assert_eq!(ds4_server::decode_budget(true, -5, def), 0); + assert_eq!(c(true, -5, def), 0); +} + +#[test] +fn reasoning_effort_names_match_c() { + fn c(name: &str) -> String { + let out = Command::new(require_oracle()) + .args(["effort", name]) + .output() + .unwrap(); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout).into_owned() + } + let rows = [ + ("max", Some(ds4_server::ThinkMode::Max)), + ("high", Some(ds4_server::ThinkMode::High)), + ("xhigh", Some(ds4_server::ThinkMode::High)), + ("low", Some(ds4_server::ThinkMode::Low)), + ("medium", Some(ds4_server::ThinkMode::Low)), + ("minimal", Some(ds4_server::ThinkMode::Low)), + ("none", Some(ds4_server::ThinkMode::None)), + ("off", Some(ds4_server::ThinkMode::None)), + ("banana", None), + ]; + for (name, want) in rows { + assert_eq!(ds4_server::parse_reasoning_effort_name(name), want); + match want { + Some(m) => assert_eq!(c(name), (m as i32).to_string()), + None => assert_eq!(c(name), "ERROR"), + } + } + assert_eq!( + ds4_server::think_mode_from_enabled(false, ds4_server::ThinkMode::High), + ds4_server::ThinkMode::None + ); + assert_eq!( + ds4_server::think_mode_from_enabled(true, ds4_server::ThinkMode::High), + ds4_server::ThinkMode::High + ); +} diff --git a/crates/ds4-server/tests/step37_output.rs b/crates/ds4-server/tests/step37_output.rs new file mode 100644 index 0000000000000000000000000000000000000000..ac802a4804168b273da6bb5fa06560766be00f52 --- /dev/null +++ b/crates/ds4-server/tests/step37_output.rs @@ -0,0 +1,40 @@ +//! Step's official XML output envelope is independent of its Jinja input. +use ds4_server::generate::chat_format_for_syntax; +use ds4_server::tools::parse_generated_for_model_id; +use ds4_server::{syntax_for_model_id, SemAccum, ToolSchemaOrder}; + +const STEP: i32 = ds4_core::Variant::Step37Flash as i32; +const PROMPT: &[u8] = b"<|im_start|>assistant\n\n"; +const OUTPUT: &[u8] = b"Need weather.\n\n\n\n\nSeoul\n\n\n2\n\n\n"; + +#[test] +fn step_tool_output() { + let orders = [ToolSchemaOrder { + name: "weather".into(), + prop: vec!["city".into(), "days".into()], + prop_type: vec!["string".into(), "integer".into()], + ..Default::default() + }]; + let parsed = parse_generated_for_model_id(STEP, OUTPUT, true, &orders); + assert!(parsed.ok); + assert_eq!(parsed.calls.len(), 1); + assert_eq!(parsed.calls[0].name, "weather"); + assert_eq!(parsed.calls[0].arguments, r#"{"city": "Seoul", "days": 2}"#); + assert!(parsed.content.is_empty()); + assert_eq!(parsed.reasoning.trim_ascii(), b"Need weather."); +} + +#[test] +fn step_tool_stream_splits() { + let format = chat_format_for_syntax(syntax_for_model_id(STEP)); + for width in 1..=OUTPUT.len() { + let mut acc = SemAccum::init(true, true, true, format, PROMPT); + assert!(acc.thinking_inside()); + for chunk in OUTPUT.chunks(width) { + acc.feed(chunk, &[]); + } + assert!(!acc.thinking_inside(), "width {width}"); + assert!(acc.saw_tool_start && acc.saw_tool_end, "width {width}"); + assert_eq!(acc.verdict, None); + } +} diff --git a/crates/ds4-server/tests/stray_image_token.rs b/crates/ds4-server/tests/stray_image_token.rs new file mode 100644 index 0000000000000000000000000000000000000000..589f6679efe25622af052ae1a14dc3839f1788fa --- /dev/null +++ b/crates/ds4-server/tests/stray_image_token.rs @@ -0,0 +1,126 @@ +//! A user-supplied image placeholder in a textual field is refused. +//! +//! The carriage's authority (the oracle, encoding.py:355-367) refuses the +//! literal `<|deepseek_image|>` in `content` and in `reasoning_content`, and +//! refuses it inside a text content block (encoding.py:346-352, reachable +//! through nested tool-result content too). An image is an image content +//! block; the literal in text is not a way to place one. +//! +//! Without the refusal the literal tokenizes to the placeholder id +//! (crates/ds4-core/src/tok.rs:2603) and a request that carries no image never +//! reaches the V4.1 expansion that counts placeholders +//! (crates/ds4-server/src/generate.rs:3107), so the model would be handed a +//! lone image token at a text position. + +use ds4_server::render::DSML_IMAGE; +use ds4_server::{parse_anthropic_request, parse_chat_request, parse_responses_request, ParseEnv}; + +fn env() -> ParseEnv { + ParseEnv::default() +} + +fn refused(err: String, field: &str) -> bool { + err.contains(DSML_IMAGE) && err.contains(field) +} + +#[test] +fn openai_string_content_is_refused() { + let body = + format!(r#"{{"messages":[{{"role":"user","content":"look at {DSML_IMAGE} this"}}]}}"#); + let err = parse_chat_request(&env(), &body).unwrap_err(); + assert!(refused(err.clone(), "content"), "unexpected error: {err}"); +} + +#[test] +fn openai_text_block_is_refused() { + let body = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"text","text":"look at {DSML_IMAGE} this"}}]}}]}}"# + ); + let err = parse_chat_request(&env(), &body).unwrap_err(); + assert!(refused(err.clone(), "content"), "unexpected error: {err}"); +} + +#[test] +fn openai_reasoning_content_is_refused() { + let body = format!( + r#"{{"messages":[{{"role":"assistant","reasoning_content":"thought {DSML_IMAGE} more","content":"hi"}}]}}"# + ); + let err = parse_chat_request(&env(), &body).unwrap_err(); + assert!( + refused(err.clone(), "reasoning_content"), + "unexpected error: {err}" + ); +} + +#[test] +fn openai_tool_result_text_is_refused() { + let body = format!( + r#"{{"messages":[{{"role":"assistant","content":"","tool_calls":[{{"id":"c1","type":"function","function":{{"name":"shot","arguments":"{{}}"}}}}]}},{{"role":"tool","tool_call_id":"c1","content":"shot {DSML_IMAGE} taken"}}]}}"# + ); + let err = parse_chat_request(&env(), &body).unwrap_err(); + assert!(refused(err.clone(), "content"), "unexpected error: {err}"); +} + +#[test] +fn anthropic_text_block_is_refused() { + let body = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"text","text":"look at {DSML_IMAGE} this"}}]}}]}}"# + ); + let err = parse_anthropic_request(&env(), &body).unwrap_err(); + assert!(refused(err.clone(), "content"), "unexpected error: {err}"); +} + +#[test] +fn anthropic_thinking_is_refused() { + let body = format!( + r#"{{"messages":[{{"role":"assistant","content":[{{"type":"thinking","thinking":"thought {DSML_IMAGE} more"}}]}}]}}"# + ); + let err = parse_anthropic_request(&env(), &body).unwrap_err(); + assert!( + refused(err.clone(), "reasoning_content"), + "unexpected error: {err}" + ); +} + +#[test] +fn anthropic_tool_result_text_is_refused() { + let body = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"tool_result","tool_use_id":"t1","content":"shot {DSML_IMAGE} taken"}}]}}]}}"# + ); + let err = parse_anthropic_request(&env(), &body).unwrap_err(); + assert!(refused(err.clone(), "content"), "unexpected error: {err}"); +} + +#[test] +fn responses_input_text_is_refused() { + let body = format!( + r#"{{"input":[{{"type":"message","role":"user","content":[{{"type":"input_text","text":"look at {DSML_IMAGE} this"}}]}}]}}"# + ); + let err = parse_responses_request(&env(), &body).unwrap_err(); + assert!(refused(err.clone(), "content"), "unexpected error: {err}"); +} + +#[test] +fn the_servers_own_placeholders_still_parse() { + // The server writes the placeholder itself for a tool message whose image + // is the pixels (parse.rs write_tool_image_placeholders): that insertion + // happens after the client's text has been validated and must not be + // mistaken for one. + const PNG: &str = concat!( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP8", + "z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC" + ); + let body = format!( + r#"{{"messages":[{{"role":"assistant","content":"","tool_calls":[{{"id":"c1","type":"function","function":{{"name":"shot","arguments":"{{}}"}}}}]}},{{"role":"tool","tool_call_id":"c1","content":[{{"type":"text","text":"shot:"}},{{"type":"image_url","image_url":{{"url":"data:image/png;base64,{PNG}"}}}},{{"type":"text","text":"done"}}]}}]}}"# + ); + let parsed = parse_chat_request(&env(), &body).unwrap(); + assert_eq!(parsed.messages[1].content, format!("shot:{DSML_IMAGE}done")); + + // A plain text turn is untouched, and a user image block still admits. + let plain = r#"{"messages":[{"role":"user","content":"say hello"}]}"#; + assert!(parse_chat_request(&env(), plain).is_ok()); + let with_image = format!( + r#"{{"messages":[{{"role":"user","content":[{{"type":"text","text":"what colour"}},{{"type":"image_url","image_url":{{"url":"data:image/png;base64,{PNG}"}}}}]}}]}}"# + ); + assert!(parse_chat_request(&env(), &with_image).is_ok()); +} diff --git a/crates/ds4-server/tests/stream.rs b/crates/ds4-server/tests/stream.rs new file mode 100644 index 0000000000000000000000000000000000000000..fbc13a703b1e50b075438759b3111f0480b85395 --- /dev/null +++ b/crates/ds4-server/tests/stream.rs @@ -0,0 +1,863 @@ +//! C↔Rust four-surface stream projectors (tape oracle, no model). + +use ds4_server::{ + anthropic_final_response, anthropic_sse_start_live, anthropic_sse_stream_update, + final_response, openai_sse_finish_live, openai_sse_stream_update, openai_stream_start, + project_anthropic_thinking, project_openai_chat_thinking, project_openai_chat_utf8, + project_openai_completion, project_responses_thinking, responses_final_response, think_end, + utf8_stream_safe_len, ChatFormat, ModelSyntax, ReqKind, StreamReq, ThinkBlock, ThinkMode, + ToolCall, Writer, CREATED_TEST, TEST_MSG_ID, TEST_RESP_ID, TEST_RS_ID, +}; + +use std::path::PathBuf; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_STREAM_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/stream_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/stream_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_bytes(args: &[&str]) -> Vec { + let out = Command::new(require_oracle()) + .args(args) + .output() + .expect("run stream_c_oracle"); + assert!( + out.status.success(), + "oracle {:?} failed: {}", + args, + String::from_utf8_lossy(&out.stderr) + ); + out.stdout +} + +fn c_str(args: &[&str]) -> String { + String::from_utf8(c_bytes(args)).expect("oracle utf8") +} + +fn assert_bytes_eq(label: &str, rust: &[u8], c: &[u8]) { + if rust != c { + let r = String::from_utf8_lossy(rust); + let k = String::from_utf8_lossy(c); + panic!("{label} mismatch\n--- rust ---\n{r}\n--- c ---\n{k}"); + } +} + +fn count_substr(hay: &str, needle: &str) -> usize { + hay.matches(needle).count() +} + +fn sse_assert_every_data_object(out: &str, must_contain: &str, must_not: Option<&str>) { + let mut saw_done = false; + let mut records = 0; + let mut rest = out; + while let Some(i) = rest.find("data: ") { + rest = &rest[i + 6..]; + let end = rest.find("\n\n").unwrap_or(rest.len()); + let rec = &rest[..end]; + if rec == "[DONE]" { + saw_done = true; + assert!(!rest[end..].contains("data: ")); + } else { + records += 1; + assert!( + rec.contains(must_contain), + "missing {must_contain} in {rec}" + ); + if let Some(bad) = must_not { + assert!(!rec.contains(bad), "found {bad} in {rec}"); + } + } + rest = if end < rest.len() { + &rest[end + 2..] + } else { + "" + }; + } + assert!(records > 0); + assert!(saw_done); +} + +fn sse_validate_anthropic(out: &str) { + let start = out.find("event: message_start").expect("message_start"); + let stop = out.find("event: message_stop").expect("message_stop"); + assert!(start < stop); + assert_eq!( + count_substr(out, "event: content_block_start"), + count_substr(out, "event: content_block_stop") + ); + let mdelta = out.find("event: message_delta").expect("message_delta"); + assert!(mdelta < stop); + let mut rest = out; + while let Some(i) = rest.find("event: ") { + rest = &rest[i + 7..]; + let n = rest.find(['\r', '\n']).unwrap_or(rest.len()); + let name = &rest[..n]; + let data = rest.find("data: ").expect("data after event"); + let rec_end = rest[data..] + .find("\n\n") + .map(|e| data + e) + .unwrap_or(rest.len()); + let rec = &rest[data..rec_end]; + let typekey = format!("\"type\":\"{name}\""); + assert!(rec.contains(&typekey), "missing {typekey} in {rec}"); + rest = if rec_end < rest.len() { + &rest[rec_end + 2..] + } else { + "" + }; + } +} + +fn sse_validate_responses(out: &str) { + let created = out.find("\"type\":\"response.created\"").expect("created"); + let completed = out + .find("\"type\":\"response.completed\"") + .expect("completed"); + assert!(created < completed); + assert_eq!( + count_substr(out, "\"type\":\"response.output_item.added\""), + count_substr(out, "\"type\":\"response.output_item.done\"") + ); + let mut expect = 0; + let mut rest = out; + while let Some(i) = rest.find("\"sequence_number\":") { + rest = &rest[i + "\"sequence_number\":".len()..]; + let n: i32 = rest + .chars() + .take_while(|c| c.is_ascii_digit()) + .collect::() + .parse() + .unwrap(); + assert_eq!(n, expect); + expect += 1; + } + assert!(expect > 0); +} + +#[test] +fn utf8_safe_len_matches_c() { + let cases = [ + (0usize, 2usize, 0, "c3a9"), + (0, 1, 0, "c3"), + (0, 3, 0, "616263"), + (0, 4, 0, "636166c3"), + (0, 5, 0, "636166c3a9"), + ]; + for (start, limit, final_, hex) in cases { + let rust = utf8_stream_safe_len(&decode_hex(hex), start, limit, final_ != 0); + let c: usize = c_str(&[ + "utf8-safe", + &start.to_string(), + &limit.to_string(), + &final_.to_string(), + hex, + ]) + .trim() + .parse() + .unwrap(); + assert_eq!(rust, c, "utf8-safe {hex} start={start} limit={limit}"); + } +} + +fn decode_hex(hex: &str) -> Vec { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()) + .collect() +} + +#[test] +fn openai_chat_thinking_tape_matches_c() { + let rust = project_openai_chat_thinking(CREATED_TEST); + let c = c_bytes(&["openai-chat-tape"]); + assert_bytes_eq("openai-chat-tape", &rust, &c); + let out = String::from_utf8_lossy(&rust); + sse_assert_every_data_object(&out, "\"object\":\"chat.completion.chunk\"", None); + let reason = out.find("\"reasoning_content\":").expect("reason"); + let content = out.find("\"content\":\"Answer").expect("content"); + assert!(out.contains("\"delta\":{\"role\":\"assistant\"}")); + assert!(reason < content); + assert!(out.contains("\"finish_reason\":\"stop\"")); + assert!(!out.contains("")); +} + +#[test] +fn openai_tool_before_think_streams_as_call() { + let raw = b"Seoul"; + let r = StreamReq { + think_mode: ThinkMode::High, + has_tools: true, + chat_format: ChatFormat::Qwen4Exp, + syntax: ModelSyntax::Mimo2, + ..Default::default() + }; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + for n in 1..=raw.len() { + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_mimo", + &mut st, + &raw[..n], + false, + )); + } + let calls = [ToolCall { + name: "get_weather".into(), + arguments: r#"{"city":"Seoul"}"#.into(), + ..Default::default() + }]; + assert!(openai_sse_finish_live( + &mut w, + &r, + "chatcmpl_mimo", + &mut st, + raw, + "tool_calls", + 100, + 18, + &calls, + )); + let out = String::from_utf8(w.out).unwrap(); + assert!(out.contains("\"tool_calls\""), "{out}"); + assert!(!out.contains("reasoning_content"), "{out}"); +} + +#[test] +fn qwen_and_step_tool_inside_open_think_stays_reasoning() { + let raw = b"Seoul"; + for syntax in [ModelSyntax::Qwen4Exp, ModelSyntax::Step37] { + let r = StreamReq { + model: format!("{syntax:?}"), + think_mode: ThinkMode::High, + has_tools: true, + chat_format: ChatFormat::Qwen4Exp, + syntax, + ..Default::default() + }; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + for n in 1..=raw.len() { + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_qwen", + &mut st, + &raw[..n], + false, + )); + } + assert!(openai_sse_finish_live( + &mut w, + &r, + "chatcmpl_qwen", + &mut st, + raw, + "stop", + 100, + 18, + &[], + )); + let out = String::from_utf8(w.out).unwrap(); + assert!(out.contains("reasoning_content"), "{out}"); + assert!(!out.contains("tool_calls"), "{out}"); + } +} + +#[test] +fn openai_chat_utf8_tape_matches_c() { + let rust = project_openai_chat_utf8(CREATED_TEST); + let c = c_bytes(&["openai-chat-utf8"]); + assert_bytes_eq("openai-chat-utf8", &rust, &c); + let out = String::from_utf8_lossy(&rust); + assert!(out.contains("\u{e9}")); + assert!(!out.contains("\u{fffd}")); + assert!(!rust.windows(2).any(|w| w == [0xc3, b'"'])); +} + +#[test] +fn openai_completion_tape_matches_c() { + let rust = project_openai_completion(CREATED_TEST); + let c = c_bytes(&["openai-completion-tape"]); + assert_bytes_eq("openai-completion-tape", &rust, &c); + let out = String::from_utf8_lossy(&rust); + sse_assert_every_data_object( + &out, + "\"object\":\"text_completion\"", + Some("chat.completion.chunk"), + ); + assert!(out.contains("\"choices\":[{\"text\":\"Hel\"")); + assert!(out.contains("\"finish_reason\":\"stop\"")); + assert!(!out.contains("\"delta\"")); +} + +#[test] +fn anthropic_thinking_tape_matches_c() { + let rust = project_anthropic_thinking(CREATED_TEST); + let c = c_bytes(&["anthropic-tape"]); + assert_bytes_eq("anthropic-tape", &rust, &c); + let out = String::from_utf8_lossy(&rust); + sse_validate_anthropic(&out); + let thinking = out.find("\"thinking\":\"plan\"").expect("thinking"); + let signature = out.find("\"type\":\"signature_delta\"").expect("signature"); + let text = out.find("\"text\":\"Answer").expect("text"); + assert!(thinking < signature && signature < text); + assert!(out.contains("\"stop_reason\":\"end_turn\"")); + assert!(!out.contains("")); +} + +#[test] +fn responses_thinking_tape_matches_c() { + let rust = project_responses_thinking(CREATED_TEST); + let c = c_bytes(&["responses-tape"]); + assert_bytes_eq("responses-tape", &rust, &c); + let out = String::from_utf8_lossy(&rust); + sse_validate_responses(&out); + let summary = out + .find("\"type\":\"response.reasoning_summary_text.delta\"") + .expect("summary"); + let text = out + .find("\"type\":\"response.output_text.delta\"") + .expect("text"); + assert!(summary < text); + assert!(out.contains("\"status\":\"completed\"")); + assert!(!out.contains("")); +} + +/// A model can emit a byte that starts no valid UTF-8 sequence mid-text (GLM-5.3 +/// VQ wrote "서", a lone 0xEB, then "ving"). The buffered bodies are built as +/// strings, so such a byte must become U+FFFD instead of panicking the +/// request into a 500; the C reference passes the byte through raw. +#[test] +fn buffered_finals_replace_invalid_utf8() { + let text: &[u8] = b"\xec\x84\x9c\xebving"; + let want = "서\u{fffd}ving"; + let mut chat = StreamReq::default(); + chat.stream = false; + let bodies = [ + final_response( + &chat, + "chatcmpl_bad", + text, + Some(text), + "stop", + 4, + 4, + CREATED_TEST, + false, + &[], + &[], + ), + anthropic_final_response( + &chat, + "msg_bad", + text, + Some(text), + "stop", + None, + 4, + 4, + false, + &[], + ), + responses_final_response( + &chat, + text, + Some(text), + "stop", + ThinkBlock::Closed, + 4, + 4, + 0, + CREATED_TEST, + false, + TEST_RESP_ID, + TEST_RS_ID, + TEST_MSG_ID, + &[], + ), + ]; + for body in bodies { + let body = String::from_utf8(body).expect("the body is valid UTF-8"); + assert!(body.contains(want), "{body}"); + } +} + +/// Beam served through Hermes wrote two of a Hangul syllable's three bytes, +/// then English; openai-python decodes each SSE line as strict UTF-8 and +/// dropped the stream. That delta carries U+FFFD now, while a syllable cut +/// between two tokens still streams whole. +#[test] +fn stream_deltas_replace_invalid_utf8() { + let r = StreamReq { + chat_format: ChatFormat::Beam, + syntax: ModelSyntax::BeamVq, + ..Default::default() + }; + let raw: &[u8] = b"<|channel|>analysis<|message|>\xec\x84\x9c\xeb\xb2ving<|end|>"; + let head = raw.len() - b"\x9c\xeb\xb2ving<|end|>".len(); + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + for (cut, last) in [(head, false), (head + 3, false), (raw.len(), true)] { + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_bad", + &mut st, + &raw[..cut], + last, + )); + } + let out = String::from_utf8(w.out).expect("every SSE line is valid UTF-8"); + assert!(out.contains("\"reasoning_content\":\"서\""), "{out}"); + assert!( + out.contains("\"reasoning_content\":\"\u{fffd}ving\""), + "{out}" + ); +} + +/// A cut character that nothing can complete becomes U+FFFD as well: one +/// before a closer, and one left when the generation ends, streamed or +/// buffered (the C reference drops both). +#[test] +fn stranded_utf8_tails_become_replacement() { + let r = StreamReq { + chat_format: ChatFormat::Beam, + syntax: ModelSyntax::BeamVq, + ..Default::default() + }; + let raw: &[u8] = b"<|channel|>analysis<|message|>a\xeb<|end|>\ + <|start|>assistant<|channel|>final<|message|>b\xeb"; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + let n = raw.len(); + for (cut, last) in [(n - 1, false), (n, false), (n, true)] { + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_cut", + &mut st, + &raw[..cut], + last, + )); + } + let out = String::from_utf8(w.out).expect("every SSE line is valid UTF-8"); + assert!(out.contains("\"reasoning_content\":\"a\u{fffd}\""), "{out}"); + assert!(out.contains("\"content\":\"b\""), "{out}"); + assert!(out.contains("\"content\":\"\u{fffd}\""), "{out}"); + let chat = StreamReq { + stream: false, + ..Default::default() + }; + let body = final_response( + &chat, + "chatcmpl_cut", + b"b\xeb", + Some(b"a\xeb"), + "length", + 4, + 4, + CREATED_TEST, + false, + &[], + &[], + ); + let body = String::from_utf8(body).expect("the body is valid UTF-8"); + assert!( + body.contains("b\u{fffd}") && body.contains("a\u{fffd}"), + "{body}" + ); +} + +#[test] +fn buffered_finals_match_c() { + let mut chat = StreamReq::default(); + chat.stream = false; + let rust = final_response( + &chat, + "chatcmpl_buf", + b"Hello world.", + None, + "stop", + 4, + 4, + CREATED_TEST, + false, + &[], + &[], + ); + let c = c_bytes(&["final", "openai-chat", "Hello world.", "stop"]); + assert_bytes_eq("final-openai-chat", &rust, &c); + let out = String::from_utf8_lossy(&rust); + assert!(out.contains("\"object\":\"chat.completion\"")); + assert!(out.contains("\"role\":\"assistant\",\"content\":\"Hello world.\"")); + assert!(out.contains("\"finish_reason\":\"stop\"")); + + let mut comp = StreamReq::default(); + comp.kind = ReqKind::Completion; + comp.stream = false; + let rust = final_response( + &comp, + "cmpl_buf", + b"Hello world.", + None, + "length", + 4, + 4, + CREATED_TEST, + false, + &[], + &[], + ); + let c = c_bytes(&["final", "openai-completion", "Hello world.", "length"]); + assert_bytes_eq("final-completion", &rust, &c); + let out = String::from_utf8_lossy(&rust); + assert!(out.contains("\"object\":\"text_completion\"")); + assert!(out.contains("\"choices\":[{\"text\":\"Hello world.\"")); + assert!(out.contains("\"finish_reason\":\"length\"")); + + let rust = anthropic_final_response( + &chat, + "msg_buf", + b"Hello world.", + None, + "length", + None, + 4, + 4, + false, + &[], + ); + let c = c_bytes(&["final", "anthropic", "Hello world.", "length"]); + assert_bytes_eq("final-anthropic", &rust, &c); + let out = String::from_utf8_lossy(&rust); + assert!(out.contains("\"type\":\"message\"")); + assert!(out.contains("\"stop_reason\":\"max_tokens\"")); + + let rust = responses_final_response( + &chat, + b"Hello world.", + None, + "stop", + ThinkBlock::Closed, + 4, + 4, + 0, + CREATED_TEST, + false, + TEST_RESP_ID, + TEST_RS_ID, + TEST_MSG_ID, + &[], + ); + let c = c_bytes(&["final", "responses", "Hello world.", "stop"]); + assert_bytes_eq("final-responses", &rust, &c); + let out = String::from_utf8_lossy(&rust); + assert!(out.contains("\"object\":\"response\"")); + assert!(out.contains("\"status\":\"completed\"")); + assert!(out.contains("Hello world.")); +} + +#[test] +fn bare_invoke_block_holds_back_and_finishes_with_the_call() { + // The V4.1 artifact dropped the wrapper in a live open-grok session + // (01a127cf, 2026-10-11): the block opened straight with , the + // live start detector missed it, and the markup streamed as visible + // content while the finish still served the call. The detector now + // knows the invoke spellings; the block must be held back and the call + // emitted at the finish. + let raw = b"Here you go.\n\n\n\ + https://example.com\n\ + \n"; + let r = StreamReq { + think_mode: ThinkMode::None, + has_tools: true, + chat_format: ChatFormat::DeepSeek, + syntax: ModelSyntax::DeepSeek, + ..Default::default() + }; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + // "Here you go." streams as one delta; a chunk ending mid-marker is + // held back; the full text holds the block in suppress mode. + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_bare", + &mut st, + &raw[..12], + false, + )); + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_bare", + &mut st, + &raw[..17], + false, + )); + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_bare", + &mut st, + raw, + false, + )); + let calls = [ToolCall { + name: "web_fetch".into(), + arguments: r#"{"url": "https://example.com"}"#.into(), + ..Default::default() + }]; + assert!(openai_sse_finish_live( + &mut w, + &r, + "chatcmpl_bare", + &mut st, + raw, + "tool_calls", + 12, + 20, + &calls, + )); + let out = String::from_utf8(w.out).unwrap(); + assert!( + !out.contains("\n\ + DGX Spark lowest price\n\ + {SPACED_INVOKE_END}\n\ + {SPACED_INVOKE_START} name=\"web_search\">\n\ + DGX Spark second\n\ + {SPACED_INVOKE_END}\n\ + {SPACED_CALLS_END}" + ); + let r = StreamReq { + think_mode: ThinkMode::None, + has_tools: true, + chat_format: ChatFormat::DeepSeek, + syntax: ModelSyntax::DeepSeek, + ..Default::default() + }; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_bareclose", + &mut st, + raw.as_bytes(), + false, + )); + let calls = [ + ToolCall { + name: "web_search".into(), + arguments: r#"{"query": "DGX Spark lowest price"}"#.into(), + ..Default::default() + }, + ToolCall { + name: "web_search".into(), + arguments: r#"{"query": "DGX Spark second"}"#.into(), + ..Default::default() + }, + ]; + assert!(openai_sse_finish_live( + &mut w, + &r, + "chatcmpl_bareclose", + &mut st, + raw.as_bytes(), + "tool_calls", + 10, + 40, + &calls, + )); + let out = String::from_utf8(w.out).unwrap(); + assert!( + !out.contains(""), + "the bare close leaked into a value: {out}" + ); + assert!( + out.contains("DGX Spark lowest price"), + "the first value must stream: {out}" + ); + assert!( + out.contains("DGX Spark second"), + "the second value must stream: {out}" + ); +} + +#[test] +fn parameter_without_string_attr_streams_as_string() { + // Engine f302be7: a bare parameter carries no string attribute and the + // engine defaults it to string (matching the final parser). Before the + // alias the live stream failed the call at the parameter open. + let raw = format!( + "go\n\n{SPACED_CALLS_START}\n\ + {SPACED_INVOKE_START} name=\"web_search\">\n\ + DGX Spark\n\ + {SPACED_INVOKE_END}\n\ + {SPACED_CALLS_END}" + ); + let r = StreamReq { + think_mode: ThinkMode::None, + has_tools: true, + chat_format: ChatFormat::DeepSeek, + syntax: ModelSyntax::DeepSeek, + ..Default::default() + }; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_nostring", + &mut st, + raw.as_bytes(), + false, + )); + let calls = [ToolCall { + name: "web_search".into(), + arguments: r#"{"query": "DGX Spark"}"#.into(), + ..Default::default() + }]; + assert!(openai_sse_finish_live( + &mut w, + &r, + "chatcmpl_nostring", + &mut st, + raw.as_bytes(), + "tool_calls", + 10, + 20, + &calls, + )); + let out = String::from_utf8(w.out).unwrap(); + assert!(out.contains("query"), "the parameter must stream: {out}"); + assert!(out.contains("DGX Spark"), "the value must stream: {out}"); +} + +/// f302be7 (2026-10-11): in non-thinking mode the artifact sometimes samples +/// one more close marker ahead of its narration. The stream pushes the +/// narration but never the marker itself; a marker tail cut between two tokens +/// is held back until the next chunk proves it. +#[test] +fn stray_close_marker_never_streams_as_content() { + let marker = think_end(ChatFormat::DeepSeek); + let prefix = "Let me check the price."; + let suffix = "Actually, first the specs."; + let raw = format!("{prefix}{marker}{suffix}"); + let r = StreamReq { + think_mode: ThinkMode::None, + has_tools: true, + chat_format: ChatFormat::DeepSeek, + syntax: ModelSyntax::DeepSeek, + ..Default::default() + }; + let mut w = Writer::new(CREATED_TEST); + let mut st = openai_stream_start(&r); + // A chunk ending mid-marker holds the partial back. + let head = &raw[..prefix.len() + 5]; + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_stray", + &mut st, + head.as_bytes(), + false, + )); + let mid = String::from_utf8(w.out.clone()).unwrap(); + assert!(mid.contains(prefix), "the narration must stream: {mid}"); + assert!( + !mid.contains(&marker[..5]), + "a partial marker leaked: {mid}" + ); + // The completed marker is skipped; the narration after it streams. + assert!(openai_sse_stream_update( + &mut w, + &r, + "chatcmpl_stray", + &mut st, + raw.as_bytes(), + true, + )); + let out = String::from_utf8(w.out).unwrap(); + assert!(out.contains(prefix), "{out}"); + assert!(out.contains(suffix), "{out}"); + assert!(!out.contains(marker), "the marker leaked: {out}"); +} + +/// The same stray-marker suppression on the anthropic surface (f302be7, +/// server_anthropic_stream.c:200-206). +#[test] +fn anthropic_stray_close_marker_never_streams_as_content() { + let marker = think_end(ChatFormat::DeepSeek); + let prefix = "Let me check the price."; + let suffix = "Actually, first the specs."; + let raw = format!("{prefix}{marker}{suffix}"); + let r = StreamReq { + think_mode: ThinkMode::None, + has_tools: true, + chat_format: ChatFormat::DeepSeek, + syntax: ModelSyntax::DeepSeek, + ..Default::default() + }; + let mut w = Writer::new(CREATED_TEST); + let mut st = anthropic_sse_start_live(&mut w, &r, "job", 2); + assert!(anthropic_sse_stream_update( + &mut w, + &r, + "job", + &mut st, + raw.as_bytes(), + true, + )); + let out = String::from_utf8(w.out).unwrap(); + assert!(out.contains(prefix), "{out}"); + assert!(out.contains(suffix), "{out}"); + assert!(!out.contains(marker), "the marker leaked: {out}"); +} diff --git a/crates/ds4-server/tests/tool_stream.rs b/crates/ds4-server/tests/tool_stream.rs new file mode 100644 index 0000000000000000000000000000000000000000..256cfebd256d01506d2fb16b6b0a0b0bed9b2bb6 --- /dev/null +++ b/crates/ds4-server/tests/tool_stream.rs @@ -0,0 +1,167 @@ +//! C↔Rust incremental DSML tool-stream tapes (no model). + +use ds4_server::tool_stream_dump_script; +use std::path::PathBuf; +use std::process::Command; + +const DSML_TOOL_CALLS_START: &str = "<|DSML|tool_calls>"; +const DSML_PARAM_START: &str = "<|DSML|parameter"; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_TOOL_STREAM_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/tool_stream_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/tool_stream_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_bytes(name: &str) -> Vec { + let out = Command::new(require_oracle()) + .arg(name) + .output() + .expect("run tool_stream_c_oracle"); + assert!( + out.status.success(), + "oracle {name} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + out.stdout +} + +fn assert_bytes_eq(label: &str, rust: &[u8], c: &[u8]) { + if rust != c { + let r = String::from_utf8_lossy(rust); + let k = String::from_utf8_lossy(c); + panic!("{label} mismatch\n--- rust ---\n{r}\n--- c ---\n{k}"); + } +} + +fn assert_script(name: &str) -> String { + let rust = tool_stream_dump_script(name); + let c = c_bytes(name); + assert_bytes_eq(name, &rust, &c); + String::from_utf8_lossy(&rust).into_owned() +} + +fn count_substr(hay: &str, needle: &str) -> usize { + hay.matches(needle).count() +} + +#[test] +fn openai_partial_arguments_match_c() { + let out = assert_script("openai-partial"); + let text = out.find("\"content\":\"Before.\"").expect("text"); + let tool = out.find("\"tool_calls\"").expect("tool"); + let key = out.find("\\\"command\\\":\\\"").expect("key"); + let partial = out.find("\"arguments\":\"echo partial\"").expect("partial"); + let rest = out.find("\"arguments\":\" done\"").expect("rest"); + assert!(text < tool && tool < partial && partial < rest); + assert!(key < partial); + assert_eq!(count_substr(&out, "\"id\":\"call_"), 1); + assert!(out.contains("\"id\":\"call_0\"")); + assert!(!out.contains(DSML_TOOL_CALLS_START)); + assert!(!out.contains(DSML_PARAM_START)); + assert!(out.contains("data: [DONE]")); +} + +#[test] +fn openai_raw_arguments_match_c() { + let out = assert_script("openai-raw"); + assert!(out.contains("\"name\":\"edit\"")); + assert!(out.contains("\\\"edits\\\":")); + assert!(out.contains("\"arguments\":\"[1,2,3\"")); + assert!(!out.contains(DSML_TOOL_CALLS_START)); +} + +#[test] +fn openai_waits_for_incomplete_tags_match_c() { + let out = assert_script("openai-wait-tag"); + assert!(out.contains("\"name\":\"bash\"")); + assert!(!out.contains(DSML_PARAM_START)); +} + +#[test] +fn openai_holds_partial_dsml_entities_match_c() { + let out = assert_script("openai-entity"); + assert!(out.contains("\"arguments\":\"echo \"")); + assert!(out.contains("\"arguments\":\"& done\"")); + assert!(!out.contains("&")); +} + +#[test] +fn openai_think_then_tool_match_c() { + let out = assert_script("openai-think-tool"); + let role = out.find("\"role\":\"assistant\"").expect("role"); + let thinking = out + .find("\"reasoning_content\":\"need a tool\"") + .expect("thinking"); + let text = out.find("\"content\":\"Hello.\"").expect("text"); + let tool = out.find("\"tool_calls\"").expect("tool"); + let done = out.find("data: [DONE]").expect("done"); + assert!(role < thinking && thinking < text && text < tool && tool < done); + assert!(!out.contains(DSML_TOOL_CALLS_START)); + assert!(!out.contains("")); +} + +#[test] +fn openai_utf8_arguments_match_c() { + let rust = tool_stream_dump_script("openai-utf8"); + let c = c_bytes("openai-utf8"); + assert_bytes_eq("openai-utf8", &rust, &c); + let out = String::from_utf8_lossy(&rust); + assert!(out.contains("\"arguments\":\"flag \"")); + assert!(rust.windows(4).any(|w| w == [0xf0, 0x9f, 0x9a, 0xa9])); + assert!(!rust.windows(3).any(|w| w == [0xef, 0xbf, 0xbd])); +} + +#[test] +fn openai_multiple_calls_match_c() { + let out = assert_script("openai-multi"); + assert_eq!(count_substr(&out, "\"id\":\"call_"), 2); + assert!(out.contains("\"name\":\"read\"")); + assert!(out.contains("\"name\":\"bash\"")); + assert!(out.contains("\\\"path\\\":")); + assert!(out.contains("\\\"command\\\":")); +} + +#[test] +fn anthropic_partial_tool_use_match_c() { + let out = assert_script("anthropic-partial"); + let text = out.find("\"text\":\"Before.\"").expect("text"); + let tool = out.find("\"type\":\"tool_use\"").expect("tool"); + let key = out.find("\\\"command\\\":\\\"").expect("key"); + let partial = out + .find("\"partial_json\":\"echo partial\"") + .expect("partial"); + let rest = out.find("\"partial_json\":\" done\"").expect("rest"); + let stop = out.find("event: message_stop").expect("stop"); + assert!(text < tool && tool < key && key < partial && partial < rest && rest < stop); + assert_eq!(count_substr(&out, "\"type\":\"tool_use\""), 1); + assert!(out.contains("\"id\":\"toolu_0\"")); + assert!(!out.contains(DSML_TOOL_CALLS_START)); + assert!(!out.contains(DSML_PARAM_START)); +} + +#[test] +fn anthropic_think_then_tool_match_c() { + let out = assert_script("anthropic-think-tool"); + let start = out.find("event: message_start").expect("start"); + let thinking = out.find("\"thinking\":\"need a tool\"").expect("thinking"); + let signature = out.find("\"type\":\"signature_delta\"").expect("sig"); + let text = out.find("\"text\":\"Hello.\"").expect("text"); + let tool = out.find("\"type\":\"tool_use\"").expect("tool"); + let stop = out.find("event: message_stop").expect("stop"); + assert!( + start < thinking && thinking < signature && signature < text && text < tool && tool < stop + ); + assert!(!out.contains(DSML_TOOL_CALLS_START)); +} diff --git a/crates/ds4-server/tests/tools.rs b/crates/ds4-server/tests/tools.rs new file mode 100644 index 0000000000000000000000000000000000000000..b68b25862052dfbe4563e3f78603d9411b2b7241 --- /dev/null +++ b/crates/ds4-server/tests/tools.rs @@ -0,0 +1,908 @@ +//! Generated-message parse + SemAccum goldens from `ds4_server.c` unit tests. + +use ds4_server::{ + append_tool_calls_json, assign_tool_ids, parse_chat_request, parse_dsml_tolerant, + parse_generated_for_response, parse_generated_message, think_start, ChatFormat, ModelSyntax, + ParseEnv, SemAccum, ThinkMode, ToolCall, ToolSchemaOrder, +}; + +const DSML_START: &str = "<|DSML|tool_calls>"; +const DSML_END: &str = ""; +const DSML_INVOKE: &str = "<|DSML|invoke"; +const DSML_INVOKE_END: &str = ""; +const DSML_PARAM: &str = "<|DSML|parameter"; +const DSML_PARAM_END: &str = ""; + +#[test] +fn parse_dots3_tool_call_message() { + let text = b"\nplan\n\n\nChecking now.\n\ + \n\ + \n\ + \nSeoul\n\n\ + \n3\n\n\ + \n\ + "; + let p = parse_generated_message(ModelSyntax::Dots3, text, true, ChatFormat::DeepSeek, &[]); + assert!(p.ok); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "get_weather"); + assert_eq!(p.calls[0].arguments, "{\"city\": \"Seoul\", \"days\": 3}"); + assert_eq!(p.content, b"\n\nChecking now."); + assert_eq!(p.reasoning, b"\nplan\n"); + + let multi = b"\n\n\n\ + \n\n1\n\n\n\ + \n\ntwo\n\n\n\ + "; + let p = parse_generated_message(ModelSyntax::Dots3, multi, true, ChatFormat::DeepSeek, &[]); + assert!(p.ok); + assert_eq!(p.calls.len(), 2); + assert_eq!(p.calls[0].arguments, "{\"x\": 1}"); + assert_eq!(p.calls[1].arguments, "{\"y\": \"two\"}"); +} + +#[test] +fn parse_motif3_tool_call_message() { + let text = b"need weather\n\ + {\"name\": \"get_weather\", \"arguments\": \ + {\"city\": \"Seoul\"}}"; + let p = parse_generated_message(ModelSyntax::Motif3, text, true, ChatFormat::DeepSeek, &[]); + assert!(p.ok); + assert_eq!(p.reasoning, b"need weather"); + assert!(p.content.is_empty()); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "get_weather"); + assert!(p.calls[0].arguments.contains("\"city\"")); + assert!(p.raw_tool_text.starts_with("\n")); +} + +#[test] +fn parse_exaone_two_hermes_calls() { + let text = b"need weather\n\n\ + {\"name\":\"get_weather\",\"arguments\":\ + {\"city\":\"Seoul\"}}\n\ + {\"name\":\"get_time\",\"arguments\":{}}"; + let p = parse_generated_message(ModelSyntax::Exaone, text, true, ChatFormat::Exaone, &[]); + assert!(p.ok); + assert_eq!(p.reasoning, b"need weather"); + assert!(p.content.is_empty()); + assert_eq!(p.calls.len(), 2); + assert_eq!(p.calls[0].name, "get_weather"); + assert_eq!(p.calls[0].arguments, "{\"city\":\"Seoul\"}"); + assert_eq!(p.calls[1].name, "get_time"); +} + +#[test] +fn parse_k2_horizon_ifm_think_not_exaone() { + let text = b"\nplan\nhello\ + \n\ + get_weather\n\ + city\n\ + Seoul\n\ + days\n\ + 2\n\ + \n\ + get_time\n\ + \n\ + "; + let orders = [ToolSchemaOrder { + name: "get_weather".into(), + prop: vec!["city".into(), "days".into()], + prop_type: vec!["string".into(), "integer".into()], + ..Default::default() + }]; + let p = parse_generated_message( + ModelSyntax::K2Horizon, + text, + true, + ChatFormat::K2Horizon, + &orders, + ); + assert!(p.ok); + assert_eq!(p.reasoning, b"\nplan\n"); + assert_eq!(p.content, b"hello"); + assert_eq!(p.calls.len(), 2); + assert_eq!(p.calls[0].name, "get_weather"); + assert_eq!(p.calls[0].arguments, r#"{"city": "Seoul", "days": 2}"#); + assert_eq!(p.calls[1].name, "get_time"); + assert!(p.raw_dsml.starts_with("")); + assert!(!String::from_utf8_lossy(&p.content).contains("<|user|>")); +} + +#[test] +fn parse_dsml_nested_parameters() { + let generated = format!( + "review done\n\n{DSML_START}\n{DSML_INVOKE} name=\"edit\">\n\ + {DSML_PARAM} name=\"path\">/private/tmp/tetris.c{DSML_PARAM_END}\n\ + {DSML_PARAM} name=\"edits\">\n\ + {DSML_PARAM} name=\"oldText\" string=\"true\">old <text>{DSML_PARAM_END}\n\ + {DSML_PARAM} name=\"newText\" string=\"true\">new text{DSML_PARAM_END}\n\ + {DSML_INVOKE_END}\n{DSML_END}" + ); + let p = parse_generated_message( + ModelSyntax::DeepSeek, + generated.as_bytes(), + false, + ChatFormat::DeepSeek, + &[], + ); + assert!(p.ok); + assert_eq!(p.content, b"review done"); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "edit"); + assert!(p.calls[0] + .arguments + .contains("\"path\": \"/private/tmp/tetris.c\"")); + assert!(p.calls[0].arguments.contains("\"edits\": {")); + assert!( + p.calls[0].arguments.contains("\"oldText\":\"old \""), + "{}", + p.calls[0].arguments + ); + assert!(p.calls[0].arguments.contains("\"newText\":\"new text\"")); +} + +#[test] +fn parse_solar_native_tool_call() { + let generated = format!( + "I should inspect the directory.<|think:end|><|tool_call:start|>list_files\n\ + <|tool_arg:start|>path<|tool_arg:value|>/tmp<|tool_arg:end|>\n\ + <|tool_arg:start|>recursive<|tool_arg:value|>false<|tool_arg:end|>\n\ + <|tool_arg:start|>literal<|tool_arg:value|>false<|tool_arg:end|>\n\ + <|tool_call:end|>" + ); + let orders = [ToolSchemaOrder { + name: "list_files".into(), + prop: vec!["path".into(), "recursive".into(), "literal".into()], + prop_type: vec!["string".into(), "boolean".into(), "string".into()], + ..Default::default() + }]; + let p = parse_generated_message( + ModelSyntax::SolarOpen2, + generated.as_bytes(), + true, + ChatFormat::SolarOpen2, + &orders, + ); + assert!(p.ok); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "list_files"); + assert!(p.calls[0].arguments.contains("\"path\": \"/tmp\"")); + assert!(p.calls[0].arguments.contains("\"recursive\": false")); + assert!(p.calls[0].arguments.contains("\"literal\": \"false\"")); + assert!(p.content.is_empty()); + assert_eq!(p.reasoning, b"I should inspect the directory."); + assert!(p.raw_dsml.starts_with("<|tool_call:start|>")); +} + +#[test] +fn parse_qwen_native_tool_call() { + let generated = b"\nNeed weather.\n\n\n\ + \n\n\ + \nSeoul\n\n\ + \n2\n\n\ + \n"; + let orders = [ToolSchemaOrder { + name: "weather".into(), + prop: vec!["city".into(), "days".into()], + prop_type: vec!["string".into(), "integer".into()], + ..Default::default() + }]; + let p = parse_generated_message( + ModelSyntax::Qwen4Exp, + generated, + true, + ChatFormat::Qwen4Exp, + &orders, + ); + assert!(p.ok); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "weather"); + assert_eq!(p.calls[0].arguments, "{\"city\": \"Seoul\", \"days\": 2}"); + assert!(p.content.is_empty()); + assert!(p.reasoning.windows(13).any(|s| s == b"Need weather.")); + assert!(p.raw_dsml.contains("")); +} + +#[test] +fn parse_mimo_tool_without_think() { + let generated = b"Seoul"; + let orders = [ToolSchemaOrder { + name: "get_weather".into(), + prop: vec!["city".into()], + prop_type: vec!["string".into()], + ..Default::default() + }]; + let p = parse_generated_message( + ModelSyntax::Mimo2, + generated, + true, + ChatFormat::Qwen4Exp, + &orders, + ); + assert!(p.ok); + assert!(p.content.is_empty()); + assert!(p.reasoning.is_empty()); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "get_weather"); + assert_eq!(p.calls[0].arguments, r#"{"city": "Seoul"}"#); +} + +#[test] +fn parse_mimo_unclosed_think_stays_reasoning() { + let generated = b"Need weather. Seoul"; + let p = parse_generated_message( + ModelSyntax::Mimo2, + generated, + true, + ChatFormat::Qwen4Exp, + &[], + ); + assert!(p.ok); + assert!(p.calls.is_empty()); + assert!(p.content.is_empty()); + assert_eq!(p.reasoning, &generated[b"".len()..]); +} + +#[test] +fn qwen_and_step_tool_inside_open_think_stays_reasoning() { + let generated = b"Seoul"; + for syntax in [ModelSyntax::Qwen4Exp, ModelSyntax::Step37] { + let p = parse_generated_message(syntax, generated, true, ChatFormat::Qwen4Exp, &[]); + assert!(p.ok); + assert!(p.calls.is_empty(), "{syntax:?}"); + assert!(p.content.is_empty(), "{syntax:?}"); + assert_eq!(p.reasoning, generated, "{syntax:?}"); + } +} + +#[test] +fn parse_glm53_native_tool_call() { + let generated = b"need bashOK\n\n\ + bash\ + commandecho hi\ + timeout10\ + "; + let p = parse_generated_message( + ModelSyntax::Glm53, + generated, + true, + ChatFormat::DeepSeek, + &[], + ); + assert!(p.ok); + assert_eq!(p.reasoning, b"need bash"); + assert_eq!(p.content, b"OK"); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "bash"); + // With no schema at all the reference's GLM rule stands: every argument is + // text (ds4_server.c:6435, and the reference test at :19262 asserts the + // same string). A declared type is what makes a numeric argument a number. + assert_eq!( + p.calls[0].arguments, + r#"{"command": "echo hi", "timeout": "10"}"# + ); + // The sampled block rides raw_tool_text, the field the reference fills and + // its GLM renderer splices back (ds4_server.c:19288, :2979-2983). + assert!(p.raw_tool_text.starts_with("\n\nbash")); +} + +/// The harness declares Rust wire types in the request schema (open-grok +/// `search_tool` carries `limit: Option`, +/// crates/codegen/xai-grok-tools/src/implementations/search_tool/types.rs:15), +/// and rejects a numeric property that arrives as a JSON string with +/// `invalid type: string "3", expected u8`. The Glm53 branch of +/// parse_glm_generated resolves each value through the declared type, so a +/// spelling it does not know must not fall back to a string. +#[test] +fn parse_glm53_types_a_u8_property_from_the_declared_schema() { + let orders = [ToolSchemaOrder { + name: "search_tool".into(), + prop: vec!["query".into(), "limit".into()], + prop_type: vec!["string".into(), "u8".into()], + ..Default::default() + }]; + let generated = b"search_tool\ + querysolidcraft\ + limit3\ + "; + let p = parse_generated_message( + ModelSyntax::Glm53, + generated, + false, + ChatFormat::DeepSeek, + &orders, + ); + assert!(p.ok); + assert_eq!(p.calls.len(), 1); + assert_eq!( + p.calls[0].arguments, + r#"{"query": "solidcraft", "limit": 3}"# + ); +} + +/// The control for the test above: a JSON Schema spelling already resolves, so +/// the fix must not change it. +#[test] +fn parse_glm53_types_an_integer_property_from_the_declared_schema() { + let orders = [ToolSchemaOrder { + name: "search_tool".into(), + prop: vec!["limit".into()], + prop_type: vec!["integer".into()], + ..Default::default() + }]; + let generated = + b"search_toollimit3"; + let p = parse_generated_message( + ModelSyntax::Glm53, + generated, + false, + ChatFormat::DeepSeek, + &orders, + ); + assert_eq!(p.calls[0].arguments, r#"{"limit": 3}"#); +} + +/// A client that renders `Option` as `{"type":["integer","null"]}` +/// (schemars does exactly that) used to reach the engine with no usable +/// declared type, because the schema reader keeps only a string `type`. The +/// harness then rejected the stringified number -- `invalid type: string "15", +/// expected usize` -- and the model repeated the call (metasearch loop of +/// session 01a14aaa, 2026-10-17). The reader now keeps the head of the list +/// (`prop_type_first`), and the GLM path consults it before falling back to the +/// reference's text rule. +#[test] +fn parse_glm53_types_a_type_list_property_from_its_head() { + let orders = [ToolSchemaOrder { + name: "metasearch".into(), + prop: vec!["query".into(), "max_results".into()], + prop_type: vec!["string".into(), "".into()], + prop_type_first: vec!["".into(), "integer".into()], + ..Default::default() + }]; + let generated = b"metasearch\ + querysilicon photonics\ + max_results15\ + "; + let p = parse_generated_message( + ModelSyntax::Glm53, + generated, + false, + ChatFormat::DeepSeek, + &orders, + ); + assert_eq!( + p.calls[0].arguments, + r#"{"query": "silicon photonics", "max_results": 15}"# + ); +} + +/// The reader must keep the head of a `type` list while leaving `prop_type` the +/// scalar reading the tolerant paths use, so a `["string","null"]` property is +/// still readable by the GLM path. +#[test] +fn parse_chat_request_keeps_a_type_list_head() { + let env = ParseEnv { + default_model: "glm-5.3-flash".into(), + default_tokens: 2048, + default_effort: ThinkMode::None, + default_temp: ds4_server::default_temperature(), + live_ids: Vec::new(), + engine_defaults: false, + }; + let request = r#"{"messages":[{"role":"user","content":"find it"}], + "tools":[{"type":"function","function":{"name":"search_tool","parameters": + {"type":"object","properties":{"query":{"type":["string","null"]}, + "limit":{"type":["integer","null"],"format":"uint"}}}}}]}"#; + let parsed = parse_chat_request(&env, request).unwrap(); + let order = parsed + .tool_orders + .iter() + .find(|o| o.name == "search_tool") + .unwrap(); + assert_eq!(order.prop, vec!["query", "limit"]); + assert_eq!(order.prop_type, vec!["", ""]); + assert_eq!(order.prop_type_first, vec!["string", "integer"]); +} + +/// An undeclared property keeps the reference's own GLM rule +/// (ds4_server.c:6435): the value is text, never reinterpreted. +#[test] +fn parse_glm53_keeps_a_bare_word_value_text() { + let orders = [ToolSchemaOrder { + name: "metasearch".into(), + prop: vec!["query".into()], + prop_type: vec!["".into()], + ..Default::default() + }]; + let generated = b"metasearch\ + querysilicon photonics\ + "; + let p = parse_generated_message( + ModelSyntax::Glm53, + generated, + false, + ChatFormat::DeepSeek, + &orders, + ); + assert_eq!(p.calls[0].arguments, r#"{"query": "silicon photonics"}"#); +} + +#[test] +fn qwen_stream_observes_native_tool_markers_without_dsml_verdict() { + let mut acc = SemAccum::init(true, true, false, ChatFormat::Qwen4Exp, b""); + let first = acc.feed(b"\n\n", &[]); + assert!(first.entered_tool_block && acc.saw_tool_start); + let last = acc.feed(b"\n", &[]); + assert!(last.tool_block_closed && acc.saw_tool_end); + assert_eq!(acc.verdict, None); +} + +#[test] +fn k2_stream_leaves_thinking_before_observing_tool_block() { + let prompt = b"<|ifm|im_start|>assistant\n\n"; + let mut acc = SemAccum::init(true, true, true, ChatFormat::K2Horizon, prompt); + assert!(acc.thinking_inside()); + acc.feed(b"plan", &[]); + assert!(!closed.entered_tool_block); + assert!(!acc.thinking_inside()); + let first = acc.feed(b"\n{}", &[]); + assert!(first.entered_tool_block && acc.saw_tool_start); + let last = acc.feed(b"", &[]); + assert!(last.tool_block_closed && acc.saw_tool_end); +} + +#[test] +fn no_tools_does_not_extract_calls() { + let text = format!( + "hi\n\n{DSML_START}\n{DSML_INVOKE} name=\"bash\">\n\ + {DSML_PARAM} name=\"command\" string=\"true\">ls{DSML_PARAM_END}\n\ + {DSML_INVOKE_END}\n{DSML_END}" + ); + let (p, finish) = parse_generated_for_response( + ModelSyntax::DeepSeek, + text.as_bytes(), + false, + true, + false, + ChatFormat::DeepSeek, + &[], + "stop", + ); + assert!(p.ok); + assert!(p.calls.is_empty()); + assert_eq!(finish, "stop"); +} + +#[test] +fn append_tool_calls_json_uses_job_fallback_id() { + let calls = [ToolCall { + name: "edit".into(), + arguments: "{\"path\":\"/tmp\"}".into(), + ..Default::default() + }]; + let mut out = Vec::new(); + append_tool_calls_json(&mut out, &calls, "chatcmpl-1"); + let s = String::from_utf8(out).unwrap(); + assert!(s.contains("\"id\":\"chatcmpl-1_tool_0\"")); + assert!(s.contains("\"type\":\"function\"")); + assert!(s.contains("\"name\":\"edit\"")); + assert!(s.contains("\"arguments\":\"{\\\"path\\\":\\\"/tmp\\\"}\"")); +} + +#[test] +fn sem_accum_dsml_closes_and_no_tools_cuts() { + let block = format!( + "{DSML_START}\n{DSML_INVOKE} name=\"bash\">\n\ + {DSML_PARAM} name=\"command\" string=\"true\">ls{DSML_PARAM_END}\n\ + {DSML_INVOKE_END}\n{DSML_END}" + ); + let mut acc = SemAccum::init(true, true, false, ChatFormat::DeepSeek, b""); + let f = acc.feed(block.as_bytes(), &[]); + assert!(acc.saw_tool_start); + assert!(acc.saw_tool_end); + assert_eq!(acc.verdict, Some("tool_calls")); + assert!(f.tool_block_closed); + + let mut cut = SemAccum::init(true, false, false, ChatFormat::DeepSeek, b""); + let f = cut.feed(b"hello <|", &[]); + assert!(!f.hit_stop); + let f = cut.feed(format!("{DSML_START} tail").as_bytes(), &[]); + assert!(f.hit_stop); + assert!(f.tool_syntax_cut); + assert_eq!(cut.verdict, Some("stop")); + assert!(!cut + .text + .windows(DSML_START.len()) + .any(|w| w == DSML_START.as_bytes())); +} + +#[test] +fn assign_tool_ids_fills_empty() { + let mut calls = vec![ + ToolCall { + name: "a".into(), + arguments: "{}".into(), + ..Default::default() + }, + ToolCall { + id: "supplied".into(), + ..Default::default() + }, + ToolCall::default(), + ]; + assign_tool_ids(&mut calls, "call_"); + for id in [&calls[0].id, &calls[2].id] { + assert_eq!(id.len(), 37); + assert!(id.starts_with("call_")); + assert!(id[5..] + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))); + } + assert_ne!(calls[0].id, calls[2].id); + assert_eq!(calls[1].id, "supplied"); +} + +#[test] +fn parse_dsml_tolerant_parses_the_v41_mixed_block() { + // The V4.1 artifact's real output, captured 2026-10-10 (docs 6.23): + // decorated wrapper and invoke opens, plain parameter tags and a plain + // invoke close inside the same block. + let text = format!( + "I'll check the current weather in Stockholm for you.\n\n\ + {DSML_START}\n{DSML_INVOKE} name=\"get_weather\">\n\ + Stockholm\n\ + \n{DSML_END}" + ); + // The strict one-style parse rejects the mix; that is why the fallback + // exists. + let strict = parse_generated_message( + ModelSyntax::DeepSeek, + text.as_bytes(), + false, + ChatFormat::DeepSeek, + &[], + ); + assert!(!strict.ok); + assert!(strict.calls.is_empty()); + + let p = parse_dsml_tolerant(text.as_bytes(), &[], false).expect("tolerant parse"); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "get_weather"); + assert_eq!(p.calls[0].arguments, r#"{"city": "Stockholm"}"#); + assert_eq!( + p.content, + b"I'll check the current weather in Stockholm for you." + ); +} + +#[test] +fn parse_dsml_tolerant_tolerates_the_shims_variants() { + // shim.py:44-48: a short wrapper, an omitted closer, and a bare + // without the string attribute, all seen in live sessions. + let short = format!( + "working\n\n<\u{ff5c}DSML\u{ff5c}calls>\n{DSML_INVOKE} name=\"get_time\">\n\ + UTC\n\ + \n" + ); + let p = parse_dsml_tolerant(short.as_bytes(), &[], false).expect("short wrapper"); + assert_eq!(p.calls[0].name, "get_time"); + assert_eq!(p.calls[0].arguments, r#"{"zone": "UTC"}"#); + assert_eq!(p.content, b"working"); + + // Wrapper close omitted, invoke complete: still served. + let unclosed = format!( + "done\n\n{DSML_START}\n{DSML_INVOKE} name=\"ping\">\n\ + 10.0.0.1\n\ + " + ); + let p = parse_dsml_tolerant(unclosed.as_bytes(), &[], false).expect("unclosed wrapper"); + assert_eq!(p.calls[0].name, "ping"); + assert_eq!(p.calls[0].arguments, r#"{"host": "10.0.0.1"}"#); + + // A truncated stream (a value running to EOF with no boundary) is + // refused rather than served half an argument. + let truncated = format!( + "{DSML_START}\n{DSML_INVOKE} name=\"bash\">\n\ + rm -rf /tmp/fo" + ); + assert!(parse_dsml_tolerant(truncated.as_bytes(), &[], false).is_none()); +} + +#[test] +fn parse_dsml_tolerant_coerces_to_the_declared_schema_type() { + let orders = [ToolSchemaOrder { + name: "search".into(), + prop: vec!["limit".into(), "query".into()], + prop_type: vec!["integer".into(), "string".into()], + ..Default::default() + }]; + let text = format!( + "{DSML_START}\n{DSML_INVOKE} name=\"search\">\n\ + 5\n\ + rtx 5090\n\ + \n{DSML_END}" + ); + let p = parse_dsml_tolerant(text.as_bytes(), &orders, false).expect("tolerant parse"); + assert_eq!(p.calls[0].arguments, r#"{"limit": 5, "query": "rtx 5090"}"#); +} + +#[test] +fn parse_dsml_tolerant_skips_undeclared_names() { + let orders = [ToolSchemaOrder { + name: "declared".into(), + ..Default::default() + }]; + let text = format!( + "{DSML_START}\n{DSML_INVOKE} name=\"ghost\">\n\ + 1\n\n\ + {DSML_INVOKE} name=\"declared\">\n\ + 2\n\n{DSML_END}" + ); + let p = parse_dsml_tolerant(text.as_bytes(), &orders, false).expect("tolerant parse"); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "declared"); +} + +#[test] +fn parse_dsml_tolerant_nests_parameters_like_the_shim() { + let text = format!( + "go\n\n{DSML_START}\n{DSML_INVOKE} name=\"use_tool\">\n\ + \n\ + https://www.amazon.se\n\ + \n\ + \n{DSML_END}" + ); + let p = parse_dsml_tolerant(text.as_bytes(), &[], false).expect("tolerant parse"); + assert_eq!(p.calls[0].name, "use_tool"); + assert!( + p.calls[0].arguments.contains("\"tool_input\": {"), + "{}", + p.calls[0].arguments + ); + assert!(p.calls[0] + .arguments + .contains("\"url\":\"https://www.amazon.se\"")); +} + +#[test] +fn parse_dsml_tolerant_binds_a_bare_child_of_a_declared_property() { + // The artifact's bare-child shape (Spark tool-call gate requests 8/23, + // 2026-10-11): spaced opens, the property written as a plain child + // element, no wrapper close. The engine rejects the block + // (server_dsml_parse.c:365-369) and the shim yields {} (shim.py:59-61); + // the schema declares `city`, so the recovery binds it (plan 6.23). + let orders = [ToolSchemaOrder { + name: "get_weather".into(), + prop: vec!["city".into()], + prop_type: vec!["string".into()], + ..Default::default() + }]; + let text = format!( + "I'll check Tokyo's weather for you.\n\n{DSML_SPACED_START}\n\ + {DSML_SPACED_INVOKE} name=\"get_weather\">\n\ + Tokyo\n{DSML_SPACED_INVOKE_END}" + ); + let p = parse_dsml_tolerant(text.as_bytes(), &orders, false).expect("tolerant parse"); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "get_weather"); + assert_eq!(p.calls[0].arguments, r#"{"city": "Tokyo"}"#); + assert_eq!(p.content, b"I'll check Tokyo's weather for you."); +} + +#[test] +fn parse_dsml_tolerant_keeps_an_undeclared_child_unbound() { + let orders = [ToolSchemaOrder { + name: "get_weather".into(), + prop: vec!["town".into()], + prop_type: vec!["string".into()], + ..Default::default() + }]; + let text = format!( + "{DSML_SPACED_START}\n{DSML_SPACED_INVOKE} name=\"get_weather\">\n\ + Tokyo\n{DSML_SPACED_INVOKE_END}" + ); + let p = parse_dsml_tolerant(text.as_bytes(), &orders, false).expect("tolerant parse"); + assert_eq!(p.calls[0].arguments, "{}"); +} + +#[test] +fn parse_dsml_tolerant_coerces_a_child_like_a_parameter() { + let orders = [ToolSchemaOrder { + name: "search".into(), + prop: vec!["limit".into()], + prop_type: vec!["integer".into()], + ..Default::default() + }]; + let text = format!( + "{DSML_SPACED_START}\n{DSML_SPACED_INVOKE} name=\"search\">\n\ + 5\n{DSML_SPACED_INVOKE_END}" + ); + let p = parse_dsml_tolerant(text.as_bytes(), &orders, false).expect("tolerant parse"); + assert_eq!(p.calls[0].arguments, r#"{"limit": 5}"#); +} + +#[test] +fn parse_dsml_tolerant_binds_a_tool_name_root_with_bare_children() { + // The artifact's tool-name-root shape (Spark session 01a13136, + // 2026-10-12, bytes captured from the open-grok transcript): the call + // opens with the TOOL NAME as the root element and the parameters as + // plain children, with no wrapper and no invoke anywhere -- + // + // .. + // 8 + // + // The engine keys every arm on the wrapper/invoke opens + // (server_dsml_parse.c find_tool_block) and the shim's scanner on the + // same starts (shim.py:492-494), so the whole block falls back to + // assistant text and the calls never run. The tolerant recovery binds + // it (plan 6.23, the named deviation): the root must name a declared + // tool, every child goes through the bare-child binding, the close + // must match the root. + let orders = [ToolSchemaOrder { + name: "search_tool".into(), + prop: vec!["query".into(), "limit".into()], + prop_type: vec!["string".into(), "integer".into()], + ..Default::default() + }]; + let text = "\n\n\n\ + health-second-opinion-mcp search_literature differential_diagnosis corpus_stats\n\ + 8\n\ + \n\n\ + \n\ + ebook-knowledge query_book\n\ + 3\n\ + \n\nWait for results."; + let p = parse_dsml_tolerant(text.as_bytes(), &orders, false).expect("tolerant parse"); + assert_eq!(p.calls.len(), 2); + assert_eq!(p.calls[0].name, "search_tool"); + assert_eq!( + p.calls[0].arguments, + r#"{"query": "health-second-opinion-mcp search_literature differential_diagnosis corpus_stats", "limit": 8}"# + ); + assert_eq!(p.calls[1].name, "search_tool"); + assert_eq!( + p.calls[1].arguments, + r#"{"query": "ebook-knowledge query_book", "limit": 3}"# + ); + assert_eq!(p.content, b""); +} + +#[test] +fn parse_dsml_tolerant_ignores_a_block_inside_closed_thinking() { + // The engine refuses DSML inside a closed reasoning section: with + // thinking on, the tool search starts after the LAST + // (server_dsml_parse.c:291-316; "DSML inside reasoning is just model + // text ... 思考里的工具块不可执行"). Before the scope fix the fallback + // scanned the whole text and served the call with a mangled split. + let orders = [ToolSchemaOrder { + name: "get_weather".into(), + prop: vec!["city".into()], + prop_type: vec!["string".into()], + ..Default::default() + }]; + let think_open = think_start(ChatFormat::DeepSeek); + let text = format!( + "{think_open}the user wants the weather; emit the tool now\n\ + {DSML_SPACED_START}\n{DSML_SPACED_INVOKE} name=\"get_weather\">\n\ + {DSML_SPACED_PARAM} name=\"city\" string=\"true\">Paris{DSML_SPACED_PARAM_END}\n\ + {DSML_SPACED_INVOKE_END}\n{DSML_SPACED_END}\n\n\n\nHello." + ); + assert!(parse_dsml_tolerant(text.as_bytes(), &orders, true).is_none()); + // Thinking off: the engine searches the whole text and the block counts. + let p = parse_dsml_tolerant(text.as_bytes(), &orders, false).expect("thinking off"); + assert_eq!(p.calls[0].arguments, r#"{"city": "Paris"}"#); +} + +#[test] +fn parse_dsml_tolerant_recovers_a_block_after_closed_thinking() { + let orders = [ToolSchemaOrder { + name: "get_weather".into(), + prop: vec!["city".into()], + prop_type: vec!["string".into()], + ..Default::default() + }]; + let think_open = think_start(ChatFormat::DeepSeek); + let text = format!( + "{think_open}check the weather tool\n\nI'll check.\n\n\ + {DSML_SPACED_START}\n{DSML_SPACED_INVOKE} name=\"get_weather\">\n\ + {DSML_SPACED_PARAM} name=\"city\" string=\"true\">Paris{DSML_SPACED_PARAM_END}\n\ + {DSML_SPACED_INVOKE_END}" + ); + let p = parse_dsml_tolerant(text.as_bytes(), &orders, true).expect("tolerant parse"); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].arguments, r#"{"city": "Paris"}"#); + assert_eq!(p.reasoning, b"check the weather tool"); + assert!( + p.content.ends_with(b"I'll check."), + "{:?}", + String::from_utf8_lossy(&p.content) + ); +} + +#[test] +fn parse_dsml_tolerant_refuses_a_block_in_unclosed_thinking() { + // No think close: the engine does no tool search at all -- the text is + // returned as reasoning with empty content (server_dsml_parse.c:299-313). + let orders = [ToolSchemaOrder { + name: "get_weather".into(), + prop: vec!["city".into()], + prop_type: vec!["string".into()], + ..Default::default() + }]; + let think_open = think_start(ChatFormat::DeepSeek); + let text = format!( + "{think_open}I'll emit the tool inside the open thinking\n\ + {DSML_SPACED_START}\n{DSML_SPACED_INVOKE} name=\"get_weather\">\n\ + {DSML_SPACED_PARAM} name=\"city\" string=\"true\">Paris{DSML_SPACED_PARAM_END}\n\ + {DSML_SPACED_INVOKE_END}\n{DSML_SPACED_END}" + ); + assert!(parse_dsml_tolerant(text.as_bytes(), &orders, true).is_none()); +} + +const DSML_SPACED_START: &str = "<|DSML| calls>"; +const DSML_SPACED_END: &str = ""; +const DSML_SPACED_INVOKE: &str = "<|DSML| invoke"; +const DSML_SPACED_INVOKE_END: &str = ""; +const DSML_SPACED_PARAM: &str = "<|DSML| parameter"; +const DSML_SPACED_PARAM_END: &str = ""; + +/// The engine's `test_parse_spaced_dsml_tool_calls` vector (4294136): the +/// official generation's block parses; values are raw (no entity unescape -- +/// "<b>" in the value stays exactly that). +#[test] +fn parse_dsml_spaced_official_block() { + let text = format!( + "Sure.\n\n{DSML_SPACED_START}\n{DSML_SPACED_INVOKE} name=\"bash\">\n\ + {DSML_SPACED_PARAM} name=\"command\" string=\"true\">echo '<b>' && ls{DSML_SPACED_PARAM_END}\n\ + {DSML_SPACED_PARAM} name=\"n\" string=\"false\">3{DSML_SPACED_PARAM_END}\n\ + {DSML_SPACED_INVOKE_END}\n{DSML_SPACED_END}" + ); + let p = parse_generated_message( + ModelSyntax::DeepSeek, + text.as_bytes(), + false, + ChatFormat::DeepSeek, + &[], + ); + assert!(p.ok); + assert_eq!(p.content, b"Sure."); + assert_eq!(p.calls.len(), 1); + assert_eq!(p.calls[0].name, "bash"); + assert_eq!( + p.calls[0].arguments, + "{\"command\": \"echo '<b>' && ls\", \"n\": 3}" + ); + assert!( + p.raw_dsml.starts_with(&format!("\n\n{DSML_SPACED_START}")), + "raw_dsml must carry the separator: {:?}", + p.raw_dsml + ); +} + +/// The render-side `<` guard before a closing tag inside a value is +/// restored; a value that genuinely contains "<" keeps it (the engine's +/// dsml_value_unescape, server_dsml_stream.c:37-53). The renderer writes +/// "<" + the closing tag minus its leading '<' (append_dsml_parameter_text). +#[test] +fn parse_dsml_spaced_value_guard_round_trip() { + let guard = format!("<{}", &DSML_SPACED_PARAM_END[1..]); + let text = format!( + "ok\n\n{DSML_SPACED_START}\n{DSML_SPACED_INVOKE} name=\"write\">\n\ + {DSML_SPACED_PARAM} name=\"body\" string=\"true\">x{guard}y & z{DSML_SPACED_PARAM_END}\n\ + {DSML_SPACED_INVOKE_END}\n{DSML_SPACED_END}" + ); + let p = parse_generated_message( + ModelSyntax::DeepSeek, + text.as_bytes(), + false, + ChatFormat::DeepSeek, + &[], + ); + assert_eq!(p.calls.len(), 1); + assert_eq!( + p.calls[0].arguments, + format!("{{\"body\": \"x{DSML_SPACED_PARAM_END}y & z\"}}") + ); +} diff --git a/crates/ds4-sys/Cargo.toml b/crates/ds4-sys/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..3a20f97196ae6943bcb439d813ac1135a7d05c5c --- /dev/null +++ b/crates/ds4-sys/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "ds4-sys" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "Unsafe FFI bindings to native/bridge/ds4_bridge.h" + +[lib] +path = "src/lib.rs" + +[dependencies] +libc = "0.2" diff --git a/crates/ds4-sys/src/lib.rs b/crates/ds4-sys/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..e5562b2c1b3f49184c6a9882714e2d02e25e2538 --- /dev/null +++ b/crates/ds4-sys/src/lib.rs @@ -0,0 +1,1383 @@ +//! The only unsafe Rust boundary onto the existing ds4 native runtime. +//! +//! Bindings cover `native/bridge/ds4_bridge.h` plus the platform libc numeric +//! parsers needed to preserve frozen C environment-variable semantics. Do not +//! bindgen `ds4.h`. + +#![allow(non_camel_case_types)] + +use std::ffi::CString; +use std::os::raw::{c_char, c_float, c_int, c_void}; +#[cfg(unix)] +use std::sync::atomic::{AtomicBool, Ordering}; +#[cfg(unix)] +use std::{io, net::TcpStream, os::fd::AsRawFd}; + +unsafe extern "C" { + fn atoi(value: *const c_char) -> c_int; + fn strtoull(value: *const c_char, end: *mut *mut c_char, base: c_int) -> u64; + fn atof(value: *const c_char) -> f64; + #[cfg(unix)] + fn signal(sig: c_int, handler: usize) -> usize; + #[cfg(unix)] + fn _exit(status: c_int) -> !; +} + +#[cfg(unix)] +static STOP_REQUESTED: AtomicBool = AtomicBool::new(false); + +#[cfg(unix)] +extern "C" fn stop_signal_handler(_signal: c_int) { + if STOP_REQUESTED.swap(true, Ordering::Relaxed) { + unsafe { _exit(130) } + } +} + +#[cfg(unix)] +pub fn install_stop_handlers() -> bool { + STOP_REQUESTED.store(false, Ordering::Relaxed); + unsafe { + let handler = stop_signal_handler as *const () as usize; + signal(2, handler) != usize::MAX && signal(15, handler) != usize::MAX + } +} + +#[cfg(not(unix))] +pub fn install_stop_handlers() -> bool { + true +} + +#[cfg(unix)] +pub fn stop_requested() -> bool { + STOP_REQUESTED.load(Ordering::Relaxed) +} + +#[cfg(not(unix))] +pub fn stop_requested() -> bool { + false +} + +pub fn libc_atoi(value: &[u8]) -> i32 { + CString::new(value).map_or(0, |value| unsafe { atoi(value.as_ptr()) }) +} + +pub fn libc_strtoull10(value: &[u8]) -> u64 { + CString::new(value).map_or(0, |value| unsafe { + strtoull(value.as_ptr(), std::ptr::null_mut(), 10) + }) +} + +pub fn libc_atof(value: &[u8]) -> f64 { + CString::new(value).map_or(0.0, |value| unsafe { atof(value.as_ptr()) }) +} + +#[cfg(unix)] +pub fn set_socket_send_buffer(stream: &TcpStream, bytes: i32) -> io::Result<()> { + if bytes <= 0 { + return Ok(()); + } + let rc = unsafe { + libc::setsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_SNDBUF, + (&bytes as *const i32).cast(), + std::mem::size_of::() as libc::socklen_t, + ) + }; + if rc == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +#[cfg(not(unix))] +pub fn set_socket_send_buffer(_stream: &std::net::TcpStream, _bytes: i32) -> std::io::Result<()> { + Ok(()) +} + +#[repr(C)] +pub struct ds4_bridge_model { + _opaque: [u8; 0], +} + +#[repr(C)] +pub struct ds4_bridge_session { + _opaque: [u8; 0], +} + +#[repr(C)] +pub struct ds4_bridge_snapshot { + _opaque: [u8; 0], +} + +#[repr(C)] +pub struct ds4_bridge_batch_ctx { + _opaque: [u8; 0], +} + +pub type ds4_bridge_prefill_fn = Option; + +/* V4.1 (ds41) callbacks (native/bridge/ds4_bridge.h): the rows provider fills + * the pinned raw rows for the exact block the forward runs (dst[k] is engram + * layer k's buffer); emit returns nonzero to stop; progress returns nonzero + * to abort the prefill. */ +pub type ds4_bridge_v41_rows_fn = + Option c_int>; +/* P2: the per-layer collect hook — the native forward calls it at each + * engram layer (the engine's v41_ejob_wait call site, core_v41_engram.c:441) + * and only then uploads the layer's rows; nonzero refuses the block. */ +pub type ds4_bridge_v41_wait_fn = Option c_int>; +/* The cross-block prefetch hook: the chunk loop declares the next prefill + * block before running the current one (ds41_forward.h's + * ds41_engram_feed.prefetch_next); nonzero refuses the run. */ +pub type ds4_bridge_v41_next_fn = + Option c_int>; +/* Join/drain all provider writes before native buffers are released, even + * after a forward error. Nonzero reports failure after writes have ended. */ +pub type ds4_bridge_v41_drain_fn = Option c_int>; +pub type ds4_bridge_v41_emit_fn = Option c_int>; +pub type ds4_bridge_v41_progress_fn = + Option c_int>; + +/* The decode-sampling face (native/bridge/ds4_bridge.h): the engine's + * ds4_decode_sampling layout (ds4_v41_api.h:69-75). All-zero = bare argmax; + * a NULL pointer to the setter resets to that. greedy_fn/greedy_ud: the + * engine's tool-syntax greedy hook (the engine's c6e2706; the Rust host + * answers through the ds4-server accumulator). */ +pub type ds4_bridge_v41_greedy_fn = + Option c_int>; +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub struct ds4_bridge_v41_sampling { + pub temperature: f32, + pub top_p: f32, + pub min_p: f32, + pub top_k: c_int, + pub seed: u64, + pub freq_penalty: f32, + pub presence_penalty: f32, + pub dry_multiplier: f32, + pub dry_base: f32, + pub dry_allowed_length: c_int, + pub greedy_fn: ds4_bridge_v41_greedy_fn, + pub greedy_ud: *mut c_void, +} + +/* The request's images (native/bridge/ds4_bridge.h; the engine's + * ds4_v41_images, core_v41.h:409-417). The Rust host owns preprocessing and + * prompt expansion; patches and row_of are borrowed for the duration of the + * generate call only. A text request passes a NULL pointer. */ +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_v41_image { + pub patches: *const f32, + pub patches_len: u64, + pub n_vit_h: u32, + pub n_vit_w: u32, + pub n_llm_h: u32, + pub n_llm_w: u32, +} +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_v41_images { + pub img: *const ds4_bridge_v41_image, + pub n: c_int, + pub row_of: *const i32, + pub sig_of: *const u64, + pub np: u32, + pub n_rows: u32, +} + +pub type ds4_bridge_backend = c_int; + +pub const DS4_BRIDGE_BACKEND_CUDA: ds4_bridge_backend = 0; +pub const DS4_BRIDGE_BACKEND_METAL: ds4_bridge_backend = 1; +pub const DS4_BRIDGE_BACKEND_CPU: ds4_bridge_backend = 2; + +pub const DS4_BRIDGE_DISTRIBUTED_NONE: c_int = 0; +pub const DS4_BRIDGE_DISTRIBUTED_COORDINATOR: c_int = 1; +pub const DS4_BRIDGE_DISTRIBUTED_WORKER: c_int = 2; + +pub const DS4_BRIDGE_MAX_DIMS: usize = 8; +pub const DS4_BRIDGE_MEMC_COUNT: usize = 17; +pub const DS4_BRIDGE_MEMD_COUNT: usize = 2; + +/// Which checkpoint a V4.1 disk payload carries; the native v41_kv_class +/// (ds4_ds41_kvreuse.inc), pinned by the C gate's static assert. +/// DS4_BRIDGE_V41_KV_CLASS_PROMPT is the reference's boundary checkpoint +/// (its reasons "cold" and "continued"); DS4_BRIDGE_V41_KV_CLASS_TURN is the +/// live frontier the run left, the class the reference stores under "evict" +/// and "shutdown". +pub const DS4_BRIDGE_V41_KV_CLASS_PROMPT: c_int = 0; +pub const DS4_BRIDGE_V41_KV_CLASS_TURN: c_int = 1; + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_token_score { + pub id: i32, + pub logit: c_float, + pub logprob: c_float, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_distributed_options { + pub role: i32, + pub layer_start: u32, + pub layer_end: u32, + pub has_output: i32, + pub listen_host: *const c_char, + pub listen_port: i32, + pub coordinator_host: *const c_char, + pub coordinator_port: i32, + pub prefill_chunk: u32, + pub prefill_window: u32, + pub activation_bits: u32, + pub replay_check: i32, + pub debug: i32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_spec_metrics { + pub drafts: u64, + pub hits: u64, + pub quench: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_mem_cell { + pub requested: u64, + pub committed: u64, + pub freed_requested: u64, + pub freed_committed: u64, + pub alloc_calls: u64, + pub free_calls: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_mem_census { + pub supported: i32, + pub faults: u64, + pub epoch: u64, + pub torn_fallbacks: u64, + pub cells: [[ds4_bridge_mem_cell; DS4_BRIDGE_MEMD_COUNT]; DS4_BRIDGE_MEMC_COUNT], +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_mem_observe { + pub status: i32, + pub source: i32, + pub free_bytes: u64, + pub total_bytes: u64, + pub cuda_free_bytes: u64, + pub meminfo_avail_bytes: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_qwen_image_input { + pub data: *const u8, + pub data_len: usize, + pub token_offset: u32, + pub grid_h: u32, + pub grid_w: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ds4_bridge_qwen_image_info { + pub source_width: u32, + pub source_height: u32, + pub resized_width: u32, + pub resized_height: u32, + pub grid_h: u32, + pub grid_w: u32, + pub token_count: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_vision_input { + pub data: *const u8, + pub data_len: usize, + pub token_offset: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_inkling_pixels { + pub pixels: *const f32, + pub pixel_count: u64, + pub token_offset: u32, + pub token_count: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_step37_pixels { + pub pixels: *const f32, + pub pixel_count: u64, + pub token_offset: u32, + pub token_count: u32, + pub edge: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_inkling_audio { + pub codes: *const i32, + pub code_count: u64, + pub token_offset: u32, + pub token_count: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ds4_bridge_vision_info { + pub source_width: u32, + pub source_height: u32, + pub content_width: u32, + pub content_height: u32, + pub padded_width: u32, + pub padded_height: u32, + pub grid_height: u32, + pub grid_width: u32, + pub token_count: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ds4_bridge_graph_fit_quote { + pub fits: i32, + pub fail_open: i32, + pub need_bytes: u64, + pub headroom_bytes: u64, + pub avail_bytes: u64, + pub deficit_bytes: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_bind_slot { + pub name: *const c_char, + pub required: u32, + pub ndim: u32, + pub dim: [u64; DS4_BRIDGE_MAX_DIMS], + pub r#type: u32, + pub rel_offset: u64, + pub abs_offset: u64, + pub bytes: u64, + pub shard: u32, + pub found: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_shard { + pub path: *const c_char, + pub size: u64, + pub base: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_bind_plan { + pub n_slots: u32, + pub slots: *const ds4_bridge_bind_slot, + pub n_shards: u32, + pub shards: *const ds4_bridge_shard, + pub data_pos: u64, + pub alignment: u64, + pub page: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_model_open_options { + pub model_path: *const c_char, + pub vision_path: *const c_char, + pub backend: c_int, + pub n_threads: c_int, + pub defer_boot_prewarm: c_int, + pub power_percent: i32, + pub warm_weights: i32, + pub quality: i32, + pub ssd_streaming: i32, + pub ssd_streaming_cold: i32, + pub ssd_streaming_cache_experts: u32, + pub ssd_streaming_cache_bytes: u64, + pub plan: *const ds4_bridge_bind_plan, + pub tensors: *const ds4_host_tensor_dir, + pub shape: *const ds4_host_shape, + pub vocab: *const ds4_host_vocab, + pub bind: *const ds4_host_bind_map, + pub mtp_path: *const c_char, + pub dspark_path: *const c_char, + pub mtp_bind: *const ds4_host_bind_map, + pub dspark_bind: *const ds4_host_bind_map, + pub mtp_draft_tokens: i32, + pub mtp_margin: c_float, + pub directional_steering_file: *const c_char, + pub directional_steering_attn: c_float, + pub directional_steering_ffn: c_float, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_host_shape { + pub variant: u32, + pub n_compress: u32, + pub compress: *const u32, + /* DeepSeek V4.1 wiring; mirrors native/bridge/ds4_host_load.h. Borrowed, + * valid for the open call; NULL/0 for every other variant. */ + pub v41_kv_source: *const u8, + pub v41_index_source: *const u8, + pub v41_kv_source_of: *const i16, + pub v41_index_source_of: *const i16, + pub v41_engram_index_of: *const i16, + pub v41_n_engram: u32, + pub v41_engram_layers: *const i32, + pub v41_engram_max_ngram: u32, + pub v41_engram_heads: u32, + pub v41_engram_head_dim: u32, + pub v41_engram_pad: u32, + pub v41_candidate_source_layer: i32, + pub v41_candidate_topk_blocks: i32, + pub v41_candidate_block_size: i32, + pub v41_mtp_towers: u32, + pub v41_mtp_experts: u32, + /* P4-3: per-engram-layer table wiring (row count, the two plane offsets + * in the shard, the shard path). Borrowed for the open call. */ + pub v41_engram_rows: *const u64, + pub v41_engram_weight_off: *const u64, + pub v41_engram_scale_off: *const u64, + pub v41_engram_table_path: *const *const c_char, + /* P5: the draft parameters; block == 0 = the drafter stays unarmed and + * the rest is ignored. v41_mtp_target is borrowed, v41_n_mtp_target + * entries. */ + pub v41_mtp_block: u32, + pub v41_mtp_used: u32, + pub v41_mtp_noise_id: u32, + pub v41_mtp_markov_rank: u32, + pub v41_n_mtp_target: u32, + pub v41_mtp_target: *const i16, + /* P5.4: context length + the engram vocab pair (deepseek4.context_length, + * .engram.vocab_size, .engram.compressed_vocab_size; the native refuses + * ctx == 0). */ + pub v41_ctx: u32, + pub v41_engram_vocab: u32, + pub v41_engram_cvocab: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_host_str { + pub ptr: *const c_char, + pub len: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_host_vocab { + pub n_vocab: u32, + pub tokens: *const ds4_host_str, + pub n_merges: u32, + pub merges: *const ds4_host_str, + pub n_user_defined: u32, + pub user_defined: *const i32, + pub user_defined_max_len: u32, + pub user_defined_first: [u8; 256], + pub motif3_added_first: [u8; 256], + pub bos_id: i32, + pub eos_id: i32, + pub system_id: i32, + pub eot_id: i32, + pub im_start_id: i32, + pub im_content_id: i32, + pub im_end_id: i32, + pub user_id: i32, + pub assistant_id: i32, + pub start_of_turn_id: i32, + pub end_of_turn_id: i32, + pub tool_id: i32, + pub reference_id: i32, + pub plan_start_id: i32, + pub plan_end_id: i32, + pub observation_id: i32, + pub sop_id: i32, + pub think_start_id: i32, + pub think_end_id: i32, + pub tool_call_start_id: i32, + pub tool_call_end_id: i32, + pub tool_response_start_id: i32, + pub tool_response_end_id: i32, + pub arg_key_start_id: i32, + pub arg_key_end_id: i32, + pub arg_value_start_id: i32, + pub latent_start_id: i32, + pub latent_pad_id: i32, + pub latent_end_id: i32, + pub tool_schema_start_id: i32, + pub tool_schema_end_id: i32, + pub dsml_id: i32, + pub image_id: i32, + pub dots3_endofsystem_id: i32, + pub dots3_endofuser_id: i32, + pub dots3_endoftext_id: i32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_host_bind_look { + pub name: *const c_char, + pub required: u32, + pub found: u32, + pub index: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_host_bind_map { + pub n: u32, + pub v: *const ds4_host_bind_look, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_host_tensor { + pub name: *const c_char, + pub ndim: u32, + pub dim: [u64; DS4_BRIDGE_MAX_DIMS], + pub r#type: u32, + pub rel_offset: u64, + pub abs_offset: u64, + pub bytes: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_host_shard { + pub path: *const c_char, + pub size: u64, + pub base: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_host_tensor_dir { + pub n: u32, + pub v: *const ds4_host_tensor, + pub data_pos: u64, + pub alignment: u64, + pub n_shards: u32, + pub shards: *const ds4_host_shard, +} + +extern "C" { + pub fn ds4_bridge_bind_plan_check( + plan: *const ds4_bridge_bind_plan, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_bind_plan_match( + host: *const ds4_bridge_bind_plan, + native: *const ds4_bridge_bind_plan, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_model_open( + out: *mut *mut ds4_bridge_model, + opt: *const ds4_bridge_model_open_options, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_model_open_distributed( + out: *mut *mut ds4_bridge_model, + opt: *const ds4_bridge_model_open_options, + distributed: *const ds4_bridge_distributed_options, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_model_run_distributed_worker( + m: *mut ds4_bridge_model, + ctx_size: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_model_boot_prewarm(m: *mut ds4_bridge_model); + + pub fn ds4_bridge_model_free(m: *mut ds4_bridge_model); + + pub fn ds4_bridge_drafter_shared(m: *mut ds4_bridge_model) -> c_int; + + pub fn ds4_bridge_session_create( + out: *mut *mut ds4_bridge_session, + m: *mut ds4_bridge_model, + ctx_size: c_int, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_free(s: *mut ds4_bridge_session); + + pub fn ds4_bridge_session_sync( + s: *mut ds4_bridge_session, + tokens: *const i32, + n_tokens: c_int, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_model_vision_probe( + m: *mut ds4_bridge_model, + data: *const u8, + data_len: usize, + info: *mut ds4_bridge_vision_info, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_sync_vision( + s: *mut ds4_bridge_session, + tokens: *const i32, + n_tokens: c_int, + images: *const ds4_bridge_vision_input, + image_count: u32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_sync_step37( + s: *mut ds4_bridge_session, + tokens: *const i32, + n_tokens: c_int, + crops: *const ds4_bridge_step37_pixels, + crop_count: u32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_sync_inkling( + s: *mut ds4_bridge_session, + tokens: *const i32, + n_tokens: c_int, + images: *const ds4_bridge_inkling_pixels, + image_count: u32, + audios: *const ds4_bridge_inkling_audio, + audio_count: u32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_sync_cb( + s: *mut ds4_bridge_session, + tokens: *const i32, + n_tokens: c_int, + progress: ds4_bridge_prefill_fn, + ud: *mut c_void, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_eval( + s: *mut ds4_bridge_session, + token: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_inkling_trial( + s: *mut ds4_bridge_session, + first: i32, + max_tokens: i32, + tokens: *mut i32, + target: *mut i32, + cap: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_glm53_trial( + s: *mut ds4_bridge_session, + first: i32, + max_tokens: i32, + tokens: *mut i32, + target: *mut i32, + cap: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_glm53_commit( + s: *mut ds4_bridge_session, + keep: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_inkling_commit( + s: *mut ds4_bridge_session, + keep: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_step37_trial( + s: *mut ds4_bridge_session, + first: i32, + max_tokens: i32, + tokens: *mut i32, + target: *mut i32, + cap: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_step37_commit( + s: *mut ds4_bridge_session, + keep: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_naive_trial( + s: *mut ds4_bridge_session, + first: i32, + max_tokens: i32, + tokens: *mut i32, + target: *mut i32, + cap: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_naive_commit( + s: *mut ds4_bridge_session, + keep: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_beam_trial( + s: *mut ds4_bridge_session, + first: i32, + max_tokens: i32, + tokens: *mut i32, + target: *mut i32, + cap: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_beam_commit( + s: *mut ds4_bridge_session, + keep: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_iquest_trial( + s: *mut ds4_bridge_session, + first: i32, + max_tokens: i32, + tokens: *mut i32, + target: *mut i32, + cap: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_iquest_commit( + s: *mut ds4_bridge_session, + keep: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_mimo2_trial( + s: *mut ds4_bridge_session, + first: i32, + max_tokens: i32, + tokens: *mut i32, + target: *mut i32, + cap: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_mimo2_commit( + s: *mut ds4_bridge_session, + keep: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_mimo2_encode_vision( + s: *mut ds4_bridge_session, + patches: *const f32, + n_patches: u32, + grid_h: u32, + grid_w: u32, + out: *mut f32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_mimo2_encode_audio( + s: *mut ds4_bridge_session, + mel: *const f32, + frames: u32, + out: *mut f32, + cap: u32, + rows: *mut u32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_mimo2_sync_media( + s: *mut ds4_bridge_session, + tokens: *const i32, + n_tokens: i32, + starts: *const u32, + counts: *const u32, + rows: *const *const f32, + nspans: u32, + tag: u64, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_dots3_enabled(s: *mut ds4_bridge_session) -> c_int; + + pub fn ds4_bridge_dots3_trial( + s: *mut ds4_bridge_session, + first: i32, + max_tokens: i32, + tokens: *mut i32, + target: *mut i32, + cap: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_dots3_commit( + s: *mut ds4_bridge_session, + keep: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_eval_speculative_argmax( + s: *mut ds4_bridge_session, + first_token: i32, + max_tokens: i32, + eos_token: i32, + accepted: *mut i32, + accepted_cap: i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + /* V4.1 (ds41) run surface (P5): the engine's one-shot generate with the + * prefill-progress and emit callbacks, plus the three process switches. + * All callbacks run on the calling thread. */ + pub fn ds4_bridge_v41_set_dspark(mode: c_int); + pub fn ds4_bridge_v41_set_graph(on: c_int); + pub fn ds4_bridge_v41_set_emit_trace(on: c_int); + pub fn ds4_bridge_v41_set_prof(on: c_int); + /* The V4.1 KV prefix reuse (U-B): --no-kv-reuse, the last run's restored + * token count, the peek that seeds the host-side engram history, and the + * mounted-amplifier identity the native drops the checkpoint on. */ + pub fn ds4_bridge_v41_kvreuse_set(on: c_int); + pub fn ds4_bridge_v41_kvreuse_last_reused() -> c_int; + pub fn ds4_bridge_v41_kvreuse_peek(prompt: *const i32, n: c_int, sig: *const u64) -> u32; + pub fn ds4_bridge_v41_kvreuse_note_mount(has_mount: c_int, hash: u64); + /* T's key (the prompt it was saved with, then the ids the state + * committed after it), which is the host store's turn entry text. Answers + * the count, filling `out` when it covers `cap`; 0 = no turn checkpoint. */ + pub fn ds4_bridge_v41_kvreuse_turn_key(out: *mut i32, cap: u32) -> u32; + /* The disk half (item 4): the payload export/import the Rust store + * (crates/ds4-kv) drives; both are gated by --no-kv-reuse and the class's + * slot being valid. cls is DS4_BRIDGE_V41_KV_CLASS_* for the export; the + * import takes the class from the payload's own meta. */ + pub fn ds4_bridge_v41_kvreuse_export(path: *const c_char, cls: c_int) -> c_int; + pub fn ds4_bridge_v41_kvreuse_import( + path: *const c_char, + offset: u64, + bytes: u64, + tokens: *const i32, + n: u32, + ) -> c_int; + pub fn ds4_bridge_v41_last_spec_stats( + rounds: *mut c_int, + offered: *mut c_int, + accepted: *mut c_int, + ); + /* The per-request decode-sampling face (the engine's + * ds4_engine_set_decode_sampling, core_v41_api.c:10-13). NULL resets to + * the all-zero face = bare argmax. */ + pub fn ds4_bridge_v41_set_sampling(sp: *const ds4_bridge_v41_sampling); + /* --mem-budget-mb (unit Z): the engine's budget global + L1 load gate + * (core_util.c:198-256); call before opening the model. */ + pub fn ds4_bridge_set_mem_budget_mb(mb: c_int); + /* The V4.1 zchain sidecar mount (unit Z): the device side the Rust + * reader (zchain.rs, P2/P3) drives, 1:1 with v41_amp_load's per-state + * writes (core_v41_amp.c:227-270). All return 1 on success, 0 on + * refusal; rb_clear is the engine's process-wide unload first + * (core_v41_amp.c:114). */ + pub fn ds4_bridge_v41_zchain_gr_set( + m: *mut ds4_bridge_model, + il: u32, + data: *const f32, + n_expert: u32, + n_embd: u32, + ) -> c_int; + pub fn ds4_bridge_v41_zchain_rb_set( + m: *mut ds4_bridge_model, + il: u32, + delta: *const f32, + n_expert: u32, + ) -> c_int; + pub fn ds4_bridge_v41_zchain_rb_clear(); + /* The serving admission's byte account (server_sched_v41.c:58). */ + pub fn ds4_bridge_v41_prefill_bytes(m: *mut ds4_bridge_model, n_prompt: c_int) -> u64; + pub fn ds4_bridge_v41_images_bytes(images: *const ds4_bridge_v41_images) -> u64; + /* The golden gate's tower entry (core_v41_vision_gate.c:45/80). */ + pub fn ds4_bridge_v41_has_vision(m: *mut ds4_bridge_model) -> c_int; + pub fn ds4_bridge_v41_vision_run( + m: *mut ds4_bridge_model, + patches: *const f32, + patches_len: u64, + n_vit_h: u32, + n_vit_w: u32, + n_llm_h: *mut u32, + n_llm_w: *mut u32, + ) -> *mut f32; + pub fn ds4_bridge_v41_vision_rows_free(rows: *mut f32); + pub fn ds4_bridge_v41_generate( + m: *mut ds4_bridge_model, + prompt: *const i32, + n_prompt: c_int, + n_predict: c_int, + no_engram: c_int, + verify_k: c_int, + images: *const ds4_bridge_v41_images, + rows: ds4_bridge_v41_rows_fn, + rows_ud: *mut c_void, + wait: ds4_bridge_v41_wait_fn, + next: ds4_bridge_v41_next_fn, + drain: ds4_bridge_v41_drain_fn, + emit: ds4_bridge_v41_emit_fn, + emit_ud: *mut c_void, + progress: ds4_bridge_v41_progress_fn, + progress_ud: *mut c_void, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_eval_layer_slice( + s: *mut ds4_bridge_session, + tokens: *const i32, + n_tokens: u32, + pos0: u32, + layer_start: u32, + layer_end: u32, + input_hc: *const f32, + output_hc: *mut f32, + output_logits: i32, + logits: *mut f32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_layer_slice_reset( + s: *mut ds4_bridge_session, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_argmax(s: *mut ds4_bridge_session) -> c_int; + + pub fn ds4_bridge_session_argmax_excluding( + s: *mut ds4_bridge_session, + excluded_id: i32, + ) -> c_int; + + pub fn ds4_bridge_session_pos(s: *mut ds4_bridge_session) -> c_int; + + pub fn ds4_bridge_session_ctx(s: *mut ds4_bridge_session) -> c_int; + + pub fn ds4_bridge_session_graph_pending(s: *mut ds4_bridge_session) -> c_int; + + pub fn ds4_bridge_session_power(s: *mut ds4_bridge_session) -> c_int; + + pub fn ds4_bridge_session_set_power(s: *mut ds4_bridge_session, power_percent: c_int) -> c_int; + + pub fn ds4_bridge_session_rewind(s: *mut ds4_bridge_session, pos: c_int); + + pub fn ds4_bridge_session_invalidate(s: *mut ds4_bridge_session); + + pub fn ds4_bridge_session_generation(s: *mut ds4_bridge_session) -> u64; + + pub fn ds4_bridge_session_prefill_cap(s: *mut ds4_bridge_session) -> c_int; + + pub fn ds4_bridge_session_exaone_rewind_span(s: *mut ds4_bridge_session) -> c_int; + + pub fn ds4_bridge_session_distributed_route_ready( + s: *mut ds4_bridge_session, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_sample( + s: *mut ds4_bridge_session, + temperature: c_float, + top_k: c_int, + top_p: c_float, + min_p: c_float, + rng: *mut u64, + ) -> c_int; + + pub fn ds4_bridge_session_sample_excluding( + s: *mut ds4_bridge_session, + temperature: c_float, + top_k: c_int, + top_p: c_float, + min_p: c_float, + rng: *mut u64, + excluded_id: c_int, + ) -> c_int; + + pub fn ds4_bridge_session_save_payload( + s: *mut ds4_bridge_session, + path: *const c_char, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_load_payload( + s: *mut ds4_bridge_session, + path: *const c_char, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_load_payload_range( + s: *mut ds4_bridge_session, + path: *const c_char, + offset: u64, + length: u64, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_save_layer_payload( + s: *mut ds4_bridge_session, + path: *const c_char, + layer_start: u32, + layer_end: u32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_load_layer_payload( + s: *mut ds4_bridge_session, + path: *const c_char, + payload_bytes: u64, + tokens: *const i32, + n_tokens: u32, + layer_start: u32, + layer_end: u32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_snapshot_create( + out: *mut *mut ds4_bridge_snapshot, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_snapshot_free(snap: *mut ds4_bridge_snapshot); + + pub fn ds4_bridge_snapshot_len(snap: *const ds4_bridge_snapshot) -> u64; + + pub fn ds4_bridge_session_save_snapshot( + s: *mut ds4_bridge_session, + snap: *mut ds4_bridge_snapshot, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_load_snapshot( + s: *mut ds4_bridge_session, + snap: *const ds4_bridge_snapshot, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_tokenize_text( + m: *mut ds4_bridge_model, + text: *const c_char, + out: *mut i32, + cap: c_int, + n_out: *mut c_int, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_tokenize_rendered_chat( + m: *mut ds4_bridge_model, + text: *const c_char, + out: *mut i32, + cap: c_int, + n_out: *mut c_int, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_token_text( + m: *mut ds4_bridge_model, + token: i32, + out: *mut c_char, + cap: usize, + n_out: *mut usize, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_token_eos(m: *mut ds4_bridge_model) -> c_int; + + pub fn ds4_bridge_token_is_stop(m: *mut ds4_bridge_model, token: i32) -> c_int; + + pub fn ds4_bridge_model_id(m: *mut ds4_bridge_model) -> c_int; + + pub fn ds4_bridge_model_routed_quant_bits(m: *mut ds4_bridge_model) -> c_int; + + pub fn ds4_bridge_session_graph_fit_quote( + m: *mut ds4_bridge_model, + ctx_size: c_int, + q: *mut ds4_bridge_graph_fit_quote, + ) -> c_int; + + pub fn ds4_bridge_encode_chat_prompt( + m: *mut ds4_bridge_model, + system: *const c_char, + prompt: *const c_char, + think_mode: c_int, + out: *mut i32, + cap: c_int, + n_out: *mut c_int, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_copy_logits( + s: *mut ds4_bridge_session, + out: *mut c_float, + cap: c_int, + ) -> c_int; + + pub fn ds4_bridge_session_output_head_bench( + s: *mut ds4_bridge_session, + iters: c_int, + path: *const c_char, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_session_top_logprobs( + s: *mut ds4_bridge_session, + out: *mut ds4_bridge_token_score, + k: c_int, + ) -> c_int; + + pub fn ds4_bridge_mem_census_snap(out: *mut ds4_bridge_mem_census) -> c_int; + + pub fn ds4_bridge_spec_snapshot(out: *mut ds4_bridge_spec_metrics); + + pub fn ds4_bridge_mem_observe_snap(out: *mut ds4_bridge_mem_observe) -> c_int; + + pub fn ds4_bridge_mem_substrate_outstanding() -> u64; + + pub fn ds4_bridge_qwen_image_probe( + data: *const u8, + data_len: usize, + info: *mut ds4_bridge_qwen_image_info, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_qwen_image_pixel_hash( + data: *const u8, + data_len: usize, + hash: *mut u64, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_batch_ctx_create_fit( + m: *mut ds4_bridge_model, + ctx_size: c_int, + max_seq: c_int, + max_total_tokens: c_int, + out: *mut *mut ds4_bridge_batch_ctx, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_batch_ctx_destroy(c: *mut ds4_bridge_batch_ctx); + + pub fn ds4_bridge_batch_ctx_max_seq(c: *mut ds4_bridge_batch_ctx) -> c_int; + + pub fn ds4_bridge_batch_ctx_raw_cap(c: *mut ds4_bridge_batch_ctx) -> c_int; + + pub fn ds4_bridge_batch_ctx_seq_cap(c: *mut ds4_bridge_batch_ctx) -> c_int; + + pub fn ds4_bridge_batch_ctx_supports_partial_reuse(c: *mut ds4_bridge_batch_ctx) -> c_int; + + pub fn ds4_bridge_batch_ctx_trim_free(c: *mut ds4_bridge_batch_ctx, want_bytes: u64) -> u64; + + pub fn ds4_bridge_batch_ctx_generate_static( + c: *mut ds4_bridge_batch_ctx, + prompt_tokens: *const *const i32, + prompt_lengths: *const i32, + max_new_tokens: *const i32, + eos_ids: *const i32, + n: i32, + out_tokens: *mut i32, + out_tokens_cap: i32, + out_lengths: *mut i32, + out_finish: *mut i32, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_batch_ctx_bank_snapshot( + c: *mut ds4_bridge_batch_ctx, + bank: i32, + tokens: *mut i32, + cap: i32, + n_tokens: *mut i32, + generation: *mut u64, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_batch_ctx_bank_save_payload( + c: *mut ds4_bridge_batch_ctx, + bank: i32, + path: *const c_char, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_batch_ctx_bank_load_payload_range( + c: *mut ds4_bridge_batch_ctx, + bank: i32, + path: *const c_char, + offset: u64, + length: u64, + err: *mut c_char, + errlen: usize, + ) -> c_int; + + pub fn ds4_bridge_continuous_generate( + c: *mut ds4_bridge_batch_ctx, + admit: Option c_int>, + on_token: Option c_int>, + on_done: Option< + unsafe extern "C" fn( + *mut c_void, + *mut c_void, + *const i32, + i32, + i32, + *const ds4_bridge_cont_stats, + ), + >, + ud: *mut c_void, + err: *mut c_char, + errlen: usize, + ) -> c_int; +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct ds4_bridge_cont_stats { + pub decode_ms: f64, + pub decode_tokens: u32, + pub decode_steps: u32, + /// Draft rows this sequence ran; 0 is ordinary decode. + pub spec_drafts: u64, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ds4_bridge_cont_request { + pub tokens: *const i32, + pub n: i32, + pub images: [ds4_bridge_qwen_image_input; 4], + pub image_count: u32, + pub max_new: i32, + pub eos: i32, + pub user: *mut c_void, + pub temperature: c_float, + pub top_k: i32, + pub top_p: c_float, + pub min_p: c_float, + pub seed: u64, + pub sample_override: Option c_int>, + pub sample_exclude: Option c_int>, + pub step_accept: Option c_int>, + pub alive: Option c_int>, + pub on_admitted: + Option c_int>, + pub place_bank: i32, + pub n_cached: i32, + pub bank_used: *mut i32, + pub fork_bank: i32, + pub checkpoint_at: i32, + pub on_checkpoint: Option, +} + +#[cfg(all(test, unix))] +mod socket_tests { + use super::set_socket_send_buffer; + use std::net::{TcpListener, TcpStream}; + use std::os::fd::AsRawFd; + + #[test] + fn send_buffer_adapter_accepts_a_connected_socket() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + + set_socket_send_buffer(&server, 32 * 1024).unwrap(); + let mut actual = 0i32; + let mut len = std::mem::size_of::() as libc::socklen_t; + let rc = unsafe { + libc::getsockopt( + server.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_SNDBUF, + (&mut actual as *mut i32).cast(), + &mut len, + ) + }; + assert_eq!(rc, 0); + assert!(actual >= 32 * 1024, "SO_SNDBUF={actual}"); + drop(client); + } +} diff --git a/crates/ds4-web/Cargo.toml b/crates/ds4-web/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..70a2d71e22e8dad4b098a93aef68bd972e3987af --- /dev/null +++ b/crates/ds4-web/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ds4-web" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true +description = "Agent web utility (Phase 5). Blocking I/O; no Tokio." + +[lib] +path = "src/lib.rs" diff --git a/crates/ds4-web/src/cdp.rs b/crates/ds4-web/src/cdp.rs new file mode 100644 index 0000000000000000000000000000000000000000..a9387eb5ac63351b7567e3285a616c5142434efa --- /dev/null +++ b/crates/ds4-web/src/cdp.rs @@ -0,0 +1,143 @@ +//! Chrome DevTools Protocol over the C WebSocket client. + +use std::thread; +use std::time::Duration; + +use crate::encode::{json_get_string, json_id_matches}; +use crate::js; +use crate::net::Ws; +use crate::wire::{cdp_request, create_target_params, eval_params, navigate_params}; + +pub fn cdp_call(ws: &mut Ws, method: &str, params: Option<&str>) -> Result { + let id = ws.next_id; + ws.next_id += 1; + let wire = cdp_request(id, method, params); + ws.send_text(&wire)?; + loop { + let msg = ws.read_message()?; + if json_id_matches(&msg, id) { + return Ok(msg); + } + } +} + +pub fn cdp_call_optional(ws: &mut Ws, method: &str, params: &str) { + let _ = cdp_call(ws, method, Some(params)); +} + +pub fn cdp_eval_string(ws: &mut Ws, expr: &str) -> Result { + let params = eval_params(expr); + let resp = cdp_call(ws, "Runtime.evaluate", Some(¶ms))?; + if resp.contains("\"exceptionDetails\"") { + return Err("JavaScript evaluation failed".into()); + } + json_get_string(&resp, "value").ok_or_else(|| "Runtime.evaluate did not return a string".into()) +} + +pub fn wait_ready(ws: &mut Ws) -> Result<(), String> { + for _ in 0..80 { + match cdp_eval_string(ws, js::READY_STATE) { + Ok(state) if state == "complete" || state == "interactive" => { + thread::sleep(Duration::from_millis(800)); + return Ok(()); + } + _ => thread::sleep(Duration::from_millis(250)), + } + } + Ok(()) +} + +pub fn cdp_navigate(ws: &mut Ws, url: &str) -> Result<(), String> { + let params = navigate_params(url); + cdp_call(ws, "Page.navigate", Some(¶ms))?; + Ok(()) +} + +fn page_probe(ws: &mut Ws) -> Result<(String, String, i64), String> { + let probe = cdp_eval_string(ws, js::PAGE_PROBE)?; + let mut lines = probe.splitn(3, '\n'); + let href = lines + .next() + .ok_or_else(|| "page readiness probe returned malformed data".to_string())?; + let ready = lines + .next() + .ok_or_else(|| "page readiness probe returned malformed data".to_string())?; + let rest = lines + .next() + .ok_or_else(|| "page readiness probe returned malformed data".to_string())?; + let text_len = rest.parse::().unwrap_or_else(|_| { + rest.split(|c: char| !c.is_ascii_digit() && c != '-') + .next() + .and_then(|s| s.parse().ok()) + .unwrap_or(0) + }); + Ok((href.to_string(), ready.to_string(), text_len)) +} + +pub fn wait_navigated_ready(ws: &mut Ws) -> Result<(), String> { + let mut last_len: i64 = -1; + let mut stable = 0; + let mut saw_real_url = false; + for i in 0..100 { + let (href, ready, text_len) = match page_probe(ws) { + Ok(v) => v, + Err(_) => { + thread::sleep(Duration::from_millis(250)); + continue; + } + }; + let real_url = !href.is_empty() && href != "about:blank" && !href.starts_with("chrome://"); + let ready_state = ready == "complete" || ready == "interactive"; + if real_url { + saw_real_url = true; + } + if text_len > 0 && text_len == last_len { + stable += 1; + } else { + stable = 0; + } + last_len = text_len; + if saw_real_url && ready_state && text_len > 0 && stable >= 2 { + thread::sleep(Duration::from_millis(500)); + return Ok(()); + } + if saw_real_url && ready_state && i >= 24 { + return Ok(()); + } + thread::sleep(Duration::from_millis(250)); + } + Ok(()) +} + +pub fn cdp_prepare_page(ws: &mut Ws) -> Result<(), String> { + cdp_call(ws, "Page.enable", Some("{}"))?; + cdp_call(ws, "Runtime.enable", Some("{}"))?; + cdp_call_optional( + ws, + "Emulation.setFocusEmulationEnabled", + "{\"enabled\":true}", + ); + cdp_call_optional( + ws, + "Emulation.setDeviceMetricsOverride", + "{\"width\":1365,\"height\":900,\"deviceScaleFactor\":1,\"mobile\":false}", + ); + wait_ready(ws) +} + +pub fn scroll_dynamic_page(ws: &mut Ws) { + let _ = cdp_eval_string(ws, js::SCROLL_DYNAMIC); +} + +pub fn create_target(ws: &mut Ws, url: &str) -> Result { + let params = create_target_params(url); + let resp = cdp_call(ws, "Target.createTarget", Some(¶ms))?; + json_get_string(&resp, "targetId") + .ok_or_else(|| "Chrome did not return a page target id".into()) +} + +pub fn browser_ws_url(port: i32) -> Result { + let body = crate::net::http_local("GET", port, "/json/version")?; + json_get_string(&body, "webSocketDebuggerUrl") + .ok_or_else(|| "Chrome did not return a browser WebSocket URL".into()) +} diff --git a/crates/ds4-web/src/chrome.rs b/crates/ds4-web/src/chrome.rs new file mode 100644 index 0000000000000000000000000000000000000000..d0843f74e2ce7de5e88b83cbaa251167c8f43e05 --- /dev/null +++ b/crates/ds4-web/src/chrome.rs @@ -0,0 +1,192 @@ +//! Chrome discovery and spawn. Matches `ds4_web.c`; does not kill the +//! process when the `Web` handle is dropped. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; + +use crate::net::http_local; + +pub const DEFAULT_PORT: i32 = 9333; +pub const CONFIRM_PROMPT: &str = + "The web tool wants to start a visible Chrome browser. Allow? (y/n) "; + +const LINUX_PATHS: &[&str] = &[ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + "/opt/google/chrome/chrome", +]; + +const PATH_NAMES: &[&str] = &[ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", +]; + +pub fn profile_dir(home: &Path) -> PathBuf { + home.join(".ds4").join("browser") +} + +pub fn effective_uid_is_root() -> bool { + if let Ok(status) = fs::read_to_string("/proc/self/status") { + for line in status.lines() { + if let Some(rest) = line.strip_prefix("Uid:") { + let mut fields = rest.split_whitespace(); + let _real = fields.next(); + if let Some(effective) = fields.next() { + return effective == "0"; + } + } + } + } + false +} + +pub fn chrome_executable() -> PathBuf { + if let Ok(env) = std::env::var("DS4_CHROME") { + if !env.is_empty() { + return PathBuf::from(env); + } + } + #[cfg(target_os = "macos")] + { + let mac = [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + ]; + for p in mac { + if is_executable(p) { + return PathBuf::from(p); + } + } + } + for p in LINUX_PATHS { + if is_executable(p) { + return PathBuf::from(p); + } + } + if let Ok(pathenv) = std::env::var("PATH") { + for dir in pathenv.split(':') { + let dir = if dir.is_empty() { "." } else { dir }; + for name in PATH_NAMES { + let candidate = Path::new(dir).join(name); + if is_executable(&candidate) { + return candidate; + } + } + } + } + PathBuf::from("google-chrome") +} + +fn is_executable(path: impl AsRef) -> bool { + let path = path.as_ref(); + if !path.is_file() { + return false; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path) + .map(|m| m.permissions().mode() & 0o111 != 0) + .unwrap_or(false) + } + #[cfg(not(unix))] + { + true + } +} + +pub fn linux_chrome_args(port: i32, profile: &Path, root: bool) -> Vec { + let mut args = vec![ + format!("--remote-debugging-port={port}"), + "--remote-allow-origins=*".into(), + format!("--user-data-dir={}", profile.display()), + "--no-first-run".into(), + "--no-default-browser-check".into(), + "--disable-sync".into(), + "--password-store=basic".into(), + ]; + if root { + args.push("--no-sandbox".into()); + } + args.push("--mute-audio".into()); + args.push("about:blank".into()); + args +} + +pub fn mkdir_p(path: &Path) -> Result<(), String> { + let mut cur = PathBuf::new(); + for part in path.components() { + cur.push(part); + if cur.as_os_str().is_empty() || cur == Path::new("/") { + continue; + } + match fs::create_dir(&cur) { + Ok(()) => { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = fs::set_permissions(&cur, fs::Permissions::from_mode(0o700)); + } + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => { + return Err(format!( + "failed to create Chrome profile dir {}: {e}", + path.display() + )); + } + } + } + Ok(()) +} + +pub fn cdp_alive(port: i32) -> bool { + match http_local("GET", port, "/json/version") { + Ok(body) => body.contains("webSocketDebuggerUrl"), + Err(_) => false, + } +} + +pub fn spawn_chrome(port: i32, profile: &Path) -> Result { + mkdir_p(profile)?; + let exe = chrome_executable(); + let args = linux_chrome_args(port, profile, effective_uid_is_root()); + #[cfg(target_os = "macos")] + { + let _ = &args; + // macOS `open -na` path is compiled but DFM gate is Linux. + } + Command::new(&exe) + .args(&args) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| format!("failed to fork Chrome: {e}")) +} + +pub fn wait_cdp_ready(port: i32, child: &mut Child) -> Result<(), String> { + for _ in 0..80 { + if cdp_alive(port) { + return Ok(()); + } + match child.try_wait() { + Ok(Some(_)) => { + return Err("Chrome exited before CDP became ready".into()); + } + _ => std::thread::sleep(std::time::Duration::from_millis(250)), + } + } + Err(format!("Chrome did not expose CDP on port {port}")) +} + +/// Forget a live child so `Drop` does not wait for Chrome (C never reaps +/// on `ds4_web_free`). +pub fn detach(child: Child) { + std::mem::forget(child); +} diff --git a/crates/ds4-web/src/encode.rs b/crates/ds4-web/src/encode.rs new file mode 100644 index 0000000000000000000000000000000000000000..cecb5a0048ad65aa9123b2638adb8e83c654b848 --- /dev/null +++ b/crates/ds4-web/src/encode.rs @@ -0,0 +1,300 @@ +//! Byte-identical ports of the encode/JSON helpers in `ds4_web.c`. + +pub fn url_encode(s: &str) -> String { + url_encode_bytes(s.as_bytes()) +} + +pub fn url_encode_bytes(s: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + let mut out = String::new(); + for &c in s { + if c.is_ascii_alphanumeric() || c == b'-' || c == b'_' || c == b'.' || c == b'~' { + out.push(c as char); + } else { + out.push('%'); + out.push(HEX[(c >> 4) as usize] as char); + out.push(HEX[(c & 15) as usize] as char); + } + } + out +} + +pub fn base64(data: &[u8]) -> String { + const TAB: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::new(); + let mut i = 0; + while i < data.len() { + let mut v = u32::from(data[i]) << 16; + if i + 1 < data.len() { + v |= u32::from(data[i + 1]) << 8; + } + if i + 2 < data.len() { + v |= u32::from(data[i + 2]); + } + out.push(TAB[((v >> 18) & 63) as usize] as char); + out.push(TAB[((v >> 12) & 63) as usize] as char); + out.push(if i + 1 < data.len() { + TAB[((v >> 6) & 63) as usize] as char + } else { + '=' + }); + out.push(if i + 2 < data.len() { + TAB[(v & 63) as usize] as char + } else { + '=' + }); + i += 3; + } + out +} + +pub fn json_quote(s: &str) -> String { + json_quote_bytes(s.as_bytes()) +} + +pub fn json_quote_bytes(s: &[u8]) -> String { + let mut out = String::from("\""); + for &c in s { + match c { + b'\\' => out.push_str("\\\\"), + b'"' => out.push_str("\\\""), + b'\n' => out.push_str("\\n"), + b'\r' => out.push_str("\\r"), + b'\t' => out.push_str("\\t"), + c if c < 0x20 => out.push_str(&format!("\\u{c:04x}")), + c => out.push(c as char), + } + } + out.push('"'); + out +} + +fn hex4(p: &[u8]) -> Option { + if p.len() < 4 { + return None; + } + let mut v = 0u32; + for i in 0..4 { + let c = p[i]; + let x = match c { + b'0'..=b'9' => c - b'0', + b'a'..=b'f' => c - b'a' + 10, + b'A'..=b'F' => c - b'A' + 10, + _ => return None, + }; + v = (v << 4) | u32::from(x); + } + Some(v) +} + +fn utf8_append_bytes(out: &mut Vec, code: u32) { + if code <= 0x7f { + out.push(code as u8); + } else if code <= 0x7ff { + out.push((0xc0 | (code >> 6)) as u8); + out.push((0x80 | (code & 0x3f)) as u8); + } else if code <= 0xffff { + out.push((0xe0 | (code >> 12)) as u8); + out.push((0x80 | ((code >> 6) & 0x3f)) as u8); + out.push((0x80 | (code & 0x3f)) as u8); + } else { + out.push((0xf0 | (code >> 18)) as u8); + out.push((0x80 | ((code >> 12) & 0x3f)) as u8); + out.push((0x80 | ((code >> 6) & 0x3f)) as u8); + out.push((0x80 | (code & 0x3f)) as u8); + } +} + +pub fn json_parse_string_at(input: &[u8]) -> Option<(Vec, usize)> { + if input.first().copied() != Some(b'"') { + return None; + } + let mut i = 1; + let mut out = Vec::new(); + while i < input.len() && input[i] != b'"' { + if input[i] != b'\\' { + out.push(input[i]); + i += 1; + continue; + } + i += 1; + if i >= input.len() { + break; + } + match input[i] { + b'"' => { + out.push(b'"'); + i += 1; + } + b'\\' => { + out.push(b'\\'); + i += 1; + } + b'/' => { + out.push(b'/'); + i += 1; + } + b'b' => { + out.push(b'\x08'); + i += 1; + } + b'f' => { + out.push(b'\x0c'); + i += 1; + } + b'n' => { + out.push(b'\n'); + i += 1; + } + b'r' => { + out.push(b'\r'); + i += 1; + } + b't' => { + out.push(b'\t'); + i += 1; + } + b'u' => { + let v = hex4(input.get(i + 1..i + 5)?)?; + i += 5; + if (0xd800..=0xdbff).contains(&v) + && i + 1 < input.len() + && input[i] == b'\\' + && input[i + 1] == b'u' + { + if let Some(lo) = hex4(input.get(i + 2..i + 6)?) { + if (0xdc00..=0xdfff).contains(&lo) { + let code = 0x10000 + (((v - 0xd800) << 10) + (lo - 0xdc00)); + utf8_append_bytes(&mut out, code); + i += 6; + continue; + } + } + } + utf8_append_bytes(&mut out, v); + } + other => { + out.push(other); + i += 1; + } + } + } + if i >= input.len() || input[i] != b'"' { + return None; + } + Some((out, i + 1)) +} + +pub fn json_get_string_bytes(json: &str, key: &str) -> Option> { + let pat = format!("\"{key}\""); + let mut search = json; + while let Some(idx) = search.find(&pat) { + let mut p = &search[idx + pat.len()..]; + p = p.trim_start_matches([' ', '\t', '\r', '\n']); + if !p.starts_with(':') { + search = &search[idx + pat.len()..]; + continue; + } + p = p[1..].trim_start_matches([' ', '\t', '\r', '\n']); + if p.starts_with('"') { + let (bytes, _) = json_parse_string_at(p.as_bytes())?; + return Some(bytes); + } + search = &search[idx + pat.len()..]; + } + None +} + +pub fn json_get_string(json: &str, key: &str) -> Option { + json_get_string_bytes(json, key).map(|b| String::from_utf8_lossy(&b).into_owned()) +} + +/// C `web_json_id_matches`: skip only space/tab after `:`, then `atoi`. +pub fn json_id_matches(json: &str, id: i32) -> bool { + let Some(p) = json.find("\"id\"") else { + return false; + }; + let rest = &json[p + 4..]; + let Some(colon) = rest.find(':') else { + return false; + }; + let mut s = rest[colon + 1..].trim_start_matches([' ', '\t']); + let mut sign = 1i32; + if let Some(rest) = s.strip_prefix('-') { + sign = -1; + s = rest; + } else if let Some(rest) = s.strip_prefix('+') { + s = rest; + } + let mut n = 0i32; + let mut any = false; + for c in s.chars() { + let Some(d) = c.to_digit(10) else { break }; + n = n.wrapping_mul(10).wrapping_add(d as i32); + any = true; + } + let v = if any { n.wrapping_mul(sign) } else { 0 }; + v == id +} + +pub fn random_bytes(len: usize) -> Vec { + let mut buf = vec![0u8; len]; + if let Ok(mut f) = std::fs::File::open("/dev/urandom") { + use std::io::Read; + if f.read_exact(&mut buf).is_ok() { + return buf; + } + } + let mut x = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + ^ (u64::from(std::process::id()) << 32); + for b in &mut buf { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1); + *b = (x >> 32) as u8; + } + buf +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn url_encode_unreserved_and_space() { + assert_eq!(url_encode("abc-_.~"), "abc-_.~"); + assert_eq!(url_encode("a b"), "a%20b"); + assert_eq!(url_encode("q=1&x"), "q%3D1%26x"); + } + + #[test] + fn base64_padding() { + assert_eq!(base64(b""), ""); + assert_eq!(base64(b"a"), "YQ=="); + assert_eq!(base64(b"ab"), "YWI="); + assert_eq!(base64(b"abc"), "YWJj"); + } + + #[test] + fn json_quote_escapes() { + assert_eq!(json_quote("ab"), "\"ab\""); + assert_eq!(json_quote("a\"b"), "\"a\\\"b\""); + assert_eq!(json_quote("a\nb"), "\"a\\nb\""); + assert_eq!(json_quote("\u{01}"), "\"\\u0001\""); + } + + #[test] + fn json_get_string_first_value() { + let json = r#"{"id":1,"value":"hello\n","value":"x"}"#; + assert_eq!(json_get_string(json, "value").as_deref(), Some("hello\n")); + assert!(json_id_matches(json, 1)); + assert!(!json_id_matches(json, 2)); + } + + #[test] + fn json_surrogate_pair() { + let json = r#"{"t":"\uD83D\uDE00"}"#; + assert_eq!(json_get_string(json, "t").as_deref(), Some("😀")); + } +} diff --git a/crates/ds4-web/src/js.rs b/crates/ds4-web/src/js.rs new file mode 100644 index 0000000000000000000000000000000000000000..c5b041a11bc6ea37c508fa28e4cb05c2cecec3c2 --- /dev/null +++ b/crates/ds4-web/src/js.rs @@ -0,0 +1,9 @@ +//! Page extractors copied from `ds4_web.c`. The files under `js/` are the +//! exact C string contents (not the source-escaped form). + +pub const CLICK_GOOGLE_CONSENT: &str = include_str!("js/consent.js"); +pub const EXTRACT_SEARCH: &str = include_str!("js/search.js"); +pub const EXTRACT_PAGE: &str = include_str!("js/page.js"); +pub const SCROLL_DYNAMIC: &str = include_str!("js/scroll.js"); +pub const PAGE_PROBE: &str = include_str!("js/probe.js"); +pub const READY_STATE: &str = "document.readyState"; diff --git a/crates/ds4-web/src/js/consent.js b/crates/ds4-web/src/js/consent.js new file mode 100644 index 0000000000000000000000000000000000000000..24e5006521662bf2dd5c5fe7aad04fa867fb0d25 --- /dev/null +++ b/crates/ds4-web/src/js/consent.js @@ -0,0 +1 @@ +(() => {const clean=s=>(s||'').replace(/\s+/g,' ').trim();const pats=[/accept all/i,/i agree/i,/agree/i,/accetta tutto/i,/tout accepter/i,/aceptar todo/i,/alle akzeptieren/i];const els=[...document.querySelectorAll('button,[role=button],input[type=submit],a')];for (const el of els){const t=clean(el.innerText||el.value||el.textContent);if(!t)continue; if(pats.some(p=>p.test(t))){el.click(); return 'clicked '+t;}}return '';})() \ No newline at end of file diff --git a/crates/ds4-web/src/js/page.js b/crates/ds4-web/src/js/page.js new file mode 100644 index 0000000000000000000000000000000000000000..338e8457660c8074821ea0a46a3d190b7b0b8365 --- /dev/null +++ b/crates/ds4-web/src/js/page.js @@ -0,0 +1 @@ +(() => {const clean=s=>(s||'').replace(/\s+/g,' ').trim();const esc=s=>clean(s).replace(/\\/g,'\\\\').replace(/\[/g,'\\[').replace(/\]/g,'\\]').replace(/\n/g,' ');const visible=el=>{const r=el.getBoundingClientRect();const st=getComputedStyle(el);return r.width>0&&r.height>0&&st.display!=='none'&&st.visibility!=='hidden'&&st.opacity!=='0';};const inline=n=>{if(!n)return'';if(n.nodeType===3)return n.nodeValue;if(n.nodeType!==1)return'';const el=n;if(el.tagName==='SCRIPT'||el.tagName==='STYLE'||el.tagName==='NOSCRIPT')return'';if(el.tagName==='A'){const t=esc(el.innerText||el.textContent);const h=el.href||'';return t&&h?`[${t}](${h})`:t;}if(el.tagName==='CODE')return '`'+clean(el.innerText||el.textContent).replace(/`/g,'\\`')+'`';return [...el.childNodes].map(inline).join('');};const lines=[`# ${clean(document.title)||location.href}`,'',`URL: ${location.href}`,'','## Content'];const blocks=[...document.body.querySelectorAll('h1,h2,h3,h4,h5,h6,p,li,pre,blockquote,td,th,[id="content-text"],[class*="comment-body"],[class*="comment-content"],[data-testid*="comment-text"]')];const seen=new Set();for(const el of blocks){if(!visible(el))continue;let s='';const tag=el.tagName;if(/^H[1-6]$/.test(tag)){s='#'.repeat(Number(tag[1]))+' '+inline(el);}else if(tag==='LI'){s='- '+inline(el);}else if(tag==='PRE'){s='```\n'+(el.innerText||el.textContent||'').trimEnd()+'\n```';}else if(tag==='BLOCKQUOTE'){s='> '+clean(el.innerText||el.textContent);}else{s=inline(el);}s=s.trim();if(!s||seen.has(s))continue;seen.add(s);lines.push('',s);if(lines.join('\n').length>900000){lines.push('','[Content truncated by browser extractor.]');break;}}lines.push('','## Visible links');let n=0;const linkSeen=new Set();for(const a of document.querySelectorAll('a[href]')){if(!visible(a))continue;const t=esc(a.innerText||a.textContent);if(t.length<3)continue;let u;try{u=new URL(a.href);}catch{continue;}if(!/^https?:$/.test(u.protocol)||linkSeen.has(u.href))continue;linkSeen.add(u.href);lines.push(`- [${t.slice(0,160)}](${u.href})`);if(++n>=80)break;}return lines.join('\n');})() \ No newline at end of file diff --git a/crates/ds4-web/src/js/probe.js b/crates/ds4-web/src/js/probe.js new file mode 100644 index 0000000000000000000000000000000000000000..f5b3135de765adb2ec6f35e2e4ac6b7f0bd49c31 --- /dev/null +++ b/crates/ds4-web/src/js/probe.js @@ -0,0 +1 @@ +location.href+'\n'+document.readyState+'\n'+((document.body&&document.body.innerText)||'').length \ No newline at end of file diff --git a/crates/ds4-web/src/js/scroll.js b/crates/ds4-web/src/js/scroll.js new file mode 100644 index 0000000000000000000000000000000000000000..d908535ba0843fc9c2a373900ef485f9db5f95fb --- /dev/null +++ b/crates/ds4-web/src/js/scroll.js @@ -0,0 +1 @@ +(() => new Promise(resolve => {const root=()=>document.scrollingElement||document.documentElement||document.body;const blockSel='h1,h2,h3,h4,h5,h6,p,li,pre,blockquote,td,th,[id="content-text"],[class*="comment-body"],[class*="comment-content"],[data-testid*="comment-text"]';const lazySel='[onscroll],[loading="lazy"],[data-src],[data-lazy],[class*="lazy"],[class*="infinite"],[class*="virtual"],[role="feed"],[id*="comment"],[class*="comment"],[data-testid*="comment"]';const hookCount=()=>{let n=0;try{if(window.onscroll)n++;if(document.onscroll)n++;if(document.body&&document.body.onscroll)n++;}catch(e){}try{if(typeof getEventListeners==='function'){for(const o of [window,document,document.body]){if(!o)continue;const ev=getEventListeners(o);if(ev&&ev.scroll)n+=ev.scroll.length;}}}catch(e){}try{n+=document.querySelectorAll(lazySel).length;}catch(e){}return n;};const metrics=()=>{const r=root();return {height:r?r.scrollHeight:0,view:innerHeight||900,y:scrollY||(r&&r.scrollTop)||0,text:((document.body&&document.body.innerText)||'').length,links:document.links?document.links.length:0,blocks:document.body?document.body.querySelectorAll(blockSel).length:0,hooks:hookCount()};};const sig=m=>[m.height,m.text,m.links,m.blocks].join('|');const grew=(a,b)=>b.height>a.height+20||b.text>a.text+200||b.links>a.links+2||b.blocks>a.blocks+2;const scrollOnce=()=>{const r=root();if(!r)return;const h=Math.max(700,Math.floor((innerHeight||900)*0.85));window.scrollTo(0,Math.min(r.scrollHeight,(scrollY||r.scrollTop||0)+h));};let last=metrics(),lastSig=sig(last),same=0,steps=0;const scrollable=last.height>last.view*1.35;if(!scrollable||last.hooks===0){resolve('scroll skipped hooks='+last.hooks+' text='+last.text);return;}const tick=()=>{if(steps>=28){resolve('scrolled '+steps+' text='+last.text);return;}const before=last;scrollOnce();steps++;setTimeout(()=>{const now=metrics(),nowSig=sig(now);if(nowSig===lastSig)same++;else same=0;const loaded=grew(before,now);last=now;lastSig=nowSig;if(steps===1&&!loaded){resolve('scroll probe unchanged text='+now.text);return;}const atBottom=now.y+now.view+20>=now.height;if(same>=4||(atBottom&&same>=1)){resolve('scrolled '+steps+' text='+now.text);return;}tick();},900);};tick();}))() \ No newline at end of file diff --git a/crates/ds4-web/src/js/search.js b/crates/ds4-web/src/js/search.js new file mode 100644 index 0000000000000000000000000000000000000000..b00680ead61f6d53a95af075fdcf04febdbd5ac3 --- /dev/null +++ b/crates/ds4-web/src/js/search.js @@ -0,0 +1 @@ +(() => {const clean=s=>(s||'').replace(/\s+/g,' ').trim();const esc=s=>clean(s).replace(/\\/g,'\\\\').replace(/\[/g,'\\[').replace(/\]/g,'\\]').replace(/\n/g,' ');const visible=el=>{const r=el.getBoundingClientRect();const st=getComputedStyle(el);return r.width>0&&r.height>0&&st.display!=='none'&&st.visibility!=='hidden'&&st.opacity!=='0';};const bad=h=>(/(^|\.)google\./.test(h)||/(^|\.)gstatic\./.test(h)||/(^|\.)googleusercontent\./.test(h));const lines=['# Google search results','',`URL: ${location.href}`,'','## Visible links'];const seen=new Set();for(const a of document.querySelectorAll('a[href]')){if(!visible(a))continue;let href=a.href||'';try{const u=new URL(href);if(u.pathname==='/url'&&u.searchParams.get('q'))href=u.searchParams.get('q');}catch{}let u;try{u=new URL(href);}catch{continue;}if(!/^https?:$/.test(u.protocol))continue;if(bad(u.hostname))continue;const text=esc(a.innerText||a.textContent);if(text.length<3)continue;if(seen.has(u.href))continue;seen.add(u.href);lines.push(`- [${text.slice(0,180)}](${u.href})`);if(seen.size>=20)break;}lines.push('','## Text snapshot',clean(document.body.innerText).slice(0,1200));return lines.join('\n');})() \ No newline at end of file diff --git a/crates/ds4-web/src/lib.rs b/crates/ds4-web/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..a7542b15bef5935c084565dd4b298bf678d1264e --- /dev/null +++ b/crates/ds4-web/src/lib.rs @@ -0,0 +1,207 @@ +//! Agent web utility. Blocking I/O port of `ds4_web.c`. No Tokio. + +use std::path::PathBuf; +use std::process::Child; +use std::thread; +use std::time::Duration; + +mod cdp; +mod chrome; +mod encode; +mod js; +mod net; +mod wire; + +pub use chrome::{chrome_executable, linux_chrome_args, profile_dir, CONFIRM_PROMPT, DEFAULT_PORT}; +pub use encode::{ + base64, json_get_string, json_get_string_bytes, json_id_matches, json_parse_string_at, + json_quote, json_quote_bytes, url_encode, url_encode_bytes, +}; +pub use js::{ + CLICK_GOOGLE_CONSENT, EXTRACT_PAGE, EXTRACT_SEARCH, PAGE_PROBE, READY_STATE, SCROLL_DYNAMIC, +}; +pub use net::http_local; +pub use wire::{ + cdp_request, close_path, create_target_params, eval_params, google_search_url, http_request, + navigate_params, page_ws_url, ws_handshake, ws_pong_frame, ws_text_frame, +}; + +pub const CONNECT_TIMEOUT_MS: u64 = 3000; +pub const CDP_TIMEOUT_MS: u64 = 20000; +pub const MAX_RESULT_BYTES: usize = 1024 * 1024; + +pub struct Config { + pub home_dir: Option, + pub port: i32, + pub confirm: Option Result>>, + pub log: Option>, +} + +impl Default for Config { + fn default() -> Self { + Self { + home_dir: None, + port: 0, + confirm: None, + log: None, + } + } +} + +pub struct Web { + profile_dir: PathBuf, + port: i32, + chrome: Option, + browser_allowed: bool, + confirm: Option Result>>, + log: Option>, +} + +impl Web { + pub fn new(cfg: Config) -> Self { + let home = cfg.home_dir.unwrap_or_else(|| { + std::env::var_os("HOME") + .map(PathBuf::from) + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| PathBuf::from(".")) + }); + Self { + profile_dir: chrome::profile_dir(&home), + port: if cfg.port > 0 { cfg.port } else { DEFAULT_PORT }, + chrome: None, + browser_allowed: false, + confirm: cfg.confirm, + log: cfg.log, + } + } + + pub fn port(&self) -> i32 { + self.port + } + + pub fn profile_dir(&self) -> &std::path::Path { + &self.profile_dir + } + + fn log(&self, msg: &str) { + if let Some(log) = &self.log { + log(msg); + } + } + + fn ensure_browser(&mut self) -> Result<(), String> { + if chrome::cdp_alive(self.port) { + return Ok(()); + } + if let Some(mut child) = self.chrome.take() { + let _ = child.try_wait(); + if child.try_wait().ok().flatten().is_none() { + chrome::detach(child); + } + } + if !self.browser_allowed { + let Some(confirm) = &self.confirm else { + return Err( + "starting a visible Chrome browser requires interactive approval".into(), + ); + }; + match confirm(CONFIRM_PROMPT) { + Ok(true) => self.browser_allowed = true, + Ok(false) => return Err("user denied Chrome browser start".into()), + Err(e) => { + if e.is_empty() { + return Err("user denied Chrome browser start".into()); + } + return Err(e); + } + } + } + let mut child = chrome::spawn_chrome(self.port, &self.profile_dir)?; + chrome::wait_cdp_ready(self.port, &mut child)?; + self.log("Chrome browser session is ready"); + self.chrome = Some(child); + Ok(()) + } + + fn close_tab(&self, target_id: &str) { + let path = close_path(target_id); + match net::http_local("GET", self.port, &path) { + Ok(_) => {} + Err(e) => self.log(&e), + } + } + + fn open_tab(&self, url: &str) -> Result<(String, String), String> { + let browser_url = cdp::browser_ws_url(self.port)?; + let mut browser = net::Ws::connect(&browser_url)?; + let id = cdp::create_target(&mut browser, url)?; + let ws_url = page_ws_url(self.port, &id); + Ok((id, ws_url)) + } + + fn run_page_js(&mut self, url: &str, js: &str, dynamic_scroll: bool) -> Result { + self.ensure_browser()?; + let (tab_id, tab_ws) = self.open_tab("about:blank")?; + let mut ws = match net::Ws::connect(&tab_ws) { + Ok(ws) => ws, + Err(e) => { + self.close_tab(&tab_id); + return Err(e); + } + }; + let fail = |web: &mut Web, tab_id: &str, e: String| { + web.close_tab(tab_id); + Err(e) + }; + if let Err(e) = cdp::cdp_prepare_page(&mut ws) { + return fail(self, &tab_id, e); + } + if let Err(e) = cdp::cdp_navigate(&mut ws, url) { + return fail(self, &tab_id, e); + } + if let Err(e) = cdp::wait_navigated_ready(&mut ws) { + return fail(self, &tab_id, e); + } + match cdp::cdp_eval_string(&mut ws, crate::js::CLICK_GOOGLE_CONSENT) { + Ok(clicked) if !clicked.is_empty() => { + self.log(&clicked); + thread::sleep(Duration::from_millis(1500)); + let _ = cdp::wait_navigated_ready(&mut ws); + } + _ => {} + } + if dynamic_scroll { + cdp::scroll_dynamic_page(&mut ws); + } + let out = match cdp::cdp_eval_string(&mut ws, js) { + Ok(v) => v, + Err(e) => return fail(self, &tab_id, e), + }; + self.close_tab(&tab_id); + Ok(out) + } + + pub fn google_search(&mut self, query: &str) -> Result { + if query.is_empty() { + return Err("google_search requires query".into()); + } + let url = google_search_url(query); + self.run_page_js(&url, crate::js::EXTRACT_SEARCH, false) + } + + pub fn visit_page(&mut self, url: &str) -> Result { + if url.is_empty() { + return Err("visit_page requires url".into()); + } + self.run_page_js(url, crate::js::EXTRACT_PAGE, true) + } +} + +impl Drop for Web { + fn drop(&mut self) { + // Do not kill Chrome. The browser profile is user-visible state. + if let Some(child) = self.chrome.take() { + chrome::detach(child); + } + } +} diff --git a/crates/ds4-web/src/net.rs b/crates/ds4-web/src/net.rs new file mode 100644 index 0000000000000000000000000000000000000000..0699f93d0ede9bcbfcff8082ebbb8b1a4c13f895 --- /dev/null +++ b/crates/ds4-web/src/net.rs @@ -0,0 +1,232 @@ +//! Blocking TCP / HTTP / WebSocket matching `ds4_web.c` poll + read loops. + +use std::io::{Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::time::Duration; + +use crate::encode::{base64, random_bytes}; +use crate::wire::{http_request, ws_handshake, ws_pong_frame, ws_text_frame}; +use crate::{CDP_TIMEOUT_MS, CONNECT_TIMEOUT_MS, MAX_RESULT_BYTES}; + +pub fn tcp_connect(host: &str, port: i32, timeout_ms: u64) -> Result { + let timeout = Duration::from_millis(timeout_ms); + let addrs = match (host, port as u16).to_socket_addrs() { + Ok(a) => a, + Err(e) => return Err(format!("getaddrinfo {host}: {e}")), + }; + let mut last = format!("connect {host}:{port} failed: connection refused"); + for addr in addrs { + match TcpStream::connect_timeout(&addr, timeout) { + Ok(s) => return Ok(s), + Err(e) => last = format!("connect {host}:{port} failed: {e}"), + } + } + Err(last) +} + +fn write_all(stream: &mut TcpStream, buf: &[u8]) -> Result<(), String> { + stream.write_all(buf).map_err(|e| e.to_string()) +} + +fn read_some(stream: &mut TcpStream, buf: &mut [u8], timeout_ms: u64) -> Result { + stream + .set_read_timeout(Some(Duration::from_millis(timeout_ms))) + .map_err(|e| e.to_string())?; + match stream.read(buf) { + Ok(n) => Ok(n), + Err(e) + if e.kind() == std::io::ErrorKind::TimedOut + || e.kind() == std::io::ErrorKind::WouldBlock => + { + Ok(0) + } + Err(e) => Err(e.to_string()), + } +} + +/// GET/POST-style request to 127.0.0.1. Status code is ignored; body is +/// everything after the first `\r\n\r\n`, matching C `web_http_request`. +pub fn http_local(method: &str, port: i32, path: &str) -> Result { + let mut fd = tcp_connect("127.0.0.1", port, CONNECT_TIMEOUT_MS)?; + let req = http_request(method, port, path); + if write_all(&mut fd, req.as_bytes()).is_err() { + return Err(format!( + "write HTTP request failed: {}", + std::io::Error::last_os_error() + )); + } + let mut resp = Vec::new(); + let mut tmp = [0u8; 4096]; + loop { + let n = match read_some(&mut fd, &mut tmp, CONNECT_TIMEOUT_MS) { + Ok(n) => n, + Err(_) => { + return Err(format!( + "read HTTP response failed: {}", + std::io::Error::last_os_error() + )) + } + }; + if n == 0 { + break; + } + resp.extend_from_slice(&tmp[..n]); + } + if resp.is_empty() { + return Err("empty HTTP response".into()); + } + let Some(pos) = find_header_end(&resp) else { + return Err("malformed HTTP response".into()); + }; + Ok(String::from_utf8_lossy(&resp[pos + 4..]).into_owned()) +} + +fn find_header_end(buf: &[u8]) -> Option { + buf.windows(4).position(|w| w == b"\r\n\r\n") +} + +pub struct Ws { + stream: TcpStream, + pub next_id: i32, +} + +impl Ws { + pub fn connect(ws_url: &str) -> Result { + if !ws_url.starts_with("ws://") { + return Err(format!("unsupported websocket URL: {ws_url}")); + } + let rest = &ws_url[5..]; + let Some(slash) = rest.find('/') else { + return Err("malformed websocket URL".into()); + }; + let mut hostport = rest[..slash].to_string(); + if hostport.len() > 255 { + hostport.truncate(255); + } + let (host, port) = if let Some(colon) = hostport.rfind(':') { + let port = hostport[colon + 1..].parse::().unwrap_or(0); + (hostport[..colon].to_string(), port) + } else { + (hostport, 80) + }; + let path = &rest[slash..]; + let mut fd = tcp_connect(&host, port, CONNECT_TIMEOUT_MS)?; + let rnd = random_bytes(16); + let key = base64(&rnd); + let req = ws_handshake(path, &host, port, &key); + if write_all(&mut fd, req.as_bytes()).is_err() { + return Err("websocket handshake write failed".into()); + } + let mut resp = Vec::new(); + let mut tmp = [0u8; 1024]; + while find_header_end(&resp).is_none() { + let n = match read_some(&mut fd, &mut tmp, CONNECT_TIMEOUT_MS) { + Ok(n) if n > 0 => n, + _ => return Err("websocket handshake read failed".into()), + }; + resp.extend_from_slice(&tmp[..n]); + if resp.len() > 8192 { + break; + } + } + let text = String::from_utf8_lossy(&resp); + if !text.contains(" 101 ") { + return Err("websocket handshake rejected".into()); + } + Ok(Self { + stream: fd, + next_id: 1, + }) + } + + pub fn send_text(&mut self, text: &str) -> Result<(), String> { + let mut mask = [0u8; 4]; + mask.copy_from_slice(&random_bytes(4)); + let frame = ws_text_frame(text.as_bytes(), mask); + write_all(&mut self.stream, &frame).map_err(|_| { + format!( + "websocket write failed: {}", + std::io::Error::last_os_error() + ) + }) + } + + fn send_pong(&mut self, payload: &[u8]) -> Result<(), String> { + let mut mask = [0u8; 4]; + mask.copy_from_slice(&random_bytes(4)); + let frame = ws_pong_frame(payload, mask); + write_all(&mut self.stream, &frame) + } + + fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), String> { + let mut off = 0; + while off < buf.len() { + let n = match read_some(&mut self.stream, &mut buf[off..], CDP_TIMEOUT_MS) { + Ok(n) if n > 0 => n, + Ok(_) => return Err("websocket read timeout".into()), + Err(_) => return Err("websocket frame read failed".into()), + }; + off += n; + } + Ok(()) + } + + pub fn read_message(&mut self) -> Result { + let mut msg = Vec::new(); + loop { + let mut h = [0u8; 2]; + if self.read_exact(&mut h).is_err() { + return Err("websocket read timeout".into()); + } + let fin = (h[0] & 0x80) != 0; + let opcode = h[0] & 0x0f; + let masked = (h[1] & 0x80) != 0; + let mut len = u64::from(h[1] & 0x7f); + if len == 126 { + let mut x = [0u8; 2]; + self.read_exact(&mut x) + .map_err(|_| "websocket frame read failed")?; + len = (u64::from(x[0]) << 8) | u64::from(x[1]); + } else if len == 127 { + let mut x = [0u8; 8]; + self.read_exact(&mut x) + .map_err(|_| "websocket frame read failed")?; + len = 0; + for b in x { + len = (len << 8) | u64::from(b); + } + } + let mut mask = [0u8; 4]; + if masked { + self.read_exact(&mut mask) + .map_err(|_| "websocket frame read failed")?; + } + if len > MAX_RESULT_BYTES as u64 * 4 { + return Err("websocket message too large".into()); + } + let mut payload = vec![0u8; len as usize]; + if len > 0 { + self.read_exact(&mut payload) + .map_err(|_| "websocket frame read failed")?; + } + if masked { + for i in 0..payload.len() { + payload[i] ^= mask[i & 3]; + } + } + match opcode { + 0x8 => return Err("websocket closed".into()), + 0x9 => { + let _ = self.send_pong(&payload); + } + 0x1 | 0x0 => { + msg.extend_from_slice(&payload); + if fin { + return Ok(String::from_utf8_lossy(&msg).into_owned()); + } + } + _ => {} + } + } + } +} diff --git a/crates/ds4-web/src/wire.rs b/crates/ds4-web/src/wire.rs new file mode 100644 index 0000000000000000000000000000000000000000..06e212e7e88cb264efdefa22b72e20f8c3f00066 --- /dev/null +++ b/crates/ds4-web/src/wire.rs @@ -0,0 +1,102 @@ +//! Request builders that match `ds4_web.c` snprintf / concat order. + +use crate::encode::{json_quote, url_encode}; + +pub fn http_request(method: &str, port: i32, path: &str) -> String { + format!("{method} {path} HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n") +} + +pub fn ws_handshake(path: &str, host: &str, port: i32, key: &str) -> String { + format!( + "GET {path} HTTP/1.1\r\n\ + Host: {host}:{port}\r\n\ + Upgrade: websocket\r\n\ + Connection: Upgrade\r\n\ + Sec-WebSocket-Key: {key}\r\n\ + Sec-WebSocket-Version: 13\r\n\r\n" + ) +} + +pub fn cdp_request(id: i32, method: &str, params: Option<&str>) -> String { + let mut out = format!("{{\"id\":{id},\"method\":"); + out.push_str(&json_quote(method)); + if let Some(params) = params { + if !params.is_empty() { + out.push_str(",\"params\":"); + out.push_str(params); + } + } + out.push('}'); + out +} + +pub fn eval_params(expr: &str) -> String { + let mut out = String::from("{\"expression\":"); + out.push_str(&json_quote(expr)); + out.push_str(",\"returnByValue\":true,\"awaitPromise\":true,\"includeCommandLineAPI\":true}"); + out +} + +pub fn navigate_params(url: &str) -> String { + let mut out = String::from("{\"url\":"); + out.push_str(&json_quote(url)); + out.push('}'); + out +} + +pub fn create_target_params(url: &str) -> String { + let mut out = String::from("{\"url\":"); + out.push_str(&json_quote(url)); + out.push_str(",\"background\":true,\"newWindow\":false}"); + out +} + +pub fn google_search_url(query: &str) -> String { + let mut out = String::from("https://www.google.com/search?q="); + out.push_str(&url_encode(query)); + out +} + +pub fn close_path(target_id: &str) -> String { + let mut out = String::from("/json/close/"); + out.push_str(&url_encode(target_id)); + out +} + +pub fn page_ws_url(port: i32, target_id: &str) -> String { + format!("ws://127.0.0.1:{port}/devtools/page/{target_id}") +} + +pub fn ws_text_frame(text: &[u8], mask: [u8; 4]) -> Vec { + let len = text.len(); + let mut hdr = Vec::new(); + hdr.push(0x81); + if len < 126 { + hdr.push(0x80 | (len as u8)); + } else if len <= 0xffff { + hdr.push(0x80 | 126); + hdr.push((len >> 8) as u8); + hdr.push(len as u8); + } else { + hdr.push(0x80 | 127); + for i in (0..8).rev() { + hdr.push((len >> (i * 8)) as u8); + } + } + hdr.extend_from_slice(&mask); + let mut out = hdr; + for (i, &b) in text.iter().enumerate() { + out.push(b ^ mask[i & 3]); + } + out +} + +pub fn ws_pong_frame(payload: &[u8], mask: [u8; 4]) -> Vec { + let len = payload.len().min(125); + let mut hdr = vec![0x8a, 0x80 | (len as u8)]; + hdr.extend_from_slice(&mask); + for i in 0..len { + hdr.push(payload[i] ^ mask[i & 3]); + } + hdr +} diff --git a/crates/ds4-web/tests/encode_oracle.rs b/crates/ds4-web/tests/encode_oracle.rs new file mode 100644 index 0000000000000000000000000000000000000000..d456bd0355b71884d789f38f3f6a812b4a17ee80 --- /dev/null +++ b/crates/ds4-web/tests/encode_oracle.rs @@ -0,0 +1,161 @@ +//! Byte compare Rust helpers against tests/parity/web_c_oracle. + +use ds4_web::{ + base64, cdp_request, create_target_params, eval_params, google_search_url, http_request, + json_get_string, json_id_matches, json_quote, linux_chrome_args, navigate_params, url_encode, + ws_handshake, ws_text_frame, +}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn oracle() -> PathBuf { + if let Ok(p) = std::env::var("DS4_WEB_C_ORACLE") { + return PathBuf::from(p); + } + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/web_c_oracle") +} + +fn require_oracle() -> PathBuf { + let p = oracle(); + assert!( + p.exists(), + "build the C oracle first: make tests/parity/web_c_oracle (missing {})", + p.display() + ); + p +} + +fn c_out(args: &[&str]) -> String { + let out = Command::new(require_oracle()) + .args(args) + .output() + .expect("run web_c_oracle"); + assert!( + out.status.success(), + "oracle failed: {}\n{}", + String::from_utf8_lossy(&out.stderr), + String::from_utf8_lossy(&out.stdout) + ); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[test] +fn url_encode_matches_c() { + for s in ["", "abc-_.~", "a b", "q=1&x", "한글", "A/Z"] { + assert_eq!(url_encode(s), c_out(&["url-encode", s]), "url_encode {s:?}"); + } +} + +#[test] +fn base64_matches_c() { + for data in [b"".as_slice(), b"a", b"ab", b"abc", b"0123456789abcdef"] { + assert_eq!( + base64(data), + c_out(&["base64", &hex(data)]), + "base64 {data:?}" + ); + } +} + +#[test] +fn json_quote_matches_c() { + for s in ["", "ab", "a\"b", "a\\b", "a\nb", "a\tb", "\u{01}"] { + assert_eq!(json_quote(s), c_out(&["json-quote", s]), "json_quote {s:?}"); + } +} + +#[test] +fn json_get_and_id_match_c() { + let json = r#"{"id": 7, "value":"hello\n", "t":"\uD83D\uDE00"}"#; + assert_eq!( + json_get_string(json, "value").unwrap(), + c_out(&["json-get", json, "value"]) + ); + assert_eq!( + json_get_string(json, "t").unwrap(), + c_out(&["json-get", json, "t"]) + ); + assert_eq!( + if json_id_matches(json, 7) { + "yes" + } else { + "no" + }, + c_out(&["json-id", json, "7"]) + ); + assert_eq!( + if json_id_matches(json, 8) { + "yes" + } else { + "no" + }, + c_out(&["json-id", json, "8"]) + ); +} + +#[test] +fn request_builders_match_c() { + assert_eq!( + http_request("GET", 9333, "/json/version"), + c_out(&["http-req", "GET", "9333", "/json/version"]) + ); + assert_eq!( + ws_handshake("/devtools/browser/x", "127.0.0.1", 9333, "dGVzdA=="), + c_out(&[ + "ws-handshake", + "/devtools/browser/x", + "127.0.0.1", + "9333", + "dGVzdA==" + ]) + ); + assert_eq!( + cdp_request(3, "Runtime.evaluate", Some("{}")), + c_out(&["cdp-req", "3", "Runtime.evaluate", "{}"]) + ); + assert_eq!( + eval_params("document.readyState"), + c_out(&["eval-params", "document.readyState"]) + ); + assert_eq!( + navigate_params("https://example.com/a?b=1"), + c_out(&["navigate-params", "https://example.com/a?b=1"]) + ); + assert_eq!( + create_target_params("about:blank"), + c_out(&["create-target-params", "about:blank"]) + ); + assert_eq!( + google_search_url("rust host"), + c_out(&["search-url", "rust host"]) + ); +} + +#[test] +fn ws_frame_matches_c() { + let text = b"{\"id\":1}"; + let mask = [0x11, 0x22, 0x33, 0x44]; + assert_eq!( + hex(&ws_text_frame(text, mask)), + c_out(&["ws-frame", &hex(text), &hex(&mask)]) + ); +} + +#[test] +fn chrome_linux_args_match_c() { + let profile = Path::new("/tmp/ds4-web-profile"); + let rust = linux_chrome_args(9333, profile, false).join("\n") + "\n"; + assert_eq!( + rust, + c_out(&["chrome-linux-args", "9333", "/tmp/ds4-web-profile", "user"]) + ); + let rust_root = linux_chrome_args(9333, profile, true).join("\n") + "\n"; + assert_eq!( + rust_root, + c_out(&["chrome-linux-args", "9333", "/tmp/ds4-web-profile", "root"]) + ); +} diff --git a/crates/ds4-web/tests/http_ws.rs b/crates/ds4-web/tests/http_ws.rs new file mode 100644 index 0000000000000000000000000000000000000000..ed3fc1bfd1eea5b643d5b8250d4b02a60a0d4778 --- /dev/null +++ b/crates/ds4-web/tests/http_ws.rs @@ -0,0 +1,54 @@ +//! Local HTTP against a std thread. Status codes are ignored, matching C. + +use ds4_web::{http_local, http_request, ws_handshake}; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread; + +#[test] +fn http_local_returns_body_after_headers() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port() as i32; + let server = thread::spawn(move || { + let (mut s, _) = listener.accept().unwrap(); + let mut buf = [0u8; 2048]; + let n = s.read(&mut buf).unwrap_or(0); + let got = String::from_utf8_lossy(&buf[..n]); + assert!(got.contains("GET /json/version HTTP/1.1")); + assert!(got.contains(&format!("Host: 127.0.0.1:{port}"))); + s.write_all(b"HTTP/1.1 500 UNUSED\r\nX-A: 1\r\n\r\nBODY-OK") + .unwrap(); + }); + let body = http_local("GET", port, "/json/version").expect("http_local"); + assert_eq!(body, "BODY-OK"); + assert_eq!( + http_request("GET", port, "/json/version") + .matches("Connection: close") + .count(), + 1 + ); + server.join().unwrap(); +} + +#[test] +fn http_local_empty_and_malformed() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port() as i32; + let server = thread::spawn(move || { + let (mut s, _) = listener.accept().unwrap(); + let mut buf = [0u8; 512]; + let _ = s.read(&mut buf); + s.write_all(b"not-http").unwrap(); + }); + let err = http_local("GET", port, "/x").unwrap_err(); + assert_eq!(err, "malformed HTTP response"); + server.join().unwrap(); +} + +#[test] +fn ws_handshake_bytes() { + let req = ws_handshake("/devtools/page/1", "127.0.0.1", 9, "abcd"); + assert!(req.starts_with("GET /devtools/page/1 HTTP/1.1\r\n")); + assert!(req.contains("Sec-WebSocket-Version: 13\r\n")); + assert!(req.contains("Upgrade: websocket\r\n")); +} diff --git a/crates/ds4-web/tests/js_from_c.rs b/crates/ds4-web/tests/js_from_c.rs new file mode 100644 index 0000000000000000000000000000000000000000..23a5c6ab75a5f16c01a69565ae28800a7d352f5e --- /dev/null +++ b/crates/ds4-web/tests/js_from_c.rs @@ -0,0 +1,89 @@ +//! JS extractors must stay byte-identical to `ds4_web.c`. + +use ds4_web::{CLICK_GOOGLE_CONSENT, EXTRACT_PAGE, EXTRACT_SEARCH, PAGE_PROBE, SCROLL_DYNAMIC}; +use std::fs; +use std::path::PathBuf; + +fn unescape_c(s: &str) -> String { + let b = s.as_bytes(); + let mut out = String::new(); + let mut i = 0; + while i < b.len() { + if b[i] != b'\\' { + out.push(b[i] as char); + i += 1; + continue; + } + i += 1; + if i >= b.len() { + break; + } + match b[i] { + b'n' => out.push('\n'), + b'r' => out.push('\r'), + b't' => out.push('\t'), + b'\\' => out.push('\\'), + b'"' => out.push('"'), + b'\'' => out.push('\''), + other => out.push(other as char), + } + i += 1; + } + out +} + +fn extract_c_strings(src: &str, after: &str) -> String { + let start = src.find(after).unwrap_or_else(|| panic!("missing {after}")); + let mut i = start + after.len(); + let bytes = src.as_bytes(); + let mut out = String::new(); + while i < bytes.len() && bytes[i] != b';' { + if bytes[i] == b'"' { + i += 1; + let lit_start = i; + while i < bytes.len() { + if bytes[i] == b'\\' { + i += 2; + continue; + } + if bytes[i] == b'"' { + break; + } + i += 1; + } + out.push_str(&unescape_c(&src[lit_start..i])); + i += 1; + } else { + i += 1; + } + } + out +} + +#[test] +fn js_matches_ds4_web_c() { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../ds4_web.c"); + let src = fs::read_to_string(&path).expect("read ds4_web.c"); + assert_eq!( + CLICK_GOOGLE_CONSENT, + extract_c_strings(&src, "static const char *web_click_google_consent_js =") + ); + assert_eq!( + EXTRACT_SEARCH, + extract_c_strings(&src, "static const char *web_extract_search_js =") + ); + assert_eq!( + EXTRACT_PAGE, + extract_c_strings(&src, "static const char *web_extract_page_js =") + ); + let scroll_src = &src[src.find("static void web_scroll_dynamic_page").unwrap()..]; + assert_eq!( + SCROLL_DYNAMIC, + extract_c_strings(scroll_src, "const char *expr =") + ); + let probe_src = &src[src.find("static bool web_page_probe").unwrap()..]; + assert_eq!( + PAGE_PROBE, + extract_c_strings(probe_src, "const char *expr =") + ); +} diff --git a/crates/ds4-web/tests/search_visit.rs b/crates/ds4-web/tests/search_visit.rs new file mode 100644 index 0000000000000000000000000000000000000000..a6e9799d3c0aa3b2375450e00f68377daf680055 --- /dev/null +++ b/crates/ds4-web/tests/search_visit.rs @@ -0,0 +1,283 @@ +//! End-to-end google_search / visit_page against a blocking mock CDP. +//! Chrome is never spawned: `/json/version` is already alive. + +use ds4_web::{ + json_get_string, json_quote, Config, Web, CLICK_GOOGLE_CONSENT, EXTRACT_PAGE, EXTRACT_SEARCH, + PAGE_PROBE, READY_STATE, SCROLL_DYNAMIC, +}; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +struct MockCdp { + port: i32, + stop: Arc, + handle: Option>, +} + +impl MockCdp { + fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let port = listener.local_addr().unwrap().port() as i32; + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let handle = thread::spawn(move || loop { + if flag.load(Ordering::SeqCst) { + break; + } + match listener.accept() { + Ok((stream, _)) => { + stream.set_nonblocking(false).ok(); + thread::spawn(move || handle_conn(stream, port)); + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(_) => break, + } + }); + Self { + port, + stop, + handle: Some(handle), + } + } +} + +impl Drop for MockCdp { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + let _ = TcpStream::connect(("127.0.0.1", self.port as u16)); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} + +fn handle_conn(mut stream: TcpStream, port: i32) { + stream.set_read_timeout(Some(Duration::from_secs(5))).ok(); + let mut buf = Vec::new(); + let mut tmp = [0u8; 2048]; + loop { + match stream.read(&mut tmp) { + Ok(0) => return, + Ok(n) => buf.extend_from_slice(&tmp[..n]), + Err(_) => return, + } + if let Some(pos) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&buf[..pos]).into_owned(); + if head.to_ascii_lowercase().contains("upgrade: websocket") { + let _ = stream.write_all(b"HTTP/1.1 101 Switching Protocols\r\n\r\n"); + serve_cdp(&mut stream); + return; + } + if head.starts_with("GET /json/version") { + let body = format!( + "{{\"webSocketDebuggerUrl\":\"ws://127.0.0.1:{port}/devtools/browser/B1\"}}" + ); + let resp = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(resp.as_bytes()); + return; + } + if head.starts_with("GET /json/close/") { + let _ = stream.write_all(b"HTTP/1.1 200 OK\r\n\r\nOK"); + return; + } + let _ = stream.write_all(b"HTTP/1.1 404 Not Found\r\n\r\n"); + return; + } + if buf.len() > 8192 { + return; + } + } +} + +fn serve_cdp(stream: &mut TcpStream) { + loop { + let msg = match read_ws_text(stream) { + Ok(m) => m, + Err(_) => return, + }; + let id = extract_id(&msg); + let method = json_get_string(&msg, "method").unwrap_or_default(); + let body = if method == "Runtime.evaluate" { + let expr = json_get_string(&msg, "expression").unwrap_or_default(); + let value = eval_reply(&expr); + format!( + "{{\"id\":{id},\"result\":{{\"result\":{{\"type\":\"string\",\"value\":{}}}}}}}", + json_quote(&value) + ) + } else if method == "Target.createTarget" { + format!("{{\"id\":{id},\"result\":{{\"targetId\":\"PAGE1\"}}}}") + } else { + format!("{{\"id\":{id},\"result\":{{}}}}") + }; + if write_ws_text(stream, &body).is_err() { + return; + } + } +} + +fn extract_id(msg: &str) -> i32 { + let Some(p) = msg.find("\"id\"") else { + return 1; + }; + let rest = &msg[p + 4..]; + let Some(colon) = rest.find(':') else { + return 1; + }; + let s = rest[colon + 1..].trim_start_matches([' ', '\t']); + let end = s + .find(|c: char| c != '-' && !c.is_ascii_digit()) + .unwrap_or(s.len()); + s[..end].parse().unwrap_or(1) +} + +fn eval_reply(expr: &str) -> String { + if expr == READY_STATE { + "complete".into() + } else if expr == PAGE_PROBE { + "https://www.google.com/search?q=rust\ncomplete\n42".into() + } else if expr == CLICK_GOOGLE_CONSENT { + String::new() + } else if expr == EXTRACT_SEARCH { + "# Google search results\n\nOK-SEARCH".into() + } else if expr == EXTRACT_PAGE { + "# Example\n\nOK-PAGE".into() + } else if expr == SCROLL_DYNAMIC { + "scroll skipped hooks=0 text=42".into() + } else { + "ok".into() + } +} + +fn read_ws_text(stream: &mut TcpStream) -> Result { + let mut h = [0u8; 2]; + stream.read_exact(&mut h).map_err(|_| ())?; + let mut len = (h[1] & 0x7f) as usize; + let masked = (h[1] & 0x80) != 0; + if len == 126 { + let mut x = [0u8; 2]; + stream.read_exact(&mut x).map_err(|_| ())?; + len = ((x[0] as usize) << 8) | x[1] as usize; + } else if len == 127 { + let mut x = [0u8; 8]; + stream.read_exact(&mut x).map_err(|_| ())?; + len = 0; + for b in x { + len = (len << 8) | b as usize; + } + } + let mut mask = [0u8; 4]; + if masked { + stream.read_exact(&mut mask).map_err(|_| ())?; + } + let mut payload = vec![0u8; len]; + if len > 0 { + stream.read_exact(&mut payload).map_err(|_| ())?; + } + if masked { + for i in 0..payload.len() { + payload[i] ^= mask[i & 3]; + } + } + Ok(String::from_utf8_lossy(&payload).into_owned()) +} + +fn write_ws_text(stream: &mut TcpStream, text: &str) -> Result<(), ()> { + let bytes = text.as_bytes(); + let mut frame = Vec::new(); + frame.push(0x81); + if bytes.len() < 126 { + frame.push(bytes.len() as u8); + } else if bytes.len() <= 0xffff { + frame.push(126); + frame.push((bytes.len() >> 8) as u8); + frame.push(bytes.len() as u8); + } else { + return Err(()); + } + frame.extend_from_slice(bytes); + stream.write_all(&frame).map_err(|_| ()) +} + +fn web_on(port: i32) -> Web { + Web::new(Config { + home_dir: Some(std::env::temp_dir().join("ds4-web-test-home")), + port, + confirm: Some(Box::new(|_| Ok(true))), + log: None, + }) +} + +#[test] +fn google_search_empty_query() { + let mut web = web_on(1); + assert_eq!( + web.google_search("").unwrap_err(), + "google_search requires query" + ); +} + +#[test] +fn visit_page_empty_url() { + let mut web = web_on(1); + assert_eq!(web.visit_page("").unwrap_err(), "visit_page requires url"); +} + +#[test] +fn confirm_required_when_cdp_down() { + let mut web = Web::new(Config { + home_dir: Some(std::env::temp_dir().join("ds4-web-test-home")), + port: 1, + confirm: None, + log: None, + }); + assert_eq!( + web.google_search("q").unwrap_err(), + "starting a visible Chrome browser requires interactive approval" + ); +} + +#[test] +fn confirm_denied() { + let mut web = Web::new(Config { + home_dir: Some(std::env::temp_dir().join("ds4-web-test-home")), + port: 1, + confirm: Some(Box::new(|msg| { + assert_eq!( + msg, + "The web tool wants to start a visible Chrome browser. Allow? (y/n) " + ); + Ok(false) + })), + log: None, + }); + assert_eq!( + web.google_search("q").unwrap_err(), + "user denied Chrome browser start" + ); +} + +#[test] +fn google_search_via_mock_cdp() { + let mock = MockCdp::start(); + let mut web = web_on(mock.port); + let out = web.google_search("rust host").expect("search"); + assert!(out.contains("OK-SEARCH"), "{out}"); +} + +#[test] +fn visit_page_via_mock_cdp() { + let mock = MockCdp::start(); + let mut web = web_on(mock.port); + let out = web.visit_page("https://example.com/").expect("visit"); + assert!(out.contains("OK-PAGE"), "{out}"); +} diff --git a/cuda/beam_mma8w.cuh b/cuda/beam_mma8w.cuh new file mode 100644 index 0000000000000000000000000000000000000000..e91dbfbdbbf236903de5a2f5433fbc1a8c17b3c8 --- /dev/null +++ b/cuda/beam_mma8w.cuh @@ -0,0 +1,574 @@ +#include "hip/hip_runtime.h" +#ifndef DS4_BEAM_MMA8W_CUH +#define DS4_BEAM_MMA8W_CUH + +#include "glm_vq_mma8.cuh" + +/* Warp-specialized FP8 prefill experts for Beam's 12-bit (K4096) layers, + * computed as tokens x weights^T. A book vector holds k {4q..4q+3, + * 16+4q..16+4q+3} of one weight row: exactly the B fragment of m16n8k32 + * for column g, so one LDS.64 gather feeds the MMAs with no register + * shuffles; token (pair) rows are the A operand, one ldmatrix.x4 per 16 + * rows. glmvq_mma8_kernel used the weights as A, and ptxas rebuilt the + * interleaved A registers with MOVs before its MMAs. + * + * A work unit is 128 rows of one 128-token item. Warp w owns rows + * 16w..16w+15 (two n8 tiles) and all eight m16 token tiles. + * + * warps 0-7 consumers book gathers + ldmatrix + MMAs, then the epilogue + * warps 8-11 producers cp.async index rows and token rows into a + * three-stage ring (full/empty mbarriers), the item + * record into stage 0, and the next unit's index + * block into L2, one piece per stage + * + * Producers load the next unit's item and token rows one unit ahead; the + * consumers read the item record from the ring and load gains and pair + * scales before the K loop, so no role waits on a dependent global load at + * a unit boundary (those waits were the largest stall: -21% on gate). + * Gate/up activations carry one E4M3 scale per token row and accumulate + * across all of K; down keeps glmvq_mma8_kernel's per-128-k promotion and + * produces its output bit for bit. Real-route replay of a 2,048-row chunk + * (glmvq_mma8_kernel -> these): gate 5.04 -> 3.98 ms, up 5.69 -> 4.41 ms, + * down 6.06 -> 4.40 ms. */ +enum { BEAM_W_BM = 128, BEAM_W_BT = 128, BEAM_W_STAGES = 3, BEAM_W_CONSUMERS = 8, BEAM_W_PRODUCERS = 4, + BEAM_W_THREADS = (BEAM_W_CONSUMERS + BEAM_W_PRODUCERS) * 32, BEAM_W_PF_FIRST = 4, + BEAM_W_BOOK = 4096 * GLMVQ_VECTOR, BEAM_W_IX = BEAM_W_BM * GLMVQ_MMA8_INDEX, + BEAM_W_STAGE = BEAM_W_IX + BEAM_W_BT * GLMVQ_MMA8_ROW + 16, + BEAM_W_BYTES = BEAM_W_BOOK + BEAM_W_STAGES * BEAM_W_STAGE + BEAM_W_BT * 4 + 2 * BEAM_W_STAGES * 8 }; + +// One expert matrix for a run of at most BEAM_W_BT grouped pairs; the first +// 16 bytes travel to the consumers in the ring. +struct __align__(16) beam_w_item { uint32_t offset, tokens; const __half *gain; const uint8_t *main; uint64_t pad; }; + +__device__ __forceinline__ uint32_t beam_w_smem(const void *p) { return (uint32_t)__cvta_generic_to_shared(p); } + +// The suspend-time hint parks a waiting warp instead of spinning on try_wait. +__device__ __forceinline__ void beam_w_wait(const uint64_t *bar, uint32_t parity) { + asm volatile("{\n .reg .pred done;\n BEAM_W_WAIT%=:\n mbarrier.try_wait.parity.shared.b64 done, [%0], %1, %2;\n" + " @!done bra BEAM_W_WAIT%=;\n}" :: "r"(beam_w_smem(bar)), "r"(parity), "r"(0x989680u) : "memory"); +} + +__device__ __forceinline__ void beam_w_arrive(uint64_t *bar) { + asm volatile("{\n .reg .b64 state;\n mbarrier.arrive.shared.b64 state, [%0];\n}" :: "r"(beam_w_smem(bar)) : "memory"); +} + +__device__ __forceinline__ void beam_w_consumers_sync() { + asm volatile("bar.sync 1, %0;" :: "n"(BEAM_W_CONSUMERS * 32) : "memory"); +} + +// Token row t's 16-byte chunk c sits at chunk c ^ (t & 7): the eight rows of +// an ldmatrix 8x16B matrix hit distinct banks. +__device__ __forceinline__ uint32_t beam_w_swz(uint32_t t, uint32_t c) { return t * GLMVQ_MMA8_ROW + ((c ^ (t & 7u)) << 4); } + +// Byte position of column k inside its row in the ldmatrix k order (see +// beam_w_quant): 4-byte group m of each 32-byte block sits at (m & 1) * 16 + (m >> 1) * 4. +__host__ __device__ __forceinline__ uint32_t beam_w_perm(uint32_t k) { + const uint32_t m = (k % 32) / 4; + return k / 32 * 32 + (m & 1u) * 16 + (m >> 1) * 4 + k % 4; +} + +__device__ __forceinline__ void beam_w_ldsm4(uint32_t (&a)[4], const void *p) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];" + : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3]) : "r"(beam_w_smem(p))); +} + +__device__ __forceinline__ void beam_w_mma(float (&d)[4], const uint32_t (&a)[4], uint2 b) { + asm volatile("mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b.x), "r"(b.y)); +} + +// x = e4m3 * scale[row], one scale per token row (amax / 448), one block per +// row. Within each 32-byte k block, 4-byte group m lands at byte +// (m & 1) * 16 + (m >> 1) * 4: ldmatrix then hands lane q the bytes 8q..8q+7 +// of the block as its A registers 0 and 2, the k order of a book vector. +__global__ void __launch_bounds__(GLMVQ_THREADS) beam_w_quant(uint8_t *out, float *scales, const float *input, + uint32_t cols) { + __shared__ float peak[GLMVQ_THREADS / 32]; + const float *x = input + (uint64_t)blockIdx.x * cols; + float amax = 0.0f; + for (uint32_t i = threadIdx.x * 4; i < cols; i += GLMVQ_THREADS * 4) { + const float4 v = *(const float4 *)(x + i); + amax = fmaxf(amax, fmaxf(fmaxf(fabsf(v.x), fabsf(v.y)), fmaxf(fabsf(v.z), fabsf(v.w)))); + } + for (uint32_t stride = 16; stride; stride >>= 1) { amax = fmaxf(amax, __shfl_xor_sync(0xffffffffu, amax, stride)); } + if ((threadIdx.x & 31) == 0) { peak[threadIdx.x / 32] = amax; } + __syncthreads(); + amax = 0.0f; + for (uint32_t w = 0; w < GLMVQ_THREADS / 32; w++) { amax = fmaxf(amax, peak[w]); } + const float scale = amax / GLMVQ_E4M3_MAX; + for (uint32_t i = threadIdx.x * 4; i < cols; i += GLMVQ_THREADS * 4) { + const float4 v = *(const float4 *)(x + i); + const float a = amax > 0.0f ? __fdiv_rn(v.x, scale) : 0.0f, b = amax > 0.0f ? __fdiv_rn(v.y, scale) : 0.0f; + const float c = amax > 0.0f ? __fdiv_rn(v.z, scale) : 0.0f, d = amax > 0.0f ? __fdiv_rn(v.w, scale) : 0.0f; + const uint32_t lo = __hip_cvt_float2_to_fp8x2(make_float2(a, b), __HIP_SATFINITE, __HIP_E4M3_FNUZ); + const uint32_t hi = __hip_cvt_float2_to_fp8x2(make_float2(c, d), __HIP_SATFINITE, __HIP_E4M3_FNUZ); + const uint32_t m = (i % 32) / 4, at = i / 32 * 32 + (m & 1u) * 16 + (m >> 1) * 4; + *(uint32_t *)(out + (uint64_t)blockIdx.x * cols + at) = lo | (hi << 16); + } + if (threadIdx.x == 0) { scales[blockIdx.x] = scale; } +} + +// Per grouped pair: its token row and that row's activation scale. +__global__ void beam_w_pairs(uint32_t *token_of, float *pair_scale, const float *scale, const uint32_t *map, + uint32_t used, uint32_t pairs) { + const uint32_t p = blockIdx.x * blockDim.x + threadIdx.x; + if (p < pairs) { + const uint32_t t = map[p] / used; + token_of[p] = t; + pair_scale[p] = scale[t]; + } +} + +template +__global__ void __launch_bounds__(BEAM_W_THREADS, 1) beam_w_kernel(const beam_w_item *items, uint32_t nitems, + const uint8_t *book, const uint8_t *act, const uint32_t *token_of, const float *pair_scale, + __hip_bfloat16 *gate, uint32_t rows, uint32_t cols, uint8_t *mid8, float *mid_scale) { + static_assert(MODE == GLMVQ_MMA_GATE || MODE == GLMVQ_MMA_UP, "gate/up input rows are the token rows"); + constexpr uint32_t BM = BEAM_W_BM, BT = BEAM_W_BT, S = BEAM_W_STAGES, C = BEAM_W_CONSUMERS; + constexpr uint32_t PT = BEAM_W_PRODUCERS * 32, MT = BT / 16, STB = BEAM_W_STAGE, TQ = PT / 8, TJ = BT / TQ; + extern __shared__ __align__(16) uint8_t memory[]; + uint8_t *ring = memory + BEAM_W_BOOK; + uint32_t *amax = (uint32_t *)(ring + S * STB); + uint64_t *full = (uint64_t *)(amax + BT), *empty = full + S; + const uint32_t tid = threadIdx.x, lane = tid & 31u, warp = tid >> 5; + for (uint32_t i = tid * 16; i < BEAM_W_BOOK; i += BEAM_W_THREADS * 16) { glmvq_mma_copy<16>(memory + i, book + i); } + asm volatile("cp.async.commit_group;" ::: "memory"); + asm volatile("cp.async.wait_group 0;" ::: "memory"); + if (tid == 0) { + for (uint32_t s = 0; s < S; s++) { + asm volatile("mbarrier.init.shared.b64 [%0], %1;" :: "r"(beam_w_smem(full + s)), "r"(PT) : "memory"); + asm volatile("mbarrier.init.shared.b64 [%0], %1;" :: "r"(beam_w_smem(empty + s)), "r"(C) : "memory"); + } + } + __syncthreads(); + const uint32_t tiles = rows / BM, stages = cols / GLMVQ_MMA8_K, units = nitems * tiles; + const uint32_t stride = cols * 3 / 16, block = BM * stride; + if (warp >= C) { + const uint32_t pt = tid - C * 32, chunk = pt % 8, tq = pt / 8; + // The next unit's index block streams into L2 in sequential pieces. + auto prefetch = [&](const uint8_t *p, uint32_t piece, uint32_t pieces) { + const uint32_t step = (block / pieces + 15) / 16 * 16, off = piece * step; + if (pt == 0 && p && off < block) { glmvq_mma_prefetch(p + off, min(step, block - off)); } + }; + // cur: this unit's item index, index block and this thread's token rows. + uint32_t ci = min(blockIdx.x, units - 1) / tiles; + beam_w_item cur = items[ci]; + const uint8_t *cm = cur.main + (uint64_t)(blockIdx.x % tiles) * block; + uint32_t src[TJ]; + #pragma unroll + for (uint32_t j = 0; j < TJ; j++) { + const uint32_t t = tq + TQ * j; + src[j] = token_of[cur.offset + (t < cur.tokens ? t : 0)]; + } + for (uint32_t piece = 0; piece < BEAM_W_PF_FIRST; piece++) { + prefetch(blockIdx.x < units ? cm : nullptr, piece, BEAM_W_PF_FIRST); + } + uint32_t ringc = 0; + for (uint32_t work = blockIdx.x; work < units; work += gridDim.x) { + const uint32_t next = work + gridDim.x, nt = cur.tokens, padded = (nt + 15) / 16 * 16; + const uint32_t ni = min(next, units - 1) / tiles; + const beam_w_item nxt = items[ni]; + const uint8_t *nm = nullptr; + uint32_t nsrc[TJ]; + for (uint32_t kt = 0; kt < stages; kt++, ringc++) { + const uint32_t s = ringc % S, use = ringc / S; + if (use) { beam_w_wait(empty + s, (use - 1) & 1); } + if (kt == 1) { + // nxt arrived during stage 0: its block and token rows load now. + nm = next < units ? nxt.main + (uint64_t)(next % tiles) * block : nullptr; + #pragma unroll + for (uint32_t j = 0; j < TJ; j++) { + const uint32_t t = tq + TQ * j; + nsrc[j] = token_of[nxt.offset + (t < nxt.tokens ? t : 0)]; + } + } + if (kt >= 1) { prefetch(nm, kt - 1, stages - 1); } + uint8_t *dst = ring + s * STB; + const uint8_t *from = cm + kt * GLMVQ_MMA8_INDEX; + for (uint32_t i = pt; i < BM * 3; i += PT) { + const uint32_t r = i / 3, part = i % 3; + glmvq_mma_copy<8>(dst + r * GLMVQ_MMA8_INDEX + part * 8, from + (uint64_t)r * stride + part * 8); + } + // Zero rows pad the last m16 tile. + #pragma unroll + for (uint32_t j = 0; j < TJ; j++) { + const uint32_t t = tq + TQ * j; + if (t < padded) { + glmvq_mma_pad(dst + BEAM_W_IX + beam_w_swz(t, chunk), + act + (uint64_t)src[j] * cols + kt * GLMVQ_MMA8_K + chunk * 16, t < nt ? 16 : 0); + } + } + // The item record (offset, tokens, gain) rides in stage 0's slot. + if (kt == 0 && pt == 0) { glmvq_mma_copy<16>(dst + STB - 16, items + ci); } + asm volatile("cp.async.mbarrier.arrive.noinc.shared.b64 [%0];" :: "r"(beam_w_smem(full + s)) : "memory"); + } + cur = nxt; ci = ni; cm = nm; + #pragma unroll + for (uint32_t j = 0; j < TJ; j++) { src[j] = nsrc[j]; } + } + asm volatile("cp.async.wait_all;" ::: "memory"); + return; + } + // MMA fragment coordinates: D rows (tokens) g and g + 8, columns (weight rows) 2q, 2q + 1. + const uint32_t g = lane >> 2, q = lane & 3u, r0 = warp * 16 + g; + // ldmatrix.x4 row address of this lane: matrix lane / 8 = (token half, k half). + const uint32_t lt = (lane & 7u) + ((lane >> 3) & 1u) * 8, lk = lane >> 4; + uint32_t ringc = 0; + for (uint32_t work = blockIdx.x; work < units; work += gridDim.x) { + const uint32_t row = (work % tiles) * BM; + // The item record arrives with stage 0. + beam_w_wait(full + ringc % S, (ringc / S) & 1); + const uint4 rec = *(const uint4 *)(ring + (ringc % S) * STB + STB - 16); + const uint32_t offset = rec.x, nt = rec.y, nmt = (nt + 15) / 16; + const __half *gain = (const __half *)((uint64_t)rec.z | (uint64_t)rec.w << 32); + // Gains (this lane's columns) and pair scales (its D rows) load now + // and are first used after the K loop. + const uint32_t c0 = row + warp * 16 + 2 * q; + const __half2 hg0 = *(const __half2 *)(gain + c0), hg1 = *(const __half2 *)(gain + c0 + 8); + float sv[MT][2]; + #pragma unroll + for (uint32_t m = 0; m < MT; m++) { + #pragma unroll + for (uint32_t h = 0; h < 2; h++) { + const uint32_t t = m * 16 + h * 8 + g; + sv[m][h] = pair_scale[offset + (t < nt ? t : 0)]; + } + } + // acc[m][n]: token tile m, weight rows 16w + 8n + (0..7). + float acc[MT][2][4] = {}; + auto run = [&](auto count) { + constexpr uint32_t NM = decltype(count)::value; + for (uint32_t kt = 0; kt < stages; kt++, ringc++) { + const uint32_t s = ringc % S; + beam_w_wait(full + s, (ringc / S) & 1); + const uint8_t *stage = ring + s * STB; + uint32_t w[2][6]; + #pragma unroll + for (uint32_t n = 0; n < 2; n++) { + #pragma unroll + for (uint32_t k = 0; k < 3; k++) { + const uint2 v = *(const uint2 *)(stage + (r0 + 8 * n) * GLMVQ_MMA8_INDEX + k * 8); + w[n][2 * k] = v.x; w[n][2 * k + 1] = v.y; + } + } + const uint8_t *tokens = stage + BEAM_W_IX; + auto step = [&](auto kk) { + constexpr uint32_t K = decltype(kk)::value; + const uint2 b0 = *(const uint2 *)(memory + glmvq_mma8_code(w[0], q) * 8); + const uint2 b1 = *(const uint2 *)(memory + glmvq_mma8_code(w[1], q) * 8); + #pragma unroll + for (uint32_t m = 0; m < NM; m++) { + uint32_t a[4]; + beam_w_ldsm4(a, tokens + beam_w_swz(m * 16 + lt, 2 * K + lk)); + beam_w_mma(acc[m][0], a, b0); + beam_w_mma(acc[m][1], a, b1); + } + }; + step(std::integral_constant{}); + step(std::integral_constant{}); + step(std::integral_constant{}); + step(std::integral_constant{}); + // The stage's operands are in registers: free the slot. + __syncwarp(); + if (lane == 0) { beam_w_arrive(empty + s); } + } + }; + // Exact m16 tile counts (padding MMAs are not free). + auto pick = [&](auto self, auto n) { + constexpr uint32_t N = decltype(n)::value; + if constexpr (N >= MT) { run(std::integral_constant{}); } + else if (nmt == N) { run(n); } + else { self(self, std::integral_constant{}); } + }; + pick(pick, std::integral_constant{}); + const float2 f0 = __half22float2(hg0), f1 = __half22float2(hg1); + const float gn[2][2] = {{f0.x, f0.y}, {f1.x, f1.y}}; + if constexpr (MODE == GLMVQ_MMA_GATE) { + #pragma unroll + for (uint32_t m = 0; m < MT; m++) { + if (m >= nmt) { break; } + #pragma unroll + for (uint32_t h = 0; h < 2; h++) { + const uint32_t t = m * 16 + h * 8 + g; + if (t >= nt) { continue; } + const uint64_t base = (uint64_t)(offset + t) * rows + c0; + #pragma unroll + for (uint32_t n = 0; n < 2; n++) { + *(__hip_bfloat162 *)(gate + base + 8 * n) = __floats2bfloat162_rn( + glmvq_round(glmvq_mul(acc[m][n][2 * h] * sv[m][h], gn[n][0])), + glmvq_round(glmvq_mul(acc[m][n][2 * h + 1] * sv[m][h], gn[n][1]))); + } + } + } + } else { + // Up: SwiGLU on the BF16 gate, then E4M3 rows for down with one + // scale per (token, 128 rows). Gate pairs of four token tiles + // load before any use. + for (uint32_t i = tid; i < BT; i += C * 32) { amax[i] = 0; } + beam_w_consumers_sync(); + #pragma unroll + for (uint32_t half = 0; half < MT; half += MT / 2) { + __hip_bfloat162 gv[MT / 2][2][2]; + #pragma unroll + for (uint32_t mm = 0; mm < MT / 2; mm++) { + #pragma unroll + for (uint32_t h = 0; h < 2; h++) { + const uint32_t t = (half + mm) * 16 + h * 8 + g, tt = t < nt ? t : 0; + const uint64_t base = (uint64_t)(offset + tt) * rows + c0; + #pragma unroll + for (uint32_t n = 0; n < 2; n++) { gv[mm][h][n] = *(const __hip_bfloat162 *)(gate + base + 8 * n); } + } + } + #pragma unroll + for (uint32_t mm = 0; mm < MT / 2; mm++) { + const uint32_t m = half + mm; + if (m >= nmt) { break; } + #pragma unroll + for (uint32_t h = 0; h < 2; h++) { + float peak = 0.0f; + #pragma unroll + for (uint32_t n = 0; n < 2; n++) { + const float2 x = __bfloat1622float2(gv[mm][h][n]); + const float y0 = glmvq_round(glmvq_mul(acc[m][n][2 * h] * sv[m][h], gn[n][0])); + const float y1 = glmvq_round(glmvq_mul(acc[m][n][2 * h + 1] * sv[m][h], gn[n][1])); + const float v0 = glmvq_swiglu_value(x.x, y0, INFINITY); + const float v1 = glmvq_swiglu_value(x.y, y1, INFINITY); + acc[m][n][2 * h] = v0; + acc[m][n][2 * h + 1] = v1; + peak = fmaxf(peak, fmaxf(fabsf(v0), fabsf(v1))); + } + peak = fmaxf(peak, __shfl_xor_sync(0xffffffffu, peak, 1)); + peak = fmaxf(peak, __shfl_xor_sync(0xffffffffu, peak, 2)); + const uint32_t t = m * 16 + h * 8 + g; + if (q == 0 && t < nt) { atomicMax(amax + t, __float_as_uint(peak)); } + } + } + } + beam_w_consumers_sync(); + #pragma unroll + for (uint32_t m = 0; m < MT; m++) { + if (m >= nmt) { break; } + #pragma unroll + for (uint32_t h = 0; h < 2; h++) { + const uint32_t t = m * 16 + h * 8 + g; + if (t >= nt) { continue; } + const float peak = __uint_as_float(amax[t]), scale = peak / GLMVQ_E4M3_MAX; + #pragma unroll + for (uint32_t n = 0; n < 2; n++) { + const float a = peak > 0.0f ? __fdiv_rn(acc[m][n][2 * h], scale) : 0.0f; + const float b = peak > 0.0f ? __fdiv_rn(acc[m][n][2 * h + 1], scale) : 0.0f; + // Down reads these rows with ldmatrix: store them in its k order. + *(uint16_t *)(mid8 + (uint64_t)(offset + t) * rows + beam_w_perm(c0 + 8 * n)) = + (uint16_t)__hip_cvt_float2_to_fp8x2(make_float2(a, b), __HIP_SATFINITE, __HIP_E4M3_FNUZ); + } + if (q == 0 && warp == 0) { + mid_scale[(uint64_t)(offset + t) * (rows / GLMVQ_MMA8_K) + row / GLMVQ_MMA8_K] = scale; + } + } + } + // amax is rewritten by the next unit only after every consumer read it. + beam_w_consumers_sync(); + } + } +} + +/* Down: pairs x weights^T over the up epilogue's E4M3 rows (ldmatrix k + * order, one scale per (pair, 128 columns)). Each 128-k stage sums in a + * zeroed MMA accumulator, then joins the FP32 total through one FMA with the + * pair's scale, as glmvq_mma8_kernel does; the epilogue writes BF16 + * round(total * gain) per pair for the reduce. Same roles and ring as the + * gate/up kernel; the stage also carries the pairs' scales. */ +enum { BEAM_W_DOWN_STAGE = BEAM_W_IX + BEAM_W_BT * GLMVQ_MMA8_ROW + BEAM_W_BT * 4 + 16, + BEAM_W_DOWN_BYTES = BEAM_W_BOOK + BEAM_W_STAGES * BEAM_W_DOWN_STAGE + 2 * BEAM_W_STAGES * 8 }; + +__global__ void __launch_bounds__(BEAM_W_THREADS, 1) beam_w_down_kernel(const beam_w_item *items, uint32_t nitems, + const uint8_t *book, const uint8_t *mid8, const float *mid_scale, __hip_bfloat16 *output, + uint32_t rows, uint32_t cols) { + constexpr uint32_t BM = BEAM_W_BM, BT = BEAM_W_BT, S = BEAM_W_STAGES, C = BEAM_W_CONSUMERS; + constexpr uint32_t PT = BEAM_W_PRODUCERS * 32, MT = BT / 16, STB = BEAM_W_DOWN_STAGE, TQ = PT / 8, TJ = BT / TQ; + constexpr uint32_t ATB = BT * GLMVQ_MMA8_ROW; + extern __shared__ __align__(16) uint8_t memory[]; + uint8_t *ring = memory + BEAM_W_BOOK; + uint64_t *full = (uint64_t *)(ring + S * STB), *empty = full + S; + const uint32_t tid = threadIdx.x, lane = tid & 31u, warp = tid >> 5; + for (uint32_t i = tid * 16; i < BEAM_W_BOOK; i += BEAM_W_THREADS * 16) { glmvq_mma_copy<16>(memory + i, book + i); } + asm volatile("cp.async.commit_group;" ::: "memory"); + asm volatile("cp.async.wait_group 0;" ::: "memory"); + if (tid == 0) { + for (uint32_t s = 0; s < S; s++) { + asm volatile("mbarrier.init.shared.b64 [%0], %1;" :: "r"(beam_w_smem(full + s)), "r"(PT) : "memory"); + asm volatile("mbarrier.init.shared.b64 [%0], %1;" :: "r"(beam_w_smem(empty + s)), "r"(C) : "memory"); + } + } + __syncthreads(); + const uint32_t tiles = rows / BM, stages = cols / GLMVQ_MMA8_K, units = nitems * tiles; + const uint32_t stride = cols * 3 / 16, block = BM * stride; + if (warp >= C) { + const uint32_t pt = tid - C * 32, chunk = pt % 8, tq = pt / 8; + auto prefetch = [&](const uint8_t *p, uint32_t piece, uint32_t pieces) { + const uint32_t step = (block / pieces + 15) / 16 * 16, off = piece * step; + if (pt == 0 && p && off < block) { glmvq_mma_prefetch(p + off, min(step, block - off)); } + }; + uint32_t ci = min(blockIdx.x, units - 1) / tiles; + beam_w_item cur = items[ci]; + const uint8_t *cm = cur.main + (uint64_t)(blockIdx.x % tiles) * block; + for (uint32_t piece = 0; piece < BEAM_W_PF_FIRST; piece++) { + prefetch(blockIdx.x < units ? cm : nullptr, piece, BEAM_W_PF_FIRST); + } + uint32_t ringc = 0; + for (uint32_t work = blockIdx.x; work < units; work += gridDim.x) { + const uint32_t next = work + gridDim.x, nt = cur.tokens, padded = (nt + 15) / 16 * 16; + const uint32_t ni = min(next, units - 1) / tiles; + const beam_w_item nxt = items[ni]; + const uint8_t *nm = nullptr; + for (uint32_t kt = 0; kt < stages; kt++, ringc++) { + const uint32_t s = ringc % S, use = ringc / S; + if (use) { beam_w_wait(empty + s, (use - 1) & 1); } + if (kt == 1) { nm = next < units ? nxt.main + (uint64_t)(next % tiles) * block : nullptr; } + if (kt >= 1) { prefetch(nm, kt - 1, stages - 1); } + uint8_t *dst = ring + s * STB; + const uint8_t *from = cm + kt * GLMVQ_MMA8_INDEX; + for (uint32_t i = pt; i < BM * 3; i += PT) { + const uint32_t r = i / 3, part = i % 3; + glmvq_mma_copy<8>(dst + r * GLMVQ_MMA8_INDEX + part * 8, from + (uint64_t)r * stride + part * 8); + } + // Pair rows (zero rows pad the last m16 tile) and their stage scales. + #pragma unroll + for (uint32_t j = 0; j < TJ; j++) { + const uint32_t t = tq + TQ * j, pair = cur.offset + (t < nt ? t : 0); + if (t < padded) { + glmvq_mma_pad(dst + BEAM_W_IX + beam_w_swz(t, chunk), + mid8 + (uint64_t)pair * cols + kt * GLMVQ_MMA8_K + chunk * 16, t < nt ? 16 : 0); + } + } + for (uint32_t t = pt; t < padded; t += PT) { + asm volatile("cp.async.ca.shared.global [%0], [%1], 4, %2;" :: + "r"(beam_w_smem(dst + BEAM_W_IX + ATB + t * 4)), + "l"(mid_scale + (uint64_t)(cur.offset + (t < nt ? t : 0)) * stages + kt), "r"(t < nt ? 4 : 0) + : "memory"); + } + if (kt == 0 && pt == 0) { glmvq_mma_copy<16>(dst + STB - 16, items + ci); } + asm volatile("cp.async.mbarrier.arrive.noinc.shared.b64 [%0];" :: "r"(beam_w_smem(full + s)) : "memory"); + } + cur = nxt; ci = ni; cm = nm; + } + asm volatile("cp.async.wait_all;" ::: "memory"); + return; + } + const uint32_t g = lane >> 2, q = lane & 3u, r0 = warp * 16 + g; + const uint32_t lt = (lane & 7u) + ((lane >> 3) & 1u) * 8, lk = lane >> 4; + uint32_t ringc = 0; + for (uint32_t work = blockIdx.x; work < units; work += gridDim.x) { + const uint32_t row = (work % tiles) * BM; + beam_w_wait(full + ringc % S, (ringc / S) & 1); + const uint4 rec = *(const uint4 *)(ring + (ringc % S) * STB + STB - 16); + const uint32_t offset = rec.x, nt = rec.y, nmt = (nt + 15) / 16; + const __half *gain = (const __half *)((uint64_t)rec.z | (uint64_t)rec.w << 32); + const uint32_t c0 = row + warp * 16 + 2 * q; + const __half2 hg0 = *(const __half2 *)(gain + c0), hg1 = *(const __half2 *)(gain + c0 + 8); + float acc[MT][2][4] = {}; + auto run = [&](auto count) { + constexpr uint32_t NM = decltype(count)::value; + for (uint32_t kt = 0; kt < stages; kt++, ringc++) { + const uint32_t s = ringc % S; + beam_w_wait(full + s, (ringc / S) & 1); + const uint8_t *stage = ring + s * STB; + uint32_t w[2][6]; + #pragma unroll + for (uint32_t n = 0; n < 2; n++) { + #pragma unroll + for (uint32_t k = 0; k < 3; k++) { + const uint2 v = *(const uint2 *)(stage + (r0 + 8 * n) * GLMVQ_MMA8_INDEX + k * 8); + w[n][2 * k] = v.x; w[n][2 * k + 1] = v.y; + } + } + uint2 b[4][2]; + b[0][0] = *(const uint2 *)(memory + glmvq_mma8_code<0>(w[0], q) * 8); + b[0][1] = *(const uint2 *)(memory + glmvq_mma8_code<0>(w[1], q) * 8); + b[1][0] = *(const uint2 *)(memory + glmvq_mma8_code<1>(w[0], q) * 8); + b[1][1] = *(const uint2 *)(memory + glmvq_mma8_code<1>(w[1], q) * 8); + b[2][0] = *(const uint2 *)(memory + glmvq_mma8_code<2>(w[0], q) * 8); + b[2][1] = *(const uint2 *)(memory + glmvq_mma8_code<2>(w[1], q) * 8); + b[3][0] = *(const uint2 *)(memory + glmvq_mma8_code<3>(w[0], q) * 8); + b[3][1] = *(const uint2 *)(memory + glmvq_mma8_code<3>(w[1], q) * 8); + const uint8_t *pairs = stage + BEAM_W_IX; + const float *scale = (const float *)(stage + BEAM_W_IX + ATB); + #pragma unroll + for (uint32_t m = 0; m < NM; m++) { + float part[2][4] = {}; + #pragma unroll + for (uint32_t k = 0; k < 4; k++) { + uint32_t a[4]; + beam_w_ldsm4(a, pairs + beam_w_swz(m * 16 + lt, 2 * k + lk)); + beam_w_mma(part[0], a, b[k][0]); + beam_w_mma(part[1], a, b[k][1]); + } + const float s0 = scale[m * 16 + g], s1 = scale[m * 16 + 8 + g]; + #pragma unroll + for (uint32_t n = 0; n < 2; n++) { + acc[m][n][0] = fmaf(part[n][0], s0, acc[m][n][0]); + acc[m][n][1] = fmaf(part[n][1], s0, acc[m][n][1]); + acc[m][n][2] = fmaf(part[n][2], s1, acc[m][n][2]); + acc[m][n][3] = fmaf(part[n][3], s1, acc[m][n][3]); + } + } + // The stage's operands are in registers: free the slot. + __syncwarp(); + if (lane == 0) { beam_w_arrive(empty + s); } + } + }; + auto pick = [&](auto self, auto n) { + constexpr uint32_t N = decltype(n)::value; + if constexpr (N >= MT) { run(std::integral_constant{}); } + else if (nmt == N) { run(n); } + else { self(self, std::integral_constant{}); } + }; + pick(pick, std::integral_constant{}); + const float2 f0 = __half22float2(hg0), f1 = __half22float2(hg1); + const float gn[2][2] = {{f0.x, f0.y}, {f1.x, f1.y}}; + #pragma unroll + for (uint32_t m = 0; m < MT; m++) { + if (m >= nmt) { break; } + #pragma unroll + for (uint32_t h = 0; h < 2; h++) { + const uint32_t t = m * 16 + h * 8 + g; + if (t >= nt) { continue; } + const uint64_t base = (uint64_t)(offset + t) * rows + c0; + #pragma unroll + for (uint32_t n = 0; n < 2; n++) { + *(__hip_bfloat162 *)(output + base + 8 * n) = __floats2bfloat162_rn( + glmvq_round(glmvq_mul(acc[m][n][2 * h], gn[n][0])), + glmvq_round(glmvq_mul(acc[m][n][2 * h + 1], gn[n][1]))); + } + } + } + } +} + +static hipError_t beam_w_down_launch(const beam_w_item *items, uint32_t nitems, const uint8_t *book, const uint8_t *mid8, + const float *mid_scale, __hip_bfloat16 *output, uint32_t rows, uint32_t cols, uint32_t sms, hipStream_t stream) { + static hipError_t ready = hipFuncSetAttribute(reinterpret_cast(beam_w_down_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, + BEAM_W_DOWN_BYTES); + if (ready != hipSuccess) { return ready; } + const uint32_t units = nitems * (rows / BEAM_W_BM); + beam_w_down_kernel<<>>(items, nitems, book, mid8, mid_scale, + output, rows, cols); + return hipGetLastError(); +} + +template +static hipError_t beam_w_launch(const beam_w_item *items, uint32_t nitems, const uint8_t *book, const uint8_t *act, + const uint32_t *token_of, const float *pair_scale, __hip_bfloat16 *gate, uint32_t rows, uint32_t cols, + uint8_t *mid8, float *mid_scale, uint32_t sms, hipStream_t stream) { + static hipError_t ready = hipFuncSetAttribute(reinterpret_cast(beam_w_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, + BEAM_W_BYTES); + if (ready != hipSuccess) { return ready; } + const uint32_t units = nitems * (rows / BEAM_W_BM); + beam_w_kernel<<>>(items, nitems, book, act, token_of, + pair_scale, gate, rows, cols, mid8, mid_scale); + return hipGetLastError(); +} + +#endif diff --git a/cuda/ds41_attn.cuh b/cuda/ds41_attn.cuh new file mode 100644 index 0000000000000000000000000000000000000000..a85c7676ee08c7549131c72cbe6c3b2217cc3b7c --- /dev/null +++ b/cuda/ds41_attn.cuh @@ -0,0 +1,430 @@ +#include "hip/hip_runtime.h" +/* ds41_attn.cuh — DeepSeek V4.1 (ds41) attention primitives, ported from the C + * engine at /data/YoungAi (commit 3946dbc): + * - RoPE: src/cuda/cuda_v41_2.inc.cu:8-43 (official precompute_freqs_cis + + * apply_rotary_emb; adjacent element pairs are one complex number, YaRN + * ramp for the compressor layers); + * - the compressor pool (:49-70, official Compressor.forward ratio>1); + * - the scalar sparse attention (:132-303, official sparse_attn_kernel + * semantics: window rows in ascending order then the topk compressed + * rows, online softmax per 8-key tile, p rounded to bf16 before the v + * multiply, sink in the denominator only); + * - act_quant / the KV pack family and the pack constants: + * src/cuda/cuda_kv_pack.inc.cu (official act_quant_kernel inplace branch; + * the 288 B / 72 B group formats); + * - the SWA window ring commit: src/cuda/cuda_kv_ring.inc.cu:20-56. + * - v41_pow2_ceil_log2 and v41_win_row: src/cuda/cuda_v41_1.inc.cu:130-151. + * + * The sparse-attention tensor-core family (decode seg, prefill, split-K, + * merge) and the ds4_gpu_v41_sparse_attn_tensor entry live in + * cuda/ds41_attn_mma.cuh (P4-5); this file keeps the scalar kernel that the + * entry's dispatch falls back to (the engine's own fallback for the 9..63 + * token band and refused shapes). */ +#pragma once + +/* Packed-KV group geometry (ds4_gpu_v41.h:229-234): the main KV stores 512 + * dims as 256 B nibbles + 32 B E4M3 scales = 288 B/group; the index key + * stores 128 dims as 64 B nibbles + 4 B E8M0 scales = 68 B padded to 72 (8 + * aligned: 68 makes a warp's 64 B read straddle two 32 B sectors). The + * constants live in ds41_kvfmt.h so the host allocator uses the same copy. */ +#include "../ds41_kvfmt.h" + +/* fast_round_scale's exponent: 2^ceil(log2 v) (cuda_v41_1.inc.cu:130-133). + * act_quant and the KV pack share this one copy — a drift here would make the + * packed value differ from the f32 it replaces. */ +__device__ __forceinline__ static float v41_pow2_ceil_log2(float v) { + int e; const float m = frexpf(v, &e); /* v = m*2^e, m in [0.5,1) => log2 v in (e-1, e] */ + return ldexpf(1.0f, (m == 0.5f) ? e - 1 : e); +} + +/* Which row of the SWA window buffer absolute position a lives in + * (cuda_v41_1.inc.cu:148-151): [0, window) is the history segment, [window, + * window+n) this batch's rows. ring=1 (main path) keeps a % window; ring=0 + * (the DSpark tower) is the linear segment push_main shifts. */ +__device__ __forceinline__ static uint64_t v41_win_row(int64_t a, uint32_t pos0, uint32_t window, uint32_t ring) { + if (a >= (int64_t)pos0) return (uint64_t)((int64_t)window + a - (int64_t)pos0); + return ring ? (uint64_t)(a % (int64_t)window) + : (uint64_t)(a - ((int64_t)pos0 - (int64_t)window)); +} + +/* ---- RoPE (cuda_v41_2.inc.cu:8-43) ---- */ +__device__ __forceinline__ static float v41_rope_freq(uint32_t i, uint32_t dim, float theta, uint32_t osl, + float factor, float beta_fast, float beta_slow) { + float f = 1.0f / powf(theta, (float)(2u * i) / (float)dim); + if (osl > 0u) { /* YaRN ramp: corrected_dim(rot) = dim*ln(osl/(rot*2pi)) / (2 ln theta) */ + const float lt = 2.0f * logf(theta); + float low = floorf((float)dim * logf((float)osl / (beta_fast * 2.0f * (float)M_PI)) / lt); + float high = ceilf((float)dim * logf((float)osl / (beta_slow * 2.0f * (float)M_PI)) / lt); + low = fmaxf(low, 0.0f); high = fminf(high, (float)(dim - 1u)); + float ramp = ((float)i - low) / fmaxf(high - low, 1e-3f); + ramp = fminf(fmaxf(ramp, 0.0f), 1.0f); + const float smooth = 1.0f - ramp; + f = f / factor * (1.0f - smooth) + f * smooth; + } + return f; +} +__global__ static void v41_rope_kernel(float *x, const int32_t *pos, uint32_t n_head, uint32_t head_dim, uint32_t n_rot, + float theta, uint32_t osl, float factor, float bf, float bs, int inverse) { + v41_pdl_wait(); + const uint32_t t = blockIdx.y, h = blockIdx.x; + const uint32_t i = threadIdx.x; /* complex index i < n_rot/2 */ + if (i >= n_rot / 2u) return; + float *xr = x + ((uint64_t)t * n_head + h) * head_dim + (head_dim - n_rot) + 2u * i; + const float ang = (float)pos[t] * v41_rope_freq(i, n_rot, theta, osl, factor, bf, bs); + float c = cosf(ang), s = sinf(ang); + if (inverse) s = -s; + const float a = xr[0], b = xr[1]; + xr[0] = v41_bf16r(a * c - b * s); /* official: x.float() rotated, copy_ back into the bf16 tensor */ + xr[1] = v41_bf16r(a * s + b * c); +} +extern "C" int ds4_gpu_v41_rope_tensor(ds4_gpu_tensor *x, const ds4_gpu_tensor *pos, uint32_t n_tok, uint32_t n_head, + uint32_t head_dim, uint32_t n_rot, float theta, uint32_t original_seq_len, + float factor, float beta_fast, float beta_slow, bool inverse) { + if (!x || !pos || (n_rot & 1u) || n_rot > 128u) return 0; + v41_rope_kernel<<>>((float *)x->ptr, (const int32_t *)pos->ptr, n_head, head_dim, n_rot, + theta, original_seq_len, factor, beta_fast, beta_slow, inverse ? 1 : 0); + return cuda_ok(hipGetLastError(), "v41 rope"); +} + +/* ---- compressor pool (cuda_v41_2.inc.cu:49-70): per-dimension softmax + * weighting inside the group -> bf16 ---- */ +__global__ static void v41_compress_pool_kernel(float *out, const float *kv, const float *sc, uint32_t ratio, uint32_t dim) { + const uint32_t g = blockIdx.x; + for (uint32_t d = threadIdx.x; d < dim; d += blockDim.x) { + float mx = -INFINITY; + for (uint32_t t = 0; t < ratio; t++) mx = fmaxf(mx, sc[((uint64_t)g * ratio + t) * dim + d]); + float den = 0.f, acc = 0.f; + for (uint32_t t = 0; t < ratio; t++) { + const float e = expf(sc[((uint64_t)g * ratio + t) * dim + d] - mx); + den += e; acc += e * kv[((uint64_t)g * ratio + t) * dim + d]; + } + out[(uint64_t)g * dim + d] = v41_bf16r(acc / den); + } +} +extern "C" int ds4_gpu_v41_compress_pool_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *kv, const ds4_gpu_tensor *score, + uint32_t n_tok, uint32_t ratio, uint32_t dim) { + if (!out || !kv || !score || ratio == 0) return 0; + const uint32_t ng = n_tok / ratio; + if (ng == 0) return 1; + v41_compress_pool_kernel<<>>((float *)out->ptr, (const float *)kv->ptr, (const float *)score->ptr, ratio, dim); + return cuda_ok(hipGetLastError(), "v41 compress pool"); +} + +/* ---- compressor source, one decode step, n rows (cuda_v41_2.inc.cu:74-128) ---- + * The graph route's replacement for the direct path's three host-shaped steps + * (host-offset memcpy of the batch rows after the pending rows, a host + * decision to launch the pool, host-offset memcpy of the remainder to the + * head): one launch whose shape does not depend on the position, so a single + * captured graph replays every step. The position comes from the device slot + * (posd), the pending slot is pos0 % ratio, and every completed group pools + * with the pool kernel's exact arithmetic (same t order, same mx/den/acc, + * same bf16 round) -- the direct and graph routes are bit-identical. + * snap_kv/snap_sc non-NULL store [old pending rows | this batch's n rows] + * linearly (the direct path's snap_cpre layout, so v41_spec_rollback reads it + * unchanged); a pure-decode n=1 capture passes NULL. */ +__global__ static void v41_compress_step_n_kernel(float *pooled, int32_t *posg, float *ckv_c, float *csc_c, float *snap_kv, float *snap_sc, + const float *ckv, const float *csc, const int32_t *posd, uint32_t ratio, uint32_t dim, uint32_t n) { + v41_pdl_wait(); + const uint32_t pos0 = (uint32_t)posd[0], pend = pos0 % ratio; + if (snap_kv) + for (uint32_t r = 0; r < pend; r++) + for (uint32_t d = threadIdx.x; d < dim; d += blockDim.x) { snap_kv[(uint64_t)r * dim + d] = ckv_c[(uint64_t)r * dim + d]; snap_sc[(uint64_t)r * dim + d] = csc_c[(uint64_t)r * dim + d]; } + uint32_t j = 0; + for (uint32_t i = 0; i < n; i++) { + const uint32_t slot = (pend + i) % ratio; + __syncthreads(); /* the previous group's pool read slots 0..ratio-1 before the next row may overwrite slot 0 */ + for (uint32_t d = threadIdx.x; d < dim; d += blockDim.x) { + const float a = ckv[(uint64_t)i * dim + d], b = csc[(uint64_t)i * dim + d]; + ckv_c[(uint64_t)slot * dim + d] = a; csc_c[(uint64_t)slot * dim + d] = b; + if (snap_kv) { snap_kv[(uint64_t)(pend + i) * dim + d] = a; snap_sc[(uint64_t)(pend + i) * dim + d] = b; } + } + if (slot + 1u != ratio) continue; /* not a complete group yet: append only (slot is block-uniform, no divergence) */ + __syncthreads(); + if (threadIdx.x == 0) posg[j] = (int32_t)(pos0 + i + 1u - ratio); + for (uint32_t d = threadIdx.x; d < dim; d += blockDim.x) { + float mx = -INFINITY; + for (uint32_t t = 0; t < ratio; t++) mx = fmaxf(mx, csc_c[(uint64_t)t * dim + d]); + float den = 0.f, acc = 0.f; + for (uint32_t t = 0; t < ratio; t++) { + const float e = expf(csc_c[(uint64_t)t * dim + d] - mx); + den += e; acc += e * ckv_c[(uint64_t)t * dim + d]; + } + pooled[(uint64_t)j * dim + d] = v41_bf16r(acc / den); + } + j++; + } +} +extern "C" int ds4_gpu_v41_compress_step_n_tensor(ds4_gpu_tensor *pooled, ds4_gpu_tensor *posg, ds4_gpu_tensor *cpre_kv, ds4_gpu_tensor *cpre_sc, + ds4_gpu_tensor *snap_kv, ds4_gpu_tensor *snap_sc, const ds4_gpu_tensor *ckv, const ds4_gpu_tensor *csc, + const ds4_gpu_tensor *posd, uint32_t ratio, uint32_t dim, uint32_t n) { + if (!pooled || !posg || !cpre_kv || !cpre_sc || !ckv || !csc || !posd || ratio < 2u || n == 0u || n > 8u) return 0; + const uint32_t ngmax = (ratio - 1u + n) / ratio; + if (cpre_kv->bytes < (uint64_t)ratio * dim * 4 || cpre_sc->bytes < (uint64_t)ratio * dim * 4) return 0; + if (pooled->bytes < (uint64_t)ngmax * dim * 4 || posg->bytes < (uint64_t)ngmax * 4 || ckv->bytes < (uint64_t)n * dim * 4) return 0; + if ((snap_kv != NULL) != (snap_sc != NULL)) return 0; + if (snap_kv && (snap_kv->bytes < (uint64_t)(ratio - 1u + n) * dim * 4 || snap_sc->bytes < (uint64_t)(ratio - 1u + n) * dim * 4)) return 0; + v41_compress_step_n_kernel<<<1, 256, 0, ds4_current_stream()>>>((float *)pooled->ptr, (int32_t *)posg->ptr, (float *)cpre_kv->ptr, (float *)cpre_sc->ptr, + snap_kv ? (float *)snap_kv->ptr : NULL, snap_sc ? (float *)snap_sc->ptr : NULL, + (const float *)ckv->ptr, (const float *)csc->ptr, (const int32_t *)posd->ptr, ratio, dim, n); + return cuda_ok(hipGetLastError(), "v41 compress step (n rows)"); +} + +/* ---- act_quant inplace (cuda_kv_pack.inc.cu:107-141) ---- + * fast_round_scale(amax, inv) = 2^ceil(log2(amax*inv)); one warp per block + * (<= 32 elements), one element per thread. mode 0: fp8 e4m3 + ue8m0 scale + * (max 448); 1: fp4 + ue8m0 (max 6); 2: fp4 + e4m3 scale (max 6). */ +__global__ static void v41_act_quant_kernel(float *x, uint32_t dim, uint32_t block, int mode, uint32_t nb_row) { + v41_pdl_wait(); + const uint32_t row = blockIdx.x / nb_row, b = blockIdx.x % nb_row; + const uint32_t lane = threadIdx.x; + float *p = x + (uint64_t)row * dim + (uint64_t)b * block; + float v = lane < block ? p[lane] : 0.0f; + float a = fabsf(v); + for (int o = 16; o > 0; o >>= 1) a = fmaxf(a, __shfl_xor_sync(0xffffffffu, a, o)); + float s; + if (mode == 0) { a = fmaxf(a, 1e-4f); s = v41_pow2_ceil_log2(a / 448.0f); } + else if (mode == 1) { a = fmaxf(a, 6.0f * ldexpf(1.0f, -126)); s = v41_pow2_ceil_log2(a / 6.0f); } + else { a = fmaxf(a, 6.0f * ldexpf(1.0f, -9)); s = ds4_e4m3fn_round(a / 6.0f); } + if (lane < block) { + float q = v / s; + if (mode == 0) { q = fminf(fmaxf(q, -448.0f), 448.0f); q = ds4_e4m3fn_round(q); } + else { q = fminf(fmaxf(q, -6.0f), 6.0f); q = ds4_e2m1fn_round(q); } + p[lane] = v41_bf16r(q * s); + } +} +extern "C" int ds4_gpu_v41_act_quant_fp8_tensor(ds4_gpu_tensor *x, uint32_t n_rows, uint32_t dim, uint32_t block) { + if (!x || block == 0 || block > 32u || (dim % block)) return 0; + v41_act_quant_kernel<<<(unsigned)((uint64_t)n_rows * (dim / block)), 32, 0, ds4_current_stream()>>>((float *)x->ptr, dim, block, 0, dim / block); + return cuda_ok(hipGetLastError(), "v41 act quant fp8"); +} +extern "C" int ds4_gpu_v41_act_quant_fp4_tensor(ds4_gpu_tensor *x, uint32_t n_rows, uint32_t dim, uint32_t block, bool e4m3_scale) { + if (!x || block == 0 || block > 32u || (dim % block)) return 0; + v41_act_quant_kernel<<<(unsigned)((uint64_t)n_rows * (dim / block)), 32, 0, ds4_current_stream()>>>((float *)x->ptr, dim, block, e4m3_scale ? 2 : 1, dim / block); + return cuda_ok(hipGetLastError(), "v41 act quant fp4"); +} + +/* ---- KV pack (cuda_kv_pack.inc.cu:20-102) ---- + * The packed bytes ARE the already-quantized values at their native width: + * pack stores (nibble, scale); reading back computes bf16(nibble*scale) — + * the same expression act_quant's last line evaluates. One warp per quant + * block, the max/shuffle/scale steps verbatim from act_quant (a drift there + * would break the "packed == the f32 it replaces" identity). */ +__global__ static void v41_kv_pack_kernel(uint8_t *cache, const float *rows, uint32_t g0, + uint32_t dim, uint32_t blk, uint32_t row_bytes, + uint32_t nib_bytes, uint32_t nb_row, int mode, + const int32_t *posd, uint32_t ratio, uint32_t g_trash, uint32_t nbatch) { + v41_pdl_wait(); + const uint32_t r = blockIdx.x / nb_row, b = blockIdx.x % nb_row, lane = threadIdx.x; + /* graph path: the group number comes from the device position; rows past + * the completed groups write the trash slot (allocated past the group + * cap, never read). nbatch=1 reduces to the original expression. */ + uint32_t grp = g0 + r; + if (posd) { + const uint32_t pos = (uint32_t)posd[0], ng_new = (pos % ratio + nbatch) / ratio; + grp = r < ng_new ? pos / ratio + r : g_trash; + } + const float *p = rows + (uint64_t)r * dim + (uint64_t)b * blk; + const float v = lane < blk ? p[lane] : 0.0f; + float a = fabsf(v); + for (int o = 16; o > 0; o >>= 1) a = fmaxf(a, __shfl_xor_sync(0xffffffffu, a, o)); + float s; + if (mode == 0) { a = fmaxf(a, 6.0f * ldexpf(1.0f, -9)); s = ds4_e4m3fn_round(a / 6.0f); } + else { a = fmaxf(a, 6.0f * ldexpf(1.0f, -126)); s = v41_pow2_ceil_log2(a / 6.0f); } + uint8_t *row = cache + (uint64_t)grp * row_bytes; + if (lane == 0) row[nib_bytes + b] = mode == 0 ? ds4_e4m3fn_f32_to_byte(s) : ds4_e8m0_f32_to_byte(s); + const uint8_t nib = lane < blk ? ds4_fp4_f32_to_nibble(v / s) : 0u; + const uint8_t odd = (uint8_t)__shfl_down_sync(0xffffffffu, (uint32_t)nib, 1); + if (lane < blk && (lane & 1u) == 0u) row[(b * blk + lane) >> 1] = (uint8_t)(nib | (odd << 4)); +} +/* Unpack: cached dim d = bf16(nibble * scale) — the pack side's inverse. */ +__device__ __forceinline__ static float v41_ckv_get(const uint8_t *row, uint32_t d) { + const uint8_t by = row[d >> 1]; + const uint8_t nib = (d & 1u) ? (uint8_t)(by >> 4) : (uint8_t)(by & 0x0Fu); + return v41_bf16r(ds4_fp4_nibble_to_f32(nib) * ds4_e4m3fn_to_f32(row[DS4_V41_CKV_NIB + (d >> 4)])); +} +/* Scales decoded once into shared first (a 512-dim row has 32 scales but 512 + * elements; same value, same multiply -> bit-identical, 480 decodes saved). */ +__device__ __forceinline__ static float v41_ckv_get_s(const uint8_t *row, uint32_t d, const float *scales) { + const uint8_t by = row[d >> 1]; + const uint8_t nib = (d & 1u) ? (uint8_t)(by >> 4) : (uint8_t)(by & 0x0Fu); + return v41_bf16r(ds4_fp4_nibble_to_f32(nib) * scales[d >> 4]); +} +__device__ __forceinline__ static float v41_idxk_get(const uint8_t *row, uint32_t d) { + const uint8_t by = row[d >> 1]; + const uint8_t nib = (d & 1u) ? (uint8_t)(by >> 4) : (uint8_t)(by & 0x0Fu); + return v41_bf16r(ds4_fp4_nibble_to_f32(nib) * ds4_e8m0_to_f32(row[DS4_V41_IDXK_NIB + (d >> 5)])); +} +static int v41_kv_pack_check(const ds4_gpu_tensor *cache, uint32_t g0, uint32_t n_rows, const ds4_gpu_tensor *posd, + uint32_t ratio, uint32_t g_trash, uint32_t row_bytes, uint32_t nbatch) { + if (!posd) return cache->bytes >= (uint64_t)(g0 + n_rows) * row_bytes; + if (ratio == 0u || nbatch == 0u || n_rows > (ratio - 1u + nbatch) / ratio) return 0; + return cache->bytes >= ((uint64_t)g_trash + 1u) * row_bytes; +} +extern "C" int ds4_gpu_v41_ckv_pack_tensor(ds4_gpu_tensor *cache, uint32_t g0, const ds4_gpu_tensor *rows, uint32_t n_rows, + const ds4_gpu_tensor *posd, uint32_t ratio, uint32_t g_trash, uint32_t nbatch) { + if (!cache || !rows || !n_rows) return 0; + if (!v41_kv_pack_check(cache, g0, n_rows, posd, ratio, g_trash, DS4_V41_CKV_BYTES, nbatch)) return 0; + const uint32_t dim = DS4_V41_CKV_NIB * 2u, nb = dim / DS4_V41_CKV_BLK; + v41_kv_pack_kernel<<<(unsigned)((uint64_t)n_rows * nb), 32, 0, ds4_current_stream()>>>( + (uint8_t *)cache->ptr, (const float *)rows->ptr, g0, dim, DS4_V41_CKV_BLK, + DS4_V41_CKV_BYTES, DS4_V41_CKV_NIB, nb, 0, posd ? (const int32_t *)posd->ptr : NULL, ratio, g_trash, nbatch); + return cuda_ok(hipGetLastError(), "v41 ckv pack"); +} +extern "C" int ds4_gpu_v41_idxk_pack_tensor(ds4_gpu_tensor *cache, uint32_t g0, const ds4_gpu_tensor *rows, uint32_t n_rows, + const ds4_gpu_tensor *posd, uint32_t ratio, uint32_t g_trash, uint32_t nbatch) { + if (!cache || !rows || !n_rows) return 0; + if (!v41_kv_pack_check(cache, g0, n_rows, posd, ratio, g_trash, DS4_V41_IDXK_BYTES, nbatch)) return 0; + const uint32_t dim = DS4_V41_IDXK_NIB * 2u, nb = dim / DS4_V41_IDXK_BLK; + v41_kv_pack_kernel<<<(unsigned)((uint64_t)n_rows * nb), 32, 0, ds4_current_stream()>>>( + (uint8_t *)cache->ptr, (const float *)rows->ptr, g0, dim, DS4_V41_IDXK_BLK, + DS4_V41_IDXK_BYTES, DS4_V41_IDXK_NIB, nb, 1, posd ? (const int32_t *)posd->ptr : NULL, ratio, g_trash, nbatch); + return cuda_ok(hipGetLastError(), "v41 idxk pack"); +} + +/* ---- scalar sparse attention (cuda_v41_2.inc.cu:132-303) ---- + * One block per (query, 8-head group), 128 threads = 4 warps x 2 heads; key + * order = window rows ascending then the topk compressed rows; 8-key tiles + * through shared (the engine's flash-style online softmax; its grouping + * differs from the official 64-key tiles because GB10's opt-in shared cap is + * 99 KB, not 128). Invalid topk slots get a -inf score AND a zeroed key row: + * 0 x NaN is NaN, and the shared residue depends on block scheduling (the + * engine's 2026-09-15 conviction: same input, two runs, different results). */ +#define V41_ATTN_HEADS_PER_BLOCK 8u +#define V41_ATTN_KTILE 8u +__global__ static void v41_sparse_attn_kernel(float *o, const float *q, const float *kvw, const uint8_t *kvc, const int32_t *idx, + const float *sink, uint32_t pos0, uint32_t window, uint32_t ng, uint32_t topk, + uint32_t n_head, uint32_t hd, float scale, uint32_t full_block, uint32_t ring, + uint32_t win_lo) { + const uint32_t i = blockIdx.x, lane = threadIdx.x & 31u, warp = threadIdx.x >> 5; + const uint32_t per = hd / 32u; /* 512/32 = 16 dims per lane */ + __shared__ float ks[V41_ATTN_KTILE][512]; + __shared__ int kok[V41_ATTN_KTILE]; + __shared__ float ksc[V41_ATTN_KTILE][32]; /* the compressed row's 32 scales, decoded once per row */ + float qa[2][16], acc[2][16], mx[2], sum[2]; + for (int hh = 0; hh < 2; hh++) { + const uint32_t h = blockIdx.y * V41_ATTN_HEADS_PER_BLOCK + warp * 2u + hh; + for (uint32_t e = 0; e < per; e++) { qa[hh][e] = q[((uint64_t)i * n_head + h) * hd + lane * per + e]; acc[hh][e] = 0.f; } + mx[hh] = -1e30f; sum[hh] = 0.f; + } + const uint32_t p = pos0 + i; /* absolute position */ + /* full_block (the DSpark draft block): every position of this chunk sees + * one key set = the whole window + all n in-block positions (official + * get_dspark_topk_idxs, no causal cut). Main path (full_block=0) is + * causal: position i sees [p+1-window, p]. */ + const uint32_t last = full_block ? pos0 + full_block - 1u : p; + uint32_t lo = full_block ? (pos0 > window ? pos0 - window : 0u) + : (p + 1u > window ? p + 1u - window : 0u); + if (lo < win_lo) lo = win_lo; /* ring slots before win_lo were never written (CED) */ + const uint32_t nwin = last - lo + 1u; + const uint32_t nkeys = nwin + topk; + for (uint32_t base = 0; base < nkeys; base += V41_ATTN_KTILE) { + const uint32_t nt = (nkeys - base) < V41_ATTN_KTILE ? (nkeys - base) : V41_ATTN_KTILE; + __syncthreads(); /* previous tile read before overwrite */ + for (uint32_t t = threadIdx.x / 32u; t < nt; t += blockDim.x / 32u) { + const uint32_t kk = base + t; + /* two key sources, two storages: window rows stay f32 (2.6 MB, + * not worth packing), compressed rows are packed FP4 */ + const float *krow = NULL; const uint8_t *cpk = NULL; + if (kk < nwin) krow = kvw + v41_win_row((int64_t)lo + kk, pos0, window, ring) * hd; + else { const int32_t g = idx[(uint64_t)i * topk + (kk - nwin)]; + if (g >= 0 && (uint32_t)g < ng) cpk = kvc + (uint64_t)g * DS4_V41_CKV_BYTES; } + if (lane == 0) kok[t] = (krow || cpk) ? 1 : 0; + if (cpk) ksc[t][lane] = ds4_e4m3fn_to_f32(cpk[DS4_V41_CKV_NIB + lane]); + __syncwarp(); + /* invalid slots must be ZEROED, not left with residue: 0 x residue + * is only 0 when the residue is finite, and shared reinterpreted + * as float can be NaN/Inf (see the file header). */ + for (uint32_t d = lane; d < hd; d += 32u) + ks[t][d] = krow ? krow[d] : (cpk ? v41_ckv_get_s(cpk, d, ksc[t]) : 0.f); + } + __syncthreads(); + for (int hh = 0; hh < 2; hh++) { + float s[V41_ATTN_KTILE]; + float tm = -1e30f; + for (uint32_t t = 0; t < nt; t++) { + float d = 0.f; + for (uint32_t e = 0; e < per; e++) d += qa[hh][e] * ks[t][lane * per + e]; + for (int off = 16; off > 0; off >>= 1) d += __shfl_xor_sync(0xffffffffu, d, off); + s[t] = kok[t] ? d * scale : -1e30f; + tm = fmaxf(tm, s[t]); + } + const float nm = fmaxf(mx[hh], tm); + const float rs = expf(mx[hh] - nm); /* rescale once per tile */ + float acc_add[16]; + for (uint32_t e = 0; e < per; e++) acc_add[e] = 0.f; + float ps = 0.f; + for (uint32_t t = 0; t < nt; t++) { + const float pv = expf(s[t] - nm); + ps += pv; + const float pb = v41_bf16r(pv); /* official acc_s_cast */ + for (uint32_t e = 0; e < per; e++) acc_add[e] += pb * ks[t][lane * per + e]; + } + sum[hh] = sum[hh] * rs + ps; + for (uint32_t e = 0; e < per; e++) acc[hh][e] = acc[hh][e] * rs + acc_add[e]; + mx[hh] = nm; + } + } + for (int hh = 0; hh < 2; hh++) { + const uint32_t h = blockIdx.y * V41_ATTN_HEADS_PER_BLOCK + warp * 2u + hh; + const float den = sum[hh] + expf(sink[h] - mx[hh]); /* the sink enters the denominator only */ + for (uint32_t e = 0; e < per; e++) o[((uint64_t)i * n_head + h) * hd + lane * per + e] = v41_bf16r(acc[hh][e] / den); + } +} +/* The ds4_gpu_v41_sparse_attn_tensor entry (cuda_v41_2.inc.cu:229-303) moved + * to cuda/ds41_attn_mma.cuh (P4-5) so its dispatch can mirror the engine's + * try order: mma decode -> draft-block mma -> scalar split-K -> prefill mma + * -> this scalar kernel. */ + +/* ---- SWA window ring commit (cuda_kv_ring.inc.cu:20-56) ---- + * The batch's rows sit in [window, window+n) until committed; committing i + * rows moves them to their ring slots. Only the last window rows are + * committed when n > window: ring slots alias at +window, so the ascending + * write's net result is exactly "the last window rows in place". */ +__global__ static void v41_win_commit_kernel(float *win, uint32_t pos0, uint32_t i0, uint32_t window, uint32_t hd, + const int32_t *posd) { + v41_pdl_wait(); + const uint32_t j = blockIdx.x, i = i0 + j; + if (posd) pos0 = (uint32_t)posd[0]; + const float *src = win + (uint64_t)(window + i) * hd; + float *dst = win + (uint64_t)((pos0 + i) % window) * hd; + for (uint32_t d = threadIdx.x; d < hd; d += blockDim.x) dst[d] = src[d]; +} +extern "C" int ds4_gpu_v41_win_commit_tensor(ds4_gpu_tensor *win, uint32_t pos0, uint32_t n, uint32_t window, uint32_t head_dim, + const ds4_gpu_tensor *posd) { + if (!win || !window || !n) return 0; + if (win->bytes < (uint64_t)(window + n) * head_dim * 4) return 0; + /* graph route: pure decode 1 row or a verify batch <= 8 rows; the kernel + * lands row i at ring cell (posd[0] + i) % window (cuda_kv_ring.inc.cu:49). */ + if (posd && n > 8u) return 0; + const uint32_t i0 = n > window ? n - window : 0u, rows = n - i0; + v41_win_commit_kernel<<>>((float *)win->ptr, pos0, i0, window, head_dim, + posd ? (const int32_t *)posd->ptr : NULL); + return cuda_ok(hipGetLastError(), "v41 win commit"); +} + +/* Spec decode: save/restore the ring cells a verify batch's commit will + * overwrite (cuda_kv_ring.inc.cu:34-68). save (back=0) copies ring cell + * (pos0+i)%window into snap[i] (indexed by batch row, so the rollback can take + * the [keep, n) interval); back=1 writes them back. The block rows of a + * verify batch sit in the window buffer's block region until commit, so only + * the cells about to be overwritten need saving: 40 layers x n(<=6) rows x + * 512 x 4 B ~ 0.5 MB, against 10.5 MB for the whole 128-row window. */ +__global__ static void v41_win_ring_snap_kernel(float *win, float *snap, uint32_t pos0, uint32_t i0, + uint32_t window, uint32_t hd, uint32_t back, const int32_t *posd) { + const uint32_t j = blockIdx.x, i = i0 + j; + if (posd) pos0 = (uint32_t)posd[0]; + float *ring = win + (uint64_t)((pos0 + i) % window) * hd; + float *sp = snap + (uint64_t)i * hd; + if (back) { for (uint32_t d = threadIdx.x; d < hd; d += blockDim.x) ring[d] = sp[d]; } + else { for (uint32_t d = threadIdx.x; d < hd; d += blockDim.x) sp[d] = ring[d]; } +} +extern "C" int ds4_gpu_v41_win_ring_snap_tensor(ds4_gpu_tensor *win, ds4_gpu_tensor *snap, uint32_t pos0, + uint32_t i0, uint32_t n, uint32_t window, uint32_t head_dim, int back, + const ds4_gpu_tensor *posd) { + if (!win || !snap || !window || n <= i0) return 1; /* no rows to touch = success */ + if (snap->bytes < (uint64_t)n * head_dim * 4) return 0; + v41_win_ring_snap_kernel<<>>((float *)win->ptr, (float *)snap->ptr, + pos0, i0, window, head_dim, back ? 1u : 0u, + posd ? (const int32_t *)posd->ptr : NULL); + return cuda_ok(hipGetLastError(), back ? "v41 win ring restore" : "v41 win ring save"); +} diff --git a/cuda/ds41_attn_mma.cuh b/cuda/ds41_attn_mma.cuh new file mode 100644 index 0000000000000000000000000000000000000000..6ede00bedd4491e029edc3f83c68a6de3ac26bc8 --- /dev/null +++ b/cuda/ds41_attn_mma.cuh @@ -0,0 +1,808 @@ +#include "hip/hip_runtime.h" +/* ds41_attn_mma.cuh — DeepSeek V4.1 (ds41) sparse-attention tensor-core + * family, ported from the C engine at /data/YoungAi (commit 3946dbc): + * - cuda_sparse_attn_mma.inc.cu: the gather/scores/stats building blocks + * (:42-120) and the single-pass prefill mma kernel (:126-318, the + * 2026-09-29 form with the 520-element key stride); + * - cuda_v41_attn_split.inc.cu: the seg-length formulas (:146-171), the + * merge kernel (:184-199) and the scalar split-K decode (:50-118, :203-230); + * - cuda_v41_attn_mma_decode.inc.cu: the two-pass decode seg kernel + * (:37-132) and the v41_attn_mma_decode host dispatcher (:150-210); + * - the entry ds4_gpu_v41_sparse_attn_tensor (cuda_v41_2.inc.cu:229-303), + * now here so the dispatch can mirror the engine's try order. + * + * Numerics (the engine's own header, cuda_v41_attn_mma_decode.inc.cu:18): + * this family is NOT bit-equal to the scalar kernel — tensor-core + * accumulation order and the online-softmax grouping both differ; the gate + * is the quality scale (NLL/PPL), not cmp. It IS the engine's live decode + * path at every key count (the nkeys<64 split fallback was removed + * 2026-09-16, M1' in the file), so the port's scalar-only deviation (P4-4) + * ends here for n<=8 and n>=64; the scalar kernel stays as the engine's own + * fallback for the 9..63-token band and for shapes the family refuses. + * + * Try order mirrored from cuda_v41_2.inc.cu:229-303: mma decode first for + * !full_block (any n<=8), then the draft-block arm (full_block, linear ring, + * no compressed keys), then the scalar split-K (n==1), then the prefill mma + * (n>=64), then the scalar kernel. + * + * Invalid topk slots (idx<0) get BOTH a zeroed key row and a -inf score — + * the shared gather does both (cuda_sparse_attn_mma.inc.cu:25-27: 0*NaN is + * NaN, and residue read as float can be NaN/Inf). Do not write a second + * gather here; a second copy is how the two paths drift apart. */ +#pragma once +#include "hip_ptx_emul.cuh" /* AMD: ldmatrix / mma equivalents (empty on CUDA) */ + +/* ---- constants (cuda_sparse_attn_mma.inc.cu:29-32, cuda_v41_attn_split.inc.cu:31-47) ---- */ +#define DS4_ATTN_MMA_HEADS 16u /* heads per block = one wmma M tile */ +#define DS4_ATTN_MMA_KT 16u /* keys per shared tile = one wmma N tile */ +#define DS4_ATTN_MMA_WARPS 8u /* warps per block: pass 1/2 split K, the O step splits the output dim */ +#define DS4_ATTN_MMA_HD 512u /* head dim, bound to the v41 shape (the entry checks it) */ +#define DS4_ATTN_FA_LD 520u /* prefill kernel key-tile stride in bf16 elems (1024 B stride = 8-way bank conflict, cuda_sparse_attn_mma.inc.cu:129) */ +#define V41_ATTN_MMA_DEC_TARGET_SEG 24u /* decode target segment count (cuda_v41_attn_split.inc.cu:141) */ +#define V41_ATTN_SPLIT_MIN_KEYS 16u /* below this the split is pure merge overhead (cuda_v41_attn_split.inc.cu:31) */ +#define V41_ATTN_SPLIT_SEG_KEYS 64u /* scalar split max keys per segment */ +#define V41_ATTN_SPLIT_SEG_MIN 8u /* scalar split min = KTILE */ +#define V41_ATTN_SPLIT_TARGET_SEG 6u /* scalar split target segments (36 was measured NEGATIVE, do not raise) */ +#define V41_ATTN_SPLIT_MAX_SEG 64u /* segment cap: scratch = 64 * 64 heads * 512 * 4 B = 8 MB */ + +/* ---- building block 1: gather 16 key rows into shared, bf16 + * (cuda_sparse_attn_mma.inc.cu:42-73). Window rows first in ascending + * order, then the topk compressed rows — the scalar kernel's key order + * exactly. The compressed unpack decodes 32 scales one per lane and the 16 + * fp4 values one per lane, then resolves each element with two shuffles + * (the 2026-09-29 form; per-element v41_ckv_get was 2.4 -> 1.4 ms/step at + * 12k). Same scale, same nibble, same multiply -> bit-equal to v41_ckv_get. */ +__device__ __forceinline__ static void ds4_attn_mma_gather_keys( + __hip_bfloat16 *ks, int *valid, const float *kvw, const uint8_t *kvc, const int32_t *idx, + uint32_t i, uint32_t base, uint32_t nt, uint32_t nwin, uint32_t lo, uint32_t pos0, + uint32_t window, uint32_t ng, uint32_t topk, uint32_t ring) { + const uint32_t lane = threadIdx.x & 31u; + const float tv = ds4_fp4_nibble_to_f32((uint8_t)(lane & 15u)); /* fp4 value table: lane l holds entry l&15 */ + for (uint32_t t = threadIdx.x / 32u; t < DS4_ATTN_MMA_KT; t += blockDim.x / 32u) { + const uint32_t kk = base + t; + const float *krow = NULL; const uint8_t *cpk = NULL; /* window row f32 / compressed row packed FP4 */ + if (t < nt) { + /* ring: main-path history is the ring (1); the DSpark tower's window is a linear segment (0, cuda_v41_attn_mma_decode.inc.cu:57) */ + if (kk < nwin) krow = kvw + v41_win_row((int64_t)lo + kk, pos0, window, ring) * DS4_ATTN_MMA_HD; + else if (kvc && idx) { const int32_t g = idx[(uint64_t)i * topk + (kk - nwin)]; + if (g >= 0 && (uint32_t)g < ng) cpk = kvc + (uint64_t)g * DS4_V41_CKV_BYTES; } + } + if (lane == 0) valid[t] = (krow || cpk) ? 1 : 0; + __hip_bfloat16 *kt = ks + (size_t)t * DS4_ATTN_MMA_HD; + if (krow) { for (uint32_t d = lane; d < DS4_ATTN_MMA_HD; d += 32u) kt[d] = __float2bfloat16(krow[d]); } + else if (cpk) { + const float sc = ds4_e4m3fn_to_f32(cpk[DS4_V41_CKV_NIB + lane]); /* 32 scales for 512 elems: one per lane */ + #pragma unroll + for (uint32_t j = 0; j < DS4_ATTN_MMA_HD / 32u; j++) { + const uint32_t d = lane + 32u * j; + const uint8_t by = cpk[d >> 1]; + const uint8_t nib = (d & 1u) ? (uint8_t)(by >> 4) : (uint8_t)(by & 0x0Fu); /* same nibble take as v41_ckv_get */ + const float s = __shfl_sync(0xffffffffu, sc, (int)(d >> 4)); + kt[d] = __float2bfloat16(__shfl_sync(0xffffffffu, tv, (int)nib) * s); + } + } else { for (uint32_t d = lane; d < DS4_ATTN_MMA_HD; d += 32u) kt[d] = (__hip_bfloat16)0.0f; } + } +} + +/* ---- building block 2: the S tile [16 heads][16 keys] (cuda_sparse_attn_mma.inc.cu:76-101). + * 8 warps each take 1/8 of the K dim (4 wmma k-steps); partials land in + * shared and are added in FIXED order — same input, two runs, same result + * (the 2026-09-15 conviction). Invalid slots read -1e30 so exp is 0. ---- */ +__device__ __forceinline__ static void ds4_attn_mma_scores( + float *stile, float *spart, const __hip_bfloat16 *qs, const __hip_bfloat16 *ks, + const int *valid, uint32_t nt, float scale) { + namespace wmma = nvcuda::wmma; + const uint32_t warp = threadIdx.x >> 5; + wmma::fragment a; + wmma::fragment b; /* B[d][key] = ks[key][d] => col-major, ldm=512 */ + wmma::fragment c; + wmma::fill_fragment(c, 0.0f); + const uint32_t ksteps = DS4_ATTN_MMA_HD / 16u / DS4_ATTN_MMA_WARPS; /* 512/16/8 = 4 */ + for (uint32_t s = 0; s < ksteps; s++) { + const uint32_t d0 = (warp * ksteps + s) * 16u; + wmma::load_matrix_sync(a, qs + d0, DS4_ATTN_MMA_HD); + wmma::load_matrix_sync(b, ks + d0, DS4_ATTN_MMA_HD); + wmma::mma_sync(c, a, b, c); + } + wmma::store_matrix_sync(spart + (size_t)warp * 256u, c, 16, wmma::mem_row_major); + __syncthreads(); + for (uint32_t e = threadIdx.x; e < 256u; e += blockDim.x) { + float v = 0.f; + for (uint32_t w = 0; w < DS4_ATTN_MMA_WARPS; w++) v += spart[(size_t)w * 256u + e]; + const uint32_t k = e & 15u; /* stile is row-major [head][key] */ + stile[e] = (k < nt && valid[k]) ? v * scale : -1e30f; + } + __syncthreads(); +} + +/* ---- building block 3: the online max/sum over one tile (cuda_sparse_attn_mma.inc.cu:105-120). + * Parallel form (2026-09-29): warp w owns heads 2w (lanes 0..15) and 2w+1 + * (lanes 16..31), one key per lane. Bit-equal to the serial form: max is + * exact (order-free); the sum is rsum*expf(m-tm) then expf(s_k-tm) added in + * k = 0..15 order — same order, same value. ALL threads must call it (the + * warp shuffles are collective), never wrap it in threadIdx.x < 16. ---- */ +__device__ __forceinline__ static void ds4_attn_mma_stats(const float *stile, float *rmax, float *rsum) { + const uint32_t lane = threadIdx.x & 31u, warp = threadIdx.x >> 5, k = lane & 15u; + const uint32_t h = warp * 2u + (lane >> 4); + const float s = stile[h * 16u + k], m = rmax[h]; + float tm = fmaxf(m, s); + tm = fmaxf(tm, __shfl_xor_sync(0xffffffffu, tm, 8)); tm = fmaxf(tm, __shfl_xor_sync(0xffffffffu, tm, 4)); + tm = fmaxf(tm, __shfl_xor_sync(0xffffffffu, tm, 2)); tm = fmaxf(tm, __shfl_xor_sync(0xffffffffu, tm, 1)); + const float e = expf(s - tm); + float sm = rsum[h] * expf(m - tm); + const int b = (int)(lane & 16u); + #pragma unroll + for (int kk = 0; kk < 16; kk++) sm += __shfl_sync(0xffffffffu, e, b + kk); + if (k == 0u) { rmax[h] = tm; rsum[h] = sm; } +} + +/* ---- the single-pass prefill mma kernel (cuda_sparse_attn_mma.inc.cu:126-282). + * Four changes vs the two-pass form: q stays in registers (A fragments), + * S via mma.m16n8k16 (not wmma), single pass (O scaled by alpha and P*V + * accumulated in the same walk; the accumulator layout is documented, so + * per-head scaling is reachable), key stride 520 bf16 (bank conflicts). + * Not bit-equal to the two-pass form; the gate is five-metric/PPL. ---- */ +__device__ __forceinline__ static void ds4_fa_ldsm4(uint32_t *r, const void *p) { +#if defined(__HIP_PLATFORM_AMD__) + ds4hip_ldmatrix<4>(r, p); +#else + const uint32_t a = (uint32_t)__cvta_generic_to_shared(p); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];" : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) : "r"(a)); +#endif +} +__device__ __forceinline__ static void ds4_fa_ldsm4t(uint32_t *r, const void *p) { +#if defined(__HIP_PLATFORM_AMD__) + ds4hip_ldmatrix_trans<4>(r, p); +#else + const uint32_t a = (uint32_t)__cvta_generic_to_shared(p); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];" : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) : "r"(a)); +#endif +} +__device__ __forceinline__ static void ds4_fa_mma(float *c, const uint32_t *a, uint32_t b0, uint32_t b1) { +#if defined(__HIP_PLATFORM_AMD__) + ds4hip_mma_m16n8k16(c, a[0], a[1], a[2], a[3], b0, b1); +#else + asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};" + : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3]) : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1)); +#endif +} +__device__ __forceinline__ static uint32_t ds4_fa_bf16x2(float lo, float hi) { + return (__float_as_uint(v41_bf16r(lo)) >> 16) | (__float_as_uint(v41_bf16r(hi)) & 0xffff0000u); +} +/* gather: same decode as ds4_attn_mma_gather_keys (bit-equal values), only the row stride is DS4_ATTN_FA_LD */ +__device__ __forceinline__ static void ds4_fa_gather(__hip_bfloat16 *ks, int *valid, const float *kvw, const uint8_t *kvc, const int32_t *idx, + uint32_t i, uint32_t base, uint32_t nt, uint32_t nwin, uint32_t lo, uint32_t pos0, + uint32_t window, uint32_t ng, uint32_t topk) { + const uint32_t lane = threadIdx.x & 31u; + const float tv = ds4_fp4_nibble_to_f32((uint8_t)(lane & 15u)); + for (uint32_t t = threadIdx.x / 32u; t < DS4_ATTN_MMA_KT; t += blockDim.x / 32u) { + const uint32_t kk = base + t; + const float *krow = NULL; const uint8_t *cpk = NULL; + if (t < nt) { + if (kk < nwin) krow = kvw + v41_win_row((int64_t)lo + kk, pos0, window, 1u) * DS4_ATTN_MMA_HD; + else if (kvc && idx) { const int32_t g = idx[(uint64_t)i * topk + (kk - nwin)]; + if (g >= 0 && (uint32_t)g < ng) cpk = kvc + (uint64_t)g * DS4_V41_CKV_BYTES; } + } + if (lane == 0) valid[t] = (krow || cpk) ? 1 : 0; + __hip_bfloat16 *kt = ks + (size_t)t * DS4_ATTN_FA_LD; + if (krow) { for (uint32_t d = lane; d < DS4_ATTN_MMA_HD; d += 32u) kt[d] = __float2bfloat16(krow[d]); } + else if (cpk) { + const float sc = ds4_e4m3fn_to_f32(cpk[DS4_V41_CKV_NIB + lane]); + #pragma unroll + for (uint32_t j = 0; j < DS4_ATTN_MMA_HD / 32u; j++) { + const uint32_t d = lane + 32u * j; + const uint8_t by = cpk[d >> 1]; + const uint8_t nib = (d & 1u) ? (uint8_t)(by >> 4) : (uint8_t)(by & 0x0Fu); + const float s = __shfl_sync(0xffffffffu, sc, (int)(d >> 4)); + kt[d] = __float2bfloat16(__shfl_sync(0xffffffffu, tv, (int)nib) * s); + } + } else { for (uint32_t d = lane; d < DS4_ATTN_MMA_HD; d += 32u) kt[d] = (__hip_bfloat16)0.0f; } + } +} +/* shared: ks 16x520x2 + spart 8x16x16x4 + stile 16x16x4 + ptile 16x16x2 + rmax/rsum/alpha 3x16x4 + valid 16x4 ~= 26 KB => 3 blocks per SM */ +__global__ __launch_bounds__(256) static void ds4_sparse_attn_mma_kernel(float *o, const float *q, const float *kvw, const uint8_t *kvc, + const int32_t *idx, const float *sink, uint32_t pos0, uint32_t window, + uint32_t ng, uint32_t topk, uint32_t n_head, float scale, uint32_t win_lo) { + constexpr uint32_t KT = DS4_ATTN_MMA_KT, HB = DS4_ATTN_MMA_HEADS; + extern __shared__ __align__(16) char ds4_attn_fa_smem[]; + __hip_bfloat16 *ks = (__hip_bfloat16 *)ds4_attn_fa_smem; /* [16][520] */ + float *spart = (float *)(ks + (size_t)KT * DS4_ATTN_FA_LD); /* [8 warp][16][16] */ + float *stile = spart + 8u * HB * KT; /* [16][16] */ + __hip_bfloat16 *ptile = (__hip_bfloat16 *)(stile + HB * KT); /* [16][16] */ + float *rmax = (float *)(ptile + HB * KT), *rsum = rmax + HB, *alpha = rsum + HB; + int *valid = (int *)(alpha + HB); + const uint32_t i = blockIdx.x, h0 = blockIdx.y * HB, lane = threadIdx.x & 31u, warp = threadIdx.x >> 5; + const uint32_t p = pos0 + i; + uint32_t lo = p + 1u > window ? p + 1u - window : 0u; + if (lo < win_lo) lo = win_lo; /* ring slots before win_lo were never written (CED), do not read (official -1 mask) */ + const uint32_t nwin = p - lo + 1u, nkeys = nwin + topk; + /* q's A fragments live in registers: warp w covers dims [64w, 64w+64) as 4 k16 steps (a0: row r col c..c+1; a1: row r+8; a2: col +8; a3: row +8 col +8) */ + uint32_t qa[4][4]; + { + const uint32_t r = lane >> 2, c = (lane & 3u) * 2u; + #pragma unroll + for (uint32_t s = 0; s < 4u; s++) { + const float *q0 = q + ((uint64_t)i * n_head + h0 + r) * DS4_ATTN_MMA_HD + warp * 64u + s * 16u + c; + const float *q1 = q0 + 8u * DS4_ATTN_MMA_HD; + qa[s][0] = ds4_fa_bf16x2(q0[0], q0[1]); qa[s][1] = ds4_fa_bf16x2(q1[0], q1[1]); + qa[s][2] = ds4_fa_bf16x2(q0[8], q0[9]); qa[s][3] = ds4_fa_bf16x2(q1[8], q1[9]); + } + } + if (threadIdx.x < HB) { rmax[threadIdx.x] = -1e30f; rsum[threadIdx.x] = 0.f; } + float oacc[8][4]; /* O[16 heads][this warp's 64 dims] = 8 n8 slices */ + #pragma unroll + for (uint32_t j = 0; j < 8u; j++) { oacc[j][0] = oacc[j][1] = oacc[j][2] = oacc[j][3] = 0.f; } + __syncthreads(); + for (uint32_t base = 0; base < nkeys; base += KT) { + const uint32_t nt = (nkeys - base) < KT ? (nkeys - base) : KT; + __syncthreads(); /* previous tile's ks/ptile fully consumed */ + ds4_fa_gather(ks, valid, kvw, kvc, idx, i, base, nt, nwin, lo, pos0, window, ng, topk); + __syncthreads(); + { /* (1) S partials: this warp's 64 dims x 16 heads x 16 keys (two n8 slices) */ + float sp[2][4] = { {0.f, 0.f, 0.f, 0.f}, {0.f, 0.f, 0.f, 0.f} }; + #pragma unroll + for (uint32_t s = 0; s < 4u; s++) { + uint32_t b[4]; + const uint32_t mt = lane >> 3, key = (mt >> 1) * 8u + (lane & 7u), col = warp * 64u + s * 16u + (mt & 1u) * 8u; + ds4_fa_ldsm4(b, ks + (size_t)key * DS4_ATTN_FA_LD + col); + ds4_fa_mma(sp[0], qa[s], b[0], b[1]); ds4_fa_mma(sp[1], qa[s], b[2], b[3]); + } + #pragma unroll + for (uint32_t nb = 0; nb < 2u; nb++) { /* c0,c1 = row lane/4, cols (lane&3)*2+{0,1}; c2,c3 = row +8 */ + float *sw = spart + (size_t)warp * HB * KT + (lane >> 2) * KT + nb * 8u + (lane & 3u) * 2u; + sw[0] = sp[nb][0]; sw[1] = sp[nb][1]; sw[8u * KT] = sp[nb][2]; sw[8u * KT + 1u] = sp[nb][3]; + } + } + __syncthreads(); + /* (2) fixed-order sum + scale + mask (invalid slots -1e30 => exp 0) */ + { + const uint32_t e = threadIdx.x, k = e % KT; + float v = 0.f; + #pragma unroll + for (uint32_t w = 0; w < 8u; w++) v += spart[(size_t)w * HB * KT + e]; + stile[e] = (k < nt && valid[k]) ? v * scale : -1e30f; + } + __syncthreads(); + /* (3) online max/sum + P tile (bf16, official acc_s_cast) + alpha: one thread per head, key order serial (the two-pass serial order) */ + if (threadIdx.x < HB) { + const uint32_t h = threadIdx.x; const float *sr = stile + h * KT; const float m = rmax[h]; + float tm = m; + for (uint32_t k = 0; k < KT; k++) tm = fmaxf(tm, sr[k]); + const float al = expf(m - tm); + float sm = rsum[h] * al; + for (uint32_t k = 0; k < KT; k++) { const float pv = expf(sr[k] - tm); sm += pv; ptile[h * KT + k] = __float2bfloat16(pv); } + rmax[h] = tm; rsum[h] = sm; alpha[h] = al; + } + __syncthreads(); + /* (4) O = O*alpha + P*V: A = ptile [16 heads][16 keys], B = ks [keys][dim] via .trans; this warp covers dims [64w, 64w+64) as 8 n8 slices */ + { + const float a0 = alpha[lane >> 2], a1 = alpha[(lane >> 2) + 8u]; + #pragma unroll + for (uint32_t j = 0; j < 8u; j++) { oacc[j][0] *= a0; oacc[j][1] *= a0; oacc[j][2] *= a1; oacc[j][3] *= a1; } + uint32_t pa[4]; + { const uint32_t mt = lane >> 3, row = (lane & 7u) + (mt & 1u) * 8u, col = (mt >> 1) * 8u; + ds4_fa_ldsm4(pa, ptile + (size_t)row * KT + col); } + #pragma unroll + for (uint32_t j = 0; j < 8u; j += 2u) { + uint32_t b[4]; + const uint32_t mt = lane >> 3, key = (mt & 1u) * 8u + (lane & 7u), col = warp * 64u + j * 8u + (mt >> 1) * 8u; + ds4_fa_ldsm4t(b, ks + (size_t)key * DS4_ATTN_FA_LD + col); + ds4_fa_mma(oacc[j], pa, b[0], b[1]); ds4_fa_mma(oacc[j + 1], pa, b[2], b[3]); + } + } + } + __syncthreads(); + /* exit: divide by (sum + exp(sink - max)) (sink enters the denominator only), round bf16, write */ + { + const uint32_t hA = lane >> 2, hB = hA + 8u; + const float dA = rsum[hA] + expf(sink[h0 + hA] - rmax[hA]), dB = rsum[hB] + expf(sink[h0 + hB] - rmax[hB]); + #pragma unroll + for (uint32_t j = 0; j < 8u; j++) { + const uint32_t d = warp * 64u + j * 8u + (lane & 3u) * 2u; + float *oA = o + ((uint64_t)i * n_head + h0 + hA) * DS4_ATTN_MMA_HD + d, *oB = o + ((uint64_t)i * n_head + h0 + hB) * DS4_ATTN_MMA_HD + d; + oA[0] = v41_bf16r(oacc[j][0] / dA); oA[1] = v41_bf16r(oacc[j][1] / dA); + oB[0] = v41_bf16r(oacc[j][2] / dB); oB[1] = v41_bf16r(oacc[j][3] / dB); + } + } +} + +/* prefill kernel shared size (~26 KB); the decode seg kernel keeps its own layout below */ +static size_t ds4_attn_mma_smem_bytes(void) { + return (size_t)DS4_ATTN_MMA_KT * DS4_ATTN_FA_LD * sizeof(__hip_bfloat16) + + (size_t)DS4_ATTN_MMA_WARPS * DS4_ATTN_MMA_HEADS * DS4_ATTN_MMA_KT * sizeof(float) + + (size_t)DS4_ATTN_MMA_HEADS * DS4_ATTN_MMA_KT * (sizeof(float) + sizeof(__hip_bfloat16)) + + 3u * DS4_ATTN_MMA_HEADS * sizeof(float) + DS4_ATTN_MMA_KT * sizeof(int); +} +static size_t ds4_attn_mma_seg_smem_bytes(void) { + return (size_t)(DS4_ATTN_MMA_HEADS + DS4_ATTN_MMA_KT) * DS4_ATTN_MMA_HD * sizeof(__hip_bfloat16) + + (size_t)DS4_ATTN_MMA_WARPS * 256u * sizeof(float) + 256u * sizeof(float) + + 256u * sizeof(__hip_bfloat16) + 2u * DS4_ATTN_MMA_HEADS * sizeof(float) + + DS4_ATTN_MMA_KT * sizeof(int); +} + +/* Usable? Shape (64 heads x 512, head count divisible by 16) + big enough + * (at small n_tok the grid is n_tok x 4 blocks, 48 SMs stay idle and the + * scalar path wins) + shared can be raised. 0 = caller falls back. */ +static int ds4_sparse_attn_mma_launch(float *o, const float *q, const float *kvw, const uint8_t *kvc, + const int32_t *idx, const float *sink, uint32_t n_tok, uint32_t pos0, + uint32_t window, uint32_t ng, uint32_t topk, uint32_t n_head, + uint32_t head_dim, float scale, uint32_t win_lo) { + if (head_dim != DS4_ATTN_MMA_HD || (n_head % DS4_ATTN_MMA_HEADS) || n_tok < 64u) return 0; + static int s_ok = 0; /* 0 untried / 1 usable / -1 shared cannot be raised */ + const size_t smem = ds4_attn_mma_smem_bytes(); + if (s_ok == 0) { + (void)hipGetLastError(); /* the opt-in must not see a previously latched error (see ds41_vq_decode.cuh) */ + s_ok = hipFuncSetAttribute(reinterpret_cast(ds4_sparse_attn_mma_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, + (int)smem) == hipSuccess ? 1 : -1; + (void)hipGetLastError(); + fprintf(stderr, "ds4: [attn] tensor-core prefill shared %zu KB %s\n", smem >> 10, s_ok == 1 ? "enabled" : "*cannot be raised, back to scalar*"); + } + if (s_ok != 1) return 0; + ds4_sparse_attn_mma_kernel<<>>( + o, q, kvw, kvc, idx, sink, pos0, window, ng, topk, n_head, scale, win_lo); + return cuda_ok(hipGetLastError(), "sparse attn mma"); +} + +/* ---- seg-length formulas (cuda_v41_attn_split.inc.cu:121-171). + * The segment length may only depend on a batch-size-independent reference + * key count (2026-09-16 M1'): the same position in pure decode (n=1) and in + * a spec-verify batch (n=4) must get the same grouping, or the two paths + * diverge at temperature 0. Feeding the formula each query's OWN absolute + * position is the fix; "first query of the batch" was measured WRONG + * (2K, byte 270 diverged). ---- */ +__host__ __device__ __forceinline__ static uint32_t v41_attn_seg_keys(uint32_t p, uint32_t window, + uint32_t ratio, uint32_t topk) { + const uint32_t nwin = p + 1u > window ? window : p + 1u; + uint32_t tref = 0; + if (ratio) { const uint32_t vis = (p + 1u) / ratio; tref = topk < vis ? topk : vis; } + uint32_t seg = (nwin + tref + V41_ATTN_MMA_DEC_TARGET_SEG - 1u) / V41_ATTN_MMA_DEC_TARGET_SEG; + seg = ((seg + DS4_ATTN_MMA_KT - 1u) / DS4_ATTN_MMA_KT) * DS4_ATTN_MMA_KT; + if (seg < DS4_ATTN_MMA_KT) seg = DS4_ATTN_MMA_KT; + return seg > 64u ? 64u : seg; +} +/* full-visible block (DSpark draft tower, 2026-09-29): every in-block + * position sees one key set (whole window + all n in-block positions, no + * compressed keys); the length depends on the key count only. Drafts only + * propose, verification decides, so this path is bound by determinism, not + * by the decode-==-verify byte gate. */ +__host__ __device__ __forceinline__ static uint32_t v41_attn_fb_seg_keys(uint32_t nkeys) { + uint32_t seg = (nkeys + V41_ATTN_MMA_DEC_TARGET_SEG - 1u) / V41_ATTN_MMA_DEC_TARGET_SEG; + seg = ((seg + DS4_ATTN_MMA_KT - 1u) / DS4_ATTN_MMA_KT) * DS4_ATTN_MMA_KT; + if (seg < DS4_ATTN_MMA_KT) seg = DS4_ATTN_MMA_KT; + return seg > 64u ? 64u : seg; +} +/* segments at position p (= the eager path's grid; the graph path's merge + * recomputes the same number from the device position) */ +__host__ __device__ __forceinline__ static uint32_t v41_attn_nseg_at(uint32_t p, uint32_t window, uint32_t ratio, uint32_t topk) { + const uint32_t nw = p + 1u > window ? window : p + 1u; + uint32_t tk = topk; + if (ratio) { const uint32_t vis = (p + 1u) / ratio; if (vis < tk) tk = vis; } + const uint32_t sk = v41_attn_seg_keys(p, window, ratio, topk); + return (nw + tk + sk - 1u) / sk; +} + +/* ---- the merge (cuda_v41_attn_split.inc.cu:184-199): one block per head, + * segments merged in FIXED segment order (atomic add would make two runs of + * one input differ), sink into the denominator, divide, round bf16. + * Local tiles are laid out [(token*nseg + seg)*head + h]; n_tok=1 is the + * original [seg*head + h] untouched. The posd arm re-derives the true + * segment count from the device position and reads only real segments — + * bit-equal to the eager path (extra segments never existed there). ---- */ +__global__ static void v41_sparse_attn_merge_kernel(float *o, const float *pacc, const float *pmax, const float *psum, + const float *sink, uint32_t nseg, uint32_t n_head, uint32_t hd, + const int32_t *posd, uint32_t window, uint32_t ratio, uint32_t topk) { + v41_pdl_wait(); /* PDL: wait for the upstream as the first instruction */ + const uint32_t h = blockIdx.x, i = blockIdx.y; + const uint64_t b0 = (uint64_t)i * nseg * n_head; /* tile row stride follows the grid nseg (the cap) */ + if (posd) nseg = v41_attn_nseg_at((uint32_t)posd[0] + i, window, ratio, topk); /* row i's position = pos0 + i */ + float m = -1e30f; + for (uint32_t s = 0; s < nseg; s++) m = fmaxf(m, pmax[b0 + (uint64_t)s * n_head + h]); + float den = 0.f; + for (uint32_t s = 0; s < nseg; s++) den += psum[b0 + (uint64_t)s * n_head + h] * expf(pmax[b0 + (uint64_t)s * n_head + h] - m); + den += expf(sink[h] - m); + for (uint32_t d = threadIdx.x; d < hd; d += blockDim.x) { + float v = 0.f; + for (uint32_t s = 0; s < nseg; s++) + v += pacc[(b0 + (uint64_t)s * n_head + h) * hd + d] * expf(pmax[b0 + (uint64_t)s * n_head + h] - m); + o[((uint64_t)i * n_head + h) * hd + d] = v41_bf16r(v / den); + } +} + +/* ---- the scalar split-K decode (cuda_v41_attn_split.inc.cu:50-118): one + * block per (segment, 8-head group), 128 threads = 4 warps x 2 heads; the + * local exit keeps the denominator out (the merge owns it). Numerics: + * segmenting regroups the online softmax -> not bit-equal, quality gate. ---- */ +__global__ static void v41_sparse_attn_split_kernel(float *pacc, float *pmax, float *psum, + const float *q, const float *kvw, const uint8_t *kvc, const int32_t *idx, + uint32_t pos0, uint32_t window, uint32_t ng, uint32_t topk, + uint32_t n_head, uint32_t hd, float scale, uint32_t seg_keys) { + const uint32_t seg = blockIdx.x, lane = threadIdx.x & 31u, warp = threadIdx.x >> 5; + const uint32_t per = hd / 32u; + __shared__ float ks[V41_ATTN_KTILE][512]; + __shared__ int kok[V41_ATTN_KTILE]; + __shared__ float ksc[V41_ATTN_KTILE][32]; /* the compressed row's 32 scales, decoded once per row */ + float qa[2][16], acc[2][16], mx[2], sum[2]; + for (int hh = 0; hh < 2; hh++) { + const uint32_t h = blockIdx.y * V41_ATTN_HEADS_PER_BLOCK + warp * 2u + hh; + for (uint32_t e = 0; e < per; e++) { qa[hh][e] = q[(uint64_t)h * hd + lane * per + e]; acc[hh][e] = 0.f; } + mx[hh] = -1e30f; sum[hh] = 0.f; + } + const uint32_t p = pos0; /* decode: one query, its absolute position is pos0 */ + const uint32_t lo = p + 1u > window ? p + 1u - window : 0u; + const uint32_t nwin = p - lo + 1u; + const uint32_t nkeys = nwin + topk; + const uint32_t k0 = seg * seg_keys, k1 = (k0 + seg_keys) < nkeys ? (k0 + seg_keys) : nkeys; + for (uint32_t base = k0; base < k1; base += V41_ATTN_KTILE) { + const uint32_t nt = (k1 - base) < V41_ATTN_KTILE ? (k1 - base) : V41_ATTN_KTILE; + __syncthreads(); + for (uint32_t t = threadIdx.x / 32u; t < nt; t += blockDim.x / 32u) { + const uint32_t kk = base + t; + const float *krow = NULL; const uint8_t *cpk = NULL; /* window row f32 / compressed row packed FP4 */ + /* ring=1 holds here: this path only runs on the main route (!full_block), whose history is the ring */ + if (kk < nwin) krow = kvw + v41_win_row((int64_t)lo + kk, pos0, window, 1u) * hd; + else { const int32_t g = idx[kk - nwin]; + if (g >= 0 && (uint32_t)g < ng) cpk = kvc + (uint64_t)g * DS4_V41_CKV_BYTES; } + if (lane == 0) kok[t] = (krow || cpk) ? 1 : 0; + if (cpk) ksc[t][lane] = ds4_e4m3fn_to_f32(cpk[DS4_V41_CKV_NIB + lane]); + __syncwarp(); /* this warp writes and reads its own rows */ + for (uint32_t d = lane; d < hd; d += 32u) /* invalid slots must be ZEROED, see the file header */ + ks[t][d] = krow ? krow[d] : (cpk ? v41_ckv_get_s(cpk, d, ksc[t]) : 0.f); + } + __syncthreads(); + for (int hh = 0; hh < 2; hh++) { + float s[V41_ATTN_KTILE]; + float tm = -1e30f; + for (uint32_t t = 0; t < nt; t++) { + float d = 0.f; + for (uint32_t e = 0; e < per; e++) d += qa[hh][e] * ks[t][lane * per + e]; + for (int off = 16; off > 0; off >>= 1) d += __shfl_xor_sync(0xffffffffu, d, off); + s[t] = kok[t] ? d * scale : -1e30f; + tm = fmaxf(tm, s[t]); + } + const float nm = fmaxf(mx[hh], tm); + const float rs = expf(mx[hh] - nm); + float acc_add[16]; + for (uint32_t e = 0; e < per; e++) acc_add[e] = 0.f; + float ps = 0.f; + for (uint32_t t = 0; t < nt; t++) { + const float pv = expf(s[t] - nm); + ps += pv; + const float pb = v41_bf16r(pv); + for (uint32_t e = 0; e < per; e++) acc_add[e] += pb * ks[t][lane * per + e]; + } + sum[hh] = sum[hh] * rs + ps; + for (uint32_t e = 0; e < per; e++) acc[hh][e] = acc[hh][e] * rs + acc_add[e]; + mx[hh] = nm; + } + } + for (int hh = 0; hh < 2; hh++) { + const uint32_t h = blockIdx.y * V41_ATTN_HEADS_PER_BLOCK + warp * 2u + hh; + float *pa = pacc + ((uint64_t)seg * n_head + h) * hd; + for (uint32_t e = 0; e < per; e++) pa[lane * per + e] = acc[hh][e]; + if (lane == 0) { pmax[(uint64_t)seg * n_head + h] = mx[hh]; psum[(uint64_t)seg * n_head + h] = sum[hh]; } + } +} + +/* decode-family scratch (cuda_v41_attn_split.inc.cu:201): the engine keeps + * one triple per concurrent lane; the port is single-lane (ds41_indexer.cuh + * precedent). */ +static v41_scratch g_v41_attn_pacc, g_v41_attn_pmax, g_v41_attn_psum; + +/* 1 = the scalar split-K took this launch; 0 = shape unfit, caller falls through. */ +static int v41_sparse_attn_split(float *o, const float *q, const float *kvw, const uint8_t *kvc, const int32_t *idx, + const float *sink, uint32_t n_tok, uint32_t pos0, uint32_t window, uint32_t ng, + uint32_t topk, uint32_t n_head, uint32_t hd, float scale) { + if (n_tok != 1u || hd != 512u || (n_head % V41_ATTN_HEADS_PER_BLOCK)) return 0; + const uint32_t nwin = pos0 + 1u > window ? window : pos0 + 1u; + const uint32_t nkeys = nwin + topk; + if (nkeys < V41_ATTN_SPLIT_MIN_KEYS) return 0; + /* seg length = min(64, keys/TARGET_SEG), multiple of 8, at least KTILE. + * Keys >= 384 give 64, the behavior validated on 09-15. */ + uint32_t seg_keys = (nkeys + V41_ATTN_SPLIT_TARGET_SEG - 1u) / V41_ATTN_SPLIT_TARGET_SEG; + seg_keys = ((seg_keys + V41_ATTN_SPLIT_SEG_MIN - 1u) / V41_ATTN_SPLIT_SEG_MIN) * V41_ATTN_SPLIT_SEG_MIN; + if (seg_keys < V41_ATTN_SPLIT_SEG_MIN) seg_keys = V41_ATTN_SPLIT_SEG_MIN; + if (seg_keys > V41_ATTN_SPLIT_SEG_KEYS) seg_keys = V41_ATTN_SPLIT_SEG_KEYS; + uint32_t nseg = (nkeys + seg_keys - 1u) / seg_keys; + while (nseg > V41_ATTN_SPLIT_MAX_SEG) { seg_keys *= 2u; nseg = (nkeys + seg_keys - 1u) / seg_keys; } + const uint64_t na = (uint64_t)nseg * n_head; + float *pacc = (float *)v41_grow(&g_v41_attn_pacc, na * hd * 4, "v41 attn split acc"); + float *pmax = (float *)v41_grow(&g_v41_attn_pmax, na * 4, "v41 attn split max"); + float *psum = (float *)v41_grow(&g_v41_attn_psum, na * 4, "v41 attn split sum"); + if (!pacc || !pmax || !psum) return 0; + v41_sparse_attn_split_kernel<<>>( + pacc, pmax, psum, q, kvw, kvc, idx, pos0, window, ng, topk, n_head, hd, scale, seg_keys); + if (!cuda_ok(hipGetLastError(), "v41 sparse attn split")) return 0; + v41_pdl_register((const void *)v41_sparse_attn_merge_kernel); /* the kernel v41_pdl_wait before reading the tiles */ + v41_sparse_attn_merge_kernel<<>>(o, pacc, pmax, psum, sink, nseg, n_head, hd, + NULL, window, 0u, topk); + return cuda_ok(hipGetLastError(), "v41 sparse attn merge"); +} + +/* ---- the decode seg kernel (cuda_v41_attn_mma_decode.inc.cu:30-132): one + * segment of keys per block, full two-pass online softmax locally, local + * (max, sum, acc) out, merged in fixed segment order by the kernel above. + * For one query, S = Q[64x512] . K^T[512xnkeys] is a real GEMM; the scalar + * kernel pays 5 shuffles per 16 FMAs (69 GFLOP/s = 0.4% of the board). ---- */ +/* posd (graph route, 2026-09-18): position from the device slot, ng/topk + * derived from it; nseg is the bucket cap and segments past the true count + * return without writing — the merge reads only real segments. */ +/* Negative archive (2026-10-07): fusing the merge into this kernel via + * threadfence (each (row, head-group)'s last block merges in fixed order) + * was byte-equal but 46.33 -> 45.66 t/s — the merge tail serializes over + * n_tok x 4 = 16 blocks instead of running as its own n_head x n_tok = 256 + * block launch. Reverted to two launches; do not re-fuse. */ +__global__ static void v41_attn_mma_seg_kernel(float *pacc, float *pmax, float *psum, + const float *q, const float *kvw, const uint8_t *kvc, + const int32_t *idx, uint32_t pos0, uint32_t window, + uint32_t ng, uint32_t topk, uint32_t n_head, + float scale, uint32_t ratio, uint32_t nseg, const int32_t *posd, + uint32_t full_block, uint32_t ring) { + v41_pdl_wait(); /* PDL: wait for the upstream as the first instruction */ + /* graph: position in the device slot; at n rows (gridDim.z, spec verify batch) source groups = (pos0 + n)/ratio, the eager path's ng_src formula */ + if (posd) { pos0 = (uint32_t)posd[0]; ng = ratio ? (pos0 + gridDim.z) / ratio : 0u; if (ng < topk) topk = ng; } + namespace wmma = nvcuda::wmma; + extern __shared__ char ds4_attn_mma_smem[]; + __hip_bfloat16 *qs = (__hip_bfloat16 *)ds4_attn_mma_smem; + __hip_bfloat16 *ks = qs + DS4_ATTN_MMA_HEADS * DS4_ATTN_MMA_HD; + float *spart = (float *)(ks + DS4_ATTN_MMA_KT * DS4_ATTN_MMA_HD); + float *stile = spart + DS4_ATTN_MMA_WARPS * 256u; + __hip_bfloat16 *ptile = (__hip_bfloat16 *)(stile + 256u); + float *rmax = (float *)(ptile + 256u), *rsum = rmax + DS4_ATTN_MMA_HEADS; + int *valid = (int *)(rsum + DS4_ATTN_MMA_HEADS); + /* i = this batch's query index (decode: always 0; spec verify: 0..k). Each query sees its own key range: + * nseg is sized for the LAST query (most keys), so earlier queries have whole segments outside their + * visible range — those must still write max=-inf / sum=0 / acc=0 or the merge reads last round's residue. */ + const uint32_t seg = blockIdx.x, h0 = blockIdx.y * DS4_ATTN_MMA_HEADS, i = blockIdx.z; + const uint64_t pbase = ((uint64_t)i * nseg + seg) * n_head + h0; + const uint32_t p = pos0 + i; + /* full_block (DSpark draft block, also here since 2026-09-29): every position of the chunk sees one key + * set = the whole window before it + all n in-block positions, no causal cut (official get_dspark_topk_idxs); + * the main route (0) is causal: position i sees [p+1-window, p]. Window layout follows ring (towers are linear). */ + const uint32_t last = full_block ? pos0 + full_block - 1u : p; + const uint32_t lo = full_block ? (pos0 > window ? pos0 - window : 0u) : (p + 1u > window ? p + 1u - window : 0u); + const uint32_t nwin = last - lo + 1u, nkeys = nwin + topk; + /* seg length by this query's OWN position (see the formulas): earlier queries may have a shorter + * length and thus fewer segments than the grid's nseg — the extra segments take the empty branch, + * which is exactly neutral to the merge. */ + const uint32_t seg_keys = full_block ? v41_attn_fb_seg_keys(nkeys) : v41_attn_seg_keys(p, window, ratio, topk); + const uint32_t k0 = seg * seg_keys; + if (k0 >= nkeys) { /* empty segment: write neutral values and leave (exp(-1e30 - m) = 0 contributes nothing) */ + if (posd) return; /* graph route: the merge reads only real segments, skip the 32 KB/segment white write */ + for (uint32_t e = threadIdx.x; e < DS4_ATTN_MMA_HEADS * DS4_ATTN_MMA_HD; e += blockDim.x) + pacc[(pbase + e / DS4_ATTN_MMA_HD) * DS4_ATTN_MMA_HD + e % DS4_ATTN_MMA_HD] = 0.f; + if (threadIdx.x < DS4_ATTN_MMA_HEADS) { pmax[pbase + threadIdx.x] = -1e30f; psum[pbase + threadIdx.x] = 0.f; } + return; + } + const uint32_t k1 = (k0 + seg_keys) < nkeys ? (k0 + seg_keys) : nkeys; + for (uint32_t e = threadIdx.x; e < DS4_ATTN_MMA_HEADS * DS4_ATTN_MMA_HD; e += blockDim.x) + qs[e] = __float2bfloat16(q[((uint64_t)i * n_head + h0 + e / DS4_ATTN_MMA_HD) * DS4_ATTN_MMA_HD + e % DS4_ATTN_MMA_HD]); + if (threadIdx.x < DS4_ATTN_MMA_HEADS) { rmax[threadIdx.x] = -1e30f; rsum[threadIdx.x] = 0.f; } + __syncthreads(); + + for (uint32_t base = k0; base < k1; base += DS4_ATTN_MMA_KT) { /* pass 1: this segment's max and exp sum */ + const uint32_t nt = (k1 - base) < DS4_ATTN_MMA_KT ? (k1 - base) : DS4_ATTN_MMA_KT; + __syncthreads(); + ds4_attn_mma_gather_keys(ks, valid, kvw, kvc, idx, i, base, nt, nwin, lo, pos0, window, ng, topk, ring); + __syncthreads(); + ds4_attn_mma_scores(stile, spart, qs, ks, valid, nt, scale); + ds4_attn_mma_stats(stile, rmax, rsum); /* online max/sum (parallel form, bit-equal to serial) */ + } + __syncthreads(); + + const uint32_t warp = threadIdx.x >> 5; + wmma::fragment oacc[4]; + for (int j = 0; j < 4; j++) wmma::fill_fragment(oacc[j], 0.0f); + for (uint32_t base = k0; base < k1; base += DS4_ATTN_MMA_KT) { /* pass 2: recompute S -> P(bf16) -> O */ + const uint32_t nt = (k1 - base) < DS4_ATTN_MMA_KT ? (k1 - base) : DS4_ATTN_MMA_KT; + __syncthreads(); + ds4_attn_mma_gather_keys(ks, valid, kvw, kvc, idx, i, base, nt, nwin, lo, pos0, window, ng, topk, ring); + __syncthreads(); + ds4_attn_mma_scores(stile, spart, qs, ks, valid, nt, scale); + for (uint32_t e = threadIdx.x; e < 256u; e += blockDim.x) + ptile[e] = __float2bfloat16(expf(stile[e] - rmax[e >> 4])); + __syncthreads(); + wmma::fragment pa; + wmma::fragment vb; + wmma::load_matrix_sync(pa, ptile, 16); + for (int j = 0; j < 4; j++) { + wmma::load_matrix_sync(vb, ks + warp * 64u + (uint32_t)j * 16u, DS4_ATTN_MMA_HD); + wmma::mma_sync(oacc[j], pa, vb, oacc[j]); + } + } + + /* exit: local acc/max/sum only. Two slices per round (2026-09-29, micro-bench V7; the old per-j + * rounds borrowed 8 KB of spart): 8 warps x 4 slices = 32 KB does not fit shared, but after the last + * P*V the 16 KB of ks is free — exactly 8 warps x 2 slices. Each slice keeps its warp offset + * (a shared slab without offsets is the 09-15 bug that made PPL 280k). */ + float *otile = (float *)ks; + for (int r = 0; r < 2; r++) { + __syncthreads(); + wmma::store_matrix_sync(otile + (size_t)warp * 512u, oacc[2 * r], 16, wmma::mem_row_major); + wmma::store_matrix_sync(otile + (size_t)warp * 512u + 256u, oacc[2 * r + 1], 16, wmma::mem_row_major); + __syncthreads(); + for (uint32_t e = threadIdx.x; e < DS4_ATTN_MMA_WARPS * 512u; e += blockDim.x) { + const uint32_t w = e >> 9, rr = e & 511u, jj = rr >> 8, t = rr & 255u, h = t >> 4, d16 = t & 15u; + pacc[(pbase + h) * DS4_ATTN_MMA_HD + w * 64u + (uint32_t)(2 * r) * 16u + jj * 16u + d16] = otile[e]; + } + } + __syncthreads(); + if (threadIdx.x < DS4_ATTN_MMA_HEADS) { + pmax[pbase + threadIdx.x] = rmax[threadIdx.x]; + psum[pbase + threadIdx.x] = rsum[threadIdx.x]; + } +} + +/* Scratch pre-grow for graph capture (cuda_v41_attn_mma_decode.inc.cu:141-146): + * v41_grow inside a capture does hipMalloc + a device sync, both of which + * void the capture — so grow the local tiles to cap-segment x graph-rows + * BEFORE capturing, grow-only. n_tok must be the graph's row count (1 for + * pure decode, 1+k for the spec batch); one row short and the verify batch + * re-grows inside the capture and voids the graph. n_tok <= 8 => <= 64 MB. */ +extern "C" int ds4_gpu_v41_attn_scratch_prepare(uint32_t n_tok, uint32_t n_head, uint32_t head_dim) { + const uint64_t na = (uint64_t)V41_ATTN_SPLIT_MAX_SEG * n_tok * n_head; + return v41_grow(&g_v41_attn_pacc, na * head_dim * 4, "v41 attn mma acc") && + v41_grow(&g_v41_attn_pmax, na * 4, "v41 attn mma max") && + v41_grow(&g_v41_attn_psum, na * 4, "v41 attn mma sum") ? 1 : 0; +} + +/* posd / pos_cap: graph route (cuda_v41_attn_mma_decode.inc.cu:148-150): + * pos0..pos_cap is the graph's valid position range; the grid's segment + * count is the max over that range (the count is not monotonic in position — + * it drops when the seg-length steps — so scan every position, not the ends). */ +static int v41_attn_mma_decode(float *o, const float *q, const float *kvw, const uint8_t *kvc, const int32_t *idx, + const float *sink, uint32_t n_tok, uint32_t pos0, uint32_t window, uint32_t ng, + uint32_t topk, uint32_t ratio, uint32_t n_head, uint32_t hd, float scale, + const int32_t *posd, uint32_t pos_cap, uint32_t full_block, uint32_t ring) { + /* n_tok 1 = pure decode; 2..8 = spec verify batch — both MUST run this + * kernel family (mtp.md M1): at temperature 0 a speculated token has to + * be byte-equal to pure decode, and two kernels with different + * accumulation orders can never be. */ + if (n_tok == 0u || n_tok > 8u || hd != DS4_ATTN_MMA_HD || (n_head % DS4_ATTN_MMA_HEADS)) return 0; + const uint32_t plast = pos0 + n_tok - 1u; /* segment count by the query with the most keys (earlier queries' extras are empty) */ + const uint32_t nwin = plast + 1u > window ? window : plast + 1u; + const uint32_t nkeys = nwin + topk; + /* Few keys still come here, no scalar fallback (2026-09-16 M1'): the old + * nkeys < 64 -> split hand-off only accepted n_tok == 1, so the first + * steps ran split while the verify batch ran the prefill kernel — not + * even the same kernel, same-track impossible. Few keys = one segment, + * the cost is one q load; no second route. */ + static int s_ok = 0; /* 0 untried / 1 usable / -1 cannot be raised */ + const size_t smem = ds4_attn_mma_seg_smem_bytes(); /* the seg kernel keeps the old qs+ks two-slab layout; the prefill kernel's shared account differs */ + if (s_ok == 0) { + (void)hipGetLastError(); /* the opt-in must not see a previously latched error (see ds41_vq_decode.cuh) */ + s_ok = hipFuncSetAttribute(reinterpret_cast(v41_attn_mma_seg_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, + (int)smem) == hipSuccess ? 1 : -1; + (void)hipGetLastError(); + fprintf(stderr, "ds4: [attn] decode tensor-core shared %zu KB %s\n", smem >> 10, s_ok == 1 ? "enabled" : "*cannot be raised, back to scalar*"); + } + if (s_ok != 1) return 0; + /* grid segments = the max of the per-query segment counts (their lengths may differ); the kernel + * recomputes each length from its own position — the host only sizes the grid. */ + uint32_t nseg = 1u; + if (full_block) { /* draft block: n positions share one key set (window + the n in-block positions), length from the count only */ + const uint32_t lo = pos0 > window ? pos0 - window : 0u, nk = pos0 + n_tok - lo; + nseg = (nk + v41_attn_fb_seg_keys(nk) - 1u) / v41_attn_fb_seg_keys(nk); + } else if (posd) { /* graph: max over each position in the bucket (the kernel computes the true count; extras run empty); with n rows the last position is pos_cap + n - 1 */ + if (pos_cap < pos0) return 0; + for (uint32_t p = pos0; p <= pos_cap + n_tok - 1u; p++) { + const uint32_t ns = v41_attn_nseg_at(p, window, ratio, topk); + if (ns > nseg) nseg = ns; + } + } else + for (uint32_t i = 0; i < n_tok; i++) { + const uint32_t p = pos0 + i; + const uint32_t nw = p + 1u > window ? window : p + 1u; + const uint32_t nk = nw + topk; + const uint32_t sk = v41_attn_seg_keys(p, window, ratio, topk); + const uint32_t ns = (nk + sk - 1u) / sk; + if (ns > nseg) nseg = ns; + } + if (nseg > V41_ATTN_SPLIT_MAX_SEG) { /* key cap = window + indexer top-k cap, unreachable */ + fprintf(stderr, "ds4: *[attn] %u keys need %u segments, over the cap %u — another kernel ran, same-track no longer holds*\n", + nkeys, nseg, (unsigned)V41_ATTN_SPLIT_MAX_SEG); + return 0; + } + const uint64_t na = (uint64_t)nseg * n_tok * n_head; + float *pacc = (float *)v41_grow(&g_v41_attn_pacc, na * hd * 4, "v41 attn mma acc"); + float *pmax = (float *)v41_grow(&g_v41_attn_pmax, na * 4, "v41 attn mma max"); + float *psum = (float *)v41_grow(&g_v41_attn_psum, na * 4, "v41 attn mma sum"); + if (!pacc || !pmax || !psum) return 0; + v41_pdl_register((const void *)v41_attn_mma_seg_kernel); /* the kernel v41_pdl_wait before reading upstream output */ + v41_attn_mma_seg_kernel<<>>( + pacc, pmax, psum, q, kvw, kvc, idx, pos0, window, ng, topk, n_head, scale, ratio, nseg, posd, full_block, ring); + if (!cuda_ok(hipGetLastError(), "v41 attn mma seg")) return 0; + /* the merge reuses the scalar split's launch (fixed segment order + sink in the denominator + divide + bf16 round), identical semantics */ + v41_pdl_register((const void *)v41_sparse_attn_merge_kernel); + v41_sparse_attn_merge_kernel<<>>(o, pacc, pmax, psum, sink, nseg, n_head, hd, + posd, window, ratio, topk); + return cuda_ok(hipGetLastError(), "v41 attn mma merge"); +} + +/* ---- the entry (cuda_v41_2.inc.cu:229-303): the engine's try order. ---- + * The clamp only exists in the prefill kernel; at decode (n<=8) it must be a + * no-op — refuse rather than silently read dirty ring slots. The generation + * route leaves the last window positions to the last block (core_v41_api.c), + * so at decode p+1-window >= win_lo always holds. */ +extern "C" int ds4_gpu_v41_sparse_attn_tensor(ds4_gpu_tensor *o, const ds4_gpu_tensor *q, const ds4_gpu_tensor *kv_win, + const ds4_gpu_tensor *kv_comp, const ds4_gpu_tensor *idx, + const void *model_map, uint64_t model_size, uint64_t sink_offset, + uint32_t n_tok, uint32_t pos0, uint32_t window, uint32_t ng, uint32_t topk, + uint32_t ratio, + uint32_t n_head, uint32_t head_dim, float scale, int full_block, int ring, + uint32_t win_lo, const ds4_gpu_tensor *posd, uint32_t pos_cap) { + if (!o || !q || !kv_win || n_head != 64u || head_dim != 512u) { fprintf(stderr, "ds4: [v41] sparse attn only implements 64 heads x 512\n"); return 0; } + const uint32_t lo_first = pos0 + 1u > window ? pos0 + 1u - window : 0u; /* this batch's first query's natural lower bound (later ones only grow) */ + const int clamp_matters = win_lo > lo_first; + if (clamp_matters && n_tok <= 8u && !full_block) { + fprintf(stderr, "ds4: *[v41] sparse attn: the decode kernels have no window clamp, but pos0 %u's window lower bound %u < win_lo %u — did the last block not keep a full window?*\n", + pos0, lo_first, win_lo); + return 0; + } + /* The graph route (posd) only accepts the decode tensor-core path; the + * scalar split sizes its segments on the host and cannot enter one + * captured graph. The tensor-core path failing to raise shared fails + * HERE, loudly — never a silent switch to another kernel: a different + * kernel is a different accumulation order and the byte gate would fork + * (engine cuda_v41_2.inc.cu:246-260). */ + if (posd) { + if (full_block || !ring || n_tok > 8u) { fprintf(stderr, "ds4: [ds41] sparse attn: the graph route takes only the main path's n<=8 shape\n"); return 0; } + const float *sink = (const float *)cuda_model_range_ptr(model_map, sink_offset, (uint64_t)n_head * 4, "v41 sink"); + if (!sink) return 0; + const int hasc = kv_comp && idx; /* pure-window layers (ratio 0) come here too: the window range moves with the position as well */ + if (!v41_attn_mma_decode((float *)o->ptr, (const float *)q->ptr, (const float *)kv_win->ptr, + hasc ? (const uint8_t *)kv_comp->ptr : NULL, hasc ? (const int32_t *)idx->ptr : NULL, + sink, n_tok, pos0, window, hasc ? ng : 0u, hasc ? topk : 0u, hasc ? ratio : 0u, n_head, head_dim, scale, + (const int32_t *)posd->ptr, pos_cap, 0u, 1u)) { + fprintf(stderr, "ds4: [ds41] sparse attn: the decode tensor-core attention the graph route needs is unavailable\n"); return 0; + } + return 1; + } + if (kv_win->bytes < (uint64_t)(window + n_tok) * head_dim * 4) { fprintf(stderr, "ds4: [v41] window buffer short %u+%u rows\n", window, n_tok); return 0; } + const float *sink = (const float *)cuda_model_range_ptr(model_map, sink_offset, (uint64_t)n_head * 4, "v41 sink"); + if (!sink) return 0; + /* The split-K and mma fast routes only serve the main route (history is + * the ring): their call condition is !full_block, and full_block only + * ever comes from the DSpark draft tower — so ring is 1 whenever they + * run, and the kernels compute ring directly. A non-draft full_block or + * a non-ring main route trips this check instead of silently reading + * wrong rows. */ + if (!full_block && !ring) { fprintf(stderr, "ds4: [v41] sparse attn: the main path's window must be a ring\n"); return 0; } + /* Decode first tries the tensor-core family (decode.md D2): for one + * query S = Q[64x512] . K^T[512xnkeys] is a real GEMM; the scalar kernel + * pays 5 shuffles per 16 FMAs (69 GFLOP/s = 0.4% of the board's peak). + * Few keys / unfit shapes return 0 here by themselves. */ + if (!full_block && + v41_attn_mma_decode((float *)o->ptr, (const float *)q->ptr, (const float *)kv_win->ptr, + kv_comp ? (const uint8_t *)kv_comp->ptr : NULL, idx ? (const int32_t *)idx->ptr : NULL, + sink, n_tok, pos0, window, ng, (kv_comp && idx) ? topk : 0u, + (kv_comp && idx) ? ratio : 0u, n_head, head_dim, scale, NULL, 0u, 0u, 1u)) + return 1; + /* The DSpark draft block runs the tensor-core family too (2026-09-29): + * the tower's attention has window keys only, in-block full visibility, + * and a linear window segment (ring=0). It used to fall to the scalar + * kernel: 1.6 ms per 3-tower round (0.5 ms per launch for 5 positions x + * 133 keys), the draft step's third-largest item. Drafts only propose, + * verification decides the output, so the gate is the acceptance + * histogram, not the text. Few keys / unfit shapes return 0 and fall + * through to the scalar kernel. */ + if (full_block && !ring && !kv_comp && n_tok <= 8u && + v41_attn_mma_decode((float *)o->ptr, (const float *)q->ptr, (const float *)kv_win->ptr, NULL, NULL, + sink, n_tok, pos0, window, 0u, 0u, 0u, n_head, head_dim, scale, NULL, 0u, n_tok, 0u)) + return 1; + if (!full_block && + v41_sparse_attn_split((float *)o->ptr, (const float *)q->ptr, (const float *)kv_win->ptr, + kv_comp ? (const uint8_t *)kv_comp->ptr : NULL, idx ? (const int32_t *)idx->ptr : NULL, + sink, n_tok, pos0, window, ng, (kv_comp && idx) ? topk : 0u, n_head, head_dim, scale)) + return 1; + /* Prefill (n >= 64) takes the single-pass mma kernel; it checks shape, + * size and shared itself and returns 0 to fall through. */ + if (!full_block && ds4_sparse_attn_mma_launch((float *)o->ptr, (const float *)q->ptr, (const float *)kv_win->ptr, + kv_comp ? (const uint8_t *)kv_comp->ptr : NULL, idx ? (const int32_t *)idx->ptr : NULL, + sink, n_tok, pos0, window, ng, (kv_comp && idx) ? topk : 0u, n_head, head_dim, scale, win_lo)) + return 1; + v41_sparse_attn_kernel<<>>( + (float *)o->ptr, (const float *)q->ptr, (const float *)kv_win->ptr, + kv_comp ? (const uint8_t *)kv_comp->ptr : NULL, idx ? (const int32_t *)idx->ptr : NULL, sink, pos0, window, ng, + (kv_comp && idx) ? topk : 0u, n_head, head_dim, scale, full_block ? n_tok : 0u, ring ? 1u : 0u, win_lo); + return cuda_ok(hipGetLastError(), "v41 sparse attn"); +} diff --git a/cuda/ds41_dense.cuh b/cuda/ds41_dense.cuh new file mode 100644 index 0000000000000000000000000000000000000000..8e706d883324d80dbceca2a621c1071039f28a4b --- /dev/null +++ b/cuda/ds41_dense.cuh @@ -0,0 +1,377 @@ +#include "hip/hip_runtime.h" +/* ds41_dense.cuh — DeepSeek V4.1 (ds41) dense small-batch GEMVs and the tiny + * elementwise kernels, ported from the C engine at /data/YoungAi (commit + * 3946dbc): + * - the f32 / bf16 GEMVs and launchers: src/cuda/cuda_v41_gemv_highprec.inc.cu:19-191 + * (the file the engine marks as transitional: f32 for the mHC fn pair, + * bf16 for the router gate / compressor / indexer projections); + * - their entries: src/cuda/cuda_v41_1.inc.cu:275-322; + * - the embedding row fetch (fp4x32 arm): :365-391; + * - rms_norm (:394-411), add (:413-421) and expand_hc (:423-433). + * + * Numerics: both GEMVs accumulate f32 with the same warp/ksplit shape as the + * engine's (a lane's float4/uint4 weights times the activations, xor-shuffle + * tree, then the ksplit parts summed in k order); bf16 weights go up to f32 + * without touching the bf16 grid, so "the same weights stored f32" differ + * only in the accumulation order — the engine's gate for those is the main + * NLL/PPL scale, not cmp (cuda_v41_gemv_highprec.inc.cu:11-14). + * + * Scope: the n <= 8 GEMV arms (P4-4) and the n > 8 cuBLAS prefill arms + * (cuda_v41_1.inc.cu:291-296 f32, :308-330 bf16). The fp4x32 skeleton arm of + * the embedding is ported for completeness; the artifact carries none. */ +#pragma once + +/* ---- f32 weights GEMV (clear.md C2 (3): the decode path's hipblasSgemm) ---- + * Consumers: the router gate (f32 form), the compressor kv/gate and the + * indexer wk/proj — the matrices still f32 in the engine's precision table. + * in_dim must be a multiple of 128 (one warp step). */ +template +__global__ static void v41_f32_gemv_kernel(float *out, const float *w, const float *x, uint32_t in_dim, + uint32_t out_dim, uint32_t x_stride, uint32_t out_stride, uint32_t ksplit) { + const uint32_t warp = threadIdx.x >> 5, lane = threadIdx.x & 31u; + const uint32_t rows_per_block = 8u / ksplit, rloc = warp / ksplit, kpart = warp % ksplit; + const uint32_t r = blockIdx.x * rows_per_block + rloc; + __shared__ float red[8][NT]; + float acc[NT]; + #pragma unroll + for (uint32_t t = 0; t < NT; t++) acc[t] = 0.f; + if (r < out_dim) { + const float *wr = w + (uint64_t)r * in_dim; + for (uint32_t c = kpart * 128u + lane * 4u; c < in_dim; c += ksplit * 128u) { + const float4 wv = *(const float4 *)(wr + c); + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + const float4 xv = *(const float4 *)(x + (uint64_t)t * x_stride + c); + acc[t] += wv.x * xv.x + wv.y * xv.y + wv.z * xv.z + wv.w * xv.w; + } + } + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + float v = acc[t]; + for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o); + acc[t] = v; + } + } + if (ksplit > 1u) { + if (lane == 0) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) red[warp][t] = acc[t]; + } + __syncthreads(); + if (kpart == 0 && lane == 0 && r < out_dim) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + float v = 0.f; + for (uint32_t k = 0; k < ksplit; k++) v += red[rloc * ksplit + k][t]; + out[(uint64_t)t * out_stride + r] = v; + } + } + } else if (lane == 0 && r < out_dim) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) out[(uint64_t)t * out_stride + r] = acc[t]; + } +} +static int v41_f32_gemv(const float *w, uint64_t in_dim, uint64_t out_dim, const float *x, float *out, + uint32_t n_tok, const char *what) { + if ((in_dim % 128u) != 0u || n_tok == 0 || n_tok > V41_GEMV_MAX_TOK) return 0; + uint32_t ksplit = 1; + while (ksplit < 8u && out_dim * ksplit < 32768u) ksplit <<= 1; + const uint32_t nseg = (uint32_t)(in_dim / 128u); /* K split unit = 128 elements */ + while (ksplit > 1u && ksplit > nseg) ksplit >>= 1; + const uint32_t rpb = 8u / ksplit; + const dim3 grid((unsigned)((out_dim + rpb - 1u) / rpb), 1); + #define V41_F32GEMV_LAUNCH(NT) v41_f32_gemv_kernel<<>>( \ + out, w, x, (uint32_t)in_dim, (uint32_t)out_dim, (uint32_t)in_dim, (uint32_t)out_dim, ksplit) + switch (n_tok) { + case 1: V41_F32GEMV_LAUNCH(1u); break; case 2: V41_F32GEMV_LAUNCH(2u); break; + case 3: V41_F32GEMV_LAUNCH(3u); break; case 4: V41_F32GEMV_LAUNCH(4u); break; + case 5: V41_F32GEMV_LAUNCH(5u); break; case 6: V41_F32GEMV_LAUNCH(6u); break; + case 7: V41_F32GEMV_LAUNCH(7u); break; default: V41_F32GEMV_LAUNCH(8u); break; + } + #undef V41_F32GEMV_LAUNCH + return cuda_ok(hipGetLastError(), what); +} + +/* ---- bf16 weights GEMV (clear.md C1: router gate / compressor / indexer) ---- + * One 128-bit read takes 8 elements, so a warp step is 256 elements and + * in_dim must be a multiple of 256 (5120 / 512 in the engine's callers). + * The skip pointer is the markov-bias-cache exit (NULL everywhere until the + * sidecar store lands; the port keeps the parameter shape). */ +template +__global__ static void v41_bf16_gemv_kernel(float *out, const __hip_bfloat16 *w, const float *x, uint32_t in_dim, + uint32_t out_dim, uint32_t x_stride, uint32_t out_stride, uint32_t ksplit, const int32_t *skip) { + if (skip && *skip >= 0) return; + const uint32_t warp = threadIdx.x >> 5, lane = threadIdx.x & 31u; + const uint32_t rows_per_block = 8u / ksplit, rloc = warp / ksplit, kpart = warp % ksplit; + const uint32_t r = blockIdx.x * rows_per_block + rloc; + __shared__ float red[8][NT]; + float acc[NT]; + #pragma unroll + for (uint32_t t = 0; t < NT; t++) acc[t] = 0.f; + /* Weights first, upstream second (PDL, 2026-09-23): this lane's first PF + * weight segments (constants) land in registers before v41_pdl_wait, and + * only then x is touched. Segment order and multiply-add form unchanged + * -> bit-identical. */ + constexpr uint32_t PF = 4u; + const __hip_bfloat16 *wr = w + (uint64_t)r * in_dim; + const uint32_t c0 = kpart * 256u + lane * 8u, cst = ksplit * 256u; + uint4 wpre[PF]; + #pragma unroll + for (uint32_t j = 0; j < PF; j++) { + wpre[j] = make_uint4(0u, 0u, 0u, 0u); + if (r < out_dim && c0 + j * cst < in_dim) wpre[j] = *(const uint4 *)(wr + c0 + j * cst); + } + v41_pdl_wait(); + if (r < out_dim) { + auto seg = [&](const uint4 raw, uint32_t c) { /* one 8-element segment: the original loop body verbatim */ + __hip_bfloat162 wb[4]; + memcpy(wb, &raw, 16); + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + const float4 x0 = *(const float4 *)(x + (uint64_t)t * x_stride + c); + const float4 x1 = *(const float4 *)(x + (uint64_t)t * x_stride + c + 4u); + const float2 w0 = __bfloat1622float2(wb[0]), w1 = __bfloat1622float2(wb[1]); + const float2 w2 = __bfloat1622float2(wb[2]), w3 = __bfloat1622float2(wb[3]); + acc[t] += w0.x * x0.x + w0.y * x0.y + w1.x * x0.z + w1.y * x0.w + + w2.x * x1.x + w2.y * x1.y + w3.x * x1.z + w3.y * x1.w; + } + }; + #pragma unroll + for (uint32_t k = 0; k < PF; k++) if (c0 + k * cst < in_dim) seg(wpre[k], c0 + k * cst); + for (uint32_t c = c0 + PF * cst; c < in_dim; c += cst) seg(*(const uint4 *)(wr + c), c); + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + float v = acc[t]; + for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o); + acc[t] = v; + } + } + if (ksplit > 1u) { + if (lane == 0) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) red[warp][t] = acc[t]; + } + __syncthreads(); + if (kpart == 0 && lane == 0 && r < out_dim) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + float v = 0.f; + for (uint32_t k = 0; k < ksplit; k++) v += red[rloc * ksplit + k][t]; + out[(uint64_t)t * out_stride + r] = v; + } + } + } else if (lane == 0 && r < out_dim) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) out[(uint64_t)t * out_stride + r] = acc[t]; + } +} +static int v41_bf16_gemv(const __hip_bfloat16 *w, uint64_t in_dim, uint64_t out_dim, const float *x, float *out, + uint32_t n_tok, const char *what, const int32_t *skip) { + if ((in_dim % 256u) != 0u || n_tok == 0 || n_tok > V41_GEMV_MAX_TOK) return 0; + uint32_t ksplit = 1; + while (ksplit < 8u && out_dim * ksplit < 32768u) ksplit <<= 1; + const uint32_t nseg = (uint32_t)(in_dim / 256u); + while (ksplit > 1u && ksplit > nseg) ksplit >>= 1; + const uint32_t rpb = 8u / ksplit; + const dim3 grid((unsigned)((out_dim + rpb - 1u) / rpb), 1); + v41_pdl_register((const void *)v41_bf16_gemv_kernel<1u>); /* the kernel v41_pdl_wait before reading x */ + #define V41_BFGEMV_LAUNCH(NT) v41_bf16_gemv_kernel<<>>( \ + out, w, x, (uint32_t)in_dim, (uint32_t)out_dim, (uint32_t)in_dim, (uint32_t)out_dim, ksplit, skip) + switch (n_tok) { + case 1: V41_BFGEMV_LAUNCH(1u); break; case 2: V41_BFGEMV_LAUNCH(2u); break; + case 3: V41_BFGEMV_LAUNCH(3u); break; case 4: V41_BFGEMV_LAUNCH(4u); break; + case 5: V41_BFGEMV_LAUNCH(5u); break; case 6: V41_BFGEMV_LAUNCH(6u); break; + case 7: V41_BFGEMV_LAUNCH(7u); break; default: V41_BFGEMV_LAUNCH(8u); break; + } + #undef V41_BFGEMV_LAUNCH + return cuda_ok(hipGetLastError(), what); +} + +/* ---- the two matmul entries (cuda_v41_1.inc.cu:275-322) ---- */ +extern "C" int ds4_gpu_v41_matmul_f32_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, + const ds4_gpu_tensor *x, uint32_t n_tok) { + if (!out || !x || !g_cublas_ready || n_tok == 0) return 0; + const uint64_t wbytes = in_dim * out_dim * 4; + if (weight_offset > model_size || wbytes > model_size - weight_offset) return 0; + const float *W = (const float *)cuda_model_range_ptr(model_map, weight_offset, wbytes, "v41 f32 w"); + if (!W) return 0; + if (n_tok <= V41_GEMV_MAX_TOK && (in_dim % 128u) == 0u) + return v41_f32_gemv(W, in_dim, out_dim, (const float *)x->ptr, (float *)out->ptr, n_tok, "v41 f32 gemv"); + /* Prefill (cuda_v41_1.inc.cu:291-296): one Sgemm straight off the mmap'd + * f32 table. The GEMV above is the decode/small-batch arm because cuBLAS + * splits an n=1 Sgemm into 200+ small kernels (the GEMV header's account). */ + (void)hipblasSetStream(g_cublas, ds4_current_stream()); /* the engine's call (cuda_v41_1.inc.cu:292): cuBLAS keys its algorithm on the bound workspace, and this call resets it to the default pool (docs 2.4.7) -- the state the engine's GEMMs actually run in */ + cuda_cublas_ws_prep(ds4_current_stream()); + const float alpha = 1.0f, beta = 0.0f; + hipblasStatus_t st = hipblasSgemm(g_cublas, HIPBLAS_OP_T, HIPBLAS_OP_N, (int)out_dim, (int)n_tok, (int)in_dim, &alpha, + W, (int)in_dim, (const float *)x->ptr, (int)in_dim, &beta, (float *)out->ptr, (int)out_dim); + return cublas_ok(st, "v41 f32 gemm"); +} +/* Prefill activation scratch (the engine's shared g_v41_xbf; each port family + * owns its own slot, ds41_fp8blk.cuh precedent). */ +static v41_scratch g_v41_dense_xbf; + +/* The decode-only bf16 GEMV with a device skip flag (cuda_v41_1.inc.cu:326-336): + * the markov-bias-cache hit makes the whole grid return (the bias comes from + * the cache slot). Decode batches only (n <= 8), never the prefill GEMM. */ +extern "C" int ds4_gpu_v41_matmul_bf16_skip_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, + const ds4_gpu_tensor *x, uint32_t n_tok, const ds4_gpu_tensor *skip) { + if (!out || !x || !skip || n_tok == 0 || n_tok > V41_GEMV_MAX_TOK || (in_dim % 256u) != 0u) return 0; + const uint64_t wbytes = in_dim * out_dim * 2; + if (weight_offset > model_size || wbytes > model_size - weight_offset) return 0; + const __hip_bfloat16 *W = (const __hip_bfloat16 *)cuda_model_range_ptr(model_map, weight_offset, wbytes, "v41 bf16 w(skip)"); + if (!W) return 0; + return v41_bf16_gemv(W, in_dim, out_dim, (const float *)x->ptr, (float *)out->ptr, n_tok, "v41 bf16 gemv(skip)", + (const int32_t *)skip->ptr); +} +/* Shared GEMM/GEMV body (the engine's v41_bf16_matmul_ptr, cuda_v41_1.inc.cu: + * 302-316): W is already a device-readable pointer -- in the model mapping for + * the trunk, or the vision tower's per-image device copy. Split out so + * ds4_gpu_v41_matmul_bf16_dev_tensor (cuda/ds41_vision.cuh) shares it, exactly + * as the engine split it for the vision unit. */ +static int v41_bf16_matmul_ptr(const __hip_bfloat16 *W, uint64_t in_dim, uint64_t out_dim, + const ds4_gpu_tensor *x, ds4_gpu_tensor *out, uint32_t n_tok) { + if (!out || !x || !W || !g_cublas_ready || n_tok == 0) return 0; + if (n_tok <= V41_GEMV_MAX_TOK && (in_dim % 256u) == 0u) + return v41_bf16_gemv(W, in_dim, out_dim, (const float *)x->ptr, (float *)out->ptr, n_tok, "v41 bf16 gemv", NULL); + /* Prefill (cuda_v41_1.inc.cu:308-330): activations to bf16 (they already + * sit on the bf16 grid — the conversion changes no value), then one + * bf16 x bf16 -> f32 GEMM straight off the mmap'd table. */ + const uint64_t xn = (uint64_t)n_tok * in_dim; + __hip_bfloat16 *xb = (__hip_bfloat16 *)v41_grow(&g_v41_dense_xbf, xn * sizeof(__hip_bfloat16), "v41 bf16 x"); + if (!xb) return 0; + v41_x_to_bf16_kernel<<<(unsigned)((xn + 255) / 256), 256, 0, ds4_current_stream()>>>(xb, (const float *)x->ptr, xn); + if (!cuda_ok(hipGetLastError(), "v41 x->bf16")) return 0; + (void)hipblasSetStream(g_cublas, ds4_current_stream()); /* the engine's call (cuda_v41_1.inc.cu:318): resets the workspace to the default pool (docs 2.4.7) */ + cuda_cublas_ws_prep(ds4_current_stream()); + const float alpha = 1.0f, beta = 0.0f; + hipblasStatus_t st = hipblasGemmEx(g_cublas, HIPBLAS_OP_T, HIPBLAS_OP_N, (int)out_dim, (int)n_tok, (int)in_dim, &alpha, + W, HIP_R_16BF, (int)in_dim, xb, HIP_R_16BF, (int)in_dim, &beta, + (float *)out->ptr, HIP_R_32F, (int)out_dim, HIP_R_32F, HIPBLAS_GEMM_DEFAULT); + return cublas_ok(st, "v41 bf16 gemm"); +} +extern "C" int ds4_gpu_v41_matmul_bf16_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, + const ds4_gpu_tensor *x, uint32_t n_tok) { + if (!out || !x || !g_cublas_ready || n_tok == 0) return 0; + const uint64_t wbytes = in_dim * out_dim * 2; + if (weight_offset > model_size || wbytes > model_size - weight_offset) return 0; + const __hip_bfloat16 *W = (const __hip_bfloat16 *)cuda_model_range_ptr(model_map, weight_offset, wbytes, "v41 bf16 w"); + if (!W) return 0; + return v41_bf16_matmul_ptr(W, in_dim, out_dim, x, out, n_tok); +} + +/* ---- embedding row fetch, fp4x32 arm (cuda_v41_1.inc.cu:365-391) ---- + * The artifact carries no fp4x32 tensor; ported for the dispatch's old-GGUF + * arm (v41_embed). One block per token row, 32 elements per 17-byte block. */ +__global__ static void v41_embed_kernel(float *out, const int32_t *tok, const uint8_t *w, uint32_t n_vocab, uint32_t n_embd) { + const uint32_t t = blockIdx.x, nb = n_embd / 32u; + int32_t id = tok[t]; if (id < 0 || (uint32_t)id >= n_vocab) id = 0; + for (uint32_t b = threadIdx.x; b < nb; b += blockDim.x) { + const uint8_t *p = w + ((uint64_t)id * nb + b) * 17u; + const float s = ds4_e8m0_to_f32(p[16]); + float *o = out + (uint64_t)t * n_embd + b * 32u; + for (int j = 0; j < 16; j++) { + o[2 * j] = v41_bf16r(ds4_fp4_nibble_to_f32(p[j] & 0x0F) * s); + o[2 * j + 1] = v41_bf16r(ds4_fp4_nibble_to_f32(p[j] >> 4) * s); + } + } +} +extern "C" int ds4_gpu_v41_embed_fp4x32_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *tokens, const void *model_map, + uint64_t model_size, uint64_t weight_offset, uint32_t n_vocab, + uint32_t n_tok, uint32_t n_embd) { + if (!out || !tokens || n_tok == 0 || (n_embd % 32u)) return 0; + const uint64_t wbytes = (uint64_t)n_vocab * (n_embd / 32u) * 17u; + if (weight_offset > model_size || wbytes > model_size - weight_offset) return 0; + const uint8_t *w = (const uint8_t *)cuda_model_range_ptr(model_map, weight_offset, wbytes, "v41 embed"); + if (!w || out->bytes < (uint64_t)n_tok * n_embd * 4) return 0; + v41_embed_kernel<<>>((float *)out->ptr, (const int32_t *)tokens->ptr, w, n_vocab, n_embd); + return cuda_ok(hipGetLastError(), "v41 embed"); +} + +/* ---- rms_norm with weight -> bf16 (official RMSNorm: f32 compute, bf16 + * weight values, result .to(bf16)) — cuda_v41_1.inc.cu:394-411 ---- */ +__global__ static void v41_rms_norm_kernel(float *out, const float *x, const float *w, uint32_t dim, float eps) { + v41_pdl_wait(); + const uint32_t r = blockIdx.x; const float *xr = x + (uint64_t)r * dim; float *o = out + (uint64_t)r * dim; + float s = 0.f; + for (uint32_t i = threadIdx.x; i < dim; i += blockDim.x) s += xr[i] * xr[i]; + __shared__ float sh[256]; sh[threadIdx.x] = s; __syncthreads(); + for (uint32_t k = blockDim.x / 2; k > 0; k >>= 1) { if (threadIdx.x < k) sh[threadIdx.x] += sh[threadIdx.x + k]; __syncthreads(); } + const float inv = rsqrtf(sh[0] / (float)dim + eps); + for (uint32_t i = threadIdx.x; i < dim; i += blockDim.x) o[i] = v41_bf16r(w[i] * (xr[i] * inv)); +} +extern "C" int ds4_gpu_v41_rms_norm_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *x, const void *model_map, + uint64_t model_size, uint64_t weight_offset, uint32_t dim, uint32_t n_tok, float eps) { + if (!out || !x) return 0; + const float *w = (const float *)cuda_model_range_ptr(model_map, weight_offset, (uint64_t)dim * 4, "v41 norm w"); + if (!w) return 0; + v41_rms_norm_kernel<<>>((float *)out->ptr, (const float *)x->ptr, w, dim, eps); + return cuda_ok(hipGetLastError(), "v41 rms norm"); +} + +/* ---- directional steering for the V4.1 forward (2026-10-23; the DS4 Metal graph's + * edit, ported to the ds41 path): y[row] -= scale * d * dot(d, y[row]) with one + * normalized E-wide direction per layer. One block per token row; the dot is a + * block reduction, then every element moves. Zero scale = untouched. ---- */ +__global__ static void v41_dir_steer_kernel(float *y, const float *d, uint32_t E, float scale) { + const uint32_t r = blockIdx.x; + float *yr = y + (uint64_t)r * E; + float s = 0.f; + for (uint32_t i = threadIdx.x; i < E; i += blockDim.x) s += yr[i] * d[i]; + __shared__ float sh[256]; + sh[threadIdx.x] = s; __syncthreads(); + for (uint32_t k = blockDim.x / 2; k > 0; k >>= 1) { if (threadIdx.x < k) sh[threadIdx.x] += sh[threadIdx.x + k]; __syncthreads(); } + const float c = scale * sh[0]; + for (uint32_t i = threadIdx.x; i < E; i += blockDim.x) yr[i] -= c * d[i]; +} +extern "C" int ds4_gpu_v41_dir_steer_tensor(ds4_gpu_tensor *y, const float *d_dev, uint32_t E, uint32_t n_tok, float scale) { + if (!y || !d_dev || n_tok == 0 || scale == 0.f) return 1; + v41_dir_steer_kernel<<>>((float *)y->ptr, d_dev, E, scale); + return cuda_ok(hipGetLastError(), "v41 dir steer"); +} +extern "C" float *ds4_gpu_v41_dir_steer_upload(const float *host, uint64_t n) { /* the direction table, device resident */ + float *d = NULL; + if (hipMalloc((void **)&d, n * 4) != hipSuccess) { (void)hipGetLastError(); return NULL; } + if (hipMemcpy(d, host, n * 4, hipMemcpyHostToDevice) != hipSuccess) { (void)hipGetLastError(); (void)hipFree(d); return NULL; } + return d; +} + +/* ---- add + expand_hc (cuda_v41_1.inc.cu:413-433) ---- */ +__global__ static void v41_add_kernel(float *a, const float *b, uint64_t n) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) a[i] += b[i]; +} +extern "C" int ds4_gpu_v41_add_tensor(ds4_gpu_tensor *a, const ds4_gpu_tensor *b, uint64_t n) { + if (!a || !b) return 0; + v41_add_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>((float *)a->ptr, (const float *)b->ptr, n); + return cuda_ok(hipGetLastError(), "v41 add"); +} +__global__ static void v41_expand_hc_kernel(float *hc, const float *x, uint32_t n_embd, uint32_t n_hc) { + const uint32_t n = blockIdx.y, d = blockIdx.x * blockDim.x + threadIdx.x; + if (d >= n_embd) return; + const float v = x[(uint64_t)n * n_embd + d]; + for (uint32_t c = 0; c < n_hc; c++) hc[((uint64_t)n * n_hc + c) * n_embd + d] = v; +} +extern "C" int ds4_gpu_v41_expand_hc_tensor(ds4_gpu_tensor *hc, const ds4_gpu_tensor *x, uint32_t n_embd, uint32_t n_hc, uint32_t n_tok) { + if (!hc || !x) return 0; + v41_expand_hc_kernel<<>>((float *)hc->ptr, (const float *)x->ptr, n_embd, n_hc); + return cuda_ok(hipGetLastError(), "v41 expand hc"); +} + +/* ---- in-place x = bf16(x*s) (cuda_v41_4.inc.cu:336-346) ---- + * The indexer weights scale (softmax_scale * n_heads^-1/2): the official + * model multiplies on bf16, so the multiply lands on the bf16 grid. Used to + * be a host round trip (517 KB read back per layer). */ +__global__ static void v41_scale_round_kernel(float *x, uint64_t n, float s) { + v41_pdl_wait(); /* PDL: first instruction waits for the upstream; no-op when not launched through PDL */ + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) x[i] = v41_bf16r(x[i] * s); +} +extern "C" int ds4_gpu_v41_scale_round_tensor(ds4_gpu_tensor *x, uint64_t n, float s) { + if (!x || x->bytes < n * 4) return 0; + v41_scale_round_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>((float *)x->ptr, n, s); + return cuda_ok(hipGetLastError(), "v41 scale round"); +} diff --git a/cuda/ds41_draft.cuh b/cuda/ds41_draft.cuh new file mode 100644 index 0000000000000000000000000000000000000000..5c301d2ea66a87e8b693f4c3616d39db8d6afbe2 --- /dev/null +++ b/cuda/ds41_draft.cuh @@ -0,0 +1,307 @@ +#include "hip/hip_runtime.h" +/* cuda/ds41_draft.cuh — DeepSeek V4.1 (ds41) DSpark draft-tower device + * pieces, ported from the C engine at /data/YoungAi (commit 3946dbc), file + * `src/cuda/cuda_v41_draft.inc.cu` (every entry cites its engine line). + * + * The draft towers differ from a normal layer in exactly two places; the rest + * (hc / norm / attention / shared experts / head) reuses the main path's + * kernels: + * 1. Their experts are either one VQ blob per tower (this artifact; the + * main fused decode kernels take it with layer = DS4_N_LAYER + tower) or + * per-expert fp4x32 tensors, which need the dense MoE below (the weight + * base comes from the routed expert id, so it cannot be compile-time). + * 2. The main model's attention input (the hc four-route mean) is captured + * inside the main forward into an absolute-position ring -> one mean + * kernel (v41_hc_mean_kernel). + * The markov head also gathers a bf16/f32 table row by a DEVICE token id + * (a host round-trip per position would cost the whole speculation gain). + * + * Failure modes (the engine's warnings, kept): a wrong main_hidden capture + * point does not error, it just drops acceptance to about 1; a dense-MoE sel + * out of range reads another expert's bytes silently, so the kernels range- + * check e. + * + * Not ported here (named): nothing — the amp/distillation tables + * (core_v41_draft_amp.c) are a separate sidecar unit. + */ +#pragma once +#include +#include +#include +#include + +#include + +/* One warp computes the dot of an fp4x32 weight row with the activation x + * (f32 accumulate, activations read on the bf16 grid; engine + * cuda_v41_draft.inc.cu:16-31). Layout as the skeleton GEMV: 4 lanes share a + * 32-element block, each lane takes 4 consecutive bytes = 8 elements (one + * scale decode amortized over 8). Returns the full warp sum (lanes 0..31 + * reduced). */ +__device__ __forceinline__ static float v41_fp4_row_dot(const uint8_t *wr, const float *x, uint32_t nblk) { + const uint32_t lane = threadIdx.x & 31u, q = lane & 3u, sub = lane >> 2; + float acc = 0.f; + const uint32_t ngrp = (nblk + 7u) / 8u; + for (uint32_t gi = 0; gi < ngrp; gi++) { + const uint32_t b = gi * 8u + sub; + if (b >= nblk) continue; /* tail group holes: other lanes still have work, no break */ + const uint8_t *p = wr + (uint64_t)b * 17u; + const float sc = ds4_e8m0_to_f32(p[16]); + const float *xb = x + b * 32u + q * 8u; + #pragma unroll + for (uint32_t j = 0; j < 4u; j++) { + const uint8_t by = p[q * 4u + j]; + acc += ds4_fp4_nibble_to_f32(by & 0x0Fu) * sc * v41_bf16r(xb[2u * j]); + acc += ds4_fp4_nibble_to_f32(by >> 4) * sc * v41_bf16r(xb[2u * j + 1u]); + } + } + for (int o = 16; o > 0; o >>= 1) acc += __shfl_xor_sync(0xffffffffu, acc, o); + return acc; +} + +/* gate/up share one kernel (official Expert: w1/w3 -> bf16 -> swiglu -> bf16). + * One warp per row, 8 rows per block. grid (ceil(MID/8), n_tok*K): blockIdx.y + * is one (token, k-th selected expert) pair. Engine :38-52. */ +__global__ static void v41_mtp_gateup_kernel(float *h, const uint8_t *const *wg, const uint8_t *const *wu, + const int32_t *sel, const float *x, uint32_t IN, uint32_t MID, + uint32_t K, uint32_t n_expert, float clamp) { + const uint32_t pair = blockIdx.y, t = pair / K, warp = threadIdx.x >> 5; + const int32_t e = sel[pair]; + if (e < 0 || (uint32_t)e >= n_expert) return; + const uint32_t r = blockIdx.x * 8u + warp; + if (r >= MID) return; + const uint32_t nblk = IN / 32u; + const float *xs = x + (uint64_t)t * IN; + float gv = v41_bf16r(v41_fp4_row_dot(wg[e] + (uint64_t)r * nblk * 17u, xs, nblk)); + float uv = v41_bf16r(v41_fp4_row_dot(wu[e] + (uint64_t)r * nblk * 17u, xs, nblk)); + if (clamp > 0.f) { if (gv > clamp) gv = clamp; if (uv > clamp) uv = clamp; if (uv < -clamp) uv = -clamp; } + if ((threadIdx.x & 31u) == 0) h[(uint64_t)pair * MID + r] = v41_bf16r((gv / (1.0f + expf(-gv))) * uv); +} + +/* down: partial[pair][OUT] = bf16(W2·h). Engine :54-67. */ +__global__ static void v41_mtp_down_kernel(float *partial, const uint8_t *const *wd, const int32_t *sel, + const float *h, uint32_t MID, uint32_t OUT, uint32_t K, uint32_t n_expert) { + const uint32_t pair = blockIdx.y, warp = threadIdx.x >> 5; + const int32_t e = sel[pair]; + const uint32_t r = blockIdx.x * 8u + warp; + if (r >= OUT) return; + if (e < 0 || (uint32_t)e >= n_expert) { /* unselected slots must write 0: the reduce reads the whole row */ + if ((threadIdx.x & 31u) == 0) partial[(uint64_t)pair * OUT + r] = 0.f; + return; + } + const uint32_t nblk = MID / 32u; + const float y = v41_fp4_row_dot(wd[e] + (uint64_t)r * nblk * 17u, h + (uint64_t)pair * MID, nblk); + if ((threadIdx.x & 31u) == 0) partial[(uint64_t)pair * OUT + r] = v41_bf16r(y); +} + +/* Per-tower expert device pointer table: n_expert x 3 matrices, resolved once. + * Why a table: the kernels pick the base by sel, and every expert is its own + * GGUF tensor (discontinuous offsets) — resolving 384 ranges per step is pure + * host waste. Engine :75-99. */ +typedef struct { const uint8_t **dev_g, **dev_u, **dev_d; uint32_t n; } v41_mtp_ptrs; +static v41_mtp_ptrs g_mtp_ptrs[8]; /* tower-count slots (official 3); values from the caller */ +static v41_scratch g_mtp_h, g_mtp_part; + +static int v41_mtp_bind(uint32_t tower, const void *model_map, const uint64_t *off, uint32_t n_expert) { + if (tower >= 8u) return 0; + v41_mtp_ptrs *P = &g_mtp_ptrs[tower]; + if (P->n == n_expert && P->dev_g) return 1; + const uint8_t **host = (const uint8_t **)malloc(sizeof(void *) * 3u * n_expert); + if (!host) return 0; + for (uint32_t i = 0; i < 3u * n_expert; i++) { + /* bytes 1: only the base address is wanted; bounds are the kernels' row + * math against the GGUF registration (rows x blocks x 17 B). */ + const char *p = cuda_model_range_ptr(model_map, off[i], 1, "mtp expert"); + if (!p) { free((void *)host); return 0; } + host[i] = (const uint8_t *)p; + } + void *dev = NULL; + if (hipMalloc(&dev, sizeof(void *) * 3u * n_expert) != hipSuccess) { (void)hipGetLastError(); free((void *)host); return 0; } + const hipError_t st = hipMemcpy(dev, host, sizeof(void *) * 3u * n_expert, hipMemcpyHostToDevice); + free((void *)host); + if (st != hipSuccess) { (void)hipGetLastError(); (void)hipFree(dev); return 0; } + P->dev_g = (const uint8_t **)dev; + P->dev_u = P->dev_g + n_expert; + P->dev_d = P->dev_g + 2u * n_expert; + P->n = n_expert; + return 1; +} + +/* The dense per-expert MoE (engine :101-141). The engine's union-by-expert + * negative result (2026-09-17) is archived there and NOT re-tried: the kernel + * is wavefront-bound (activations are 4 of every 5 wavefronts), not weight- + * byte-bound. */ +extern "C" int ds4_gpu_v41_mtp_moe_tensor(ds4_gpu_tensor *out, const void *model_map, uint32_t tower, const uint64_t *exp_off, + uint32_t in_dim, uint32_t mid_dim, uint32_t out_dim, + const ds4_gpu_tensor *selected, const ds4_gpu_tensor *weights, + uint32_t n_expert, uint32_t topk, float clamp, + const ds4_gpu_tensor *x, uint32_t n_tok) { + if (!out || !selected || !weights || !x || !exp_off || !n_tok || !topk) return 0; + if ((in_dim % 32u) || (mid_dim % 32u)) return 0; + if (!v41_mtp_bind(tower, model_map, exp_off, n_expert)) return 0; + const v41_mtp_ptrs *P = &g_mtp_ptrs[tower]; + const uint64_t np = (uint64_t)n_tok * topk; + float *h = (float *)v41_grow(&g_mtp_h, np * mid_dim * 4, "mtp h"); + float *part = (float *)v41_grow(&g_mtp_part, np * out_dim * 4, "mtp partial"); + if (!h || !part) return 0; + v41_mtp_gateup_kernel<<>>( + h, P->dev_g, P->dev_u, (const int32_t *)selected->ptr, (const float *)x->ptr, in_dim, mid_dim, topk, n_expert, clamp); + if (!cuda_ok(hipGetLastError(), "v41 mtp gateup")) return 0; + v41_mtp_down_kernel<<>>( + part, P->dev_d, (const int32_t *)selected->ptr, h, mid_dim, out_dim, topk, n_expert); + if (!cuda_ok(hipGetLastError(), "v41 mtp down")) return 0; + v41_vq_reduce_kernel<<>>( + (float *)out->ptr, part, (const float *)weights->ptr, topk, out_dim); + return cuda_ok(hipGetLastError(), "v41 mtp reduce"); +} + +/* hc four-route mean -> main_hidden slot (official Transformer.forward: + * main_hiddens.append(h.mean(dim=2))). Captured at the engram-to-layer edge + * (core_v41_forward.c:200), NOT at the layer output — the wrong point does + * not error, acceptance just collapses to about 1. + * + * Only the last n_rows rows move (a prefill block has thousands of tokens and + * the drafter only eats the last few confirmed positions), and the landing + * slot is an absolute-position ring: row t -> cell ((pos + t) % cap) + * (engine :143-166). The graph path passes posd (the device position slot); + * the eager path passes the position directly. */ +__global__ static void v41_hc_mean_kernel(float *out, const float *hc, uint32_t E, uint32_t n_hc, uint32_t n_rows, + uint32_t src_row0, uint32_t slot, uint32_t n_slot, + uint32_t dst_pos0, uint32_t cap, const int32_t *posd) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= (uint64_t)n_rows * E) return; + const uint32_t t = (uint32_t)(i / E), d = (uint32_t)(i % E); + const float *h = hc + (uint64_t)(src_row0 + t) * n_hc * E + d; + float s = 0.f; + for (uint32_t c = 0; c < n_hc; c++) s += h[(uint64_t)c * E]; + const uint32_t pos = posd ? (uint32_t)posd[0] + src_row0 : dst_pos0; + out[(uint64_t)((pos + t) % cap) * n_slot * E + (uint64_t)slot * E + d] = s / (float)n_hc; +} +extern "C" int ds4_gpu_v41_hc_mean_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *hc, uint32_t n_embd, uint32_t n_hc, + uint32_t n_rows, uint32_t src_row0, uint32_t slot, uint32_t n_slot, + uint32_t dst_pos0, uint32_t cap, const ds4_gpu_tensor *posd) { + if (!out || !hc || !n_rows || !cap || n_rows > cap) return 0; + const uint64_t n = (uint64_t)n_rows * n_embd; + if (out->bytes < (uint64_t)cap * n_slot * n_embd * 4) return 0; + v41_hc_mean_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>( + (float *)out->ptr, (const float *)hc->ptr, n_embd, n_hc, n_rows, src_row0, slot, n_slot, + dst_pos0, cap, posd ? (const int32_t *)posd->ptr : NULL); + return cuda_ok(hipGetLastError(), "v41 hc mean"); +} + +/* Ring -> contiguous: dst[t] = ring[(first_row + t) % cap]. One block per + * row, floats in steps (row_floats = n_target * E = 15360). firstd (non-NULL + * = the graph path) reads the start row from a device int: the fill start + * moves with the previous round's accepted count, so a baked value is wrong + * (silently wrong rows -> dropping acceptance). Engine :169-189. */ +__global__ static void v41_ring_rows_kernel(float *dst, const float *ring, uint32_t row_floats, uint32_t cap, + uint32_t first_row, uint32_t count, const int32_t *firstd) { + const uint32_t t = blockIdx.x; + if (t >= count) return; + if (firstd) first_row = (uint32_t)firstd[0]; + const float *src = ring + (uint64_t)((first_row + t) % cap) * row_floats; + float *d = dst + (uint64_t)t * row_floats; + for (uint32_t j = threadIdx.x; j < row_floats; j += blockDim.x) d[j] = src[j]; +} +extern "C" int ds4_gpu_v41_ring_rows_tensor(ds4_gpu_tensor *dst, const ds4_gpu_tensor *ring, uint32_t row_floats, uint32_t cap, + uint32_t first_row, uint32_t count, const ds4_gpu_tensor *firstd) { + if (!dst || !ring || !count || !cap || count > cap) return 0; + if (dst->bytes < (uint64_t)count * row_floats * 4 || ring->bytes < (uint64_t)cap * row_floats * 4) return 0; + v41_ring_rows_kernel<<>>((float *)dst->ptr, (const float *)ring->ptr, row_floats, cap, first_row, count, + firstd ? (const int32_t *)firstd->ptr : NULL); + return cuda_ok(hipGetLastError(), "v41 ring rows"); +} + +/* markov embed: gather a table row by a DEVICE token id -> f32 (bf16 grid). + * No host readback: the block's positions are sequentially dependent (the + * bias of position i needs the token sampled at i), so a D2H per position + * would cost the whole gain. Engine :191-215. */ +__global__ static void v41_row_gather_kernel(float *out, const uint8_t *tab, const int32_t *ids, + uint32_t which, uint32_t dim, uint32_t out_row, uint32_t is_f32) { + const uint32_t d = blockIdx.x * blockDim.x + threadIdx.x; + if (d >= dim) return; + const int32_t id = ids[which]; + float v = 0.f; + if (id >= 0) { + if (is_f32) memcpy(&v, tab + ((uint64_t)id * dim + d) * 4u, 4); + else { __hip_bfloat16 h; memcpy(&h, tab + ((uint64_t)id * dim + d) * 2u, 2); v = __bfloat162float(h); } + } + out[(uint64_t)out_row * dim + d] = v; +} +/* elem_bytes comes from the GGUF-registered tensor type (4 = f32, 2 = bf16) — + * the engine never assumes it, and guessing does not error, it reads garbage + * (the 2026-09-15 crash: a bf16 read of the f32 markov table, acceptance + * 0.14/5). */ +extern "C" int ds4_gpu_v41_row_gather_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, + uint64_t tab_offset, uint64_t n_rows, uint32_t dim, uint32_t elem_bytes, + const ds4_gpu_tensor *ids, uint32_t which, uint32_t out_row) { + if (!out || !ids || (elem_bytes != 2u && elem_bytes != 4u)) return 0; + const uint64_t bytes = n_rows * dim * elem_bytes; + if (tab_offset > model_size || bytes > model_size - tab_offset) return 0; + const uint8_t *tab = (const uint8_t *)cuda_model_range_ptr(model_map, tab_offset, bytes, "markov embed"); + if (!tab) return 0; + v41_row_gather_kernel<<<(dim + 255u) / 256u, 256, 0, ds4_current_stream()>>>( + (float *)out->ptr, tab, (const int32_t *)ids->ptr, which, dim, out_row, elem_bytes == 4u); + return cuda_ok(hipGetLastError(), "v41 row gather"); +} + +/* logits row += bias row (markov bias; official logits[:, i].add_(logits_bias)). + * Engine :218-229. */ +__global__ static void v41_row_add_kernel(float *dst, const float *src, uint64_t n) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) dst[i] += src[i]; +} +extern "C" int ds4_gpu_v41_row_add_tensor(ds4_gpu_tensor *dst, uint64_t dst_row, const ds4_gpu_tensor *src, uint64_t n) { + if (!dst || !src) return 0; + v41_row_add_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>( + (float *)dst->ptr + dst_row * n, (const float *)src->ptr, n); + return cuda_ok(hipGetLastError(), "v41 row add"); +} + +/* ---- markov bias cache (2026-10-07; interface semantics in the engine's + * ds4_gpu_v41.h). Only the original bf16 markov_head path uses it; the + * distilled bias table (mkH_dev) does not. ---- + * lookup: one block, slot = thread; one id lives in at most one slot (the + * claim records the id, a hit never claims), so at most one thread hits. + * Miss = round-robin evict: the old slot's id becomes the new id on the spot, + * and the bias lands in add — nobody reads that slot in between (same-stream + * order: lookup -> gather -> GEMV -> add -> next position's lookup). + * Engine :232-253. */ +__global__ static void v41_mkcache_lookup_kernel(int32_t *hit, int32_t *cache_ids, uint32_t *next, const int32_t *ids, uint32_t which, uint32_t n_slots) { + __shared__ int found; + if (threadIdx.x == 0) found = -1; + __syncthreads(); + const int32_t id = ids[which]; + if (threadIdx.x < n_slots && cache_ids[threadIdx.x] == id) found = (int)threadIdx.x; + __syncthreads(); + if (threadIdx.x == 0) { + if (found >= 0) { hit[0] = found; return; } + const uint32_t s = *next % n_slots; + *next = s + 1u; + cache_ids[s] = id; + hit[0] = -(int32_t)s - 2; + } +} +extern "C" int ds4_gpu_v41_mkcache_lookup_tensor(ds4_gpu_tensor *hit, ds4_gpu_tensor *cache_ids, ds4_gpu_tensor *next, const ds4_gpu_tensor *ids, + uint32_t which, uint32_t n_slots) { + if (!hit || !cache_ids || !next || !ids || n_slots == 0u || n_slots > 1024u) return 0; + v41_mkcache_lookup_kernel<<<1, (unsigned)n_slots, 0, ds4_current_stream()>>>( + (int32_t *)hit->ptr, (int32_t *)cache_ids->ptr, (uint32_t *)next->ptr, (const int32_t *)ids->ptr, which, n_slots); + return cuda_ok(hipGetLastError(), "v41 mkcache lookup"); +} +/* add: hit => += cache[slot]; miss => += bias and store it. Same addition as + * v41_row_add_kernel (dst += the same f32) => bit-identical. Engine :255-267. */ +__global__ static void v41_mkcache_add_kernel(float *dst, const float *bias, float *cache, const int32_t *hit, uint64_t n) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + const int32_t h = hit[0]; + if (h >= 0) dst[i] += cache[(uint64_t)h * n + i]; + else { const float b = bias[i]; dst[i] += b; cache[(uint64_t)(-h - 2) * n + i] = b; } +} +extern "C" int ds4_gpu_v41_mkcache_add_tensor(ds4_gpu_tensor *logits, uint64_t row, const ds4_gpu_tensor *bias, ds4_gpu_tensor *cache, + const ds4_gpu_tensor *hit, uint64_t n) { + if (!logits || !bias || !cache || !hit) return 0; + v41_mkcache_add_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>( + (float *)logits->ptr + row * n, (const float *)bias->ptr, (float *)cache->ptr, (const int32_t *)hit->ptr, n); + return cuda_ok(hipGetLastError(), "v41 mkcache add"); +} diff --git a/cuda/ds41_engram.cuh b/cuda/ds41_engram.cuh new file mode 100644 index 0000000000000000000000000000000000000000..0e482c2fa457add5024cc79da0646145e6862ed2 --- /dev/null +++ b/cuda/ds41_engram.cuh @@ -0,0 +1,144 @@ +#include "hip/hip_runtime.h" +/* ds41_engram.cuh — DeepSeek V4.1 (ds41) engram device pieces, ported from the + * C engine at /data/YoungAi (commit 3946dbc): + * - the gate kernel + entry: src/cuda/cuda_v41_3.inc.cu:109-141; + * - the read-path primitives: src/cuda/cuda_decode_graph.inc.cu:118-171 + * (the host-flag spin kernel, the pinned->device zero-copy upload) and + * src/cuda/cuda_graphcap.inc.cu:254-259 (pinned mapped host memory). + * + * The host half of the read path (shard open, O_DIRECT pread, the per-layer + * rounds) lives in the Rust host (crates/ds4-core EngramShard/EngramHash, P2, + * §6.6.2); these entries are the device side the forward wires: host allocates + * pinned rows, uploads them zero-copy, the graph's spin kernel waits on the + * per-layer flag, and the rows dequant (ds41_fp8blk.cuh) turns them into the + * bf16 grid the wkv matmul eats. + * + * Numerics of the gate (official Engram.forward): per (token, hc row) the + * block computes sh = sum h^2, sk = sum key^2, sd = sum h*(qw*kw)*key in f32, + * then rstd = rsqrt(sh/E + eps) * rsqrt(sk/E + eps), dot = sd * rstd * + * rsqrt(E), mag = sqrt(max(|dot|, 1e-6)), z = copysign(mag, dot), + * gate = sigmoid(z), h[d] = bf16r(h[d] + gate*val[d]). The f32 reduction + * order (per-thread stride + shared-memory tree) and the fast-math expf + * differ from a host emulation; the output is bf16-quantized, so the compare + * is at bf16-ulp distance. */ +#pragma once + +/* engram gate: one block per (token, hc row), 256 threads, three tree + * reductions in shared memory (cuda_v41_3.inc.cu:109-134). */ +__global__ static void v41_engram_gate_kernel(float *hc, const float *kv, const float *qw, const float *kw, + uint32_t E, uint32_t n_hc, float eps) { + v41_pdl_wait(); /* PDL: first instruction waits for the upstream; no-op when not launched through PDL */ + const uint32_t t = blockIdx.y, c = blockIdx.x; + float *h = hc + ((uint64_t)t * n_hc + c) * E; + const float *key = kv + (uint64_t)t * (n_hc + 1u) * E + (uint64_t)c * E; + const float *val = kv + (uint64_t)t * (n_hc + 1u) * E + (uint64_t)n_hc * E; + float sh = 0.f, sk = 0.f, sd = 0.f; + for (uint32_t d = threadIdx.x; d < E; d += blockDim.x) { + const float hv = h[d], kv_ = key[d], w = qw[c * E + d] * kw[c * E + d]; + sh += hv * hv; sk += kv_ * kv_; sd += hv * w * kv_; + } + __shared__ float r[3][256]; + r[0][threadIdx.x] = sh; r[1][threadIdx.x] = sk; r[2][threadIdx.x] = sd; __syncthreads(); + for (uint32_t k = blockDim.x / 2; k > 0; k >>= 1) { + if (threadIdx.x < k) { r[0][threadIdx.x] += r[0][threadIdx.x + k]; r[1][threadIdx.x] += r[1][threadIdx.x + k]; r[2][threadIdx.x] += r[2][threadIdx.x + k]; } + __syncthreads(); + } + const float rstd = rsqrtf(r[0][0] / (float)E + eps) * rsqrtf(r[1][0] / (float)E + eps); + const float dot = r[2][0] * rstd * rsqrtf((float)E); + const float mag = sqrtf(fmaxf(fabsf(dot), 1e-6f)); + const float z = copysignf(mag, dot); + const float gate = 1.0f / (1.0f + expf(-z)); + for (uint32_t d = threadIdx.x; d < E; d += blockDim.x) h[d] = v41_bf16r(h[d] + gate * val[d]); +} +extern "C" int ds4_gpu_v41_engram_gate_tensor(ds4_gpu_tensor *hc, const ds4_gpu_tensor *kv, const void *model_map, uint64_t model_size, + uint64_t q_w_offset, uint64_t k_w_offset, uint32_t n_embd, uint32_t n_hc, uint32_t n_tok, float eps) { + if (!hc || !kv) return 0; + const float *qw = (const float *)cuda_model_range_ptr(model_map, q_w_offset, (uint64_t)n_hc * n_embd * 4, "v41 engram q"); + const float *kw = (const float *)cuda_model_range_ptr(model_map, k_w_offset, (uint64_t)n_hc * n_embd * 4, "v41 engram k"); + if (!qw || !kw) return 0; + v41_engram_gate_kernel<<>>((float *)hc->ptr, (const float *)kv->ptr, qw, kw, n_embd, n_hc, eps); + return cuda_ok(hipGetLastError(), "v41 engram gate"); +} + +/* ---- the read path's device primitives (cuda_decode_graph.inc.cu) ---- */ + +/* Pinned, device-mapped host memory (cuda_graphcap.inc.cu:254-259): the + * upload kernels address it directly, and a pageable hipMemcpyAsync would + * degrade to a sync copy (and void a capture). */ +extern "C" void *ds4_gpu_host_alloc(uint64_t bytes) { + void *p = NULL; + if (hipHostAlloc(&p, (size_t)bytes, hipHostMallocMapped) != hipSuccess) { (void)hipGetLastError(); return NULL; } + return p; +} +extern "C" void ds4_gpu_host_free(void *p) { if (p) (void)hipHostFree(p); } + +/* GPU spin on a host flag, replacing graph host nodes: on GB10 every host + * node leaves a 0.6-0.9 ms hole (the driver callback wakeup), while the + * 1-thread spin kernel resumes in ~1 us. Semantics: return once flag >= want + * (both pinned; want is this step's sequence, written by the host before the + * launch; the sequence only grows, so no ABA). If the host's read failed and + * never sets the flag, the ~5 s timeout writes err and lets the kernel + * through; the host checks err after the sync and stops. __ldcv + * (ld.global.cv) re-fetches at each coherence point — plain reads saw stale + * L2 lines on GB10 (a whole token of lag, hit in the V4 era). */ +__global__ static void decode_flagwait_kernel(const int32_t *flag, const int32_t *want, int32_t *err) { + const int32_t w = __ldcv(want); + for (uint32_t i = 0; ; i++) { + if (__ldcv(flag) >= w) return; + if (i >= 5000000u) { *err = 1; return; } + __nanosleep(1000); + } +} +extern "C" int ds4_gpu_host_flag_wait(const void *flag_pinned, const void *want_pinned, void *err_pinned) { + void *f = NULL, *w = NULL, *e = NULL; + if (hipHostGetDevicePointer(&f, (void *)flag_pinned, 0) != hipSuccess || hipHostGetDevicePointer(&w, (void *)want_pinned, 0) != hipSuccess || + hipHostGetDevicePointer(&e, err_pinned, 0) != hipSuccess || !f || !w || !e) { (void)hipGetLastError(); return 0; } + decode_flagwait_kernel<<<1, 1, 0, ds4_current_stream()>>>((const int32_t *)f, (const int32_t *)w, (int32_t *)e); + return cuda_ok(hipGetLastError(), "host flag wait"); +} + +/* Zero-copy upload from pinned host memory (cuda_decode_graph.inc.cu:147-171): + * a graph memcpy node costs ~170 us on GB10; a 256-thread kernel reading the + * pinned memory through its device address costs 2-3 us. __ldcv is mandatory + * — GB10's L2 caches host-memory lines and plain reads return the previous + * step's values. The launch stream is ds4_current_stream() so an eager call + * orders with the eager neighbors and a captured one records the capture + * stream (the engine's 2026-09-30 fix for concurrent lanes). */ +__global__ static void decode_hostcopy_kernel(uint32_t *dst, const uint32_t *src_host, uint32_t nwords) { + const uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < nwords) dst[i] = __ldcv(src_host + i); +} +extern "C" int ds4_gpu_tensor_write_zerocopy(ds4_gpu_tensor *t, uint64_t offset, const void *pinned, uint64_t bytes) { + if (!t || !pinned || offset > t->bytes || bytes > t->bytes - offset || (bytes & 3u) || (offset & 3u)) return 0; + if (bytes == 0) return 1; + void *dev = NULL; + if (hipHostGetDevicePointer(&dev, (void *)pinned, 0) != hipSuccess || !dev) { + (void)hipGetLastError(); + fprintf(stderr, "ds4: [ds41] pinned memory has no device address (needs hipHostMallocMapped)\n"); + return 0; + } + const uint32_t nw = (uint32_t)(bytes / 4u); + decode_hostcopy_kernel<<<(nw + 255u) / 256u, 256, 0, ds4_current_stream()>>>((uint32_t *)((char *)t->ptr + offset), + (const uint32_t *)dev, nw); + return cuda_ok(hipGetLastError(), "tensor write zerocopy"); +} + +/* The reverse direction (cuda_decode_graph.inc.cu:172-183): a kernel writes a + * few device words into the mapped pinned buffer (the graph's per-row argmax + * landing), saving the ~170 us D2H memcpy node; the host reads the pinned + * buffer after the step's sync. Same stream rule as the upload: the capture + * stream during capture, so the node lands in the graph. */ +__global__ static void decode_hoststore_kernel(uint32_t *dst_host, const uint32_t *src, uint32_t nwords) { + const uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < nwords) dst_host[i] = src[i]; +} +extern "C" int ds4_gpu_tensor_read_zerocopy(void *pinned, const ds4_gpu_tensor *t, uint64_t offset, uint64_t bytes) { + if (!t || !pinned || offset > t->bytes || bytes > t->bytes - offset || (bytes & 3u) || (offset & 3u)) return 0; + if (bytes == 0) return 1; + void *dev = NULL; + if (hipHostGetDevicePointer(&dev, pinned, 0) != hipSuccess || !dev) { (void)hipGetLastError(); return 0; } + const uint32_t nw = (uint32_t)(bytes / 4u); + decode_hoststore_kernel<<<(nw + 255u) / 256u, 256, 0, ds4_current_stream()>>>((uint32_t *)dev, + (const uint32_t *)((const char *)t->ptr + offset), nw); + return cuda_ok(hipGetLastError(), "tensor read zerocopy"); +} diff --git a/cuda/ds41_fp8blk.cuh b/cuda/ds41_fp8blk.cuh new file mode 100644 index 0000000000000000000000000000000000000000..3a04b610f7d2416edd01e45d21caa32d2d19619d --- /dev/null +++ b/cuda/ds41_fp8blk.cuh @@ -0,0 +1,430 @@ +#include "hip/hip_runtime.h" +/* ds41_fp8blk.cuh — DeepSeek V4.1 (ds41) fp8_32x32 decode family, ported from + * the C engine at /data/YoungAi (commit 3946dbc): + * - primitives: src/common/ds4_fp8.h:33-50 (E4M3FN bit decode, E8M0 scale); + * - the GEMV family: src/cuda/cuda_v41_gemv_highprec.inc.cu:194-341 (the + * kernel, the grouped launcher) and :397-434 (the two tensor entries); + * - the single-group entry and the >8 arm: src/cuda/cuda_v41_1.inc.cu:154-167 + * (x->bf16, round-bf16) and :338-364 (wkv entry, to-bf16 + cuBLAS); + * - the engram row dequant: src/cuda/cuda_v41_4.inc.cu:302-315. + * + * The on-disk format (ds4_quantfmt.h:36-38): a rows*cols E4M3 plane followed + * by ceil(rows/32)*ceil(cols/32) E8M0 scale bytes, one scale per 32x32 tile. + * Value = e4m3 * 2^(e-127); e=0 decodes to the 2^-127 bit pattern (ds4_fp8.h:46). + * This is NOT a per-row block format, so it never enters the gguf block table: + * the geometry here reads the two planes directly. + * + * Consumers (the engine's): the engram wkv [6144][25600] (ds4_gpu_v41_matmul_ + * fp8blk_tensor, core_v41_engram.c:468) and the 25 tower dense tensors + * (ds4_gpu_v41_matmul_fp8blk_round_tensor / _grouped_, core_v41_attn.c:208,224). + * + * Numerics: the GEMV decodes each weight element as (e4m3 * scale) in f32 and + * accumulates f32 per token; n>1 tokens read activations already rounded to + * bf16 (XB=1, the engine's same knife as the skeleton GEMV). The >8 arm + * expands the weight to bf16 and runs cuBLAS — a different numeric path by + * design (prefill), named in the engine at cuda_v41_1.inc.cu:350-364. */ +#pragma once + +/* E4M3FN bit decode (ds4_fp8.h:33-42): abs==0x7f is NaN (E4M3FN has no Inf); + * exp==0 is the subnormal man*2^-9; else (1+man/8)*2^(exp-7). */ +__device__ __forceinline__ static float ds4_e4m3fn_to_f32(uint8_t x) { + const uint8_t abs = x & 0x7f; + const bool sign = (x & 0x80) != 0; + if (abs == 0) return sign ? -0.0f : 0.0f; + if (abs == 0x7f) return NAN; + const int exp = (x >> 3) & 0x0f; + const int man = x & 0x07; + float value = exp == 0 ? ldexpf((float)man, -9) + : ldexpf(1.0f + (float)man / 8.0f, exp - 7); + return sign ? -value : value; +} + +/* E8M0 scale byte -> f32 (ds4_fp8.h:46-50): e=0 is the 0x00400000 bit pattern + * (2^-127), else 2^(e-127). */ +__device__ __forceinline__ static float ds4_e8m0_to_f32(uint8_t e) { + const uint32_t bits = e == 0 ? 0x00400000u : ((uint32_t)e << 23); + float result; + memcpy(&result, &bits, sizeof(result)); + return result; +} + +/* ---- the encode-side primitives (ds4_fp8.h:52-164) ---- + * P4-4's act_quant / KV pack need the round and encode halves, not just the + * decoders: the FP4/E4M3 magnitude tables, nearest-rounding (ties to even + * mantissa), the nibble map and the byte encoders. One copy each, shared by + * act_quant and the pack kernels (a second copy would drift, and the packed + * bytes must equal what act_quant writes). */ +__device__ __forceinline__ static float ds4_e4m3fn_value(int i) { + static const float exp_scale[16] = { + 0.0f, 0.015625f, 0.03125f, 0.0625f, + 0.125f, 0.25f, 0.5f, 1.0f, + 2.0f, 4.0f, 8.0f, 16.0f, + 32.0f, 64.0f, 128.0f, 256.0f, + }; + const int exp = (i >> 3) & 0x0f; + const int mant = i & 0x07; + return exp == 0 + ? (float)mant * 0.001953125f + : (1.0f + (float)mant * 0.125f) * exp_scale[exp]; +} +__device__ __forceinline__ static float ds4_e4m3fn_round(float x) { + const float sign = x < 0.0f ? -1.0f : 1.0f; + const float ax = fminf(fabsf(x), 448.0f); + int lo = 0; + int hi = 126; + while (lo < hi) { + const int mid = (lo + hi + 1) >> 1; + if (ds4_e4m3fn_value(mid) <= ax) lo = mid; + else hi = mid - 1; + } + int best = lo; + if (best < 126) { + const float best_diff = fabsf(ax - ds4_e4m3fn_value(best)); + const float next_diff = fabsf(ax - ds4_e4m3fn_value(best + 1)); + if (next_diff < best_diff || + (next_diff == best_diff && ((best + 1) & 1) == 0 && (best & 1) != 0)) { + best++; + } + } + return sign * ds4_e4m3fn_value(best); +} +__device__ __forceinline__ static float ds4_e2m1fn_value(int i) { + static const float values[8] = { + 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, + }; + return values[i & 7]; +} +__device__ __forceinline__ static float ds4_e2m1fn_round(float x) { + const float sign = x < 0.0f ? -1.0f : 1.0f; + const float ax = fminf(fabsf(x), 6.0f); + int best = 0; + float best_diff = fabsf(ax - ds4_e2m1fn_value(0)); + for (int i = 1; i < 8; i++) { + const float diff = fabsf(ax - ds4_e2m1fn_value(i)); + if (diff < best_diff || (diff == best_diff && (i & 1) == 0 && (best & 1) != 0)) { + best = i; + best_diff = diff; + } + } + return sign * ds4_e2m1fn_value(best); +} +/* FP4 nibble (with sign, 0..15) -> f32 (ds4_fp8.h:122-127; slot 8 is -0.0f). + * The engine's measured verdict stands: the table beats a bit-trick version + * (the decode kernels are memory-latency bound and the LDC replays are hidden; + * the bit version measured 8% slower decode, 2026-09-15). */ +__device__ __forceinline__ static float ds4_fp4_nibble_to_f32(uint8_t n) { + static const float t[16] = { + 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, + -0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f, + }; + return t[n & 15]; +} +__device__ __forceinline__ static uint8_t ds4_fp4_f32_to_nibble(float x) { + const float v = ds4_e2m1fn_round(x); + const float av = fabsf(v); + uint8_t n = 0; + for (int i = 1; i < 8; i++) { + if (av == ds4_e2m1fn_value(i)) { n = (uint8_t)i; break; } /* exact table values, equality is safe */ + } + return n == 0 ? (uint8_t)0 : (uint8_t)(v < 0.0f ? (n | 8u) : n); +} +__device__ __forceinline__ static uint8_t ds4_e8m0_f32_to_byte(float s) { + uint32_t bits; + memcpy(&bits, &s, sizeof(bits)); + if (bits == 0x00400000u) return 0; + return (uint8_t)((bits >> 23) & 0xffu); +} +__device__ __forceinline__ static uint8_t ds4_e4m3fn_f32_to_byte(float x) { + const float v = ds4_e4m3fn_round(x); + const float av = fabsf(v); + int lo = 0, hi = 126; + while (lo < hi) { + const int mid = (lo + hi + 1) >> 1; + if (ds4_e4m3fn_value(mid) <= av) lo = mid; + else hi = mid - 1; + } + return (uint8_t)((v < 0.0f && av != 0.0f) ? (lo | 0x80) : lo); +} + +/* f32 -> bf16 (cuda_v41_1.inc.cu:154-157) and the RNE round-back kernel + * (:158-162; the port's v41_bf16r lives in ds41_primitives.cuh:53). */ +__global__ static void v41_x_to_bf16_kernel(__hip_bfloat16 *out, const float *x, uint64_t n) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) out[i] = __float2bfloat16(x[i]); +} +__global__ static void v41_round_bf16_kernel(float *x, uint64_t n) { + v41_pdl_wait(); /* PDL: waits for the upstream when launched through PDL; a no-op otherwise */ + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) x[i] = v41_bf16r(x[i]); +} +extern "C" int ds4_gpu_v41_round_bf16_tensor(ds4_gpu_tensor *x, uint64_t n) { + if (!x || x->bytes < n * 4) return 0; + v41_round_bf16_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>((float *)x->ptr, n); + return cuda_ok(hipGetLastError(), "v41 round bf16"); +} + +/* ---- FP8(e4m3 + 32x32 block ue8m0) weight GEMV: engram wkv + the towers ---- + * Layout: w[0..rows*cols) is e4m3; then ceil(rows/32)*ceil(cols/32) ue8m0 + * bytes, tile (r/32, c/32). One lane eats 16 consecutive elements per step + * (one 128-bit read), so its 16 elements always sit in one column tile and + * the scale is looked up once. in_dim % 32 == 0 is the only shape rule + * (relaxed 2026-09-17 from 512; the 512 gate now lives only on the wkv + * entry's GEMV-vs-cuBLAS choice, cuda_v41_1.inc.cu:348). */ +/* XB=1: activations are read from the bf16 buffer — the same knife and the + * same account as the skeleton GEMV (see the engine's kernel comment). */ +/* grid.y = the block-diagonal group (2026-09-17): the towers' wo_a is 8 block + * groups and its row segment [g*out_dim, (g+1)*out_dim) is contiguous in BOTH + * planes (out_dim is a multiple of 32), so the group is just a row offset. + * n_groups=1 for every non-grouped call (offset is always 0, byte-for-byte + * the pre-grouping instruction stream). */ +template +__global__ static void v41_fp8blk_gemv_kernel(float *out, const uint8_t *w, const uint8_t *sc, const float *x, + const __hip_bfloat16 *x16, + uint32_t in_dim, uint32_t out_dim, uint32_t sbc, uint32_t ksplit, + uint32_t x_stride, uint32_t out_stride, + uint32_t x_gstride, uint32_t out_gstride) { + v41_pdl_wait(); /* PDL: first instruction waits for the upstream; no-op when not launched through PDL */ + const uint32_t g = blockIdx.y; + w += (uint64_t)g * in_dim * out_dim; + sc += (uint64_t)g * ((out_dim + 31u) / 32u) * sbc; + x += (uint64_t)g * x_gstride; out += (uint64_t)g * out_gstride; + if (XB) x16 += (uint64_t)g * x_gstride; + const uint32_t warp = threadIdx.x >> 5, lane = threadIdx.x & 31u; + const uint32_t rows_per_block = 8u / ksplit, rloc = warp / ksplit, kpart = warp % ksplit; + const uint32_t r = blockIdx.x * rows_per_block + rloc; + __shared__ float red[8][NT]; + float acc[NT]; + #pragma unroll + for (uint32_t t = 0; t < NT; t++) acc[t] = 0.f; + if (r < out_dim) { + const uint8_t *wr = w + (uint64_t)r * in_dim; + const uint8_t *scr = sc + (uint64_t)(r >> 5) * sbc; + for (uint32_t c = kpart * 512u + lane * 16u; c < in_dim; c += ksplit * 512u) { + const uint4 raw = *(const uint4 *)(wr + c); + uint8_t b[16]; + memcpy(b, &raw, 16); + const float s = ds4_e8m0_to_f32(scr[c >> 5]); + float wv[16]; + #pragma unroll + for (uint32_t j = 0; j < 16u; j++) wv[j] = ds4_e4m3fn_to_f32(b[j]) * s; + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + /* The two arms write their own loop bodies on purpose: a shared + * float xv[16] intermediate changes the whole kernel's register + * allocation (measured 5% slower decode on the skeleton GEMV); + * the XB=0 arm is the pre-change instruction stream verbatim. */ + if (XB) { /* 16 bf16 = 32 B contiguous, two uint4; the shift restores f32 exactly */ + const uint4 *xp = (const uint4 *)(x16 + (uint64_t)t * x_stride + c); + const uint4 x0 = xp[0], x1 = xp[1]; + const uint32_t xu[8] = { x0.x, x0.y, x0.z, x0.w, x1.x, x1.y, x1.z, x1.w }; + #pragma unroll + for (uint32_t j = 0; j < 8u; j++) { + acc[t] += wv[2u * j] * __uint_as_float(xu[j] << 16); + acc[t] += wv[2u * j + 1u] * __uint_as_float(xu[j] & 0xffff0000u); + } + } else { + const float *xt = x + (uint64_t)t * x_stride + c; + #pragma unroll + for (uint32_t j = 0; j < 16u; j++) acc[t] += wv[j] * xt[j]; + } + } + } + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + float v = acc[t]; + for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o); + acc[t] = v; + } + } + if (ksplit > 1u) { + if (lane == 0) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) red[warp][t] = acc[t]; + } + __syncthreads(); + if (kpart == 0 && lane == 0 && r < out_dim) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + float v = 0.f; + for (uint32_t k = 0; k < ksplit; k++) v += red[rloc * ksplit + k][t]; + out[(uint64_t)t * out_stride + r] = v; + } + } + } else if (lane == 0 && r < out_dim) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) out[(uint64_t)t * out_stride + r] = acc[t]; + } +} + +/* Prefill (n > 8): expand this table to bf16 once, then one cuBLAS call. + * Only allowed to spill here: e4m3 with 2-D block scales has no cuBLASLt FP8 + * algorithm (it eats one scale per tensor), and the table is 0.157 GB/layer + * with only two layers, so the expansion amortizes over the whole token + * block. The decode path above spills nothing. */ +__global__ static void v41_fp8blk_to_bf16_kernel(__hip_bfloat16 *o, const uint8_t *w, const uint8_t *sc, + uint32_t in_dim, uint32_t sbc, uint64_t n) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + const uint32_t r = (uint32_t)(i / in_dim), c = (uint32_t)(i % in_dim); + o[i] = __float2bfloat16(ds4_e4m3fn_to_f32(w[i]) * ds4_e8m0_to_f32(sc[(uint64_t)(r >> 5) * sbc + (c >> 5)])); +} + +/* in_dim % 32 is the shape rule (a lane's 16 elements and the 32-wide scale + * tile need it); a ragged last warp is fine, only idle lanes. */ +static v41_scratch g_v41_fp8_xb; +static v41_scratch g_v41_fp8_wbf, g_v41_fp8_xbf; +static int v41_fp8blk_gemv_g(const uint8_t *w, const uint8_t *sc, uint64_t in_dim, uint64_t out_dim, + const float *x, float *out, uint32_t n_tok, uint32_t n_groups, + uint32_t x_stride, uint32_t out_stride, uint32_t x_gstride, uint32_t out_gstride, + const char *what) { + if ((in_dim % 32u) != 0u || n_tok == 0 || n_groups == 0) return 0; + if (n_tok > V41_GEMV_MAX_TOK) { + /* Bigger batches loop in 8-row segments (the engine's 2026-09-18 fix: + * the draft tower's 128-position window rebuild needs one call; the + * weight is re-read per segment, decode keeps single-issue rows). */ + for (uint32_t off = 0; off < n_tok; off += V41_GEMV_MAX_TOK) { + const uint32_t nn = n_tok - off < V41_GEMV_MAX_TOK ? n_tok - off : V41_GEMV_MAX_TOK; + if (!v41_fp8blk_gemv_g(w, sc, in_dim, out_dim, x + (uint64_t)off * x_stride, out + (uint64_t)off * out_stride, + nn, n_groups, x_stride, out_stride, x_gstride, out_gstride, what)) return 0; + } + return 1; + } + uint32_t ksplit = 1; + while (ksplit < 8u && out_dim * ksplit * n_groups < 32768u) ksplit <<= 1; + const uint32_t nseg = (uint32_t)((in_dim + 511u) / 512u); + while (ksplit > 1u && ksplit > nseg) ksplit >>= 1; + const uint32_t rpb = 8u / ksplit, sbc = (uint32_t)((in_dim + 31u) / 32u); + const dim3 grid((unsigned)((out_dim + rpb - 1u) / rpb), n_groups); + /* n>1 converts the activations to bf16 (the same knife as the skeleton + * GEMV); n=1 does not (that path is byte-bound). */ + const __hip_bfloat16 *x16 = NULL; + if (n_tok > 1u) { + const uint64_t xn = (uint64_t)n_tok * x_stride; + __hip_bfloat16 *xbuf = (__hip_bfloat16 *)v41_grow(&g_v41_fp8_xb, xn * sizeof(__hip_bfloat16), "v41 fp8blk x bf16"); + if (!xbuf) return 0; + v41_x_to_bf16_kernel<<<(unsigned)((xn + 255) / 256), 256, 0, ds4_current_stream()>>>(xbuf, x, xn); + if (!cuda_ok(hipGetLastError(), "v41 fp8blk x bf16")) return 0; + x16 = xbuf; + } + #define V41_FP8GEMV_LAUNCH(NT, XB) v41_fp8blk_gemv_kernel<<>>( \ + out, w, sc, x, x16, (uint32_t)in_dim, (uint32_t)out_dim, sbc, ksplit, \ + x_stride, out_stride, x_gstride, out_gstride) + switch (n_tok) { + case 1: V41_FP8GEMV_LAUNCH(1u, 0u); break; case 2: V41_FP8GEMV_LAUNCH(2u, 1u); break; + case 3: V41_FP8GEMV_LAUNCH(3u, 1u); break; case 4: V41_FP8GEMV_LAUNCH(4u, 1u); break; + case 5: V41_FP8GEMV_LAUNCH(5u, 1u); break; case 6: V41_FP8GEMV_LAUNCH(6u, 1u); break; + case 7: V41_FP8GEMV_LAUNCH(7u, 1u); break; default: V41_FP8GEMV_LAUNCH(8u, 1u); break; + } + #undef V41_FP8GEMV_LAUNCH + return cuda_ok(hipGetLastError(), what); +} + +/* Single-group entry (the engine's v41_fp8blk_gemv): groups 1, strides = dims. */ +static int v41_fp8blk_gemv(const uint8_t *w, const uint8_t *sc, uint64_t in_dim, uint64_t out_dim, + const float *x, float *out, uint32_t n_tok, const char *what) { + return v41_fp8blk_gemv_g(w, sc, in_dim, out_dim, x, out, n_tok, 1u, + (uint32_t)in_dim, (uint32_t)out_dim, 0u, 0u, what); +} + +/* Expand one whole fp8 tensor to bf16 (cuda_v41_gemv_highprec.inc.cu:344-348). */ +static int v41_fp8blk_to_bf16(__hip_bfloat16 *o, const uint8_t *w, const uint8_t *sc, uint64_t in_dim, uint64_t rows) { + const uint64_t n = in_dim * rows; + const uint32_t sbc = (uint32_t)((in_dim + 31u) / 32u); + v41_fp8blk_to_bf16_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>(o, w, sc, (uint32_t)in_dim, sbc, n); + return cuda_ok(hipGetLastError(), "v41 fp8blk->bf16"); +} + +/* engram wkv entry (cuda_v41_1.inc.cu:338-364): n <= 8 and in_dim % 512 == 0 + * take the fused GEMV; otherwise expand to bf16 + one cuBLAS call. */ +extern "C" int ds4_gpu_v41_matmul_fp8blk_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, + const ds4_gpu_tensor *x, uint32_t n_tok) { + if (!out || !x || n_tok == 0) return 0; + const uint64_t sbc = (in_dim + 31u) / 32u, sbr = (out_dim + 31u) / 32u; + const uint64_t wbytes = in_dim * out_dim + sbr * sbc; + if (weight_offset > model_size || wbytes > model_size - weight_offset) return 0; + const uint8_t *W = (const uint8_t *)cuda_model_range_ptr(model_map, weight_offset, wbytes, "v41 fp8blk w"); + if (!W) return 0; + const uint8_t *SC = W + in_dim * out_dim; + if (n_tok <= V41_GEMV_MAX_TOK && (in_dim % 512u) == 0u) + return v41_fp8blk_gemv(W, SC, in_dim, out_dim, (const float *)x->ptr, (float *)out->ptr, n_tok, "v41 fp8blk gemv"); + if (!g_cublas_ready) return 0; + const uint64_t wn = in_dim * out_dim; + __hip_bfloat16 *wb = (__hip_bfloat16 *)v41_grow(&g_v41_fp8_wbf, wn * sizeof(__hip_bfloat16), "v41 wkv bf16"); + if (!wb || !v41_fp8blk_to_bf16(wb, W, SC, in_dim, out_dim)) return 0; + const uint64_t xn = (uint64_t)n_tok * in_dim; + __hip_bfloat16 *xb = (__hip_bfloat16 *)v41_grow(&g_v41_fp8_xbf, xn * sizeof(__hip_bfloat16), "v41 wkv x bf16"); + if (!xb) return 0; + v41_x_to_bf16_kernel<<<(unsigned)((xn + 255) / 256), 256, 0, ds4_current_stream()>>>(xb, (const float *)x->ptr, xn); + if (!cuda_ok(hipGetLastError(), "v41 wkv x->bf16")) return 0; + /* The engine calls hipblasSetStream before every V4.1 GEMM + * (cuda_v41_1.inc.cu:360 here), and per the cuBLAS docs (2.4.7) that call + * unconditionally resets the bound workspace to the default pool — so the + * engine's GEMMs actually run on the default pool, not on its 32 MB user + * buffer. Mirror the call (the port's stream is the legacy default, so + * ordering is unchanged); cuda_cublas_ws_prep (the port's S1.1a rule, + * ds4_cuda.cu) then only touches the now-unbound user buffer. */ + (void)hipblasSetStream(g_cublas, ds4_current_stream()); + cuda_cublas_ws_prep(ds4_current_stream()); + const float alpha = 1.0f, beta = 0.0f; + hipblasStatus_t st = hipblasGemmEx(g_cublas, HIPBLAS_OP_T, HIPBLAS_OP_N, (int)out_dim, (int)n_tok, (int)in_dim, &alpha, + wb, HIP_R_16BF, (int)in_dim, xb, HIP_R_16BF, (int)in_dim, &beta, + (float *)out->ptr, HIP_R_32F, (int)out_dim, HIP_R_32F, HIPBLAS_GEMM_DEFAULT); + return cublas_ok(st, "v41 wkv bf16 gemm"); +} + +/* ---- the towers' fp8 entries (mtp-1.md M6 root fix, 2026-09-17) ---- + * DSpark's towers are FP8 in the original; the converter used to compress + * them to FP4 with the skeleton and halved the drafter's precision (first- + * position agreement 0.50). Both entries round the output to bf16 once + * when round_out is set — the fp4 arm folds that rounding into its kernel, + * and the two must share one numeric contract. */ +extern "C" int ds4_gpu_v41_matmul_fp8blk_round_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, + const ds4_gpu_tensor *x, uint32_t n_tok, int round_out) { + if (!out || !x || n_tok == 0) return 0; + const uint64_t sbc = (in_dim + 31u) / 32u, sbr = (out_dim + 31u) / 32u; + const uint64_t wbytes = in_dim * out_dim + sbr * sbc; + if (weight_offset > model_size || wbytes > model_size - weight_offset) return 0; + const uint8_t *W = (const uint8_t *)cuda_model_range_ptr(model_map, weight_offset, wbytes, "v41 mtp fp8 w"); + if (!W) return 0; + if (!v41_fp8blk_gemv_g(W, W + in_dim * out_dim, in_dim, out_dim, (const float *)x->ptr, (float *)out->ptr, + n_tok, 1u, (uint32_t)in_dim, (uint32_t)out_dim, 0u, 0u, "v41 mtp fp8 gemv")) return 0; + return round_out ? ds4_gpu_v41_round_bf16_tensor(out, (uint64_t)n_tok * out_dim) : 1; +} +extern "C" int ds4_gpu_v41_grouped_matmul_fp8blk_tensor(ds4_gpu_tensor *low, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint32_t n_groups, uint64_t group_dim, + uint64_t rank, const ds4_gpu_tensor *heads, uint32_t n_tok, int round_out) { + if (!low || !heads || n_tok == 0 || n_groups == 0) return 0; + const uint64_t in_all = (uint64_t)n_groups * group_dim, out_all = (uint64_t)n_groups * rank; + const uint64_t sbc = (group_dim + 31u) / 32u, sbr = (rank + 31u) / 32u; + /* On disk: the whole [n_groups*rank][group_dim] e4m3 plane plus the same + * shape's scale plane — group g's row segment is contiguous, so the + * kernel only needs the grid.y row offset (see the kernel comment). */ + const uint64_t wbytes = out_all * group_dim + n_groups * sbr * sbc; + if (weight_offset > model_size || wbytes > model_size - weight_offset) return 0; + const uint8_t *W = (const uint8_t *)cuda_model_range_ptr(model_map, weight_offset, wbytes, "v41 mtp wo_a fp8"); + if (!W) return 0; + if (!v41_fp8blk_gemv_g(W, W + out_all * group_dim, group_dim, rank, (const float *)heads->ptr, + (float *)low->ptr, n_tok, n_groups, (uint32_t)in_all, (uint32_t)out_all, + (uint32_t)group_dim, (uint32_t)rank, "v41 mtp wo_a fp8 gemv")) return 0; + return round_out ? ds4_gpu_v41_round_bf16_tensor(low, (uint64_t)n_tok * out_all) : 1; +} + +/* ---- engram table rows: fp8 (e4m3 x ue8m0/32) -> bf16 grid ---- + * One thread per element (cuda_v41_4.inc.cu:302-308): row stride is + * head_dim + head_dim/32 (the scale plane of one row is its own tail). */ +__global__ static void v41_engram_rows_kernel(float *out, const uint8_t *raw, uint32_t n_rows, uint32_t hd) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= (uint64_t)n_rows * hd) return; + const uint32_t r = (uint32_t)(i / hd), d = (uint32_t)(i % hd), stride = hd + hd / 32u; + const uint8_t *row = raw + (uint64_t)r * stride; + out[i] = v41_bf16r(ds4_e4m3fn_to_f32(row[d]) * ds4_e8m0_to_f32(row[hd + d / 32u])); +} +extern "C" int ds4_gpu_v41_engram_rows_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *raw, uint32_t n_rows, uint32_t head_dim) { + if (!out || !raw || (head_dim % 32u)) return 0; + const uint64_t n = (uint64_t)n_rows * head_dim; + if (out->bytes < n * 4 || raw->bytes < (uint64_t)n_rows * (head_dim + head_dim / 32u)) return 0; + v41_engram_rows_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>((float *)out->ptr, (const uint8_t *)raw->ptr, n_rows, head_dim); + return cuda_ok(hipGetLastError(), "v41 engram rows"); +} diff --git a/cuda/ds41_graph.cuh b/cuda/ds41_graph.cuh new file mode 100644 index 0000000000000000000000000000000000000000..5378d81744bce3483a4f25712649ea39d6b323bf --- /dev/null +++ b/cuda/ds41_graph.cuh @@ -0,0 +1,217 @@ +/* cuda/ds41_graph.cuh — decode-step CUDA graph primitives, ported from the C + * engine at /data/YoungAi (commit 3946dbc), src/cuda/cuda_decode_graph.inc.cu. + * + * The engine captures a whole decode step (~1500 kernels) as one graph and + * replays it with a single hipGraphLaunch; the orchestration (when to + * capture, bucket bounds, device positions, accounting) lives host-side in + * ds4_ds41_graph.inc. This file is only the primitive layer: begin/end + * capture, launch, free, and the PDL edge rewrite. + * + * Port adaptations, named: + * - Stream. The engine compiles -default-stream per-thread and captures on + * hipStreamPerThread (every launch lands there). The port launches on + * ds4_current_stream(), which is the legacy default stream (0) outside a + * capture -- and capture on stream 0 is forbidden by CUDA. So the port + * captures on ONE dedicated stream created with default (blocking) flags + * and routes ds4_current_stream() onto it for the capture duration via the + * existing ds4_capture_set_stream. The blocking flag is load-bearing, not + * cosmetic: the legacy default stream implicitly synchronizes with every + * blocking stream in BOTH directions (CUDA Programming Guide, implicit + * synchronization), so eager work on stream 0 -- notably v41_spec_rollback + * -- is ordered against a graph launched on this stream without extra + * events. A nonblocking stream would NOT get that ordering and the next + * graph could read pre-rollback state. Replay uses the same stream. + * - The routing must be restored on EVERY exit (failed begin, failed end, + * failed instantiate); a leaked routing would send later eager launches to + * the graph stream (the port's tensor-copy wrapper picks hipMemcpyAsync on + * ds4_current_stream() while the routing is active). + * - Nested capture: the port's V4-era layer graphs share the same thread-local + * routing, so capture_begin refuses while any capture is active (the engine + * only ever had its own stream to check). + * - PDL edge rewrite is skipped below compute capability 9 (the engine only + * ever runs sm_121; on older devices the driver's error for a programmatic + * edge is not necessarily hipErrorNotSupported, which the engine's + * per-edge fallback does not catch, and PDL does not exist there anyway). + * On sm_121 the rewrite and its fail-closed handling are the engine's. + */ +#pragma once +#include +#include + +#include + +static void v41_pdl_register_small(void); /* defined at the aggregation root ds4_ds41_gpu.cuh (the engine's ds4_cuda.cu:110-124) */ + +/* One blocking stream, created on first capture and kept for the process + * (graphs replay on it; see the header comment for why it must be blocking). */ +static hipStream_t g_ds41_graph_stream = (hipStream_t)0; + +extern "C" int ds4_gpu_decode_graph_capture_begin(void) { + v41_pdl_register_small(); + if (!g_ds41_graph_stream && hipStreamCreate(&g_ds41_graph_stream) != hipSuccess) { + (void)hipGetLastError(); + fprintf(stderr, "ds4: [graph] capture stream create failed\n"); + return 0; + } + if (ds4_capture_active()) return 0; /* another capture (the V4-era layer graphs) owns the thread-local routing: never nest */ + hipStreamCaptureStatus cs = hipStreamCaptureStatusNone; + if (hipStreamIsCapturing(g_ds41_graph_stream, &cs) != hipSuccess || cs != hipStreamCaptureStatusNone) { + (void)hipGetLastError(); + return 0; + } + /* ThreadLocal (the engine's choice): an unsafe call from THIS thread voids + * the capture and capture_end reports NULL -- that is the safety net: the + * caller falls back to direct dispatch instead of replaying a graph that + * silently lost a step. */ + if (hipStreamBeginCapture(g_ds41_graph_stream, hipStreamCaptureModeThreadLocal) != hipSuccess) { + fprintf(stderr, "ds4: [graph] capture begin failed: %s\n", hipGetErrorString(hipGetLastError())); + return 0; + } + ds4_capture_set_stream(g_ds41_graph_stream); /* route every ds4_current_stream() launch into the capture */ + return 1; +} + +/* The engine's per-edge fallback discipline (cuda_decode_graph.inc.cu:33-74), + * fail-closed: a library kernel whose params cannot be read keeps its ordinary + * edge; hipErrorNotSupported removing or adding an edge restores/keeps the + * ordinary edge; any OTHER failure discards the whole graph -- a graph that + * instantiates without a required dependency is worse than a loud direct + * fallback. */ +static uint64_t g_ds41_pdl_skipped = 0; +static int g_ds41_pdl_arch_ok = -1; /* -1 untested / 0 skip the rewrite / 1 rewrite (device major >= 9) */ +static int ds41_graph_pdl_edges(hipGraph_t graph) { + if (g_v41_pdl_n == 0) return 0; + if (g_ds41_pdl_arch_ok < 0) { + int major = 0; + g_ds41_pdl_arch_ok = hipDeviceGetAttribute(&major, hipDeviceAttributeComputeCapabilityMajor, 0) == hipSuccess && major >= 9 ? 1 : 0; + if (!g_ds41_pdl_arch_ok) { + (void)hipGetLastError(); + fprintf(stderr, "ds4: [graph] PDL edge rewrite skipped (device below compute capability 9)\n"); + } + } + if (!g_ds41_pdl_arch_ok) return 0; + size_t ne = 0; + if (hipGraphGetEdges(graph, NULL, NULL, NULL, &ne) != hipSuccess) { (void)hipGetLastError(); return -1; } + if (ne == 0) return 0; + hipGraphNode_t *from = (hipGraphNode_t *)malloc(ne * sizeof *from), *to = (hipGraphNode_t *)malloc(ne * sizeof *to); + hipGraphEdgeData *ed = (hipGraphEdgeData *)malloc(ne * sizeof *ed); + int n = 0, bad = 0, skipped = 0; + if (!from || !to || !ed || hipGraphGetEdges(graph, from, to, ed, &ne) != hipSuccess) bad = 1; + for (size_t i = 0; !bad && i < ne; i++) { + if (ed[i].type != hipGraphDependencyTypeDefault || ed[i].from_port != hipGraphKernelNodePortDefault) continue; + hipGraphNodeType tf, tt; + if (hipGraphNodeGetType(from[i], &tf) != hipSuccess || hipGraphNodeGetType(to[i], &tt) != hipSuccess) { bad = 1; break; } + if (tf != hipGraphNodeTypeKernel || tt != hipGraphNodeTypeKernel) continue; + hipKernelNodeParams kp; memset(&kp, 0, sizeof kp); + /* A node whose params cannot be read is a library kernel (cuBLAS): it + * cannot be in the PDL registry, so its in-edges stay ordinary + * (engine 2026-10-01: cuBLAS Sgemm nodes reported invalid device + * function and dropped whole graphs until skipped here). */ + if (hipGraphKernelNodeGetParams(to[i], &kp) != hipSuccess) { (void)hipGetLastError(); continue; } + if (!v41_pdl_is_ready(kp.func)) continue; + hipGraphEdgeData pe; memset(&pe, 0, sizeof pe); + pe.from_port = hipGraphKernelNodePortLaunchCompletion; pe.type = hipGraphDependencyTypeProgrammatic; + hipError_t er = hipGraphRemoveDependencies(graph, &from[i], &to[i], &ed[i], 1); + if (er == hipErrorNotSupported) { (void)hipGetLastError(); skipped++; continue; } + if (er != hipSuccess) { bad = 1; break; } + const hipError_t ea = hipGraphAddDependencies(graph, &from[i], &to[i], &pe, 1); + if (ea == hipErrorNotSupported) { + (void)hipGetLastError(); + if (hipGraphAddDependencies(graph, &from[i], &to[i], &ed[i], 1) != hipSuccess) { bad = 1; break; } /* put the original edge back */ + skipped++; continue; + } + if (ea != hipSuccess) { bad = 1; break; } + n++; + } + free(from); free(to); free(ed); + if (bad) { fprintf(stderr, "ds4: [graph] PDL edge rewrite failed: %s\n", hipGetErrorString(hipGetLastError())); return -1; } + if (skipped) g_ds41_pdl_skipped += skipped; + return n; +} + +/* Node-level dump for the decode-capture diff (DS41_GRAPH_NODE_DUMP=): + * one line per captured node -- sequence, kind, kernel name, launch geometry -- + * so the port's verify capture can be diffed against the engine's node by node + * (docs/ds41-perf-2026-10-11.md, "The decode regime, resolved": 1788-1792 + * against 1460-1464). Debug only: nothing is written unless the variable is + * set, and captures are one-off per bucket. */ +static void ds41_graph_node_dump(hipGraph_t graph, const char *prefix) { + static unsigned seq = 0; + char path[512]; + snprintf(path, sizeof path, "%s.%u.txt", prefix, seq++); + FILE *f = fopen(path, "w"); + if (!f) { return; } + size_t nn = 0; + if (hipGraphGetNodes(graph, NULL, &nn) != hipSuccess || nn == 0) { (void)hipGetLastError(); fclose(f); return; } + hipGraphNode_t *nodes = (hipGraphNode_t *)malloc(nn * sizeof *nodes); + if (!nodes || hipGraphGetNodes(graph, nodes, &nn) != hipSuccess) { (void)hipGetLastError(); free(nodes); fclose(f); return; } + for (size_t i = 0; i < nn; i++) { + hipGraphNodeType t = hipGraphNodeTypeEmpty; + if (hipGraphNodeGetType(nodes[i], &t) != hipSuccess) { (void)hipGetLastError(); } + if (t == hipGraphNodeTypeKernel) { + hipKernelNodeParams kp; memset(&kp, 0, sizeof kp); + const char *nm = "?"; + if (hipGraphKernelNodeGetParams(nodes[i], &kp) == hipSuccess) { (void)cudaFuncGetName(&nm, kp.func); } + else { (void)hipGetLastError(); } + fprintf(f, "%zu\tkernel\t%s\tg%ux%ux%u b%ux%ux%u s%u\n", i, nm, + kp.gridDim.x, kp.gridDim.y, kp.gridDim.z, + kp.blockDim.x, kp.blockDim.y, kp.blockDim.z, kp.sharedMemBytes); + } else if (t == hipGraphNodeTypeMemcpy) { + hipMemcpy3DParms mp; memset(&mp, 0, sizeof mp); + unsigned long long bytes = 0; + if (hipGraphMemcpyNodeGetParams(nodes[i], &mp) == hipSuccess) { + bytes = (unsigned long long)mp.extent.width * (mp.extent.height ? mp.extent.height : 1u) * + (mp.extent.depth ? mp.extent.depth : 1u); + } else { (void)hipGetLastError(); } + fprintf(f, "%zu\tmemcpy\t%llu\n", i, bytes); + } else if (t == hipGraphNodeTypeMemset) { + fprintf(f, "%zu\tmemset\n", i); + } else if (t == hipGraphNodeTypeHost) { + fprintf(f, "%zu\thost\n", i); + } else if (t == hipGraphNodeTypeGraph) { + fprintf(f, "%zu\tsubgraph\n", i); + } else { + fprintf(f, "%zu\tother%d\n", i, (int)t); + } + } + free(nodes); + fclose(f); +} + +extern "C" void *ds4_gpu_decode_graph_capture_end(void) { + ds4_capture_set_stream((hipStream_t)0); /* restore on every exit: a leaked routing misdirects later eager launches */ + hipGraph_t graph = NULL; + const hipError_t e = hipStreamEndCapture(g_ds41_graph_stream, &graph); + if (e != hipSuccess || !graph) { + fprintf(stderr, "ds4: [graph] capture voided: %s\n", hipGetErrorString(e)); + (void)hipGetLastError(); + if (graph) (void)hipGraphDestroy(graph); + return NULL; + } + size_t n_nodes = 0; + (void)hipGraphGetNodes(graph, NULL, &n_nodes); + const char *dump = getenv("DS41_GRAPH_NODE_DUMP"); + if (dump && *dump) { ds41_graph_node_dump(graph, dump); } + const int n_pdl = ds41_graph_pdl_edges(graph); + if (n_pdl < 0) { (void)hipGraphDestroy(graph); return NULL; } + hipGraphExec_t exec = NULL; + const hipError_t ei = hipGraphInstantiate(&exec, graph, 0); + (void)hipGraphDestroy(graph); + if (ei != hipSuccess || !exec) { + fprintf(stderr, "ds4: [graph] instantiate failed (%zu nodes): %s\n", n_nodes, hipGetErrorString(ei)); + (void)hipGetLastError(); + return NULL; + } + fprintf(stderr, "ds4: [graph] decode step captured: %zu nodes (%d edges made programmatic, %llu unsupported kept ordinary)\n", + n_nodes, n_pdl, (unsigned long long)g_ds41_pdl_skipped); + return (void *)exec; +} + +extern "C" int ds4_gpu_decode_graph_launch(void *exec) { + if (!exec) return 0; + return cuda_ok(hipGraphLaunch((hipGraphExec_t)exec, g_ds41_graph_stream), "decode graph launch"); +} + +extern "C" void ds4_gpu_decode_graph_free(void *exec) { + if (exec) (void)hipGraphExecDestroy((hipGraphExec_t)exec); +} diff --git a/cuda/ds41_hc.cuh b/cuda/ds41_hc.cuh new file mode 100644 index 0000000000000000000000000000000000000000..3e96ca81ae13c6ae714ca39629333d8f1e4788f9 --- /dev/null +++ b/cuda/ds41_hc.cuh @@ -0,0 +1,392 @@ +#include "hip/hip_runtime.h" +/* ds41_hc.cuh — DeepSeek V4.1 (ds41) hyper-connection (mHC) family, ported from + * the C engine at /data/YoungAi (commit 3946dbc), file + * src/cuda/cuda_v41_hc.inc.cu (mix -> sinkhorn split -> hc_pre -> hc_post). + * + * The official model.py hc_mixes / hc_split_sinkhorn / hc_pre / hc_post are + * the ground truth; f32 compute with explicit bf16 rounding at the official + * bf16 module boundaries. The fused variants (mix's three launches folded + * into the GEMV, split+pre+norm folded into one) are the engine's measured + * mega-mHC first cut — their accumulation shapes are copied verbatim from the + * unfused kernels, which is why the engine gates them by NLL/top-1 where a + * thread count changed and by byte-compare where it did not. + * + * Port notes: launches on ds4_current_stream() (the engine's g_cur_stream); + * the fallback of the mix entry needs the dense f32 GEMV (ds41_dense.cuh), + * which is included before this file. */ +#pragma once + +/* ---- hc_mix: rsqrt(mean(flat^2)+eps) per row, multiplied onto the GEMM + * result (cuda_v41_hc.inc.cu:12-22) ---- */ +__global__ static void v41_row_rsqrt_kernel(float *inv, const float *x, uint32_t dim, float eps) { + const uint32_t r = blockIdx.x; const float *xr = x + (uint64_t)r * dim; + float s = 0.f; + for (uint32_t i = threadIdx.x; i < dim; i += blockDim.x) s += xr[i] * xr[i]; + __shared__ float sh[256]; sh[threadIdx.x] = s; __syncthreads(); + for (uint32_t k = blockDim.x / 2; k > 0; k >>= 1) { if (threadIdx.x < k) sh[threadIdx.x] += sh[threadIdx.x + k]; __syncthreads(); } + if (threadIdx.x == 0) inv[r] = rsqrtf(sh[0] / (float)dim + eps); +} +__global__ static void v41_scale_rows_kernel(float *y, const float *inv, uint32_t cols, uint64_t n) { + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) y[i] *= inv[i / cols]; +} + +/* ---- the fused mix (cuda_v41_hc.inc.cu:38-148) ---- + * Three launches folded into one: the row rsqrt (its grid was n_tok = one + * block at decode, 1.1 ms/step of pure latency), the f32 GEMV (24 rows) and + * the scale_rows pass. V41_HC_MIX_SPLIT blocks per row share the 32 K + * segments; the last-arriving block sums part[] in segment order (atomic + * counter + threadfence) and folds the scale multiply in. The rms preamble + * and the segment dot keep the unfused kernels' accumulation shapes exactly, + * so the output is bit-identical to the three-launch form. */ +#define V41_HC_MIX_SPLIT 2u +__global__ static void v41_hc_mix_fused_kernel(float *mix, float *part, uint32_t *ctr, const float *w, const float *hc, + uint32_t dim, uint32_t out_dim, float eps, uint32_t ksplit, uint32_t nsplit, uint32_t n_tok) { + const uint32_t warp = threadIdx.x >> 5, lane = threadIdx.x & 31u; + const uint32_t r = blockIdx.x / nsplit, sp = blockIdx.x % nsplit, kper = ksplit / nsplit; + __shared__ float sh[1024]; + __shared__ uint32_t s_last; + /* Weights first, rms preamble second (2026-09-23): W has no dependency on + * the preamble, and issuing it up front hides DRAM latency behind the + * tree reduction. Bit-identical: every item is the same float4 dot in + * ascending c, just fetched earlier. */ + constexpr uint32_t PF = 5u; + const uint32_t kpart = sp * kper + warp; + const bool act = r < out_dim && warp < kper; + const float *wr = w + (uint64_t)r * dim; + const uint32_t c0 = kpart * 128u + lane * 4u, cst = ksplit * 128u; + float4 wpre[PF]; + #pragma unroll + for (uint32_t i = 0; i < PF; i++) { + wpre[i] = make_float4(0.f, 0.f, 0.f, 0.f); + if (act && c0 + i * cst < dim) wpre[i] = *(const float4 *)(wr + c0 + i * cst); + } + v41_pdl_wait(); /* PDL: W (constants) already in flight; wait for the upstream hc here */ + for (uint32_t t = 0; t < n_tok; t++) { /* tokens in ascending order; the body below is the single-token form */ + const float *x = hc + (uint64_t)t * dim; + /* (1) the rms preamble, shape verbatim from v41_row_rsqrt_kernel; reads + * batched 10 at a time (same accumulation order, bit-identical). */ + { + float s = 0.f; + for (uint32_t i0 = threadIdx.x; i0 < dim; i0 += 10u * blockDim.x) { + float xv[10]; + #pragma unroll + for (uint32_t j = 0; j < 10u; j++) { const uint32_t i = i0 + j * blockDim.x; xv[j] = i < dim ? x[i] : 0.f; } + #pragma unroll + for (uint32_t j = 0; j < 10u; j++) if (i0 + j * blockDim.x < dim) s += xv[j] * xv[j]; + } + sh[threadIdx.x] = s; __syncthreads(); + /* The tree tail moves into warp 0 (2026-10-07): the last six levels + * keep adding sh[i] + sh[i+k] in the same left-right order, without + * the block-wide barrier per level (11 -> 6 barriers/token). */ + for (uint32_t k = blockDim.x / 2; k > 32u; k >>= 1) { if (threadIdx.x < k) sh[threadIdx.x] += sh[threadIdx.x + k]; __syncthreads(); } + if (threadIdx.x < 32u) { + float v = sh[threadIdx.x] + sh[threadIdx.x + 32u]; /* k=32 */ + #pragma unroll + for (int k = 16; k > 0; k >>= 1) v += __shfl_down_sync(0xffffffffu, v, k); /* k=16..1 */ + if (threadIdx.x == 0) sh[0] = v; + } + __syncthreads(); + } + const float inv = rsqrtf(sh[0] / (float)dim + eps); + /* (2) the GEMV ksplit branch, this block's kpart range only */ + float acc = 0.f; + if (act) { + #pragma unroll + for (uint32_t i = 0; i < PF; i++) { + const uint32_t c = c0 + i * cst; + if (c < dim) { + const float4 wv = wpre[i]; + const float4 xv = *(const float4 *)(x + c); + acc += wv.x * xv.x + wv.y * xv.y + wv.z * xv.z + wv.w * xv.w; + } + } + for (uint32_t c = c0 + PF * cst; c < dim; c += cst) { + const float4 wv = *(const float4 *)(wr + c); + const float4 xv = *(const float4 *)(x + c); + acc += wv.x * xv.x + wv.y * xv.y + wv.z * xv.z + wv.w * xv.w; + } + for (int o = 16; o > 0; o >>= 1) acc += __shfl_xor_sync(0xffffffffu, acc, o); + if (lane == 0) part[((uint64_t)t * out_dim + r) * ksplit + kpart] = acc; + } + __threadfence(); + __syncthreads(); + if (threadIdx.x == 0) s_last = (atomicAdd(&ctr[(uint64_t)t * out_dim + r], 1u) == nsplit - 1u) ? 1u : 0u; + __syncthreads(); + if (s_last && threadIdx.x == 0 && r < out_dim) { /* last block: sum 32 segments in order, fold the scale, reset */ + __threadfence(); + const float *pr = part + ((uint64_t)t * out_dim + r) * ksplit; + float v = 0.f; + for (uint32_t k = 0; k < ksplit; k++) v += __ldcg(pr + k); /* bypass L1 for other blocks' segments */ + mix[(uint64_t)t * out_dim + r] = v * inv; /* (3) the retired scale_rows pass is this multiply */ + ctr[(uint64_t)t * out_dim + r] = 0u; + } + } +} +static v41_scratch g_v41_hc_part, g_v41_hc_ctr; + +extern "C" int ds4_gpu_v41_hc_mix_tensor(ds4_gpu_tensor *mix, const ds4_gpu_tensor *hc, const void *model_map, uint64_t model_size, + uint64_t fn_offset, uint32_t n_embd, uint32_t n_hc, uint32_t n_tok, float eps) { + const uint32_t dim = n_embd * n_hc, mix_hc = 2u * n_hc + n_hc * n_hc; + if (!mix || !hc || hc->bytes < (uint64_t)n_tok * dim * 4 || mix->bytes < (uint64_t)n_tok * mix_hc * 4) return 0; + /* The fused kernel only takes the shape whose GEMV would pick ksplit=8, + * one row per block — the live 24 x 20480. Any other shape (other hc + * widths, prefill batches) walks the original three-launch path so an + * accumulation order cannot silently change. */ + const uint64_t wbytes = (uint64_t)dim * mix_hc * 4; + const uint32_t nseg = dim / 128u; + if (n_tok <= V41_GEMV_MAX_TOK && (dim % 128u) == 0u && mix_hc * 8u < 32768u && nseg >= 32u && + fn_offset <= model_size && wbytes <= model_size - fn_offset) { + const float *W = (const float *)cuda_model_range_ptr(model_map, fn_offset, wbytes, "v41 hc fn"); + if (W) { + /* 1024 threads = 32 warps (the ncu account in the kernel header): + * ksplit must equal the warp count, and the K segment unit is 128 + * elements, so dim >= 32*128 = 4096 (20480 live). */ + const uint64_t npart = (uint64_t)n_tok * mix_hc; + const int fresh = g_v41_hc_ctr.cap < npart * 4; + float *part = (float *)v41_grow(&g_v41_hc_part, npart * 32u * 4, "v41 hc mix part"); + uint32_t *ctr = (uint32_t *)v41_grow(&g_v41_hc_ctr, npart * 4, "v41 hc mix ctr"); + if (!part || !ctr) return 0; + if (fresh && !cuda_ok(hipMemsetAsync(ctr, 0, (size_t)npart * 4, ds4_current_stream()), "v41 hc mix ctr zero")) return 0; + v41_pdl_register((const void *)v41_hc_mix_fused_kernel); /* waits before touching hc/part/ctr */ + v41_hc_mix_fused_kernel<<>>( + (float *)mix->ptr, part, ctr, W, (const float *)hc->ptr, dim, mix_hc, eps, 32u, V41_HC_MIX_SPLIT, n_tok); + return cuda_ok(hipGetLastError(), "v41 hc mix fused"); + } + } + static v41_scratch g_v41_hc_inv; + float *inv = (float *)v41_grow(&g_v41_hc_inv, (uint64_t)n_tok * 4, "v41 hc inv"); + if (!inv) return 0; + v41_row_rsqrt_kernel<<>>(inv, (const float *)hc->ptr, dim, eps); + if (!cuda_ok(hipGetLastError(), "v41 hc rsqrt")) return 0; + if (!ds4_gpu_v41_matmul_f32_tensor(mix, model_map, model_size, fn_offset, dim, mix_hc, hc, n_tok)) return 0; + const uint64_t n = (uint64_t)n_tok * mix_hc; + v41_scale_rows_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>((float *)mix->ptr, inv, mix_hc, n); + return cuda_ok(hipGetLastError(), "v41 hc mix scale"); +} + +/* ---- sinkhorn split (cuda_v41_hc.inc.cu:188-230): one block per row ---- */ +__global__ static void v41_hc_split_kernel(float *pre, float *post, float *comb, const float *mix, const float *scale, + const float *base, uint32_t hc, uint32_t iters, float eps) { + const uint32_t n = blockIdx.x, mix_hc = 2u * hc + hc * hc; + const float *m = mix + (uint64_t)n * mix_hc; + __shared__ float c[64]; + if (threadIdx.x < hc) { + pre[n * hc + threadIdx.x] = 1.f / (1.f + expf(-(m[threadIdx.x] * scale[0] + base[threadIdx.x]))) + eps; + post[n * hc + threadIdx.x] = 2.f / (1.f + expf(-(m[hc + threadIdx.x] * scale[1] + base[hc + threadIdx.x]))); + } + if (threadIdx.x < hc * hc) c[threadIdx.x] = m[2u * hc + threadIdx.x] * scale[2] + base[2u * hc + threadIdx.x]; + __syncthreads(); + if (threadIdx.x == 0) { /* the 4x4 sinkhorn runs serially in the reference's order */ + for (uint32_t j = 0; j < hc; j++) { /* softmax(-1) + eps */ + float mx = -INFINITY; for (uint32_t k = 0; k < hc; k++) mx = fmaxf(mx, c[j * hc + k]); + float s = 0.f; for (uint32_t k = 0; k < hc; k++) { c[j * hc + k] = expf(c[j * hc + k] - mx); s += c[j * hc + k]; } + for (uint32_t k = 0; k < hc; k++) c[j * hc + k] = c[j * hc + k] / s + eps; + } + for (uint32_t k = 0; k < hc; k++) { /* / (sum(-2) + eps) */ + float s = 0.f; for (uint32_t j = 0; j < hc; j++) s += c[j * hc + k]; + for (uint32_t j = 0; j < hc; j++) c[j * hc + k] /= (s + eps); + } + for (uint32_t it = 1; it < iters; it++) { + for (uint32_t j = 0; j < hc; j++) { float s = 0.f; for (uint32_t k = 0; k < hc; k++) s += c[j * hc + k]; for (uint32_t k = 0; k < hc; k++) c[j * hc + k] /= (s + eps); } + for (uint32_t k = 0; k < hc; k++) { float s = 0.f; for (uint32_t j = 0; j < hc; j++) s += c[j * hc + k]; for (uint32_t j = 0; j < hc; j++) c[j * hc + k] /= (s + eps); } + } + } + __syncthreads(); + if (threadIdx.x < hc * hc) comb[(uint64_t)n * hc * hc + threadIdx.x] = c[threadIdx.x]; +} +extern "C" int ds4_gpu_v41_hc_split_tensor(ds4_gpu_tensor *pre, ds4_gpu_tensor *post, ds4_gpu_tensor *comb, + const ds4_gpu_tensor *mix, const void *model_map, uint64_t model_size, + uint64_t scale_offset, uint64_t base_offset, uint32_t n_hc, uint32_t iters, + float eps, uint32_t n_tok) { + if (!pre || !post || !comb || !mix || n_hc > 8u) return 0; + const uint32_t mix_hc = 2u * n_hc + n_hc * n_hc; + const float *sc = (const float *)cuda_model_range_ptr(model_map, scale_offset, 12, "v41 hc scale"); + const float *bs = (const float *)cuda_model_range_ptr(model_map, base_offset, (uint64_t)mix_hc * 4, "v41 hc base"); + if (!sc || !bs) return 0; + v41_hc_split_kernel<<>>((float *)pre->ptr, (float *)post->ptr, (float *)comb->ptr, + (const float *)mix->ptr, sc, bs, n_hc, iters, eps); + return cuda_ok(hipGetLastError(), "v41 hc split"); +} + +/* ---- hc_pre (cuda_v41_hc.inc.cu:233-245) ---- */ +__global__ static void v41_hc_pre_kernel(float *out, const float *hc, const float *pre, uint32_t n_embd, uint32_t n_hc) { + const uint32_t n = blockIdx.y; + const uint32_t d = blockIdx.x * blockDim.x + threadIdx.x; + if (d >= n_embd) return; + float s = 0.f; + for (uint32_t c = 0; c < n_hc; c++) s += pre[n * n_hc + c] * hc[((uint64_t)n * n_hc + c) * n_embd + d]; + out[(uint64_t)n * n_embd + d] = v41_bf16r(s); +} +extern "C" int ds4_gpu_v41_hc_pre_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *hc, const ds4_gpu_tensor *pre, + uint32_t n_embd, uint32_t n_hc, uint32_t n_tok) { + if (!out || !hc || !pre) return 0; + v41_hc_pre_kernel<<>>((float *)out->ptr, (const float *)hc->ptr, (const float *)pre->ptr, n_embd, n_hc); + return cuda_ok(hipGetLastError(), "v41 hc pre"); +} + +/* ---- the fused split + hc_pre + rms_norm (cuda_v41_hc.inc.cu:246-404) ---- + * ★pre_in and pre are two different things★: the split inside produces THIS + * layer's pre (for the lower half), while hc_pre must use the pre_in handed + * down from the previous layer (the official Block.forward pre_mix). Using + * the same slot here does not error — every layer's residual mixing + * coefficients shift one layer and the output still looks plausible. */ +__global__ static void v41_hc_fused_kernel(float *pre, float *post, float *comb, float *x, float *xn, + const float *mix, const float *hc, const float *pre_in, + const float *scale, const float *base, + const float *nw, uint32_t n_embd, uint32_t n_hc, uint32_t iters, + float hc_eps, float norm_eps) { + v41_pdl_wait(); + const uint32_t n = blockIdx.x, mix_hc = 2u * n_hc + n_hc * n_hc; + const float *m = mix + (uint64_t)n * mix_hc; + __shared__ float c[64], sh[1024]; /* sh slots = blockDim */ + /* The whole sinkhorn lives in warp 0 (2026-09-18): a 4x4 matrix does not + * need 1024 threads through 80 block-wide barriers; the per-element + * accumulation order is unchanged (row/column sums run in ascending + * j/k inside one thread) -> bit-identical. n_hc <= 5 keeps n_hc^2 <= 32. */ + const uint32_t warp = threadIdx.x >> 5; + if (warp == 0) { + if (threadIdx.x < n_hc) { + pre[n * n_hc + threadIdx.x] = 1.f / (1.f + expf(-(m[threadIdx.x] * scale[0] + base[threadIdx.x]))) + hc_eps; + post[n * n_hc + threadIdx.x] = 2.f / (1.f + expf(-(m[n_hc + threadIdx.x] * scale[1] + base[n_hc + threadIdx.x]))); + } + if (threadIdx.x < n_hc * n_hc) c[threadIdx.x] = m[2u * n_hc + threadIdx.x] * scale[2] + base[2u * n_hc + threadIdx.x]; + __syncwarp(); + const uint32_t tj = threadIdx.x / n_hc, tk = threadIdx.x % n_hc; + const bool act = threadIdx.x < n_hc * n_hc; + float nv = 0.f; + if (act) { /* (1) row softmax + eps */ + float mx = -INFINITY; + for (uint32_t k = 0; k < n_hc; k++) mx = fmaxf(mx, c[tj * n_hc + k]); + float sum = 0.f; + for (uint32_t k = 0; k < n_hc; k++) sum += expf(c[tj * n_hc + k] - mx); + nv = expf(c[tj * n_hc + tk] - mx) / sum + hc_eps; + } + __syncwarp(); + if (act) c[tj * n_hc + tk] = nv; + __syncwarp(); + if (act) { /* (2) column normalization */ + float cs = 0.f; + for (uint32_t j = 0; j < n_hc; j++) cs += c[j * n_hc + tk]; + nv = c[tj * n_hc + tk] / (cs + hc_eps); + } + __syncwarp(); + if (act) c[tj * n_hc + tk] = nv; + __syncwarp(); + for (uint32_t it = 1; it < iters; it++) { + if (act) { /* row normalization */ + float rs = 0.f; + for (uint32_t k = 0; k < n_hc; k++) rs += c[tj * n_hc + k]; + nv = c[tj * n_hc + tk] / (rs + hc_eps); + } + __syncwarp(); + if (act) c[tj * n_hc + tk] = nv; + __syncwarp(); + if (act) { /* column normalization */ + float cs = 0.f; + for (uint32_t j = 0; j < n_hc; j++) cs += c[j * n_hc + tk]; + nv = c[tj * n_hc + tk] / (cs + hc_eps); + } + __syncwarp(); + if (act) c[tj * n_hc + tk] = nv; + __syncwarp(); + } + if (act) comb[(uint64_t)n * n_hc * n_hc + threadIdx.x] = c[threadIdx.x]; + } + /* hc_pre does not wait on the sinkhorn (2026-09-23): it consumes pre_in + * (the previous layer's), independent of this layer's warp-0 work. */ + float acc = 0.f; + const float *hcn = hc + (uint64_t)n * n_hc * n_embd; + const float *pin = pre_in + (uint64_t)n * n_hc; + float *xr = x + (uint64_t)n * n_embd; + /* The n_hc == 4, n_embd <= 5*blockDim specialization reads this thread's 5 + * d's x 4 paths in one batch (same values and order as the loop below). */ + constexpr uint32_t DP = 5u; + float *on = xn + (uint64_t)n * n_embd; + if (n_hc == 4u && n_embd <= DP * blockDim.x) { + float hv[DP][4], vv[DP], wv[DP]; + #pragma unroll + for (uint32_t j = 0; j < DP; j++) { + const uint32_t d = threadIdx.x + j * blockDim.x; + #pragma unroll + for (uint32_t k = 0; k < 4u; k++) hv[j][k] = d < n_embd ? hcn[(uint64_t)k * n_embd + d] : 0.f; + wv[j] = d < n_embd ? nw[d] : 0.f; + } + const float p0 = pin[0], p1 = pin[1], p2 = pin[2], p3 = pin[3]; + #pragma unroll + for (uint32_t j = 0; j < DP; j++) { + const uint32_t d = threadIdx.x + j * blockDim.x; + if (d < n_embd) { + float v = 0.f; + v += p0 * hv[j][0]; v += p1 * hv[j][1]; v += p2 * hv[j][2]; v += p3 * hv[j][3]; + v = v41_bf16r(v); + xr[d] = v; vv[j] = v; + acc += v * v; + } + } + sh[threadIdx.x] = acc; + __syncthreads(); + for (uint32_t k = blockDim.x / 2; k > 0; k >>= 1) { if (threadIdx.x < k) sh[threadIdx.x] += sh[threadIdx.x + k]; __syncthreads(); } + const float inv = rsqrtf(sh[0] / (float)n_embd + norm_eps); + #pragma unroll + for (uint32_t j = 0; j < DP; j++) { const uint32_t d = threadIdx.x + j * blockDim.x; if (d < n_embd) on[d] = v41_bf16r(wv[j] * (vv[j] * inv)); } + return; + } + for (uint32_t d = threadIdx.x; d < n_embd; d += blockDim.x) { + float v = 0.f; + for (uint32_t k = 0; k < n_hc; k++) v += pin[k] * hcn[(uint64_t)k * n_embd + d]; + v = v41_bf16r(v); + xr[d] = v; + acc += v * v; + } + sh[threadIdx.x] = acc; + __syncthreads(); + for (uint32_t k = blockDim.x / 2; k > 0; k >>= 1) { if (threadIdx.x < k) sh[threadIdx.x] += sh[threadIdx.x + k]; __syncthreads(); } + const float inv = rsqrtf(sh[0] / (float)n_embd + norm_eps); + for (uint32_t d = threadIdx.x; d < n_embd; d += blockDim.x) on[d] = v41_bf16r(nw[d] * (xr[d] * inv)); +} +extern "C" int ds4_gpu_v41_hc_fused_tensor(ds4_gpu_tensor *pre, ds4_gpu_tensor *post, ds4_gpu_tensor *comb, + ds4_gpu_tensor *x, ds4_gpu_tensor *xn, const ds4_gpu_tensor *mix, + const ds4_gpu_tensor *hc, const ds4_gpu_tensor *pre_in, + const void *model_map, uint64_t model_size, + uint64_t scale_offset, uint64_t base_offset, uint64_t norm_offset, + uint32_t n_embd, uint32_t n_hc, uint32_t iters, float hc_eps, float norm_eps, + uint32_t n_tok) { + if (!pre || !post || !comb || !x || !xn || !mix || !hc || !pre_in || n_hc > 5u) return 0; /* n_hc^2 in one warp */ + const uint32_t mix_hc = 2u * n_hc + n_hc * n_hc; + const float *sc = (const float *)cuda_model_range_ptr(model_map, scale_offset, 12, "v41 hc scale"); + const float *bs = (const float *)cuda_model_range_ptr(model_map, base_offset, (uint64_t)mix_hc * 4, "v41 hc base"); + const float *nw = (const float *)cuda_model_range_ptr(model_map, norm_offset, (uint64_t)n_embd * 4, "v41 norm w"); + if (!sc || !bs || !nw) return 0; + /* 1024 threads (2026-09-17, ncu): the grid is n_tok — one block at decode + * — and the rms tree changes slots with the thread count, so this cut's + * gate is the quality scale, not cmp. */ + v41_hc_fused_kernel<<>>((float *)pre->ptr, (float *)post->ptr, (float *)comb->ptr, + (float *)x->ptr, (float *)xn->ptr, (const float *)mix->ptr, (const float *)hc->ptr, + (const float *)pre_in->ptr, sc, bs, nw, + n_embd, n_hc, iters, hc_eps, norm_eps); + return cuda_ok(hipGetLastError(), "v41 hc fused"); +} + +/* ---- hc_post (cuda_v41_hc.inc.cu:409-430): out[k][d] = post[k]*y[d] + sum_j comb[j][k]*res[j][d] -> bf16 ---- */ +__global__ static void v41_hc_post_kernel(float *out, const float *y, const float *res, const float *post, const float *comb, + uint32_t n_embd, uint32_t n_hc) { + v41_pdl_wait(); + const uint32_t n = blockIdx.y; + const uint32_t d = blockIdx.x * blockDim.x + threadIdx.x; + if (d >= n_embd) return; + const float yv = y[(uint64_t)n * n_embd + d]; + for (uint32_t k = 0; k < n_hc; k++) { + float s = post[n * n_hc + k] * yv; + for (uint32_t j = 0; j < n_hc; j++) s += comb[(uint64_t)n * n_hc * n_hc + j * n_hc + k] * res[((uint64_t)n * n_hc + j) * n_embd + d]; + out[((uint64_t)n * n_hc + k) * n_embd + d] = v41_bf16r(s); + } +} +extern "C" int ds4_gpu_v41_hc_post_tensor(ds4_gpu_tensor *out_hc, const ds4_gpu_tensor *y, const ds4_gpu_tensor *res, + const ds4_gpu_tensor *post, const ds4_gpu_tensor *comb, + uint32_t n_embd, uint32_t n_hc, uint32_t n_tok) { + if (!out_hc || !y || !res || !post || !comb || out_hc->ptr == res->ptr) return 0; /* no in-place: every k reads all j */ + v41_hc_post_kernel<<>>((float *)out_hc->ptr, (const float *)y->ptr, (const float *)res->ptr, + (const float *)post->ptr, (const float *)comb->ptr, n_embd, n_hc); + return cuda_ok(hipGetLastError(), "v41 hc post"); +} diff --git a/cuda/ds41_indexer.cuh b/cuda/ds41_indexer.cuh new file mode 100644 index 0000000000000000000000000000000000000000..ad46247a6ce13e8da181d81b195279c811c4b158 --- /dev/null +++ b/cuda/ds41_indexer.cuh @@ -0,0 +1,446 @@ +#include "hip/hip_runtime.h" +/* ds41_indexer.cuh — DeepSeek V4.1 (ds41) indexer three-piece family, ported + * from the C engine at /data/YoungAi (commit 3946dbc), file + * `src/cuda/cuda_v41_indexer.inc.cu`: + * - the score kernel and its entry (:25-122, official Indexer.forward: + * score[i][g] = bf16(sum_h bf16(relu(bf16(q_h·k_g))·w[i][h])) with + * invisible groups at -inf; V41_IDX_G groups per warp); + * - the candidate-block selector (:132-238, official + * select_candidate_blocks: block score = in-block max, the block holding + * the query's newest position pinned +inf, topk_blocks blocks kept; + * radix select when kk < nb) and its [nb] scratch (:145-149); + * - the topk selector (:260-406, official topk over min(index_topk, visible) + * then position-ascending, unreachable slots -1; 1024-thread radix select + * + segmented write-out). + * + * The C2 compact candidate list is the contract (ds4_gpu_v41.h:126-152): the + * candidate kernel writes list[i][0] = block count nc followed by nc + * ascending block numbers; when a list is present the score kernel writes a + * compact row [ns] with ns = v41_cand_ns(ng, bs, cap) and the topk kernel maps + * compact index c back to group cl[1 + c/bs]·bs + c%bs. The list is + * ascending, so compact order == position order and the "ties take the smaller + * index" rule holds unchanged. + * + * ★Named port deviations (P4-4)★: + * - the s8 mma score variant (cuda_v41_indexer_mma.inc.cu) is NOT ported; + * ds4_gpu_v41_set_indexer_mma records the flag and the score entry refuses + * by name while it is on (the engine keeps the switch for A/B judgment + * only, default off); + * - the engine's candidate scratch is per decode lane (DS4_GPU_MAX_LANES, + * concurrent captured decode graphs); the port has no lane concept yet — + * one static slot, named here because the day lanes land this must split. + * + * posd (the captured-graph device-position convention, ds4_gpu_v41.h:116-124) + * is kept in the kernels and entries with the engine's refusals (n_tok > 8); + * the port's eager forward passes NULL. + */ +#pragma once + +/* Groups per warp in the score kernel. The engine's archived negative result + * (2026-10-07) kept 4: 8 groups took registers 64 -> 96 (half the blocks per + * SM), A/B 46.68 -> 46.48 t/s within noise, bit-identical output. */ +#define V41_IDX_G 4u + +/* Compact-row width (cuda_v41_indexer.inc.cu:25-28): ns = min(cap·bs, ng). + * The score kernel and the topk kernel each compute it with this one copy or + * their row strides drift apart. */ +__host__ __device__ __forceinline__ static uint32_t v41_cand_ns(uint32_t ng, uint32_t cand_bs, uint32_t cand_cap) { + const uint64_t full = (uint64_t)cand_cap * cand_bs; + return full < ng ? (uint32_t)full : ng; +} + +/* ---- indexer score (cuda_v41_indexer.inc.cu:29-92) ---- + * One warp advances V41_IDX_G groups at once (2026-09-23): the per-(group, + * head) chain "4 MACs -> 5-level shfl reduce -> round" is a serial dependency; + * G chains interleave and each head's q is read once for G groups. + * Bit-identical to one-group-per-warp: the MAC order (e ascending), the xor + * reduction tree, the bf16 rounding point and the head accumulation order are + * all unchanged. grid.y covers the group dimension so a decode step is not + * one block; the kernel wraps by grid stride when grid.y runs out (65535 cap). + * The key is unpacked to registers once and reused across all 64 heads. */ +__global__ static void v41_indexer_score_kernel(float *score, const float *q, const uint8_t *k, const float *w, const int32_t *cand, + uint32_t pos0, uint32_t ng, uint32_t n_head, uint32_t dk, uint32_t ratio, + const int32_t *posd, uint32_t cand_bs, uint32_t cand_cap) { + v41_pdl_wait(); /* PDL: first instruction waits for the upstream; no-op when not launched through PDL */ + const uint32_t i = blockIdx.x, lane = threadIdx.x & 31u, warp = threadIdx.x >> 5, nwarp = blockDim.x >> 5; + if (posd) { pos0 = (uint32_t)posd[0]; ng = (pos0 + gridDim.x) / ratio; } /* graph: device position; host ng is the bucket cap */ + const uint32_t vis = (pos0 + i + 1u) / ratio; /* visible groups at this absolute position */ + const uint32_t per = dk / 32u; /* dk=128 => 4 dims per lane */ + /* C2: with a list only the compact row [ns] is scored, item c maps to group g */ + const int32_t *cl = cand ? cand + (uint64_t)i * (1u + cand_cap) : NULL; + const uint32_t nc = cl ? (uint32_t)cl[0] : 0u; + const uint32_t ns = cl ? v41_cand_ns(ng, cand_bs, cand_cap) : ng; + const uint32_t gstride = nwarp * gridDim.y * V41_IDX_G; + for (uint32_t gb = (blockIdx.y * nwarp + warp) * V41_IDX_G; gb < ns; gb += gstride) { + float kv[V41_IDX_G][4], acc[V41_IDX_G]; + bool live[V41_IDX_G]; + #pragma unroll + for (uint32_t j = 0; j < V41_IDX_G; j++) { + const uint32_t c = gb + j; + uint32_t g = c; + if (cl) { const uint32_t b = c / cand_bs; g = b < nc ? (uint32_t)cl[1u + b] * cand_bs + c % cand_bs : ng; } /* off-list item = dead group */ + live[j] = c < ns && g < ng && g < vis; + acc[j] = 0.f; + /* Unpack the packed MXFP4 index key into registers once, reuse across all heads */ + const uint8_t *kg = k + (uint64_t)(live[j] ? g : 0u) * DS4_V41_IDXK_BYTES; + #pragma unroll + for (uint32_t e = 0; e < 4u; e++) kv[j][e] = (live[j] && e < per) ? v41_idxk_get(kg, lane * per + e) : 0.f; + } + for (uint32_t h = 0; h < n_head; h++) { + const float *qh = q + ((uint64_t)i * n_head + h) * dk; + float qv[4]; + #pragma unroll + for (uint32_t e = 0; e < 4u; e++) qv[e] = e < per ? qh[lane * per + e] : 0.f; + const float wh = w[(uint64_t)i * n_head + h]; + float d[V41_IDX_G]; + #pragma unroll + for (uint32_t j = 0; j < V41_IDX_G; j++) { + d[j] = 0.f; + #pragma unroll + for (uint32_t e = 0; e < 4u; e++) if (e < per) d[j] += qv[e] * kv[j][e]; + } + #pragma unroll + for (int o = 16; o > 0; o >>= 1) { + #pragma unroll + for (uint32_t j = 0; j < V41_IDX_G; j++) d[j] += __shfl_xor_sync(0xffffffffu, d[j], o); + } + #pragma unroll + for (uint32_t j = 0; j < V41_IDX_G; j++) { + float dd = v41_bf16r(d[j]); /* einsum lands in bf16 */ + dd = fmaxf(dd, 0.0f); /* relu_ */ + acc[j] += v41_bf16r(dd * wh); /* x weights (bf16), then sum */ + } + } + if (lane == 0) { + #pragma unroll + for (uint32_t j = 0; j < V41_IDX_G; j++) if (gb + j < ns) score[(uint64_t)i * ns + gb + j] = live[j] ? v41_bf16r(acc[j]) : -INFINITY; + } + } +} + +/* The mma score switch (cuda_v41_indexer.inc.cu:95-99). Kept so the flag has + * one owner; the port has no mma variant, and the score entry below refuses by + * name while the flag is on. */ +static int g_v41_idx_mma = 0; +void ds4_gpu_v41_set_indexer_mma(int on) { g_v41_idx_mma = on ? 1 : 0; } + +extern "C" int ds4_gpu_v41_indexer_score_tensor(ds4_gpu_tensor *score, const ds4_gpu_tensor *q, const ds4_gpu_tensor *k, + const ds4_gpu_tensor *weights, const ds4_gpu_tensor *cand_list, uint32_t cand_bs, uint32_t cand_cap, + uint32_t n_tok, uint32_t pos0, uint32_t ng, uint32_t n_head, uint32_t dk, uint32_t ratio, + const ds4_gpu_tensor *posd) { + if (!score || !q || !k || !weights || (dk % 32u) || ratio == 0) return 0; + if (dk / 32u > 4u) { fprintf(stderr, "ds4: [ds41] indexer score supports dk <= 128 (<= 4 dims per lane)\n"); return 0; } + if (posd && n_tok > 8u) return 0; /* graph path: pure decode 1 row or a verify batch <= 8 rows */ + if (ng == 0) return 1; + if (cand_list && (cand_bs == 0u || cand_cap == 0u)) return 0; + if (g_v41_idx_mma) { + fprintf(stderr, "ds4: [ds41] the indexer s8 mma score variant is not ported (scalar kernel only)\n"); + return 0; + } + /* One block = 8 warps x V41_IDX_G groups; more groups = more blocks + * (65535 is CUDA's grid.y hard cap; the kernel wraps by grid stride). + * C2: with a list the grid is sized by the compact width. */ + const uint32_t ns = cand_list ? v41_cand_ns(ng, cand_bs, cand_cap) : ng; + uint32_t gblocks = (ns + 8u * V41_IDX_G - 1u) / (8u * V41_IDX_G); + if (gblocks > 65535u) gblocks = 65535u; + v41_indexer_score_kernel<<>>((float *)score->ptr, (const float *)q->ptr, (const uint8_t *)k->ptr, + (const float *)weights->ptr, cand_list ? (const int32_t *)cand_list->ptr : NULL, pos0, ng, n_head, dk, ratio, + posd ? (const int32_t *)posd->ptr : NULL, cand_bs, cand_cap); + return cuda_ok(hipGetLastError(), "v41 indexer score"); +} + +/* ---- candidate blocks (cuda_v41_indexer.inc.cu:132-238) ---- + * The engine's original was one serial O(kk x nb) thread-0 loop: at 12k + * context (nb = 1558, kk = 2048 > nb) that was 2.4M rounds = 46.4 ms/step. + * The rewrite keeps the selection set bit-identical: + * (1) kk >= nb (the live config up to ~32k): the original selects every + * block with score > -inf — written directly, no selection needed; + * (2) kk < nb: parallel radix select for the kk-th key; keys > threshold + * all selected, keys == threshold filled in ascending block order + * (the original scanned with strict > so ties take the smaller block). + * The [nb] block scores and [nb] flags live in global scratch (not shared): + * the old static shared 48 KB capped nb at 9830 blocks, which was the wall + * that pinned ctx to 32768; the row stride is the host's bucket cap nb_cap so + * two rows can never overlap when the kernel's own nb is smaller. + * float -> order-preserving u32 key, shared with topk (:137-141): + * -inf and NaN -> 0 (both originals scan with `x > bv`, bv = -inf, so those + * never win; key 0 reproduces that). Positives set the top bit; negatives are + * complemented — IEEE float order becomes unsigned key order. */ +__device__ __forceinline__ static uint32_t v41_topk_key(float f) { + if (!(f > -INFINITY)) return 0u; + uint32_t u; memcpy(&u, &f, 4); + return (u & 0x80000000u) ? ~u : (u | 0x80000000u); +} + +/* Grow-only scratch for the candidate kernel, 5 bytes per block per row + * (float score + u8 flag). Named deviation: the engine splits this per + * decode lane; the port has no lanes yet. */ +static v41_scratch g_v41_cand_blk; + +extern "C" int ds4_gpu_v41_candidate_scratch_prepare(uint32_t n_tok, uint32_t nb) { + if (n_tok == 0u || nb == 0u) return 1; + return v41_grow(&g_v41_cand_blk, (uint64_t)n_tok * nb * 5u, "v41 candidate blocks") ? 1 : 0; +} + +__global__ static void v41_candidate_kernel(int32_t *list, const float *score, uint32_t pos0, uint32_t ng, uint32_t ratio, + uint32_t topk_blocks, uint32_t bs, const int32_t *posd, + float *blk, uint32_t nb_cap) { + v41_pdl_wait(); /* PDL: first instruction waits for the upstream; no-op when not launched through PDL */ + const uint32_t i = blockIdx.x; + if (posd) { pos0 = (uint32_t)posd[0]; ng = (pos0 + gridDim.x) / ratio; } /* graph: device position; scratch sized by the bucket cap */ + const uint32_t nb = (ng + bs - 1u) / bs; + float *bsc = blk + (uint64_t)i * nb_cap; + uint8_t *sel = (uint8_t *)(blk + (uint64_t)gridDim.x * nb_cap) + (uint64_t)i * nb_cap; + __shared__ uint32_t hist[256], sh_bucket, sh_k; + const uint32_t vis = (pos0 + i + 1u) / ratio; + for (uint32_t b = threadIdx.x; b < nb; b += blockDim.x) { + float m = -INFINITY; + for (uint32_t j = b * bs; j < (b + 1u) * bs && j < ng; j++) m = fmaxf(m, score[(uint64_t)i * ng + j]); + if (vis > 0u && b == (vis - 1u) / bs) m = INFINITY; /* pin the block holding the newest position */ + bsc[b] = m; sel[b] = 0; + } + __syncthreads(); + const uint32_t kk = topk_blocks < nb ? topk_blocks : nb; + if (kk >= nb) { /* (1) select all except fully invisible blocks */ + for (uint32_t b = threadIdx.x; b < nb; b += blockDim.x) sel[b] = v41_topk_key(bsc[b]) != 0u; + } else { /* (2) parallel radix select for the threshold */ + uint32_t prefix = 0, want = kk; + for (int shift = 24; shift >= 0; shift -= 8) { + for (uint32_t t = threadIdx.x; t < 256u; t += blockDim.x) hist[t] = 0; + __syncthreads(); + for (uint32_t b = threadIdx.x; b < nb; b += blockDim.x) { + const uint32_t key = v41_topk_key(bsc[b]); + if (key == 0u || (shift < 24 && (key >> (shift + 8)) != (prefix >> (shift + 8)))) continue; + atomicAdd(&hist[(key >> shift) & 0xffu], 1u); + } + __syncthreads(); + if (threadIdx.x == 0) { /* accumulate from the top bucket down to the one holding the want-th key */ + uint32_t acc = 0; int t = 255; + for (; t > 0; t--) { if (acc + hist[t] >= want) break; acc += hist[t]; } + sh_bucket = (uint32_t)t; sh_k = want - acc; + } + __syncthreads(); + prefix |= sh_bucket << shift; + want = sh_k; + __syncthreads(); + } + for (uint32_t b = threadIdx.x; b < nb; b += blockDim.x) sel[b] = v41_topk_key(bsc[b]) > prefix; + __syncthreads(); + if (threadIdx.x == 0) { /* keys == threshold fill in ascending block order (ties take the smaller block) */ + uint32_t eq = want; + for (uint32_t b = 0; b < nb && eq; b++) if (v41_topk_key(bsc[b]) == prefix) { sel[b] = 1; eq--; } + } + } + __syncthreads(); + /* Compact the selection into an ascending list (2026-09-30 C2, contract in + * ds4_gpu_v41.h): list[i][0] = block count, list[i][1..] = ascending block + * numbers. The old form wrote an [n][nb] byte mask (268 MB for a 1M + * context prefill) and downstream still ran the head loop over the masked + * million groups. Each thread owns one contiguous block span (ascending + * within the span); span counts -> exclusive prefix -> spans written in + * order, so the list is ascending exactly like the original mask scan. */ + __shared__ uint32_t cnt[256]; /* same origin as the launched blockDim(256) */ + const uint32_t chunk = (nb + blockDim.x - 1u) / blockDim.x; + const uint32_t c0 = threadIdx.x * chunk, c1 = (c0 + chunk) < nb ? (c0 + chunk) : nb; + uint32_t mine = 0; + for (uint32_t b = c0; b < c1; b++) mine += sel[b]; + cnt[threadIdx.x] = mine; + __syncthreads(); + if (threadIdx.x == 0) { + uint32_t run = 0; + for (uint32_t t = 0; t < blockDim.x; t++) { const uint32_t v = cnt[t]; cnt[t] = run; run += v; } + list[(uint64_t)i * (1u + topk_blocks)] = (int32_t)run; /* <= kk <= topk_blocks: the list always fits */ + } + __syncthreads(); + uint32_t wr = cnt[threadIdx.x]; + for (uint32_t b = c0; b < c1; b++) if (sel[b]) list[(uint64_t)i * (1u + topk_blocks) + 1u + wr++] = (int32_t)b; +} + +extern "C" int ds4_gpu_v41_candidate_blocks_tensor(ds4_gpu_tensor *cand_list, const ds4_gpu_tensor *score, uint32_t n_tok, uint32_t pos0, + uint32_t ng, uint32_t ratio, uint32_t topk_blocks, uint32_t block_size, + const ds4_gpu_tensor *posd) { + if (!cand_list || !score || block_size == 0 || ratio == 0 || topk_blocks == 0) return 0; + if (posd && n_tok > 8u) return 0; + if (ng == 0) return 1; + const uint32_t nb = (ng + block_size - 1u) / block_size; + /* Eager path grows the scratch here; on the graph path the prepare has + * already grown it to the bucket cap and this call is a no-op. */ + float *blk = (float *)v41_grow(&g_v41_cand_blk, (uint64_t)n_tok * nb * 5u, "v41 candidate blocks"); + if (!blk) return 0; + v41_candidate_kernel<<>>((int32_t *)cand_list->ptr, (const float *)score->ptr, pos0, ng, ratio, + topk_blocks, block_size, posd ? (const int32_t *)posd->ptr : NULL, + blk, nb); + return cuda_ok(hipGetLastError(), "v41 candidate blocks"); +} + +/* ---- topk (cuda_v41_indexer.inc.cu:260-406) ---- + * Official semantics: topk over min(index_topk, visible groups), output in + * position order, unreachable slots -1. The engine's original was a serial + * O(topk·ng) thread-0 loop (44 ms per call at 3300 context, 62% of the GPU + * time in long-context decode). The rewrite is bit-equivalent to it: + * (1) parallel radix select finds the kk-th largest key (float mapped to an + * order-preserving u32, 8 bits per round, 4 rounds; 1024 threads, reads + * batched V41_TOPK_B at a time); + * (2) thread-0's ascending scan is replaced by segmented write-out: position + * = (count of > threshold before g) + min(count of == threshold before + * g, eq), segment counts + two exclusive prefix sums, segments written + * in order — the set, the order and the tie rule are all unchanged. + * Equivalence points preserved verbatim: `s[g] > bv` (strict) so ties take + * the earlier (smaller) index; -inf and NaN (key 0) are never selected. + * With a candidate list the scan runs over the compact row [ns] only and + * writes map back through v41_cand_map; the topk cap still uses ng (official + * min(index_topk, ng)). */ +#define V41_TOPK_THREADS 1024u +#define V41_TOPK_B 8u +/* C2: compact item c -> real group number (ascending list => monotone); no list => c is the group */ +__device__ __forceinline__ static int32_t v41_cand_map(const int32_t *cl, uint32_t c, uint32_t bs) { + return cl ? cl[1u + c / bs] * (int32_t)bs + (int32_t)(c % bs) : (int32_t)c; +} +__global__ static void __launch_bounds__(V41_TOPK_THREADS) v41_topk_kernel(int32_t *idx, const float *score, uint32_t ng, uint32_t topk, + uint32_t ratio, const int32_t *posd, + const int32_t *cand, uint32_t cand_bs, uint32_t cand_cap) { + v41_pdl_wait(); /* PDL: first instruction waits for the upstream; no-op when not launched through PDL */ + const uint32_t i = blockIdx.x, tid = threadIdx.x, nt = blockDim.x; + if (posd) { /* graph: ng and topk recomputed from the device position (same formula as the host's min(index_topk, ng)) */ + ng = ((uint32_t)posd[0] + gridDim.x) / ratio; + if (ng < topk) topk = ng; + } + /* C2: with a list score is the compact row [ns] the score kernel wrote; the scan covers ns items and writes map back; the topk cap still uses ng */ + const int32_t *cl = cand ? cand + (uint64_t)i * (1u + cand_cap) : NULL; + const uint32_t ns = cl ? v41_cand_ns(ng, cand_bs, cand_cap) : ng; + const float *s = score + (uint64_t)i * ns; + /* Archived negative result (2026-09-16): per-warp histograms (hist[8][256]) + * to remove shared-atomic contention measured 5.91 -> 7.15 ms (21% + * slower); contention is not this kernel's bottleneck. */ + __shared__ uint32_t hist[256], suf[257], sh_bucket; + uint32_t prefix = 0, kk = 0; + for (int shift = 24; shift >= 0; shift -= 8) { + for (uint32_t b = tid; b < 256u; b += nt) hist[b] = 0; + __syncthreads(); + for (uint32_t gb = tid; gb < ns; gb += V41_TOPK_B * nt) { + float v[V41_TOPK_B]; + #pragma unroll + for (uint32_t j = 0; j < V41_TOPK_B; j++) { const uint32_t g = gb + j * nt; v[j] = g < ns ? s[g] : -INFINITY; } + #pragma unroll + for (uint32_t j = 0; j < V41_TOPK_B; j++) { + const uint32_t key = v41_topk_key(v[j]); /* out-of-range -inf => key 0, same as "not counted" */ + /* shift=24 has no higher bits to compare (a 32-bit right shift is UB); everything counts */ + if (key == 0u || (shift < 24 && (key >> (shift + 8)) != (prefix >> (shift + 8)))) continue; + atomicAdd(&hist[(key >> shift) & 0xffu], 1u); + } + } + __syncthreads(); + if (tid < 32u) { /* suffix sums suf[b] = sum_{b' >= b} hist[b']: lane l owns buckets [8l, 8l+8), shfl for "lanes above me" */ + uint32_t loc = 0; + for (uint32_t b = 0; b < 8u; b++) loc += hist[tid * 8u + b]; + uint32_t inc = loc; + for (int o = 1; o < 32; o <<= 1) { const uint32_t t = __shfl_down_sync(0xffffffffu, inc, o); if (tid + (uint32_t)o < 32u) inc += t; } + uint32_t run = inc - loc; /* buckets above this lane */ + for (int b = 7; b >= 0; b--) { run += hist[tid * 8u + (uint32_t)b]; suf[tid * 8u + (uint32_t)b] = run; } + if (tid == 0u) { suf[256] = 0u; sh_bucket = 0u; } + } + __syncthreads(); + if (shift == 24) { /* (3) round 1's histogram covers every valid key => suf[0] = nvalid */ + const uint32_t nval = suf[0]; + kk = topk < nval ? topk : nval; + if (kk == 0u) break; /* original: kk == 0 never enters radix, prefix stays 0 (block-uniform branch) */ + } + for (uint32_t b = 1u + tid; b < 256u; b += nt) if (suf[b] >= kk) atomicMax(&sh_bucket, b); + __syncthreads(); + const uint32_t bs = sh_bucket; + prefix |= bs << shift; + kk -= suf[bs + 1u]; /* how many more to take inside this bucket */ + __syncthreads(); /* everyone must finish reading hist/suf before the next round clears them */ + } + /* Write-out segments (parallel since 2026-09-16): position = (count of > + * threshold before g) + min(count of == threshold before g, eq). ng is + * cut into nt spans, each counts (gt, eq), two prefix sums give the span + * starts, spans written ascending. Set, order and tie rule identical to + * the original. */ + __shared__ uint32_t cgt[V41_TOPK_THREADS], ceq[V41_TOPK_THREADS], sh_tgt, sh_teq; + if (tid == 0) { sh_tgt = 0; sh_teq = 0; } + __syncthreads(); + const uint32_t chunk = (ns + nt - 1u) / nt; + const uint32_t g0 = tid * chunk, g1 = (g0 + chunk) < ns ? (g0 + chunk) : ns; + { /* (1) each span counts its own > threshold / == threshold (reads batched) */ + uint32_t ngt = 0, neq = 0; + for (uint32_t gb = g0; gb < g1; gb += V41_TOPK_B) { + float v[V41_TOPK_B]; + #pragma unroll + for (uint32_t j = 0; j < V41_TOPK_B; j++) v[j] = gb + j < g1 ? s[gb + j] : -INFINITY; + #pragma unroll + for (uint32_t j = 0; j < V41_TOPK_B; j++) { + const uint32_t key = v41_topk_key(v[j]); + if (key == 0u) continue; + if (key > prefix) ngt++; + else if (key == prefix) neq++; + } + } + cgt[tid] = ngt; ceq[tid] = neq; + atomicAdd(&sh_tgt, ngt); atomicAdd(&sh_teq, neq); /* totals: (4) uses them to fill the tail */ + } + __syncthreads(); + /* (2) two exclusive prefix sums (nt entries, Hillis-Steele in place: shift + * right one, then add level by level; every level needs its __syncthreads, + * all outside branches) */ + { + uint32_t vg = tid ? cgt[tid - 1u] : 0u; + uint32_t ve = tid ? ceq[tid - 1u] : 0u; + __syncthreads(); + cgt[tid] = vg; ceq[tid] = ve; + __syncthreads(); + for (uint32_t off = 1u; off < nt; off <<= 1) { + const uint32_t ag = tid >= off ? cgt[tid - off] : 0u; + const uint32_t ae = tid >= off ? ceq[tid - off] : 0u; + __syncthreads(); + cgt[tid] += ag; ceq[tid] += ae; + __syncthreads(); + } + } + { /* (3) spans written ascending; eq = "how many == threshold still to take" from the radix tail */ + const uint32_t eq = kk; + uint32_t wgt = cgt[tid], weq = ceq[tid]; + for (uint32_t gb = g0; gb < g1; gb += V41_TOPK_B) { + float v[V41_TOPK_B]; + #pragma unroll + for (uint32_t j = 0; j < V41_TOPK_B; j++) v[j] = gb + j < g1 ? s[gb + j] : -INFINITY; + #pragma unroll + for (uint32_t j = 0; j < V41_TOPK_B; j++) { + const uint32_t g = gb + j; + const uint32_t key = v41_topk_key(v[j]); + if (key == 0u) continue; + if (key > prefix) { + const uint32_t pos = wgt + (weq < eq ? weq : eq); + if (pos < topk) idx[(uint64_t)i * topk + pos] = v41_cand_map(cl, g, cand_bs); + wgt++; + } else if (key == prefix) { + if (weq < eq) { + const uint32_t pos = wgt + weq; + if (pos < topk) idx[(uint64_t)i * topk + pos] = v41_cand_map(cl, g, cand_bs); + } + weq++; + } + } + } + } + __syncthreads(); + { /* (4) fill the tail with -1 (unreachable). Total = all > threshold + min(all == threshold, eq), same as the original's w carry. */ + uint32_t total = sh_tgt + (sh_teq < kk ? sh_teq : kk); + if (total > topk) total = topk; + for (uint32_t w = total + tid; w < topk; w += nt) idx[(uint64_t)i * topk + w] = -1; + } +} + +extern "C" int ds4_gpu_v41_indexer_topk_tensor(ds4_gpu_tensor *idx, const ds4_gpu_tensor *score, uint32_t n_tok, uint32_t ng, + uint32_t topk, uint32_t ratio, const ds4_gpu_tensor *posd, + const ds4_gpu_tensor *cand_list, uint32_t cand_bs, uint32_t cand_cap) { + if (!idx || !score || topk == 0) return 0; + if (posd && (n_tok > 8u || ratio == 0u)) return 0; + if (cand_list && (cand_bs == 0u || cand_cap == 0u)) return 0; + if (ng == 0) return 1; + /* shared is down to the fixed 256 buckets (1 KB), so the old "ng > 48K refuses" gate is gone */ + v41_topk_kernel<<>>((int32_t *)idx->ptr, (const float *)score->ptr, ng, topk, ratio, + posd ? (const int32_t *)posd->ptr : NULL, + cand_list ? (const int32_t *)cand_list->ptr : NULL, cand_bs, cand_cap); + return cuda_ok(hipGetLastError(), "v41 topk"); +} diff --git a/cuda/ds41_primitives.cuh b/cuda/ds41_primitives.cuh new file mode 100644 index 0000000000000000000000000000000000000000..1372c5ae0c10f4d06f3192d7ce06fdbde0813d0e --- /dev/null +++ b/cuda/ds41_primitives.cuh @@ -0,0 +1,93 @@ +#include "hip/hip_runtime.h" +/* ds41_primitives.cuh — DeepSeek V4.1 (ds41) decode-side primitives, ported + * from the C engine at /data/YoungAi (commit 3946dbc), file + * `src/cuda/cuda_v41_1.inc.cu` (scratch/grow lines 7-88, v41_bf16r :120) and + * `src/cuda/cuda_internal.cuh` (PDL, :204-218). + * + * Scope note: only what the VQ decode family consumes is ported here. The + * engine's scratch registry and `ds4_gpu_v41_scratch_release` stay behind + * until the ds41 state wiring lands (P4-2); the grow-only discipline and the + * generation counter are kept because captured decode graphs must be + * re-captured when a scratch pointer moves (the engine's 2026-09-19 + * conviction: a graph baked the pre-growth pointer and read a freed page). + */ +#pragma once +#include +#include +#include + +#include +#include + +/* `V41_GEMV_MAX_TOK` = `DS4_V41_GEMV_MAX_TOK` (`ds4_gpu_v41.h:265`): the largest + * token count the fused GEMV decode path takes; above it the prefill family + * runs. The Rust host splits on the same value. */ +#define V41_GEMV_MAX_TOK 8u + +/* Grow-only device scratch slot. One slot per buffer; a grow frees the old + * block and bumps the generation so a captured decode graph is re-captured. */ +typedef struct { void *p; uint64_t cap; } v41_scratch; + +static uint64_t g_v41_scratch_gen = 0; +extern "C" uint64_t ds4_gpu_v41_scratch_generation(void) { return g_v41_scratch_gen; } /* the host TU reads it (dg_launch's stale-pointer check) */ + +static void *v41_grow(v41_scratch *s, uint64_t bytes, const char *what) { + if (bytes <= s->cap) return s->p; + (void)hipDeviceSynchronize(); + if (s->p) (void)hipFree(s->p); + s->p = NULL; s->cap = 0; + if (hipMalloc(&s->p, (size_t)bytes) != hipSuccess) { + (void)hipGetLastError(); + fprintf(stderr, "ds4: [ds41] %s scratch alloc failed (%.1f MB)\n", what, (double)bytes / 1048576.0); + return NULL; + } + s->cap = bytes; + g_v41_scratch_gen++; + /* A grow mid-run re-resolves every consumer's pointer; print it so a + * reallocation can never hide inside a quiet gate log (a grow landing + * between decode steps is a determinism lead, not noise). */ + fprintf(stderr, "ds4: [ds41] grow %s -> %.1f MB (gen %llu)\n", what, (double)bytes / 1048576.0, + (unsigned long long)g_v41_scratch_gen); + return s->p; +} + +/* RNE round f32 -> bf16 -> f32 (`cuda_v41_1.inc.cu:120`). NaN/Inf pass + * through. Identity on values already on the bf16 grid, which is why the + * VQ path's "pack activations to bf16" step is lossless by contract. */ +__device__ __forceinline__ static float v41_bf16r(float x) { + uint32_t u; memcpy(&u, &x, 4); + if ((u & 0x7F800000u) == 0x7F800000u) return x; + u += 0x7FFFu + ((u >> 16) & 1u); + u &= 0xFFFF0000u; + float y; memcpy(&y, &u, 4); return y; +} + +/* Programmatic dependent launch (`cuda_internal.cuh:204-218`). A registered + * kernel waits for its upstream's completion and memory visibility before + * touching upstream outputs; when the kernel is not launched through PDL the + * wait returns immediately, so one kernel serves both paths. + * + * Portability deviation, named: the engine compiles sm_121 only and writes the + * bare asm; `griddepcontrol` is sm_90+, so this port guards it. On sm_121 the + * compiled code is identical; below sm_90 no PDL graph can exist and the wait + * is a no-op, which is exactly the engine's "not launched through PDL" + * behavior. */ +__device__ __forceinline__ static void v41_pdl_wait(void) { +#if !defined(__HIP_PLATFORM_AMD__) && (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 900) + asm volatile("griddepcontrol.wait;" ::: "memory"); +#endif /* HIP: no programmatic dependent launch, so the wait is the engine's not-PDL no-op */ +} + +#define V41_PDL_MAX 64 +static const void *g_v41_pdl_ready[V41_PDL_MAX]; +static int g_v41_pdl_n = 0; +/* Launch side: declares that this kernel calls v41_pdl_wait before reading + * upstream output. Duplicate registration is harmless. */ +static void v41_pdl_register(const void *fn) { + for (int i = 0; i < g_v41_pdl_n; i++) if (g_v41_pdl_ready[i] == fn) return; + if (g_v41_pdl_n < V41_PDL_MAX) g_v41_pdl_ready[g_v41_pdl_n++] = fn; +} +static int v41_pdl_is_ready(const void *fn) { + for (int i = 0; i < g_v41_pdl_n; i++) if (g_v41_pdl_ready[i] == fn) return 1; + return 0; +} diff --git a/cuda/ds41_q4k.cuh b/cuda/ds41_q4k.cuh new file mode 100644 index 0000000000000000000000000000000000000000..50d2941067a18dab0d1354d6711bc273ec4fc134 --- /dev/null +++ b/cuda/ds41_q4k.cuh @@ -0,0 +1,403 @@ +#include "hip/hip_runtime.h" +/* ds41_q4k.cuh — DeepSeek V4.1 (ds41) q4_K skeleton decode family, ported from + * the C engine at /data/YoungAi (commit 3946dbc), file + * src/cuda/cuda_v41_q4k.inc.cu (2026-09-19, the 100 GB recipe's skeleton). + * + * On-disk q4_K: 144 B / 256 elements — f16 d + f16 dmin + 12 B of 8 packed + * 6-bit (scale, min) pairs + 128 B nibbles. The golden decoder is + * src/common/ds4_quantfmt.c ds4_deq_q4_K (byte-exact fixture in the engine's + * unit tests); the kernels here mirror it expression for expression — the + * block header packs 8 six-bit pairs into 12 B and a one-bit packing error + * silently shifts a whole 32-element group by a constant. + * + * Lane layout (the engine's measured 64 B DRAM granule): lane l reads + * qs[4l..4l+3] so one warp step is 128 contiguous bytes; byte p feeds + * sub-block 2(p/32) (low nibble) and 2(p/32)+1 (high nibble), 32 elements + * apart. The GEMV pair (stage/pipe) stages the CTA's rows through shared + * first — byte-identical to the retired per-warp reader (benchmarked on the + * real shapes) — and selects between them by the group's row bytes. + * + * Scope: the n <= 8 GEMV arm, the embedding row fetch and the grouped (wo_a) + * arm (P4-4), plus the n > 8 prefill GEMM (P4-5: decode to bf16 in row tiles, + * then cuBLAS — cuda_v41_q4k.inc.cu:183-205 the to-bf16 kernel, :330-390 the + * GEMM). The head colnorm is not ported yet and refuses by name. */ +#pragma once +#include /* the pipe variant's cp.async intrinsics */ + +#define V41_Q4K_BLK 256u /* elements per block */ +#define V41_Q4K_BYTES 144u /* bytes per block: 2(d) + 2(dmin) + 12(scales) + 128(qs) */ +/* The GEMV's warp budget (`cuda_v41_4.inc.cu:17`); the q4k launchers were + * tuned around it (parallelism target 8192, stage/pipe selection). */ +#define V41_GEMV_WARPS 8u + +/* The j-th (scale, min) pair of a block header; expression-identical to + * ds4_quantfmt.c q4k_scale_min (cuda_v41_q4k.inc.cu:21-27). */ +__device__ __forceinline__ static void v41_q4k_sm(const uint8_t *sc, int j, float *s, float *m) { + uint8_t a, b; + if (j < 4) { a = sc[j] & 63u; b = sc[j + 4] & 63u; } + else { a = (uint8_t)((sc[j + 4] & 0xFu) | ((sc[j - 4] >> 6) << 4)); + b = (uint8_t)((sc[j + 4] >> 4) | ((sc[j - 0] >> 6) << 4)); } + *s = (float)a; *m = (float)b; +} + +/* Header already in registers (one uint4): scale byte k of the 12-byte table. + * NOT ((const uint8_t *)&h)[k] — k varies per lane and a dynamic index into a + * register array drops it into local memory; this compiles to SEL/SHF. */ +__device__ __forceinline__ static uint32_t v41_q4k_scb(const uint4 &h, uint32_t k) { + const uint32_t w = k < 4u ? h.y : (k < 8u ? h.z : h.w); + return (w >> ((k & 3u) * 8u)) & 0xFFu; +} +__device__ __forceinline__ static void v41_q4k_sm_reg(const uint4 &h, uint32_t j, float *s, float *m) { + uint32_t a, b; + if (j < 4u) { a = v41_q4k_scb(h, j) & 63u; b = v41_q4k_scb(h, j + 4u) & 63u; } + else { a = (v41_q4k_scb(h, j + 4u) & 0xFu) | ((v41_q4k_scb(h, j - 4u) >> 6) << 4); + b = (v41_q4k_scb(h, j + 4u) >> 4) | ((v41_q4k_scb(h, j) >> 6) << 4); } + *s = (float)a; *m = (float)b; +} + +/* One warp dot-products one q4_K block against NT activations, accumulating + * into acc[NT]; the header and qs word come from the caller (shared staging). + * lane l: gidx = l>>3 selects a 64-element group, q0 = (l&7)*4 four + * consecutive elements inside it; low half -> gidx*64 + q0 + i (sub-block + * 2*gidx), high half -> +32 (sub-block 2*gidx+1). */ +template +__device__ __forceinline__ static void v41_q4k_blk_acc_reg(const uint4 &h, uint32_t qw, const float *x, uint32_t x_stride, + uint32_t base, float *acc) { + const uint32_t lane = threadIdx.x & 31u, gidx = lane >> 3, q0 = (lane & 7u) * 4u; + const float d = __half2float(__ushort_as_half((unsigned short)(h.x & 0xFFFFu))); + const float dmin = __half2float(__ushort_as_half((unsigned short)(h.x >> 16))); + float s_lo, m_lo, s_hi, m_hi; + v41_q4k_sm_reg(h, gidx * 2u, &s_lo, &m_lo); + v41_q4k_sm_reg(h, gidx * 2u + 1u, &s_hi, &m_hi); + const float dl = d * s_lo, ml = dmin * m_lo, dh = d * s_hi, mh = dmin * m_hi; + float wl[4], wh[4]; + #pragma unroll + for (int i = 0; i < 4; i++) { + const uint32_t byte = (qw >> (8 * i)) & 0xFFu; + wl[i] = dl * (float)(byte & 0xFu) - ml; + wh[i] = dh * (float)(byte >> 4) - mh; + } + const uint32_t e_lo = base + gidx * 64u + q0, e_hi = e_lo + 32u; + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + const float *xt = x + (uint64_t)t * x_stride; + const float4 xl = *(const float4 *)(xt + e_lo), xh = *(const float4 *)(xt + e_hi); + acc[t] += wl[0] * xl.x + wl[1] * xl.y + wl[2] * xl.z + wl[3] * xl.w + + wh[0] * xh.x + wh[1] * xh.y + wh[2] * xh.z + wh[3] * xh.w; + } +} + +/* The stage/pipe pair (cuda_v41_q4k.inc.cu:72-179): byte-identical to the + * retired per-warp reader on all nine real shapes (the engine's benchmark); + * selection by the group's row bytes — stage for large groups (18-23 KB), + * pipe (cp.async double buffer) for 4-12 KB groups. */ +#define V41_Q4K_PIPE_MIN 4096u +#define V41_Q4K_PIPE_MAX 12288u + +template +__device__ __forceinline__ static void v41_q4k_rown_smem(const uint8_t *wr, uint32_t nblk, uint32_t ksplit, uint32_t kpart, + const float *x, uint32_t x_stride, float *acc) { + const uint32_t lane = threadIdx.x & 31u; + for (uint32_t b = kpart; b < nblk; b += ksplit) { + const uint8_t *blk = wr + b * V41_Q4K_BYTES; + v41_q4k_blk_acc_reg(*(const uint4 *)blk, *(const uint32_t *)(blk + 16u + 4u * lane), x, x_stride, b * V41_Q4K_BLK, acc); + } +} +template +__device__ __forceinline__ static void v41_q4k_finishn(float *acc, float *out, uint32_t out_stride, uint32_t r, uint32_t out_dim, + uint32_t ksplit, uint32_t rloc, uint32_t kpart, int round_out, float *red) { + const uint32_t warp = threadIdx.x >> 5, lane = threadIdx.x & 31u; + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + for (int o = 16; o; o >>= 1) acc[t] += __shfl_xor_sync(0xffffffffu, acc[t], o); + if (lane == 0) red[warp * NT + t] = acc[t]; + } + __syncthreads(); + if (r < out_dim && kpart == 0 && lane == 0) { + #pragma unroll + for (uint32_t t = 0; t < NT; t++) { + float sum = 0.f; + for (uint32_t k = 0; k < ksplit; k++) sum += red[(rloc * ksplit + k) * NT + t]; + out[(uint64_t)t * out_stride + r] = round_out ? v41_bf16r(sum) : sum; + } + } +} +template +__global__ static void __launch_bounds__(256, 4) v41_q4k_gemv1_stage_kernel(float *out, const uint8_t *w, const float *x, + uint32_t in_dim, uint32_t out_dim, uint32_t ksplit, uint64_t w_gstride, uint32_t x_gstride, uint32_t out_gstride, int round_out, + uint32_t x_stride, uint32_t out_stride) { + extern __shared__ uint4 v41_q4k_st[]; + __shared__ float red[V41_GEMV_WARPS * NT]; + const uint32_t g = blockIdx.y; + x += (uint64_t)g * x_gstride; out += (uint64_t)g * out_gstride; w += (uint64_t)g * w_gstride; + const uint32_t warp = threadIdx.x >> 5, rpb = V41_GEMV_WARPS / ksplit, rloc = warp / ksplit, kpart = warp % ksplit; + const uint32_t nblk = in_dim / V41_Q4K_BLK, r0 = blockIdx.x * rpb, r = r0 + rloc; + const uint32_t nrows = out_dim - r0 < rpb ? out_dim - r0 : rpb, n16 = nrows * nblk * (V41_Q4K_BYTES / 16u); + const uint4 *src = (const uint4 *)(w + (uint64_t)r0 * nblk * V41_Q4K_BYTES); + for (uint32_t i = threadIdx.x; i < n16; i += blockDim.x) v41_q4k_st[i] = __ldcs(src + i); /* streaming: weights read once per step */ + v41_pdl_wait(); /* PDL: weights (constants) staged first, then wait for the upstream activations */ + __syncthreads(); + float acc[NT]; + #pragma unroll + for (uint32_t t = 0; t < NT; t++) acc[t] = 0.f; + if (r < out_dim) v41_q4k_rown_smem((const uint8_t *)v41_q4k_st + (uint64_t)rloc * nblk * V41_Q4K_BYTES, nblk, ksplit, kpart, x, x_stride, acc); + v41_q4k_finishn(acc, out, out_stride, r, out_dim, ksplit, rloc, kpart, round_out, red); +} +template +__global__ static void __launch_bounds__(256, 4) v41_q4k_gemv1_pipe_kernel(float *out, const uint8_t *w, const float *x, + uint32_t in_dim, uint32_t out_dim, uint32_t ksplit, uint64_t w_gstride, uint32_t x_gstride, uint32_t out_gstride, int round_out, + uint32_t x_stride, uint32_t out_stride) { + extern __shared__ uint4 v41_q4k_st[]; + __shared__ float red[V41_GEMV_WARPS * NT]; + const uint32_t g = blockIdx.y; + x += (uint64_t)g * x_gstride; out += (uint64_t)g * out_gstride; w += (uint64_t)g * w_gstride; + const uint32_t warp = threadIdx.x >> 5, rpb = V41_GEMV_WARPS / ksplit, rloc = warp / ksplit, kpart = warp % ksplit; + const uint32_t nblk = in_dim / V41_Q4K_BLK, ngrp = (out_dim + rpb - 1u) / rpb, grp16 = rpb * nblk * (V41_Q4K_BYTES / 16u); + uint32_t rg = blockIdx.x, buf = 0; + #define V41_Q4K_ISSUE(RG, BUF) do { \ + const uint32_t r0_ = (RG) * rpb, nr_ = out_dim - r0_ < rpb ? out_dim - r0_ : rpb; \ + const uint32_t n16_ = nr_ * nblk * (V41_Q4K_BYTES / 16u); \ + const uint4 *src_ = (const uint4 *)(w + (uint64_t)r0_ * nblk * V41_Q4K_BYTES); \ + uint4 *dst_ = v41_q4k_st + (uint64_t)(BUF) * grp16; \ + for (uint32_t i_ = threadIdx.x; i_ < n16_; i_ += blockDim.x) __pipeline_memcpy_async(dst_ + i_, src_ + i_, 16); \ + } while (0) + if (rg < ngrp) V41_Q4K_ISSUE(rg, 0u); + __pipeline_commit(); + v41_pdl_wait(); /* PDL: first group already in flight, then wait for the upstream activations */ + for (; rg < ngrp; rg += gridDim.x, buf ^= 1u) { + if (rg + gridDim.x < ngrp) V41_Q4K_ISSUE(rg + gridDim.x, buf ^ 1u); + __pipeline_commit(); /* an empty commit counts too: wait_prior(1) counts commits */ + __pipeline_wait_prior(1); /* this group's half arrived (the next is still in flight) */ + __syncthreads(); + const uint32_t r = rg * rpb + rloc; + float acc[NT]; + #pragma unroll + for (uint32_t t = 0; t < NT; t++) acc[t] = 0.f; + if (r < out_dim) v41_q4k_rown_smem((const uint8_t *)(v41_q4k_st + (uint64_t)buf * grp16) + (uint64_t)rloc * nblk * V41_Q4K_BYTES, + nblk, ksplit, kpart, x, x_stride, acc); + v41_q4k_finishn(acc, out, out_stride, r, out_dim, ksplit, rloc, kpart, round_out, red); + __syncthreads(); /* this half is overwritten next round: everyone done reading */ + } + #undef V41_Q4K_ISSUE +} + +/* The q4_K GEMV launcher (cuda_v41_q4k.inc.cu:232-277): alignment hard-fails + * (a uint32 qs read needs 4-byte alignment; block length 144 is a multiple of + * 16 and GGUF data starts 32 B aligned, so it holds — refuse rather than run + * a silent slow path), ksplit target 8192, stage/pipe by group bytes. */ +static int v41_q4k_gemv(const void *model_map, uint64_t model_size, uint64_t off, uint64_t in_dim, uint64_t out_dim, + const float *x, uint32_t x_stride, float *out, uint32_t out_stride, uint32_t n_tok, + uint32_t n_groups, uint32_t x_gstride, uint32_t out_gstride, int round_out, const char *what) { + if ((in_dim % V41_Q4K_BLK) != 0u || n_tok == 0 || n_tok > V41_GEMV_MAX_TOK || n_groups == 0) return 0; + const uint64_t wg = out_dim * (in_dim / V41_Q4K_BLK) * V41_Q4K_BYTES, wbytes = wg * n_groups; + if (off > model_size || wbytes > model_size - off) return 0; + const uint8_t *w = (const uint8_t *)cuda_model_range_ptr(model_map, off, wbytes, what); + if (!w) return 0; + if (((uintptr_t)w & 15u) != 0u) { fprintf(stderr, "ds4: %s q4_K tensor base is not 16-byte aligned\n", what); return 0; } + uint32_t ksplit = 1; + const uint32_t ptarget = v41_knob(5) > 0 ? (uint32_t)v41_knob(5) : 8192u; + while (ksplit < V41_GEMV_WARPS && out_dim * ksplit * n_groups < ptarget) ksplit <<= 1; + const uint32_t nb = (uint32_t)(in_dim / V41_Q4K_BLK); + while (ksplit > 1u && ksplit > nb) ksplit >>= 1; + const uint32_t rpb1 = V41_GEMV_WARPS / ksplit, ngrp = (uint32_t)((out_dim + rpb1 - 1u) / rpb1); + const uint32_t grp = rpb1 * nb * V41_Q4K_BYTES; + uint32_t gx = 0; + const int arm = v41_knob(4); + if (arm == 2 || (arm != 1 && grp > V41_Q4K_PIPE_MIN && grp <= V41_Q4K_PIPE_MAX)) { + static int s_sm = 0; + if (!s_sm && hipDeviceGetAttribute(&s_sm, hipDeviceAttributeMultiprocessorCount, 0) != hipSuccess) return cuda_ok(hipGetLastError(), what); + gx = (uint32_t)s_sm * (v41_knob(3) > 0 ? (uint32_t)v41_knob(3) : 4u) / n_groups; /* 4 = the CTA/SM count __launch_bounds__ pins */ + if (gx == 0u) gx = 1u; + if (gx > ngrp) gx = ngrp; + } + /* Largest groups are the head / wo_b kind: 8 rows x 20 blocks or 4 x 32 = 23 KB, inside the 48 KB dynamic shared default. */ + #define V41_Q4K_LAUNCH(NT) do { \ + v41_pdl_register((const void *)v41_q4k_gemv1_stage_kernel); /* both kernels v41_pdl_wait before x/out */ \ + v41_pdl_register((const void *)v41_q4k_gemv1_pipe_kernel); \ + if (gx) v41_q4k_gemv1_pipe_kernel<<>>( \ + out, w, x, (uint32_t)in_dim, (uint32_t)out_dim, ksplit, wg, x_gstride, out_gstride, round_out, x_stride, out_stride); \ + else v41_q4k_gemv1_stage_kernel<<>>( \ + out, w, x, (uint32_t)in_dim, (uint32_t)out_dim, ksplit, wg, x_gstride, out_gstride, round_out, x_stride, out_stride); \ + } while (0) + switch (n_tok) { + case 1: V41_Q4K_LAUNCH(1u); break; case 2: V41_Q4K_LAUNCH(2u); break; + case 3: V41_Q4K_LAUNCH(3u); break; case 4: V41_Q4K_LAUNCH(4u); break; + case 5: V41_Q4K_LAUNCH(5u); break; case 6: V41_Q4K_LAUNCH(6u); break; + case 7: V41_Q4K_LAUNCH(7u); break; default: V41_Q4K_LAUNCH(8u); break; + } + #undef V41_Q4K_LAUNCH + return cuda_ok(hipGetLastError(), what); +} + +/* Embedding row fetch: token id -> that row as f32 (one warp per block). */ +__global__ static void v41_q4k_embed_kernel(float *out, const int32_t *tok, const uint8_t *w, + uint32_t n_vocab, uint32_t dim) { + const uint32_t t = blockIdx.y, nblk = dim / V41_Q4K_BLK; + const uint32_t b = blockIdx.x * blockDim.y + threadIdx.y; + if (b >= nblk) return; + const int32_t id = tok[t]; + if (id < 0 || (uint32_t)id >= n_vocab) return; + const uint8_t *blk = w + ((uint64_t)id * nblk + b) * V41_Q4K_BYTES; + const uint32_t lane = threadIdx.x & 31u, gidx = lane >> 3, q0 = (lane & 7u) * 4u; + uint16_t hd = (uint16_t)blk[0] | ((uint16_t)blk[1] << 8); + uint16_t hm = (uint16_t)blk[2] | ((uint16_t)blk[3] << 8); + const float d = __half2float(__ushort_as_half(hd)), dmin = __half2float(__ushort_as_half(hm)); + float s_lo, m_lo, s_hi, m_hi; + v41_q4k_sm(blk + 4, (int)(gidx * 2u), &s_lo, &m_lo); + v41_q4k_sm(blk + 4, (int)(gidx * 2u + 1u), &s_hi, &m_hi); + const uint32_t qw = *(const uint32_t *)(blk + 16u + 4u * lane); + float *o = out + (uint64_t)t * dim + b * V41_Q4K_BLK; + #pragma unroll + for (int i = 0; i < 4; i++) { + const uint32_t byte = (qw >> (8 * i)) & 0xFFu; + o[gidx * 64u + q0 + i] = d * s_lo * (float)(byte & 0xFu) - dmin * m_lo; + o[gidx * 64u + 32u + q0 + i] = d * s_hi * (float)(byte >> 4) - dmin * m_hi; + } +} + +/* q4_K -> bf16 prefill decode (cuda_v41_q4k.inc.cu:183-205): one warp per + * block, the lane layout identical to the GEMV's, so both paths decode the + * same value bits. */ +__global__ static void v41_q4k_to_bf16_kernel(__hip_bfloat16 *o, const uint8_t *w, uint64_t nblk) { + const uint64_t b = (uint64_t)blockIdx.x * blockDim.y + threadIdx.y; + if (b >= nblk) return; + const uint8_t *blk = w + b * V41_Q4K_BYTES; + const uint32_t lane = threadIdx.x & 31u, gidx = lane >> 3, q0 = (lane & 7u) * 4u; + uint16_t hd = (uint16_t)blk[0] | ((uint16_t)blk[1] << 8); + uint16_t hm = (uint16_t)blk[2] | ((uint16_t)blk[3] << 8); + const float d = __half2float(__ushort_as_half(hd)), dmin = __half2float(__ushort_as_half(hm)); + float s_lo, m_lo, s_hi, m_hi; + v41_q4k_sm(blk + 4, (int)(gidx * 2u), &s_lo, &m_lo); + v41_q4k_sm(blk + 4, (int)(gidx * 2u + 1u), &s_hi, &m_hi); + const uint32_t qw = *(const uint32_t *)(blk + 16u + 4u * lane); + __hip_bfloat16 *ob = o + b * V41_Q4K_BLK; + #pragma unroll + for (int i = 0; i < 4; i++) { + const uint32_t byte = (qw >> (8 * i)) & 0xFFu; + ob[gidx * 64u + q0 + i] = __float2bfloat16(d * s_lo * (float)(byte & 0xFu) - dmin * m_lo); + ob[gidx * 64u + 32u + q0 + i] = __float2bfloat16(d * s_hi * (float)(byte >> 4) - dmin * m_hi); + } +} + +/* Prefill GEMM (cuda_v41_q4k.inc.cu:330-390): decode the tensor to bf16 in + * row tiles bounded by V41_BF16_STAGE_ELEMS, then one cuBLAS bf16 GEMM per + * tile. Only the head (129280 rows) exceeds the cap; grouped wo_a (67 MB) + * decodes whole once and issues one GEMM per block-diagonal group. + * Deviation, named: the engine's v41_wc_* weight cache (a backward-pass + * buffer, cuda_v41_q4k.inc.cu:350-353) has no port — `hit` is NULL on every + * call here, so the cache branches are omitted. The cache stores exactly the + * bytes v41_q4k_to_bf16_kernel writes, so the arithmetic is identical. */ +#define V41_BF16_STAGE_ELEMS (200ull * 1000000ull) /* cuda_v41_1.inc.cu:27 */ +static v41_scratch g_v41_q4k_wbf, g_v41_q4k_xbf; +#if defined(__HIP_PLATFORM_AMD__) +static int v41_q4k_gemm_wmma(const uint8_t *w, const uint16_t *xb, uint32_t in_dim, uint32_t out_dim, uint32_t n_tok, uint32_t n_groups, + uint64_t w_gstride_blk, uint32_t x_stride, uint32_t out_stride, float *out, const char *what); /* ds41_q4k_wmma.cuh */ +#endif +static int v41_q4k_gemm(const void *model_map, uint64_t model_size, uint64_t off, uint64_t in_dim, + uint64_t out_dim, const float *x, float *out, uint32_t n_tok, uint32_t n_groups, + uint32_t x_stride, uint32_t out_stride, int round_out, const char *what) { + (void)round_out; /* rounding is the caller's (one whole-tensor pass when round_out) */ + const uint64_t bpr = in_dim / V41_Q4K_BLK, nblk_g = out_dim * bpr, nblk = nblk_g * n_groups; + if (off > model_size || nblk * V41_Q4K_BYTES > model_size - off) return 0; + const uint8_t *w = (const uint8_t *)cuda_model_range_ptr(model_map, off, nblk * V41_Q4K_BYTES, what); + if (!w) return 0; + if (((uintptr_t)w & 3u) != 0u) { fprintf(stderr, "ds4: %s q4_K tensor start is not 4-byte aligned\n", what); return 0; } + const uint64_t rows_cap = V41_BF16_STAGE_ELEMS / in_dim; + const uint64_t tile = n_groups > 1u ? out_dim : (rows_cap < 256u ? 256u : (rows_cap >= out_dim ? out_dim : (rows_cap & ~255ull))); + const uint64_t stage = n_groups > 1u ? nblk * V41_Q4K_BLK : tile * in_dim; + __hip_bfloat16 *wb = (__hip_bfloat16 *)v41_grow(&g_v41_q4k_wbf, stage * sizeof(__hip_bfloat16), "v41 q4k w bf16"); + if (!wb) return 0; + const uint64_t xn = (uint64_t)n_tok * x_stride; + __hip_bfloat16 *xb = (__hip_bfloat16 *)v41_grow(&g_v41_q4k_xbf, xn * sizeof(__hip_bfloat16), "v41 q4k x bf16"); + if (!xb) return 0; + v41_x_to_bf16_kernel<<<(unsigned)((xn + 255) / 256), 256, 0, ds4_current_stream()>>>(xb, x, xn); + if (!cuda_ok(hipGetLastError(), "v41 q4k x->bf16")) return 0; +#if defined(__HIP_PLATFORM_AMD__) + if (!v41_knob(12)) { /* the rocWMMA kernel reads q4_K directly (ds41_q4k_wmma.cuh); knob 12 = the hipBLAS A/B */ + const int r = v41_q4k_gemm_wmma(w, (const uint16_t *)xb, (uint32_t)in_dim, (uint32_t)out_dim, n_tok, n_groups, nblk_g, x_stride, out_stride, out, what); + if (r) return r > 0; + } +#endif + const float alpha = 1.0f, beta = 0.0f; + const dim3 dblk(32, 8); + (void)hipblasSetStream(g_cublas, ds4_current_stream()); /* the engine's call (cuda_v41_q4k.inc.cu:359): cuBLAS keys its algorithm on the bound workspace, and this call resets it to the default pool (docs 2.4.7) -- the state the engine's GEMMs actually run in */ + cuda_cublas_ws_prep(ds4_current_stream()); + if (n_groups > 1u) { + v41_q4k_to_bf16_kernel<<<(unsigned)((nblk + 7) / 8), dblk, 0, ds4_current_stream()>>>(wb, w, nblk); + if (!cuda_ok(hipGetLastError(), "v41 q4k->bf16")) return 0; + for (uint32_t g = 0; g < n_groups; g++) { + hipblasStatus_t st = hipblasGemmEx(g_cublas, HIPBLAS_OP_T, HIPBLAS_OP_N, (int)out_dim, (int)n_tok, (int)in_dim, &alpha, + wb + (uint64_t)g * nblk_g * V41_Q4K_BLK, HIP_R_16BF, (int)in_dim, + xb + (uint64_t)g * in_dim, HIP_R_16BF, (int)x_stride, &beta, + out + (uint64_t)g * out_dim, HIP_R_32F, (int)out_stride, + HIP_R_32F, HIPBLAS_GEMM_DEFAULT); + if (!cublas_ok(st, what)) return 0; + } + return 1; + } + for (uint64_t r0 = 0; r0 < out_dim; r0 += tile) { + const uint64_t rows = out_dim - r0 < tile ? out_dim - r0 : tile, tb = rows * bpr; + v41_q4k_to_bf16_kernel<<<(unsigned)((tb + 7) / 8), dblk, 0, ds4_current_stream()>>>(wb, w + r0 * bpr * V41_Q4K_BYTES, tb); + if (!cuda_ok(hipGetLastError(), "v41 q4k->bf16")) return 0; + hipblasStatus_t st = hipblasGemmEx(g_cublas, HIPBLAS_OP_T, HIPBLAS_OP_N, (int)rows, (int)n_tok, (int)in_dim, &alpha, + wb, HIP_R_16BF, (int)in_dim, xb, HIP_R_16BF, (int)x_stride, &beta, + out + r0, HIP_R_32F, (int)out_stride, HIP_R_32F, HIPBLAS_GEMM_DEFAULT); + if (!cublas_ok(st, what)) return 0; + } + return 1; +} + +/* The q4_K entries (cuda_v41_q4k.inc.cu:392-433). n <= 8 takes the GEMV, + * n > 8 the prefill GEMM above. */ +extern "C" int ds4_gpu_v41_matmul_q4k_tensor(ds4_gpu_tensor *out, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint64_t in_dim, uint64_t out_dim, + const ds4_gpu_tensor *x, uint32_t n_tok, int round_out) { + if (!out || !x || !g_cublas_ready || n_tok == 0) return 0; + if (x->bytes < (uint64_t)n_tok * in_dim * 4 || out->bytes < (uint64_t)n_tok * out_dim * 4) return 0; + if (n_tok <= V41_GEMV_MAX_TOK) { + /* The GEMV finish folds round_out in (engine cuda_v41_q4k.inc.cu:402-403 + * returns straight from this arm) -- no second round launch. This arm + * used to fall through to the trailing round below, rounding an + * already-bf16 buffer with one extra kernel per site (328 per decode + * step; the node diff against the engine, docs/ds41-perf-2026-10-11.md). */ + return v41_q4k_gemv(model_map, model_size, weight_offset, in_dim, out_dim, (const float *)x->ptr, (uint32_t)in_dim, + (float *)out->ptr, (uint32_t)out_dim, n_tok, 1u, 0u, 0u, round_out, "v41 q4k gemv"); + } + if (!v41_q4k_gemm(model_map, model_size, weight_offset, in_dim, out_dim, (const float *)x->ptr, + (float *)out->ptr, n_tok, 1u, (uint32_t)in_dim, (uint32_t)out_dim, round_out, "v41 q4k gemm")) { + return 0; + } + return round_out ? ds4_gpu_v41_round_bf16_tensor(out, (uint64_t)n_tok * out_dim) : 1; /* GEMM arm only (engine :410-412) */ +} +extern "C" int ds4_gpu_v41_grouped_matmul_q4k_tensor(ds4_gpu_tensor *low, const void *model_map, uint64_t model_size, + uint64_t weight_offset, uint32_t n_groups, uint64_t group_dim, + uint64_t rank, const ds4_gpu_tensor *heads, uint32_t n_tok, int round_out) { + if (!low || !heads || !g_cublas_ready || n_tok == 0) return 0; + const uint64_t in_all = (uint64_t)n_groups * group_dim, out_all = (uint64_t)n_groups * rank; + if (heads->bytes < (uint64_t)n_tok * in_all * 4 || low->bytes < (uint64_t)n_tok * out_all * 4) return 0; + if (n_tok <= V41_GEMV_MAX_TOK) { + if (group_dim % V41_Q4K_BLK) return 0; + /* Same early return as the single-group entry (engine :423-425). */ + return v41_q4k_gemv(model_map, model_size, weight_offset, group_dim, rank, (const float *)heads->ptr, (uint32_t)in_all, + (float *)low->ptr, (uint32_t)out_all, n_tok, n_groups, (uint32_t)group_dim, (uint32_t)rank, round_out, "v41 wo_a q4k gemv"); + } + if (!v41_q4k_gemm(model_map, model_size, weight_offset, group_dim, rank, (const float *)heads->ptr, + (float *)low->ptr, n_tok, n_groups, (uint32_t)in_all, (uint32_t)out_all, round_out, "v41 wo_a q4k gemm")) { + return 0; + } + return round_out ? ds4_gpu_v41_round_bf16_tensor(low, (uint64_t)n_tok * out_all) : 1; /* GEMM arm only */ +} +extern "C" int ds4_gpu_v41_embed_q4k_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *tokens, const void *model_map, + uint64_t model_size, uint64_t weight_offset, uint64_t n_vocab, + uint32_t n_tok, uint64_t dim) { + if (!out || !tokens || n_tok == 0 || (dim % V41_Q4K_BLK)) return 0; + const uint64_t nblk = n_vocab * (dim / V41_Q4K_BLK); + if (weight_offset > model_size || nblk * V41_Q4K_BYTES > model_size - weight_offset) return 0; + const uint8_t *w = (const uint8_t *)cuda_model_range_ptr(model_map, weight_offset, nblk * V41_Q4K_BYTES, "v41 q4k embed"); + if (!w || ((uintptr_t)w & 3u) != 0u) return 0; + const uint32_t nb = (uint32_t)(dim / V41_Q4K_BLK); + const dim3 dblk(32, 8), grid((nb + 7u) / 8u, n_tok); + v41_q4k_embed_kernel<<>>((float *)out->ptr, (const int32_t *)tokens->ptr, w, + (uint32_t)n_vocab, (uint32_t)dim); + return cuda_ok(hipGetLastError(), "v41 q4k embed"); +} diff --git a/cuda/ds41_q4k_wmma.cuh b/cuda/ds41_q4k_wmma.cuh new file mode 100644 index 0000000000000000000000000000000000000000..c4e298e8ce3ba6c422932550bd68139a90159b69 --- /dev/null +++ b/cuda/ds41_q4k_wmma.cuh @@ -0,0 +1,152 @@ +#include "hip/hip_runtime.h" +/* ds41_q4k_wmma.cuh — the q4_K skeleton prefill GEMM on rocWMMA (AMD, + * 2026-10-23). The port's n > 8 arm dequantized each tensor to bf16 in row tiles + * and called hipBLAS; on gfx1151 (Windows HIP SDK 7.2) hipBLAS's bf16 GEMM runs + * at 0.5-1.3 TFLOPS (attn_wo_b 263 ms, wo_a 223 ms, q_b 102 ms per layer at + * 1527 tokens — 0.7 s of a ~1 s layer), while the VQ expert kernel on rocWMMA + * reaches 9 TFLOPS on the same chip. This kernel keeps q4_K on the wire: one + * block owns a 128-row x 128-token tile, each round decodes a 64-element slice + * of every row's q4_K block straight into a bf16 A tile in shared (the same + * element expression as v41_q4k_blk_acc_reg / v41_q4k_to_bf16_kernel), stages + * the tokens' bf16 activations as the B tile and multiplies with wave32 + * 16x16x16 bf16 instructions. Every k16 step starts from zero and is added back + * with a plain FADD (the engine's hoist in the VQ arm). + * + * Numerics: the dequantized weight values are exactly the bf16 values the + * hipBLAS path staged; activations are the same bf16; products exact in f32; + * the K accumulation order differs from hipBLAS (which was never a byte gate). + * Grouped form (attn_output_a): grid.z = group; group g reads blocks + * [g*nblk_g, (g+1)*nblk_g), activations at column offset g*in_dim of x_stride, + * output at column offset g*out_dim of out_stride. + * + * Shared: A 128 x 72 bf16 (18 KB) + B 128 x 72 bf16 (18 KB) = 36 KB. + * Must follow ds41_q4k.cuh (v41_q4k_sm, V41_Q4K_BLK/BYTES). + */ +#pragma once +#if defined(__HIP_PLATFORM_AMD__) +#define Q4W_BM 128u +#define Q4W_BN 128u +#define Q4W_BK 64u /* one quarter of a q4_K block: 32 bytes = 32 low + 32 high nibbles */ +#define Q4W_LD (Q4W_BK + 8u) +#define Q4W_THREADS 512u /* 16 warps: 8 row tiles x 2 token halves */ + +__global__ __launch_bounds__(Q4W_THREADS, 1) static void q4w_kernel( + float *out, const uint8_t *w, const uint16_t *xb, uint32_t in_dim, uint32_t out_dim, uint32_t n_tok, + uint64_t w_gstride_blk, uint32_t x_stride, uint32_t out_stride) { + namespace wmma = nvcuda::wmma; + constexpr uint32_t TPR = Q4W_THREADS / Q4W_BM; /* 4 threads per row: 8 bytes = 16 elements each */ + constexpr uint32_t NFW = Q4W_BN / 2u / 16u; /* 4 token fragments per warp */ + constexpr uint32_t CH = (Q4W_BN * (Q4W_BK / 8u)) / Q4W_THREADS; /* 2 x 16 B chunks of B per thread per round */ + __shared__ __align__(16) uint16_t As[Q4W_BM * Q4W_LD]; + __shared__ __align__(16) uint16_t Bs[Q4W_BN * Q4W_LD]; + float *Os = (float *)As; /* exit scratch, per warp one 16x16 f32 tile */ + const uint32_t tid = threadIdx.x, lane = tid & 31u, warp = tid >> 5, wr = warp & 7u, th = warp >> 3; + const uint32_t g = blockIdx.z, bpr = in_dim / V41_Q4K_BLK; /* q4_K blocks per row */ + const uint8_t *wg = w + (uint64_t)g * w_gstride_blk * V41_Q4K_BYTES; + const uint16_t *xg = xb + (uint64_t)g * in_dim; + float *og = out + (uint64_t)g * out_dim; + const uint32_t mt = (out_dim + Q4W_BM - 1u) / Q4W_BM, nt = (n_tok + Q4W_BN - 1u) / Q4W_BN, nwork = mt * nt, nit = in_dim / Q4W_BK; + const uint32_t ar = tid / TPR, sub = tid % TPR; /* decode split: row ar, bytes 8*sub.. of the 32-byte quarter */ + for (uint32_t wk = blockIdx.x; wk < nwork; wk += gridDim.x) { + const uint32_t r0 = (wk / nt) * Q4W_BM, t0 = (wk % nt) * Q4W_BN, ntk = n_tok - t0 < Q4W_BN ? n_tok - t0 : Q4W_BN; + const uint32_t grow = r0 + ar; + const bool rv = grow < out_dim; + const uint8_t *rowp = wg + (uint64_t)(rv ? grow : 0u) * bpr * V41_Q4K_BYTES; + wmma::fragment acc[NFW]; + #pragma unroll + for (uint32_t f = 0; f < NFW; f++) wmma::fill_fragment(acc[f], 0.0f); + uint4 hdr = make_uint4(0, 0, 0, 0); uint2 qb = make_uint2(0, 0); uint4 bx[CH]; + #pragma unroll + for (uint32_t c = 0; c < CH; c++) bx[c] = make_uint4(0, 0, 0, 0); + auto load_round = [&](uint32_t i) { /* round i = elements [64i, 64i+64) = block i/4, quarter i%4 */ + const uint8_t *blk = rowp + (uint64_t)(i >> 2) * V41_Q4K_BYTES; + if (rv) { hdr = v41_glb_ld4(blk); const uint32_t *q = (const uint32_t *)(blk + 16u + (i & 3u) * 32u + sub * 8u); + qb.x = v41_glb_ld1(q); qb.y = v41_glb_ld1(q + 1); } + #pragma unroll + for (uint32_t c = 0; c < CH; c++) { + const uint32_t ch = tid + c * Q4W_THREADS, bt = ch / (Q4W_BK / 8u), bc = ch % (Q4W_BK / 8u); + if (bt < ntk) bx[c] = v41_glb_ld4(xg + (uint64_t)(t0 + bt) * x_stride + (uint64_t)i * Q4W_BK + bc * 8u); + } + }; + load_round(0); + for (uint32_t i = 0; i < nit; i++) { + { /* decode this thread's 8 bytes: low nibbles -> k 8*sub+j, high nibbles -> k 32+8*sub+j (v41_q4k_blk_acc_reg's map) */ + const uint32_t gq = i & 3u; /* the block's 64-element group = scale pairs (2gq, 2gq+1) */ + const float d = __half2float(__ushort_as_half((unsigned short)(hdr.x & 0xFFFFu))); + const float dmin = __half2float(__ushort_as_half((unsigned short)(hdr.x >> 16))); + float s_lo, m_lo, s_hi, m_hi; + v41_q4k_sm_reg(hdr, gq * 2u, &s_lo, &m_lo); + v41_q4k_sm_reg(hdr, gq * 2u + 1u, &s_hi, &m_hi); + const float dl = d * s_lo, ml = dmin * m_lo, dh = d * s_hi, mh = dmin * m_hi; + uint32_t lo[4], hi[4]; + #pragma unroll + for (int j = 0; j < 8; j += 2) { + const uint32_t word = (j < 4) ? qb.x : qb.y; + const uint32_t b0 = (word >> (8 * (j & 3))) & 0xFFu, b1 = (word >> (8 * (j & 3) + 8)) & 0xFFu; + const float l0 = dl * (float)(b0 & 0xFu) - ml, l1 = dl * (float)(b1 & 0xFu) - ml; + const float h0 = dh * (float)(b0 >> 4) - mh, h1 = dh * (float)(b1 >> 4) - mh; + lo[j / 2] = (__float_as_uint(v41_bf16r(l0)) >> 16) | (__float_as_uint(v41_bf16r(l1)) & 0xFFFF0000u); + hi[j / 2] = (__float_as_uint(v41_bf16r(h0)) >> 16) | (__float_as_uint(v41_bf16r(h1)) & 0xFFFF0000u); + } + *(uint4 *)(As + ar * Q4W_LD + sub * 8u) = make_uint4(lo[0], lo[1], lo[2], lo[3]); + *(uint4 *)(As + ar * Q4W_LD + 32u + sub * 8u) = make_uint4(hi[0], hi[1], hi[2], hi[3]); + } + #pragma unroll + for (uint32_t c = 0; c < CH; c++) { const uint32_t ch = tid + c * Q4W_THREADS; *(uint4 *)(Bs + (ch / (Q4W_BK / 8u)) * Q4W_LD + (ch % (Q4W_BK / 8u)) * 8u) = bx[c]; } + __syncthreads(); + if (i + 1u < nit) load_round(i + 1u); + #pragma unroll + for (uint32_t kk = 0; kk < Q4W_BK / 16u; kk++) { + wmma::fragment a; + wmma::load_matrix_sync(a, (const rocwmma::bfloat16_t *)(As + (wr * 16u) * Q4W_LD + kk * 16u), Q4W_LD); + #pragma unroll + for (uint32_t f = 0; f < NFW; f++) { + const uint32_t tk = (th * NFW + f) * 16u; + if (tk >= ntk) break; + wmma::fragment b; + wmma::load_matrix_sync(b, (const rocwmma::bfloat16_t *)(Bs + tk * Q4W_LD + kk * 16u), Q4W_LD); + wmma::fragment t; + wmma::fill_fragment(t, 0.0f); + wmma::mma_sync(t, a, b, t); + #pragma unroll + for (int e = 0; e < (int)t.num_elements; e++) acc[f].x[e] += t.x[e]; + } + } + __syncthreads(); + } + float *os = Os + warp * 256u; + #pragma unroll + for (uint32_t f = 0; f < NFW; f++) { + const uint32_t tk = (th * NFW + f) * 16u; + if (tk < ntk) wmma::store_matrix_sync(os, acc[f], 16, wmma::mem_row_major); + __syncthreads(); + if (tk < ntk) { + const uint32_t r = r0 + wr * 16u + (lane >> 1); + if (r < out_dim) { + #pragma unroll + for (uint32_t j = 0; j < 8u; j++) { + const uint32_t tl = (lane & 1u) * 8u + j, t = tk + tl; + if (t >= ntk) break; + og[(uint64_t)(t0 + t) * out_stride + r] = os[(lane >> 1) * 16u + tl]; + } + } + } + __syncthreads(); + } + } +} + +/* out[tok][row] (row-major with out_stride) = W (q4_K, out_dim rows x in_dim) . xb[tok] (bf16, x_stride). + * 1 = launched, 0 = shape refused (the caller falls back to hipBLAS). */ +static int v41_q4k_gemm_wmma(const uint8_t *w, const uint16_t *xb, uint32_t in_dim, uint32_t out_dim, uint32_t n_tok, uint32_t n_groups, + uint64_t w_gstride_blk, uint32_t x_stride, uint32_t out_stride, float *out, const char *what) { + static int s_nsm = 0; + if ((in_dim % V41_Q4K_BLK) != 0u || n_tok == 0u) return 0; + if (!s_nsm && hipDeviceGetAttribute(&s_nsm, hipDeviceAttributeMultiprocessorCount, 0) != hipSuccess) { (void)hipGetLastError(); return 0; } + const uint32_t mt = (out_dim + Q4W_BM - 1u) / Q4W_BM, nt = (n_tok + Q4W_BN - 1u) / Q4W_BN; + uint32_t grid = (uint32_t)s_nsm * 2u; + if (grid > mt * nt) grid = mt * nt; + q4w_kernel<<>>(out, w, xb, in_dim, out_dim, n_tok, w_gstride_blk, x_stride, out_stride); + return cuda_ok(hipGetLastError(), what) ? 1 : -1; +} +#endif diff --git a/cuda/ds41_router.cuh b/cuda/ds41_router.cuh new file mode 100644 index 0000000000000000000000000000000000000000..32d00115c5ddf9a61cbe4cdc2136676dacc40445 --- /dev/null +++ b/cuda/ds41_router.cuh @@ -0,0 +1,151 @@ +#include "hip/hip_runtime.h" +/* ds41_router.cuh — DeepSeek V4.1 (ds41) router and SwiGLU, ported from the C + * engine at /data/YoungAi (commit 3946dbc), file `src/cuda/cuda_v41_3.inc.cu`: + * - the router kernel and entry (:8-91, official Gate.forward: + * scores = linear(x, w); probs = sqrt(softplus(scores)); + * indices = topk(probs + bias); weights = probs[indices]/(sum + 1e-20) + * * route_scale. One warp per token, each lane holding 12 experts + * (384/32); one warp argmax per round, ties to the smaller expert id); + * - the SwiGLU kernel and entry (:93-107, official Expert: up clamped to + * +-limit, gate clamped to max=limit, h = silu(gate)*up rounded to bf16). + * + * The route-bias override store (ds4_gpu_v41_set_rb_override, :48-77) is the + * zchain sidecar's rb half: it keeps a per-layer device copy of (on-disk + * exp_probs_b + delta) and the router entry consults it by bias_offset. No + * mounted table, no delta — the router then reads the on-disk bias unchanged. + */ +#pragma once + +/* 12 experts per lane covers 384 experts (384/32); the entry refuses above + * that. softplus uses torch's threshold 20 (z > 20 => z, else log1pf(expf)). */ +#define V41_ROUTER_PER_LANE 12u + +__global__ static void v41_router_kernel(int32_t *sel, float *wts, const float *logits, const float *bias_txt, + const float *bias_vl, const int32_t *vl_rows, + uint32_t n_tok, uint32_t n_expert, uint32_t topk, float route_scale) { + v41_pdl_wait(); /* PDL: first instruction waits for the upstream; no-op when not launched through PDL */ + const uint32_t t = blockIdx.x * (blockDim.x >> 5) + (threadIdx.x >> 5), lane = threadIdx.x & 31u; + if (t >= n_tok) return; + const float *lg = logits + (uint64_t)t * n_expert; + /* Image positions use bias_vl (official torch.where(image_mask, bias_vl, bias)); + * all 32 lanes read the same predicate, so the warp does not diverge. */ + const float *bias = (bias_vl && vl_rows && vl_rows[t] >= 0) ? bias_vl : bias_txt; + float pr[V41_ROUTER_PER_LANE], sc[V41_ROUTER_PER_LANE]; + #pragma unroll + for (uint32_t k = 0; k < V41_ROUTER_PER_LANE; k++) { + const uint32_t e = lane + 32u * k; + if (e < n_expert) { + const float z = lg[e]; + const float sp = z > 20.0f ? z : log1pf(expf(z)); /* softplus (beta=1, threshold 20 as in torch) */ + pr[k] = sqrtf(sp); sc[k] = pr[k] + bias[e]; + } else { pr[k] = 0.f; sc[k] = -INFINITY; } + } + uint32_t used = 0; float wsum = 0.f; + for (uint32_t r = 0; r < topk; r++) { + float bv = -INFINITY; int bk = -1; + #pragma unroll + for (uint32_t k = 0; k < V41_ROUTER_PER_LANE; k++) if (!((used >> k) & 1u) && sc[k] > bv) { bv = sc[k]; bk = (int)k; } + int be = bk >= 0 ? (int)(lane + 32u * (uint32_t)bk) : -1; + for (int off = 16; off > 0; off >>= 1) { + const float ov = __shfl_xor_sync(0xffffffffu, bv, off); const int oe = __shfl_xor_sync(0xffffffffu, be, off); + if (oe >= 0 && (be < 0 || ov > bv || (ov == bv && oe < be))) { bv = ov; be = oe; } + } + float myp = 0.f; + if (be >= 0 && (uint32_t)(be & 31) == lane) { const uint32_t k = (uint32_t)be >> 5; used |= 1u << k; myp = pr[k]; } + for (int off = 16; off > 0; off >>= 1) myp += __shfl_xor_sync(0xffffffffu, myp, off); /* only one lane is nonzero */ + /* Round results land immediately (2026-09-18): the old chosen[16]/ + * chosen_p[16] arrays used runtime (topk) indices and fell into local + * memory. The unnormalized myp is stored and divided by wsum at the + * end: the formula myp/(wsum+1e-20)*route_scale is unchanged. */ + if (lane == 0) { sel[(uint64_t)t * topk + r] = be; wts[(uint64_t)t * topk + r] = myp; } + wsum += myp; + } + if (lane == 0) + for (uint32_t r = 0; r < topk; r++) wts[(uint64_t)t * topk + r] = wts[(uint64_t)t * topk + r] / (wsum + 1e-20f) * route_scale; +} + +/* Route-bias override store (cuda_v41_3.inc.cu:48-77): per layer (keyed by + * the exp_probs_b file offset — the 40 main layers and each tower have + * distinct offsets, so the router signature does not change) a device copy of + * (on-disk bias + delta). The delta moves the SELECTION score only; the + * weight score (the sqrtsoftplus probability) is untouched. host_delta NULL + * unloads: bias_offset 0 unloads all, else just that layer. The table is + * process-level, so a fresh mount unloads first (the engine's loader does). + * Mounted means 100% applied; the on-disk file is never written. */ +static struct { uint64_t off; float *dev; uint32_t n; } g_v41_rb[64]; +static uint32_t g_v41_rb_n = 0; + +extern "C" int ds4_gpu_v41_set_rb_override(const void *model_map, uint64_t model_size, uint64_t bias_offset, const float *host_delta, uint32_t n_expert) { + if (!host_delta) { /* unload: offset 0 = all layers */ + for (uint32_t i = 0; i < g_v41_rb_n;) { + if (bias_offset == 0 || g_v41_rb[i].off == bias_offset) { (void)hipFree(g_v41_rb[i].dev); g_v41_rb[i] = g_v41_rb[--g_v41_rb_n]; } + else i++; + } + return 1; + } + if (!model_map || !n_expert || bias_offset > model_size || (uint64_t)n_expert * 4 > model_size - bias_offset) return 0; + const float *base = (const float *)cuda_model_range_ptr(model_map, bias_offset, (uint64_t)n_expert * 4, "v41 rb base"); + if (!base) return 0; + float *h = (float *)malloc((size_t)n_expert * 4); + if (!h) return 0; + /* the on-disk bias is a device copy or the host mapping: hipMemcpyDefault + * reads by UVA pointer (the engine's comment, cuda_v41_3.inc.cu:64) */ + if (hipMemcpy(h, base, (size_t)n_expert * 4, hipMemcpyDefault) != hipSuccess) { (void)hipGetLastError(); free(h); return 0; } + for (uint32_t e = 0; e < n_expert; e++) h[e] += host_delta[e]; + uint32_t i = 0; + for (; i < g_v41_rb_n; i++) if (g_v41_rb[i].off == bias_offset) break; + if (i == g_v41_rb_n) { + if (g_v41_rb_n >= 64u) { free(h); return 0; } + if (hipMalloc((void **)&g_v41_rb[i].dev, (size_t)n_expert * 4) != hipSuccess) { (void)hipGetLastError(); free(h); return 0; } + g_v41_rb[i].off = bias_offset; g_v41_rb[i].n = n_expert; g_v41_rb_n++; + } else if (g_v41_rb[i].n != n_expert) { free(h); return 0; } + const int ok = hipMemcpy(g_v41_rb[i].dev, h, (size_t)n_expert * 4, hipMemcpyHostToDevice) == hipSuccess; + if (!ok) (void)hipGetLastError(); + free(h); + return ok; +} + +/* The on-disk bias (or the zchain-mounted copy) as a device pointer; the text + * bias and the image bias_vl take the same path, keyed by offset - a layer with + * no vl bias (V4 / an old GGUF) has only the text one (engine cuda_v41_3.inc.cu). */ +static const float *v41_router_bias_ptr(const void *model_map, uint64_t model_size, uint64_t off, uint32_t n_expert, const char *what) { + if (off > model_size || (uint64_t)n_expert * 4 > model_size - off) return NULL; + const float *bias = (const float *)cuda_model_range_ptr(model_map, off, (uint64_t)n_expert * 4, what); + if (!bias) return NULL; + for (uint32_t i = 0; i < g_v41_rb_n; i++) if (g_v41_rb[i].off == off && g_v41_rb[i].n == n_expert) return g_v41_rb[i].dev; + return bias; +} +extern "C" int ds4_gpu_v41_router_tensor(ds4_gpu_tensor *selected, ds4_gpu_tensor *weights, const ds4_gpu_tensor *logits, + const void *model_map, uint64_t model_size, uint64_t bias_offset, + const ds4_gpu_tensor *bias_vl, const ds4_gpu_tensor *vl_rows, + uint32_t n_tok, uint32_t n_expert, uint32_t topk, float route_scale) { + if (!selected || !weights || !logits || topk > 16u || n_expert > 32u * V41_ROUTER_PER_LANE) return 0; + const float *bias = v41_router_bias_ptr(model_map, model_size, bias_offset, n_expert, "v41 gate bias"); + if (!bias) return 0; + const bool vl = bias_vl && vl_rows && bias_vl->ptr && vl_rows->ptr; + v41_router_kernel<<<(n_tok + 7u) / 8u, 256, 0, ds4_current_stream()>>>((int32_t *)selected->ptr, (float *)weights->ptr, + (const float *)logits->ptr, bias, vl ? (const float *)bias_vl->ptr : NULL, vl ? (const int32_t *)vl_rows->ptr : NULL, + n_tok, n_expert, topk, route_scale); + return cuda_ok(hipGetLastError(), "v41 router"); +} + +/* Official Expert.forward: gate=w1(x).float(); up=w3(x).float(); + * up=clamp(+-limit); gate=clamp(max=limit); h = silu(gate)*up -> + * (the routing weight multiply happens outside) -> .to(bf16). gate/up here + * are the bf16 outputs of the fp4 linear (the caller rounded them already). */ +__global__ static void v41_swiglu_kernel(float *h, const float *g, const float *u, uint64_t n, float limit) { + v41_pdl_wait(); /* PDL: first instruction waits for the upstream; no-op when not launched through PDL */ + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + float gv = g[i], uv = u[i]; + if (limit > 0.0f) { uv = fminf(fmaxf(uv, -limit), limit); gv = fminf(gv, limit); } + h[i] = v41_bf16r((gv / (1.0f + expf(-gv))) * uv); +} + +extern "C" int ds4_gpu_v41_swiglu_tensor(ds4_gpu_tensor *h, const ds4_gpu_tensor *gate, const ds4_gpu_tensor *up, + uint32_t n_tok, uint32_t mid, float limit) { + if (!h || !gate || !up) return 0; + const uint64_t n = (uint64_t)n_tok * mid; + v41_swiglu_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>((float *)h->ptr, (const float *)gate->ptr, (const float *)up->ptr, n, limit); + return cuda_ok(hipGetLastError(), "v41 swiglu"); +} diff --git a/cuda/ds41_sample.cuh b/cuda/ds41_sample.cuh new file mode 100644 index 0000000000000000000000000000000000000000..15f1274e1c2ca7d0d871052585f7db95da670b0c --- /dev/null +++ b/cuda/ds41_sample.cuh @@ -0,0 +1,311 @@ +#include "hip/hip_runtime.h" +/* cuda/ds41_sample.cuh — the V4.1 device sampling kernel, ported from the C + * engine at /data/YoungAi (commit 3946dbc): src/cuda/cuda_v41_sample.inc.cu + * (2026-09-28). Temperature / top-k / top-p / min-p all run on the GPU, one + * block per row, each row's result lands in a 16-byte device slot (the SAME + * slot the argmax kernel uses, zero-copy readback); the speculative rejection + * sampling comes out of the same launch. + * + * Why it exists (engine header comment): after 2026-09-28 made sampling the + * default (temperature 1.0, the model-card recipe), a request without a + * temperature took the sampling path — which used to read all 129280 logits + * back to the host per step, run two expf passes there, and turn speculation + * off. Moving the pick into the decode-step graph's tail (replacing the + * argmax launch) leaves the host reading back 16 bytes; speculation stays on. + * + * The draw is Gumbel-max: argmax_i (l_i/T + g_i), g_i = -ln(-ln u_i), u_i iid + * uniform, so i is picked with probability exactly softmax(l/T)_i (an exact + * equivalence, not an approximation); restricting to the kept set K is the + * truncated-then-renormalized distribution. u_i is hashed from (seed, + * position, i, salt) with splitmix64's final mix — no dependence on thread + * scheduling, so same seed + same input always gives the same result, and the + * graph and direct routes (same kernel) emit the same token stream. + * Kept set K = {l_i >= L_cut}, L_cut = max(min_p gate, top_k gate, top_p + * gate): min_p: l >= M + T*ln(min_p) (relative to the max probability); + * top_k: the k-th largest logit (radix select over 4 x 8-bit histogram + * passes, integer counts -> deterministic); top_p: the largest-to-smallest + * cumulative probability first reaching top_p (same radix select, masses + * accumulated as 2^40 fixed-point integers -> deterministic, floating-point + * atomic-add order cannot move the threshold). Semantics match the host + * ds4_sample_logits: top_k first, top_p relative to the post-cut total mass, + * min_p relative to the max probability; ties at a boundary are all kept + * (the host's qsort cuts ties in an arbitrary order of its own). + * Speculation (rejection sampling, Leviathan 2023 / Chen 2023): row i's draft + * d = tok[i+1] (the next row's input in the verify batch). Point-mass draft + * (qlogits == NULL): accept probability = p(d), reject draws from the + * "p minus d, renormalized" residual; distribution draft (qlogits = the + * draft tower's row logits, 2026-09-29): accept = min(1, p(d)/q(d)), reject + * draws from max(0, p - q) renormalized — both emit tokens whose marginal + * distribution is exactly p; the latter's accept-rate ceiling is sum min(p,q), + * far above the point mass's p(argmax q). The accept coin and the Gumbel + * noise use different salts, and the draft draw (stream 1) and the verify + * (stream 0) use different streams, so everything is independent. The + * residual is empty (p == q, or K subset of {d}) -> accept outright. The + * last row has no draft and only emits its full-distribution sample. + * Error behavior: an all-non-finite row writes four zero ints (same fallback + * as the host sample_argmax: index 0); temperature <= 0 must not enter this + * kernel (the caller runs the argmax kernel — that is the greedy gate). */ +#ifndef DS41_SAMPLE_CUH +#define DS41_SAMPLE_CUH + +#define V41_SAMPLE_THREADS 1024u +#define V41_SAMPLE_BINS 256u +/* 2^40: fixed-point resolution relative to the max probability. Every vocab + * entry <= 1, so the total stays < 2^58 and cannot overflow u64; entries + * below 2^-40 relative probability count as 0 mass — they cannot move a + * top_p threshold anyway (host float accumulation absorbs no such terms + * either). */ +#define V41_SAMPLE_FIX 1099511627776.0 + +__device__ __forceinline__ static uint64_t v41_mix64(uint64_t x) { + x ^= x >> 30; x *= 0xBF58476D1CE4E5B9ull; x ^= x >> 27; x *= 0x94D049BB133111EBull; x ^= x >> 31; return x; +} +/* A full-precision uniform on (0,1): the binary exponent comes from the + * leading-zero count of the remaining 41 bits (geometric, P(n) = 2^-(n+1)), + * the mantissa takes the top 23 bits -> value = (1 + m*2^-23) * 2^-(n+1), + * 24-bit resolution in every binary bucket, finer toward 0 (down to 2^-42) — + * matching float's own resolution structure; upper bound 1-2^-24, never 0 or 1. + * Why this fuss (the engine's three crashes, judge = tests/cuda_sample_selftest.c + * counting low-p tokens): + * 1. 24-bit integer + 0.5 times 2^-24: at k >= 2^23, k+0.5 exceeds fp32's + * 24 significant bits, rounds to even, k = 2^24-1 rounds to 2^24 => u = + * 1.0f => -ln(-ln 1) = +inf => that token wins the argmax unconditionally. + * With top_p=1 the kept set is the whole vocab; each slot hit with + * probability 1-(1-2^-24)^129280 = 0.77%/stream — a uniformly random + * word every ~130 tokens. In a 200k-token report, 1,880 (846 verify + + * 1,034 draft); the body degraded into multilingual noise and the model + * could not close. Gate read 1,720 vs 558 expected. + * 2. 23 bits + half-cell (exact, below 1) still red at 1,363: fast-math's + * __logf has absolute error 3.6e-7 near 1, which scrambles the |ln u| + * ~1e-7 upper tail — fixed by log1pf (see the kernel). + * 3. After that the gate read 347, too few: the uniform near 0 only had + * 2^-23 absolute resolution, flattening the Gumbel upper tail, and + * probability-1e-7 tokens were sampled ~40% short. This version solves + * 3: the uniform is bucketed the way float is, upper tail fine to 2^-42. */ +__device__ __forceinline__ static float v41_u01(uint64_t seed, int32_t pos, uint32_t i, uint32_t salt) { + const uint64_t x = v41_mix64(seed ^ (0x9E3779B97F4A7C15ull * (uint64_t)(uint32_t)pos + 0xD1B54A32D192ED03ull * (uint64_t)i + + 0x8CB92BA72F3D8DD7ull * (uint64_t)(salt + 1u))); + const uint32_t m = (uint32_t)(x >> 41); /* 23-bit mantissa */ + const uint64_t r = x << 23; /* the remaining 41 bits moved high, count leading zeros */ + const int n = r ? __clzll((long long)r) : 41; /* all-zero caps at 41 => smallest bucket 2^-42, still a normal float */ + return __uint_as_float(((uint32_t)(126 - n) << 23) | m); /* exponent field 127-(n+1), mantissa field m */ +} +/* float <-> monotonic u32 key (ascending key = ascending float), finite values only */ +__device__ __forceinline__ static uint32_t v41_fkey(float f) { const uint32_t u = __float_as_uint(f); return (u & 0x80000000u) ? ~u : (u | 0x80000000u); } + +/* Block argmax (ties take the lower index; index -1 = empty, always loses) */ +__device__ static void v41_blk_argmax(float *sf, int32_t *si, float v, int32_t i, float *ov, int32_t *oi) { + sf[threadIdx.x] = v; si[threadIdx.x] = i; __syncthreads(); + for (uint32_t k = blockDim.x / 2u; k > 0u; k >>= 1) { + if (threadIdx.x < k) { + const float bv = sf[threadIdx.x + k]; const int32_t bi = si[threadIdx.x + k]; + const float av = sf[threadIdx.x]; const int32_t ai = si[threadIdx.x]; + if (bi >= 0 && (ai < 0 || bv > av || (bv == av && bi < ai))) { sf[threadIdx.x] = bv; si[threadIdx.x] = bi; } + } + __syncthreads(); + } + *ov = sf[0]; *oi = si[0]; __syncthreads(); +} +/* Block float sum: fixed tree order => deterministic (no atomic adds) */ +__device__ static float v41_blk_sumf(float *sf, float v) { + sf[threadIdx.x] = v; __syncthreads(); + for (uint32_t k = blockDim.x / 2u; k > 0u; k >>= 1) { if (threadIdx.x < k) sf[threadIdx.x] += sf[threadIdx.x + k]; __syncthreads(); } + const float s = sf[0]; __syncthreads(); return s; +} + +/* Radix select: over {finite and key >= floor_key}, accumulate weight from the + * largest key down, return the key of the first item reaching target. + * mass=0: weight = 1 (top_k, target = k); mass=1: weight = fixed-point + * probability (top_p, target = ceil(top_p*Z)). Four passes x 256 buckets + * (shared integer atomic adds, order-independent) -> scan from the high + * bucket down to the one holding the threshold -> next pass looks only inside + * that bucket. Returns the threshold item's 32-bit key (everything >= it is + * kept, ties together). When target exceeds the total (only top_k > finite + * count) it returns 0 = keep everything. */ +__device__ static uint32_t v41_radix_select(const float *l, uint32_t V, uint32_t floor_key, int mass, float M, float inv_T, + unsigned long long target, unsigned long long *hist, uint32_t *s_prefix, unsigned long long *s_target) { + uint32_t prefix = 0u; + for (int shift = 24; shift >= 0; shift -= 8) { + const uint32_t mask = shift == 24 ? 0u : (0xFFFFFFFFu << (shift + 8)); + for (uint32_t b = threadIdx.x; b < V41_SAMPLE_BINS; b += blockDim.x) hist[b] = 0ull; + __syncthreads(); + for (uint32_t i = threadIdx.x; i < V; i += blockDim.x) { + const float v = l[i]; + if (!isfinite(v)) continue; + const uint32_t k = v41_fkey(v); + if (k < floor_key || (k & mask) != prefix) continue; + unsigned long long w = 1ull; + if (mass) w = (unsigned long long)((double)expf((v - M) * inv_T) * V41_SAMPLE_FIX + 0.5); + atomicAdd(&hist[(k >> shift) & 0xFFu], w); + } + __syncthreads(); + if (threadIdx.x == 0) { + unsigned long long cum = 0ull; uint32_t sel = 0u; unsigned long long rest = target; + for (int b = (int)V41_SAMPLE_BINS - 1; b >= 0; b--) { + cum += hist[b]; + if (cum >= target) { sel = (uint32_t)b; rest = target - (cum - hist[b]); break; } + } + *s_prefix = prefix | (sel << shift); *s_target = rest; + } + __syncthreads(); + prefix = *s_prefix; target = *s_target; + __syncthreads(); + if (target == 0ull) return 0u; /* scanned every bucket without reaching target => keep everything; otherwise the remainder is >= 1 */ + } + return prefix; +} + +/* One row's three gates: returns the key floor cut (0 = keep everything), + * *M = the finite maximum (*Mi < 0 = the whole row is non-finite, caller + * falls back). */ +__device__ static uint32_t v41_sample_gate(const float *l, uint32_t V, float inv_T, float min_p, uint32_t top_k, float top_p, + float *sf, int32_t *si, unsigned long long *hist, uint32_t *s_prefix, unsigned long long *s_u64, + float *M, int32_t *Mi) { + float best = -INFINITY; int32_t bi = -1; + for (uint32_t i = threadIdx.x; i < V; i += blockDim.x) { const float v = l[i]; if (isfinite(v) && v > best) { best = v; bi = (int32_t)i; } } + v41_blk_argmax(sf, si, best, bi, M, Mi); + if (*Mi < 0) return 0u; + uint32_t cut = 0u; + if (min_p > 0.0f) cut = v41_fkey(*M + logf(min_p) / inv_T); + uint32_t kfloor = 0u; + if (top_k > 0u && top_k < V) { + kfloor = v41_radix_select(l, V, 0u, 0, *M, inv_T, (unsigned long long)top_k, hist, s_prefix, s_u64); + if (kfloor > cut) cut = kfloor; + } + if (top_p < 1.0f) { + if (threadIdx.x == 0) *s_u64 = 0ull; + __syncthreads(); + unsigned long long z = 0ull; + for (uint32_t i = threadIdx.x; i < V; i += blockDim.x) { + const float v = l[i]; + if (!isfinite(v) || v41_fkey(v) < kfloor) continue; + z += (unsigned long long)((double)expf((v - *M) * inv_T) * V41_SAMPLE_FIX + 0.5); + } + atomicAdd(s_u64, z); + __syncthreads(); + const unsigned long long Z = *s_u64; + __syncthreads(); + unsigned long long target = (unsigned long long)ceil((double)top_p * (double)Z); + if (target == 0ull) target = 1ull; + const uint32_t pfloor = v41_radix_select(l, V, kfloor, 1, *M, inv_T, target, hist, s_prefix, s_u64); + if (pfloor > cut) cut = pfloor; + } + return cut; +} + +__global__ static void v41_sample_kernel(int32_t *out, const float *logits, const float *qlogits, uint32_t V, const int32_t *pos, const int32_t *tok, + uint32_t row0, uint32_t n_rows, float inv_T, float min_p, uint32_t top_k, float top_p, + uint64_t seed, uint32_t stream) { + v41_pdl_wait(); /* upstream = the head writing logits; returns immediately when not launched with PDL */ + __shared__ float sf[V41_SAMPLE_THREADS]; __shared__ int32_t si[V41_SAMPLE_THREADS]; + __shared__ unsigned long long hist[V41_SAMPLE_BINS]; + __shared__ uint32_t s_prefix; __shared__ unsigned long long s_u64; + const uint32_t r = row0 + blockIdx.x; + const float *l = logits + (size_t)r * V; + const int32_t p = pos[r]; + const int32_t d = blockIdx.x + 1u < n_rows ? tok[r + 1u] : -1; + const float *lq = (d >= 0 && qlogits) ? qlogits + (size_t)blockIdx.x * V : NULL; + const uint32_t salt_g = 2u * stream, salt_a = 2u * stream + 1u; /* Gumbel noise / accept coin, split by stream */ + int32_t *o = out + 4u * blockIdx.x; + float M; int32_t Mi; + const uint32_t cut = v41_sample_gate(l, V, inv_T, min_p, top_k, top_p, sf, si, hist, &s_prefix, &s_u64, &M, &Mi); + if (Mi < 0) { if (threadIdx.x == 0) { o[0] = 0; o[1] = 0; o[2] = 0; o[3] = 0; } return; } + /* The draft distribution q's gates and both normalization constants (a + * point-mass draft needs none: the residual is p minus d, the ratio needs + * no normalization). */ + float Mq = 0.0f; int32_t Mqi = -1; uint32_t cutq = 0u; float Zp = 1.0f, Zq = 1.0f; + if (lq) { + cutq = v41_sample_gate(lq, V, inv_T, min_p, top_k, top_p, sf, si, hist, &s_prefix, &s_u64, &Mq, &Mqi); + if (Mqi < 0) lq = NULL; /* the q row is all non-finite: treat it as a point-mass draft */ + } + if (lq) { + float zp = 0.0f, zq = 0.0f; + for (uint32_t i = threadIdx.x; i < V; i += blockDim.x) { + const float v = l[i], vq = lq[i]; + if (isfinite(v) && v41_fkey(v) >= cut) zp += expf((v - M) * inv_T); + if (isfinite(vq) && v41_fkey(vq) >= cutq) zq += expf((vq - Mq) * inv_T); + } + Zp = v41_blk_sumf(sf, zp); Zq = v41_blk_sumf(sf, zq); + } + /* Gumbel-max over the kept set: the full-distribution sample; the residual + * sample (q: over max(0, p-q) / point mass: p minus d); and along the way + * Z_K, and the numerators of p(d) and q(d). */ + float gb = -INFINITY, gxb = -INFINITY, zk = 0.0f, md = 0.0f, mqd = 0.0f; int32_t gi = -1, gxi = -1; + for (uint32_t i = threadIdx.x; i < V; i += blockDim.x) { + const float v = l[i]; + /* q(d) must be computed even outside p's kept set (the 09-29 selftest + * conviction: the draft fell outside K_p, this used to skip it, q(d) + * stayed 0 and was read as "a draft that cannot be drawn" and + * accepted, emitting a token outside the kept set) — a draft outside + * K_p means p(d) = 0 => reject. */ + if (lq && (int32_t)i == d) { const float vq = lq[i]; mqd = (isfinite(vq) && v41_fkey(vq) >= cutq) ? expf((vq - Mq) * inv_T) : 0.0f; } + if (!isfinite(v) || v41_fkey(v) < cut) continue; + const float m = expf((v - M) * inv_T); + zk += m; + if ((int32_t)i == d) md = m; + /* Gumbel g = -ln E, E = -ln(1-u) ~ Exp(1) (u uniform => 1-u uniform). + * Why log1pf(-u) and not logf(u) (the 09-29 distribution-gate + * conviction): nvcc --use_fast_math turns logf into __logf, which has + * absolute error 2^-21.4 ~ 3.6e-7 on [0.5, 2], while the u values + * closest to 1 have |ln u| of only 0.6~3e-7 — the computed E could be + * 0 (=> g = +inf, always wins), negative (=> NaN, never wins) or off + * by a factor, and this is exactly the tail that decides whether an + * extremely low-probability word wins. With u == 1.0f fixed the gate + * stayed red (low-p tokens 1,363 vs 558 expected, 32.7 sigma) because + * of it. log1pf has no fast-math replacement and keeps relative + * accuracy across small arguments; the outer -logf(E) has E in + * [2^-24, 16.7] where __logf's error is ulp-level, ~1e-5 on g, + * harmless. */ + const float u = v41_u01(seed, p, i, salt_g); + const float g = -logf(-log1pf(-u)); + const float key = v * inv_T + g; + if (key > gb) { gb = key; gi = (int32_t)i; } + if (lq) { + const float vq = lq[i]; + const float qm = (isfinite(vq) && v41_fkey(vq) >= cutq) ? expf((vq - Mq) * inv_T) : 0.0f; + const float rres = m / Zp - qm / Zq; + if (rres > 0.0f) { const float rk = logf(rres) + g; if (rk > gxb) { gxb = rk; gxi = (int32_t)i; } } + } else if ((int32_t)i != d && key > gxb) { gxb = key; gxi = (int32_t)i; } + } + float tv; int32_t full, resid; + v41_blk_argmax(sf, si, gb, gi, &tv, &full); + v41_blk_argmax(sf, si, gxb, gxi, &tv, &resid); + const float ZK = v41_blk_sumf(sf, zk); + const float MD = v41_blk_sumf(sf, md); /* only one thread holds d's term, the sum is the value */ + const float MQD = v41_blk_sumf(sf, mqd); + if (threadIdx.x == 0) { + int32_t acc = 0; + if (d >= 0) { + const float pd = MD / ZK; /* d not in K_p => MD = 0 => reject (residual non-empty; should it ever be empty [2] falls back to the full sample, still a sample of p) */ + if (pd <= 0.0f) acc = 0; + else if (resid < 0) acc = 1; /* residual empty (K_p subset of {d}, or p == q): ratio = 1, accept */ + else if (lq) { + const float qd = MQD / Zq; /* q(d) = 0 only at a numerical edge (the draft was drawn from q): ratio inf => accept */ + acc = (qd <= 0.0f || v41_u01(seed, p, (uint32_t)d, salt_a) < pd / qd) ? 1 : 0; + } else acc = v41_u01(seed, p, (uint32_t)d, salt_a) < pd ? 1 : 0; + } + o[0] = full; o[1] = acc; o[2] = resid >= 0 ? resid : full; o[3] = Mi; // [3] = the raw argmax (the tool-syntax greedy hook, c6e2706) + } +} + +extern "C" int ds4_gpu_v41_sample_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *logits, uint32_t row0, uint32_t n_rows, uint32_t n_vocab, + const ds4_gpu_tensor *pos, const ds4_gpu_tensor *tok, const ds4_gpu_sample_params *sp, const ds4_gpu_tensor *qlogits) { + if (!out || !logits || !pos || !tok || !sp || n_rows == 0u || n_vocab == 0u) return 0; + if (!(sp->temperature > 0.0f)) return 0; /* greedy is the argmax kernel's job, this one does not pretend */ + const uint64_t last = (uint64_t)row0 + n_rows; + if (out->bytes < (uint64_t)n_rows * 16u || logits->bytes < last * n_vocab * 4u || pos->bytes < last * 4u || tok->bytes < last * 4u) return 0; + if (qlogits && (row0 != 0u || (n_rows > 1u && qlogits->bytes < (uint64_t)(n_rows - 1u) * n_vocab * 4u))) return 0; + /* Parameter conventions match the host ds4_sample_logits: top_p outside + * (0,1] counts as 1; negative min_p as 0; top_k <= 0 = no cut (the device + * has no host's 1024 stack limit). */ + float top_p = sp->top_p; if (!(top_p > 0.0f) || top_p > 1.0f) top_p = 1.0f; + const float min_p = sp->min_p > 0.0f ? sp->min_p : 0.0f; + const uint32_t top_k = sp->top_k > 0 ? (uint32_t)sp->top_k : 0u; + v41_sample_kernel<<>>((int32_t *)out->ptr, (const float *)logits->ptr, + qlogits ? (const float *)qlogits->ptr : NULL, n_vocab, + (const int32_t *)pos->ptr, (const int32_t *)tok->ptr, row0, n_rows, + 1.0f / sp->temperature, min_p, top_k, top_p, sp->seed, sp->stream); + return cuda_ok(hipGetLastError(), "v41 sample launch"); +} + +#endif /* DS41_SAMPLE_CUH */ diff --git a/cuda/ds41_vision.cuh b/cuda/ds41_vision.cuh new file mode 100644 index 0000000000000000000000000000000000000000..306f254963437a08be822696b8af000d22fb1a97 --- /dev/null +++ b/cuda/ds41_vision.cuh @@ -0,0 +1,254 @@ +#include "hip/hip_runtime.h" +/* cuda/ds41_vision.cuh — DeepSeek V4.1 vision tower (DeepSeek-ViT + aligner) + * device primitives. 1:1 port of the engine's + * src/cuda/cuda_v41_vision.inc.cu (2026-10-13). + * + * Only a handful of primitives are new: the ViT matmuls (patch_embed, wqkv, + * wo, w1/w2, both aligner layers) are BF16 and take the existing bf16 path; + * rms_norm / residual add / bf16 rounding exist. Missing: the Linear bias, + * 2D RoPE, unmasked dense attention, untruncated SwiGLU, GELU, the aligner's + * 3x3 pixel re-arrange, and the two row helpers that place the visual rows. + * Discipline: f32 compute, bf16-rounded at the official module boundaries + * (v41_bf16r), the same as the trunk. + * + * ★Weights do NOT live in the model mapping★ (core_v41_vision.c): an image + * streams them off the sidecar mmap into a temporary device tensor per block, + * so the two "dev tensor" entries below exist (ds4_gpu_v41_matmul_bf16_dev_tensor + * and ds4_gpu_v41_rms_norm_dev_tensor). The engine split the same two out of + * its model-map variants (cuda_v41_1.inc.cu:302). + * + * Getting any of this wrong does not error - the model still generates, it + * just answers about the image wrongly. The gate is tools/vision_golden.py's + * official aligner rows compared per-row by cosine (core_v41_vision_gate.c). */ + +/* ---- the two model-map entries, retargeted at a device tensor ---- */ +/* CUDA v41_bf16_matmul_ptr (cuda_v41_1.inc.cu:302-316): W is already a + * device-readable pointer. The GEMV/GEMM dispatch is preserved exactly so the + * tower's numerics match the trunk's (and the engine's golden). */ +extern "C" int ds4_gpu_v41_matmul_bf16_dev_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *W, + uint64_t in_dim, uint64_t out_dim, + const ds4_gpu_tensor *x, uint32_t n_tok) { + if (!out || !W || !x || n_tok == 0 || W->bytes < in_dim * out_dim * 2) return 0; + return v41_bf16_matmul_ptr((const __hip_bfloat16 *)W->ptr, in_dim, out_dim, x, out, n_tok); +} +extern "C" int ds4_gpu_v41_rms_norm_dev_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *x, const ds4_gpu_tensor *w, + uint32_t dim, uint32_t n_tok, float eps) { + if (!out || !x || !w || w->bytes < (uint64_t)dim * 4) return 0; + v41_rms_norm_kernel<<>>((float *)out->ptr, (const float *)x->ptr, + (const float *)w->ptr, dim, eps); + return cuda_ok(hipGetLastError(), "vision rms norm"); +} + +/* ---- the ten new primitives (cuda_v41_vision.inc.cu) ---- */ +static v41_scratch g_vis_qkv16, g_vis_s, g_vis_p; + +/* x[i] = bf16(x[i] + bias[i % dim]) — the official Linear's bias, rounded to + * bf16 in the same step (cuda_v41_vision.inc.cu:12). */ +__global__ static void v41_vision_bias_add_kernel(float *x, const float *b, uint64_t n, uint32_t dim) { + v41_pdl_wait(); + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + x[i] = v41_bf16r(x[i] + b[i % dim]); +} +extern "C" int ds4_gpu_v41_vision_bias_add_tensor(ds4_gpu_tensor *x, const ds4_gpu_tensor *bias, uint32_t n_tok, uint32_t dim) { + if (!x || !bias || !n_tok || !dim || bias->bytes < (uint64_t)dim * 4) return 0; + const uint64_t n = (uint64_t)n_tok * dim; + v41_vision_bias_add_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>( + (float *)x->ptr, (const float *)bias->ptr, n, dim); + return cuda_ok(hipGetLastError(), "vision bias add"); +} + +/* 2D RoPE (cuda_v41_vision.inc.cu:38-57): one thread per (token, head, j= total) return; + const uint32_t j = (uint32_t)(i % rd), h = (uint32_t)((i / rd) % n_head), t = (uint32_t)(i / ((uint64_t)rd * n_head)); + const uint32_t fi = j < nf ? j : j - nf; + const float pos = j < nf ? (float)(t / n_w) : (float)(t % n_w); + const float inv = 1.0f / powf(theta, (float)(2u * fi) / (float)rd); + const float f = pos * inv, c = cosf(f), s = sinf(f); + const uint64_t row = (uint64_t)t * 3u * n_head * head_dim; + #pragma unroll + for (uint32_t part = 0; part < 2u; part++) { /* 0 = q, 1 = k */ + float *b = qkv + row + (uint64_t)part * n_head * head_dim + (uint64_t)h * head_dim; + const float x1 = b[j], x2 = b[j + rd]; + b[j] = v41_bf16r(x1 * c - x2 * s); + b[j + rd] = v41_bf16r(x2 * c + x1 * s); + } +} +extern "C" int ds4_gpu_v41_vision_rope_tensor(ds4_gpu_tensor *qkv, uint32_t n_tok, uint32_t n_w, uint32_t n_head, + uint32_t head_dim, float theta) { + if (!qkv || !n_tok || !n_w || !n_head || (head_dim % 4u) != 0u) return 0; + const uint64_t total = (uint64_t)n_tok * n_head * (head_dim / 2u); + v41_vision_rope_kernel<<<(unsigned)((total + 255) / 256), 256, 0, ds4_current_stream()>>>( + (float *)qkv->ptr, n_tok, n_w, n_head, head_dim, theta); + return cuda_ok(hipGetLastError(), "vision rope"); +} + +/* Row softmax: one block per row, scale then max-subtract then exp then + * normalize, output bf16 for the P·V tensor-core product + * (cuda_v41_vision.inc.cu:66-81). */ +__global__ static void v41_vision_softmax_kernel(__hip_bfloat16 *P, const float *S, uint32_t nk, float scale) { + v41_pdl_wait(); + __shared__ float red[256]; + const float *s = S + (uint64_t)blockIdx.x * nk; + __hip_bfloat16 *p = P + (uint64_t)blockIdx.x * nk; + float m = -INFINITY; + for (uint32_t k = threadIdx.x; k < nk; k += blockDim.x) m = fmaxf(m, s[k] * scale); + red[threadIdx.x] = m; __syncthreads(); + for (uint32_t o = 128; o > 0; o >>= 1) { if (threadIdx.x < o) red[threadIdx.x] = fmaxf(red[threadIdx.x], red[threadIdx.x + o]); __syncthreads(); } + m = red[0]; __syncthreads(); + float sum = 0.f; + for (uint32_t k = threadIdx.x; k < nk; k += blockDim.x) sum += expf(s[k] * scale - m); + red[threadIdx.x] = sum; __syncthreads(); + for (uint32_t o = 128; o > 0; o >>= 1) { if (threadIdx.x < o) red[threadIdx.x] += red[threadIdx.x + o]; __syncthreads(); } + const float inv = 1.0f / red[0]; + for (uint32_t k = threadIdx.x; k < nk; k += blockDim.x) p[k] = __float2bfloat16(expf(s[k] * scale - m) * inv); +} +/* Query-tile byte cap: S(f32)+P(bf16) is H*qc*n*6 B; a 1024-token image has + * n ~ 8600 and H 16 (0.8 MB/row), so 192 MB caps qc ~ 230 and ~38 tiles per + * layer - the same order as the trunk prefill scratch + * (cuda_v41_vision.inc.cu:82-84). */ +#define V41_VIS_ATTN_BYTES (192ull << 20) + +/* Dense unmasked attention, scale = head_dim^-1/2 (official + * F.scaled_dot_product_attention; cuda_v41_vision.inc.cu:86-121). Per-head + * batched cuBLAS: S = Q·K^T (f32) -> row softmax -> bf16 P -> P·V; queries are + * tiled so the f32 scratch stays under V41_VIS_ATTN_BYTES. */ +extern "C" int ds4_gpu_v41_dense_attn_tensor(ds4_gpu_tensor *o, const ds4_gpu_tensor *qkv, uint32_t n_tok, uint32_t n_head, uint32_t head_dim) { + if (!o || !qkv || !g_cublas_ready || !n_tok || !n_head || !head_dim) return 0; + const uint32_t HD = n_head * head_dim, W = 3u * HD; + const uint64_t xn = (uint64_t)n_tok * W; + __hip_bfloat16 *q16 = (__hip_bfloat16 *)v41_grow(&g_vis_qkv16, xn * 2, "vision qkv bf16"); + if (!q16) return 0; + v41_x_to_bf16_kernel<<<(unsigned)((xn + 255) / 256), 256, 0, ds4_current_stream()>>>(q16, (const float *)qkv->ptr, xn); + if (!cuda_ok(hipGetLastError(), "vision qkv->bf16")) return 0; + uint64_t qc = V41_VIS_ATTN_BYTES / ((uint64_t)n_head * n_tok * 6u); + if (qc < 1) qc = 1; + if (qc > n_tok) qc = n_tok; + float *S = (float *)v41_grow(&g_vis_s, (uint64_t)n_head * qc * n_tok * 4, "vision attn S"); + __hip_bfloat16 *P = (__hip_bfloat16 *)v41_grow(&g_vis_p, (uint64_t)n_head * qc * n_tok * 2, "vision attn P"); + if (!S || !P) return 0; + const float alpha = 1.0f, beta = 0.0f, scale = 1.0f / sqrtf((float)head_dim); + (void)hipblasSetStream(g_cublas, ds4_current_stream()); + cuda_cublas_ws_prep(ds4_current_stream()); + for (uint32_t q0 = 0; q0 < n_tok; q0 += (uint32_t)qc) { + const uint32_t nq = (uint32_t)(n_tok - q0 < qc ? n_tok - q0 : qc); + /* S[h][q][k] = Σ_d K[k][d]·Q[q][d]: column-major this is C(nk × nq) = + * K^T(nk × D) · Q(D × nq), both strides W (the whole row). */ + hipblasStatus_t st = hipblasGemmStridedBatchedEx(g_cublas, HIPBLAS_OP_T, HIPBLAS_OP_N, (int)n_tok, (int)nq, (int)head_dim, &alpha, + q16 + HD, HIP_R_16BF, (int)W, (long long)head_dim, + q16 + (uint64_t)q0 * W, HIP_R_16BF, (int)W, (long long)head_dim, &beta, + S, HIP_R_32F, (int)n_tok, (long long)n_tok * nq, (int)n_head, + HIPBLAS_COMPUTE_32F, HIPBLAS_GEMM_DEFAULT); + if (!cublas_ok(st, "vision attn QK^T")) return 0; + v41_vision_softmax_kernel<<<(unsigned)(n_head * nq), 256, 0, ds4_current_stream()>>>(P, S, n_tok, scale); + if (!cuda_ok(hipGetLastError(), "vision attn softmax")) return 0; + /* O[h][q][d] = Σ_k V[k][d]·P[q][k]: C(D × nq) = V(D × nk) · P(nk × nq); + * O lands directly in o's [q0.., h·D..] span (row stride HD). */ + st = hipblasGemmStridedBatchedEx(g_cublas, HIPBLAS_OP_N, HIPBLAS_OP_N, (int)head_dim, (int)nq, (int)n_tok, &alpha, + q16 + 2u * HD, HIP_R_16BF, (int)W, (long long)head_dim, + P, HIP_R_16BF, (int)n_tok, (long long)n_tok * nq, &beta, + (float *)o->ptr + (uint64_t)q0 * HD, HIP_R_32F, (int)HD, (long long)head_dim, (int)n_head, + HIPBLAS_COMPUTE_32F, HIPBLAS_GEMM_DEFAULT); + if (!cublas_ok(st, "vision attn PV")) return 0; + } + return ds4_gpu_v41_round_bf16_tensor(o, (uint64_t)n_tok * HD); +} + +/* Untruncated SwiGLU on the packed [gate|up] halves: + * h = bf16(silu(gate)·up) (cuda_v41_vision.inc.cu:124-135). */ +__global__ static void v41_vision_swiglu_pair_kernel(float *h, const float *gu, uint64_t n, uint32_t mid) { + v41_pdl_wait(); + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + const uint64_t t = i / mid, j = i % mid; + const float g = gu[t * 2u * mid + j], u = gu[t * 2u * mid + mid + j]; + h[i] = v41_bf16r((g / (1.0f + expf(-g))) * u); +} +extern "C" int ds4_gpu_v41_swiglu_pair_tensor(ds4_gpu_tensor *h, const ds4_gpu_tensor *gu, uint32_t n_tok, uint32_t mid) { + if (!h || !gu || !n_tok || !mid) return 0; + const uint64_t n = (uint64_t)n_tok * mid; + v41_vision_swiglu_pair_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>( + (float *)h->ptr, (const float *)gu->ptr, n, mid); + return cuda_ok(hipGetLastError(), "vision swiglu"); +} + +/* In-place exact-erf GELU (official F.gelu default; cuda_v41_vision.inc.cu:139-149). */ +__global__ static void v41_vision_gelu_kernel(float *x, uint64_t n) { + v41_pdl_wait(); + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + const float v = x[i]; + x[i] = v41_bf16r(0.5f * v * (1.0f + erff(v * 0.70710678118654752f))); +} +extern "C" int ds4_gpu_v41_gelu_tensor(ds4_gpu_tensor *x, uint64_t n) { + if (!x || !n) return 0; + v41_vision_gelu_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>((float *)x->ptr, n); + return cuda_ok(hipGetLastError(), "vision gelu"); +} + +/* Aligner r×r pixel re-arrange (official F.pad(0 to a multiple of r) + + * F.unfold(r, stride=r) + transpose; cuda_v41_vision.inc.cu:152-169): + * x[n_h·n_w][dim] (patch grid, row-major) -> out[⌈n_h/r⌉·⌈n_w/r⌉][dim·r·r], + * column order c·r·r + kh·r + kw; outside the grid is 0. */ +__global__ static void v41_vision_unfold_kernel(float *out, const float *x, uint32_t n_h, uint32_t n_w, uint32_t dim, uint32_t r, uint32_t ow, uint64_t total) { + v41_pdl_wait(); + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= total) return; + const uint32_t cols = dim * r * r; + const uint32_t col = (uint32_t)(i % cols), cell = (uint32_t)(i / cols); + const uint32_t oi = cell / ow, oj = cell % ow; + const uint32_t c = col / (r * r), rem = col % (r * r), kh = rem / r, kw = rem % r; + const uint32_t pi = oi * r + kh, pj = oj * r + kw; + out[i] = (pi < n_h && pj < n_w) ? x[((uint64_t)pi * n_w + pj) * dim + c] : 0.0f; +} +extern "C" int ds4_gpu_v41_vision_unfold_tensor(ds4_gpu_tensor *out, const ds4_gpu_tensor *x, uint32_t n_h, uint32_t n_w, uint32_t dim, uint32_t r) { + if (!out || !x || !n_h || !n_w || !dim || !r) return 0; + const uint32_t oh = (n_h + r - 1u) / r, ow = (n_w + r - 1u) / r; + const uint64_t total = (uint64_t)oh * ow * dim * r * r; + v41_vision_unfold_kernel<<<(unsigned)((total + 255) / 256), 256, 0, ds4_current_stream()>>>( + (float *)out->ptr, (const float *)x->ptr, n_h, n_w, dim, r, ow, total); + return cuda_ok(hipGetLastError(), "vision unfold"); +} + +/* Row replace: rows[i] >= 0 => x row i = src row rows[i], else untouched - the + * visual embeddings overwrite the word embeddings at the image positions + * (official merge_image_embeddings; cuda_v41_vision.inc.cu:171-181). */ +__global__ static void v41_row_select_kernel(float *x, const float *src, const int32_t *rows, uint64_t n, uint32_t dim) { + v41_pdl_wait(); + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + const int32_t r = rows[i / dim]; + if (r >= 0) x[i] = src[(uint64_t)r * dim + i % dim]; +} +extern "C" int ds4_gpu_v41_row_select_tensor(ds4_gpu_tensor *x, const ds4_gpu_tensor *src, const ds4_gpu_tensor *rows, uint32_t n_tok, uint32_t dim) { + if (!x || !src || !rows || !n_tok || !dim) return 0; + const uint64_t n = (uint64_t)n_tok * dim; + v41_row_select_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>( + (float *)x->ptr, (const float *)src->ptr, (const int32_t *)rows->ptr, n, dim); + return cuda_ok(hipGetLastError(), "vision row select"); +} + +/* Row zero: rows[i] >= 0 => x row i = 0 - the engram contributes nothing at + * the image positions (official engram_mask; cuda_v41_vision.inc.cu:183-195). */ +__global__ static void v41_rows_zero_kernel(float *x, const int32_t *rows, uint64_t n, uint32_t dim) { + v41_pdl_wait(); + const uint64_t i = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) return; + if (rows[i / dim] >= 0) x[i] = 0.0f; +} +extern "C" int ds4_gpu_v41_rows_zero_tensor(ds4_gpu_tensor *x, const ds4_gpu_tensor *rows, uint32_t n_tok, uint32_t dim) { + if (!x || !rows || !n_tok || !dim) return 0; + const uint64_t n = (uint64_t)n_tok * dim; + v41_rows_zero_kernel<<<(unsigned)((n + 255) / 256), 256, 0, ds4_current_stream()>>>( + (float *)x->ptr, (const int32_t *)rows->ptr, n, dim); + return cuda_ok(hipGetLastError(), "vision rows zero"); +} diff --git a/cuda/ds41_vq_decode.cuh b/cuda/ds41_vq_decode.cuh new file mode 100644 index 0000000000000000000000000000000000000000..fdb4a469b9bba06c49c400c046603c93325f1527 --- /dev/null +++ b/cuda/ds41_vq_decode.cuh @@ -0,0 +1,391 @@ +#include "hip/hip_runtime.h" +/* ds41_vq_decode.cuh — VQ expert "decode is the multiply" kernels, ported from + * the C engine at /data/YoungAi (commit 3946dbc), file + * `src/cuda/cuda_vq_decode.inc.cu` (whole file). + * + * One subject: how compressed expert weights take part in the multiply + * directly. On disk (DQVL v2 blob): 16 B header + 384*3 slot table; each slot + * is a DQVQ payload = 16 B header + codebook (nc*8 f16; nc4096 = 64 KB) + row + * gains f16 + bitstream (one codeword per 8 elements). The index width comes + * from nc: nc4096 = 12 bit, nc2048 = 11 bit. The v41_vq_row_dot family is + * NBIT-templated and geometrically general: a block is always 8 rounds = + * 8*NBIT words, 3 registers per lane, row starts always whole bytes. + * + * Measured history kept from the engine: activation stored as bf16 (the + * activation was two thirds of the load traffic; 14.23 -> 13.28 ms/step, + * bit-identical because values sit on the bf16 grid); whole-block bitstream + * reads (the three earlier read schemes all requested only 48 B per round — + * DRAM fetches 64 B, half wasted). The 20 B-stride bank-conflict experiment + * and the "dedupe experts per token" batch experiment were both measured + * negative and are archived in the engine, not resurrected here. + * + * Must follow ds41_vq_row.cuh (uses its v41_vq_mat / row_dot) and + * ds41_vq_probe.cuh (the prof watchdogs); ds41_vq_group.cuh and + * ds41_vq_persist.cuh follow this file and are instantiated from here. + */ +#pragma once +#include +#include +#include + +#include "ds41_primitives.cuh" +#include "ds41_vq_row.cuh" + +/* Copy the codebook into shared with 8 loads in flight (each one previously + * waited a full L2 round trip; the copy is a full-block sync point with nothing + * to hide it). The codebook is only 8 B aligned, so uint2. bytes % 8 == 0. */ +__device__ __forceinline__ static void v41_vq_cb_to_shared(uint8_t *dst, const uint8_t *src, uint32_t bytes) { + const uint2 *s = (const uint2 *)src; uint2 *d = (uint2 *)dst; + const uint32_t n = bytes / 8u; + uint32_t i = threadIdx.x; + for (; i + 7u * blockDim.x < n; i += 8u * blockDim.x) { + uint2 t[8]; + #pragma unroll + for (int q = 0; q < 8; q++) t[q] = s[i + (uint32_t)q * blockDim.x]; + #pragma unroll + for (int q = 0; q < 8; q++) d[i + (uint32_t)q * blockDim.x] = t[q]; + } + for (; i < n; i += blockDim.x) d[i] = s[i]; +} + +/* SwiGLU exit, same form as the official Expert: clamp -> silu(g)*u -> bf16. + * g and u are already on the bf16 grid. */ +__device__ __forceinline__ static uint16_t v41_vq_swiglu(float gi, float ui, float clamp) { + if (clamp > 0.f) { if (gi > clamp) gi = clamp; if (ui > clamp) ui = clamp; if (ui < -clamp) ui = -clamp; } + const float sg = gi / (1.0f + expf(-gi)); + return (uint16_t)(__float_as_uint(v41_bf16r(sg * ui)) >> 16); +} + +/* Rows per block. One block = 32 warps x ITERS rows so the codebook is copied + * once per block; with ITERS=8 the grid degenerates to ~1 wave per SM and the + * tail dominates, with ITERS=2 (64 rows/block, 4.5 waves) the scheduler keeps + * feeding finished SMs — measured 58.7 vs 63.0 ms. gateup and down were + * re-scanned separately after the block-pipeline change (gateup 2 rows 6.72 < + * 4 rows 7.19 < 8 rows 8.77; down 4 rows 3.76 < 2 rows 4.51). */ +#define V41_VQ_ITERS_GU 2u +#define V41_VQ_ITERS_DN 4u +#define V41_VQ_WARPS 32u +#define V41_VQ_GU_ROWS (V41_VQ_WARPS * V41_VQ_ITERS_GU) +#define V41_VQ_DN_ROWS (V41_VQ_WARPS * V41_VQ_ITERS_DN) + +/* gate/up in one kernel (official Expert: w1/w3 -> bf16 -> f32 -> silu(g)*u -> + * bf16); cb_bytes > 0: codebook into shared (grid.x over 32-row units), else + * global gather (grid.x over 8-row units). x is already on the bf16 grid. + * Do not touch this kernel lightly: 1024 threads x 64 registers x 64 KB + * codebook exactly fills one SM (1 block/SM, 67% occupancy); one extra runtime + * possibility cost 21% even on the n=1 path with byte-identical output. */ +template +__global__ static void v41_vq_gateup_kernel(uint16_t *h, const uint8_t *blob, const int32_t *sel, const uint32_t *x, + uint32_t IN, uint32_t MID, uint32_t K, float clamp, uint32_t cb_bytes, const int32_t *order, + uint32_t np, uint32_t uniq_only) { + extern __shared__ __align__(16) uint8_t vqsh[]; + const uint32_t pair = SORTED ? (uint32_t)order[blockIdx.y] : blockIdx.y, t = pair / K, rows = cb_bytes ? V41_VQ_GU_ROWS : 8u; + const int32_t e = sel[pair]; + if (e < 0) return; + if (SORTED && uniq_only && + ((blockIdx.y > 0u && sel[order[blockIdx.y - 1u]] == e) || (blockIdx.y + 1u < np && sel[order[blockIdx.y + 1u]] == e))) return; + const v41_vq_mat mg = v41_vq_open(blob, e, 0, MID, IN, NULL), mu = v41_vq_open(blob, e, 1, MID, IN, NULL); + if (!mg.ok || !mu.ok) return; + const uint32_t r0 = blockIdx.x * rows + (threadIdx.x >> 5), nit = cb_bytes ? V41_VQ_ITERS_GU : 1u; + const uint32_t *xs = x + (uint64_t)t * (IN / 2u); /* two bf16 per word */ + uint16_t *hp = h + (uint64_t)pair * MID; + const bool lead = (threadIdx.x & 31u) == 0u; + if (cb_bytes) { + /* One shared buffer serves twice: gate and up each carry a codebook; + * both at once exceeds the per-block shared limit, which would push the + * whole kernel back to global gather. Load gate's, compute this block's + * rows, sync, overwrite with up's, compute u. No return inside the loop + * (a missing warp would deadlock the __syncthreads); out-of-range rows + * just skip the compute. The gate result parks as bf16 in h's own row + * slot — it is already on the bf16 grid, so the round trip is lossless + * and the same lane writes and reads it back. */ + constexpr uint32_t CW = V3 ? 8u : 16u; /* bytes per codeword: v3 = 8 E4M3, v2 = 8 f16 */ + if (mg.nc * CW != cb_bytes || mu.nc * CW != cb_bytes) return; + v41_vq_blk carry; carry.w0 = 0u; carry.w1 = 0u; carry.w2 = 0u; carry.ex = 0u; + if (r0 < MID) carry = v41_vq_row_first_blk(mg, r0); /* first block issues behind the codebook copy */ + v41_vq_cb_to_shared(vqsh, mg.cb, cb_bytes); + __syncthreads(); + for (uint32_t i = 0; i < nit; i++) { + const uint32_t r = r0 + i * V41_VQ_WARPS; + if (r >= MID) break; + const uint32_t rn = r + V41_VQ_WARPS; /* on gate's last row, preload up's first (computed after the swap) */ + const int own = (i + 1u < nit && rn < MID); + const uint32_t *next = own ? v41_vq_row_ptr(mg, rn) : v41_vq_row_ptr(mu, r0); + const uint32_t *nextex = EXT ? (own ? v41_vq_ext_ptr(mg, rn) : v41_vq_ext_ptr(mu, r0)) : NULL; + const float gv = v41_bf16r(v41_vq_row_dot(mg, r, xs, vqsh, 1, &carry, next, nextex)); + if (lead) hp[r] = (uint16_t)(__float_as_uint(gv) >> 16); + } + __syncthreads(); /* every warp done with gate's codebook before overwriting it */ + v41_vq_cb_to_shared(vqsh, mu.cb, cb_bytes); + __syncthreads(); + for (uint32_t i = 0; i < nit; i++) { + const uint32_t r = r0 + i * V41_VQ_WARPS; + if (r >= MID) break; + const uint32_t rn = r + V41_VQ_WARPS; + const int own = (i + 1u < nit && rn < MID); + const uint32_t *next = own ? v41_vq_row_ptr(mu, rn) : NULL; + const float ui = v41_bf16r(v41_vq_row_dot(mu, r, xs, vqsh, 1, &carry, next, EXT && own ? v41_vq_ext_ptr(mu, rn) : NULL)); + if (lead) hp[r] = v41_vq_swiglu(__uint_as_float((uint32_t)hp[r] << 16), ui, clamp); + } + } else if (r0 < MID) { + v41_vq_blk carry = v41_vq_row_first_blk(mg, r0); + const float gv = v41_bf16r(v41_vq_row_dot(mg, r0, xs, mg.cb, 0, &carry, + v41_vq_row_ptr(mu, r0), EXT ? v41_vq_ext_ptr(mu, r0) : NULL)); + const float ui = v41_bf16r(v41_vq_row_dot(mu, r0, xs, mu.cb, 0, &carry, NULL, NULL)); + if (lead) hp[r0] = v41_vq_swiglu(gv, ui, clamp); + } +} + +/* down: partial[pair][OUT] = bf16(W2 * h). */ +template +__global__ static void v41_vq_down_kernel(float *partial, const uint8_t *blob, const int32_t *sel, const uint32_t *h, + uint32_t MID, uint32_t OUT, uint32_t K, uint32_t cb_bytes, const float *gr, const int32_t *order, + uint32_t np, uint32_t uniq_only) { + extern __shared__ __align__(16) uint8_t vqsh[]; + const uint32_t pair = SORTED ? (uint32_t)order[blockIdx.y] : blockIdx.y, rows = cb_bytes ? V41_VQ_DN_ROWS : 8u; + const int32_t e = sel[pair]; + if (e < 0) return; + if (SORTED && uniq_only && + ((blockIdx.y > 0u && sel[order[blockIdx.y - 1u]] == e) || (blockIdx.y + 1u < np && sel[order[blockIdx.y + 1u]] == e))) return; + const v41_vq_mat md = v41_vq_open(blob, e, 2, OUT, MID, gr ? gr + (size_t)e * OUT : NULL); + const uint32_t r0 = blockIdx.x * rows + (threadIdx.x >> 5), nit = cb_bytes ? V41_VQ_ITERS_DN : 1u; + if (!md.ok) { /* bad payload: this block's rows all write 0 (not just one — the reduce would read stale values) */ + for (uint32_t i = 0; i < nit; i++) { const uint32_t r = r0 + i * V41_VQ_WARPS; if (r < OUT && (threadIdx.x & 31u) == 0) partial[(uint64_t)pair * OUT + r] = 0.f; } + return; + } + const uint8_t *cbd = md.cb; + v41_vq_blk carry; carry.w0 = 0u; carry.w1 = 0u; carry.w2 = 0u; carry.ex = 0u; + if (r0 < OUT) carry = v41_vq_row_first_blk(md, r0); + if (cb_bytes) { + if (md.nc * (V3 ? 8u : 16u) != cb_bytes) return; + v41_vq_cb_to_shared(vqsh, md.cb, cb_bytes); + __syncthreads(); + cbd = vqsh; + } + const uint32_t *hs = h + (uint64_t)pair * (MID / 2u); + for (uint32_t i = 0; i < nit; i++) { + const uint32_t r = r0 + i * V41_VQ_WARPS; + if (r >= OUT) break; + const uint32_t rn = r + V41_VQ_WARPS; + const int own = (i + 1u < nit && rn < OUT); + const uint32_t *next = own ? v41_vq_row_ptr(md, rn) : NULL; + const float y = v41_bf16r(v41_vq_row_dot(md, r, hs, cbd, cb_bytes != 0, &carry, next, + EXT && own ? v41_vq_ext_ptr(md, rn) : NULL)); + if ((threadIdx.x & 31u) == 0) partial[(uint64_t)pair * OUT + r] = y; + } + (void)K; +} + +/* out[t][o] = sum_k w[t][k] * partial[t*K+k][o] (f32; the official y += weights * expert_out). */ +__global__ static void v41_vq_reduce_kernel(float *out, const float *partial, const float *w, uint32_t K, uint32_t OUT) { + const uint32_t t = blockIdx.y, o = blockIdx.x * 256u + threadIdx.x; + if (o >= OUT) return; + float a = 0.f; + for (uint32_t k = 0; k < K; k++) a += w[(uint64_t)t * K + k] * partial[((uint64_t)t * K + k) * OUT + o]; + out[(uint64_t)t * OUT + o] = a; +} + +/* Verify-batch block order: order[q] = the pair computed by block q, sorted by + * (expert, pair); one block of np (<=48) threads counts how many precede it. */ +__global__ static void v41_vq_order_kernel(int32_t *order, const int32_t *sel, uint32_t np) { + const uint32_t p = threadIdx.x; + if (p >= np) return; + const int32_t e = sel[p]; + uint32_t rank = 0; + for (uint32_t q = 0; q < np; q++) { const int32_t eq = sel[q]; if (eq < e || (eq == e && q < p)) rank++; } + order[rank] = (int32_t)p; +} + +static v41_scratch g_v41_vq_h, g_v41_vq_part, g_v41_vq_xb, g_v41_vq_ogc, g_v41_vq_ord; + +/* Forward declarations of the family members that follow this file in the + * aggregate include order (group after decode, persist after decode, launch + * last). The group flag only affects n >= 2. */ +template +static int v41_vq_grp_launch(int stage, uint32_t n_tok, uint16_t *h, float *part, const uint8_t *blob, const int32_t *sel, const uint32_t *xb, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t K, float clamp, uint32_t cbb, const float *gr, + const int32_t *ord, uint32_t np); +extern int g_ds4_v41_vq_group; +template +static int v41_vq_persist_launch(int stage, uint16_t *h, float *part, const uint8_t *blob, const int32_t *sel, const uint32_t *xb, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t np, float clamp, uint32_t cbb, const float *gr); +template +static int v41_vq_persist_n_launch(int stage, uint32_t n_tok, uint16_t *h, float *part, const uint8_t *blob, const int32_t *sel, + const int32_t *ord, const uint32_t *xb, uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t K, + uint32_t np, float clamp, uint32_t cbb, const float *gr); + +/* The n=1..8 worker. gateup and down are gated separately: one codebook (64 KB) + * fits a block's shared, two (128 KB) do not, and a single ok-flag once pushed + * both kernels to global gather when only gateup's request failed. */ +template +static int v41_vq_fused_moe_n(float *out, const uint8_t *blob, uint32_t IN, uint32_t MID, uint32_t OUT, + const int32_t *sel, const float *w, uint32_t K, float clamp, const float *x, uint32_t n_tok, uint32_t nc, + const float *gr) { + if (n_tok == 0 || n_tok > V41_GEMV_MAX_TOK || (IN % 8u) || (MID % 8u)) return 0; + const uint64_t np = (uint64_t)n_tok * K; + /* h holds bf16 (2 B/element): only the down kernel reads it, in packed form. */ + uint16_t *h = (uint16_t *)v41_grow(&g_v41_vq_h, np * MID * 2, "v41 vq h"); + float *part = (float *)v41_grow(&g_v41_vq_part, np * OUT * 4, "v41 vq partial"); + uint16_t *xb = (uint16_t *)v41_grow(&g_v41_vq_xb, (uint64_t)n_tok * IN * 2, "v41 vq x(bf16)"); + if (!h || !part || !xb) return 0; + { /* pack the activation to bf16: once per layer, negligible next to the expert kernels */ + const uint64_t nx = (uint64_t)n_tok * IN; + v41_vq_xpack_kernel<<<(unsigned)((nx + 255) / 256), 256, 0, ds4_current_stream()>>>(xb, x, nx); + if (!cuda_ok(hipGetLastError(), "v41 vq xpack")) return 0; + extern int g_ds4_v41_prof; + if (g_ds4_v41_prof) { + uint32_t *c = (uint32_t *)v41_grow(&g_v41_vq_ogc, 4, "v41 vq offgrid"); + uint32_t hc = 0; + if (c && hipMemsetAsync(c, 0, 4, ds4_current_stream()) == hipSuccess) { + v41_vq_offgrid_kernel<<<(unsigned)((nx + 255) / 256), 256, 0, ds4_current_stream()>>>(c, x, nx); + if (hipStreamSynchronize(ds4_current_stream()) == hipSuccess && + hipMemcpy(&hc, c, 4, hipMemcpyDeviceToHost) == hipSuccess) { + static uint64_t tot = 0, bad = 0; + tot += nx; bad += hc; + if ((tot / nx) % 40u == 0u) + fprintf(stderr, "[vq-grid] activations off the bf16 grid: %llu / %llu cumulative\n", + (unsigned long long)bad, (unsigned long long)tot); + } + } + v41_f16range_probe(x, NULL, nx, 0, "x"); + } + } + const uint32_t cbb = nc * (V3 ? 8u : 16u); /* v3 codebook is 8 E4M3 per word => nc8192 is still 64 KB */ + /* The opt-in must be recorded per (kernel instance, this layer's codebook): + * hipFuncSetAttribute grants a max dynamic shared size to ONE function and + * the grant then caps every later launch. Both codebook sizes coexist + * (shallow 13-bit layers nc8192 = 64 KB, deep ones nc4096 = 32 KB), so a + * file-level "done once" flag granted the first size seen and later failed + * the other with "invalid argument". static locals in a template are per + * instantiation, which is exactly per (NBIT,V3,EXT). */ + static uint32_t s_gu_optin = 0u, s_dn_optin = 0u; + if (s_gu_optin < cbb || s_dn_optin < cbb) { + /* Clear any error a previous call left latched BEFORE the opt-in: + * the runtime reports a latched error to a later call, and a failed + * hipFuncSetAttribute silently drops this layer set to the other + * arm (global gather — a different accumulation order). The arm + * must depend on the device alone, never on prior error state. */ + (void)hipGetLastError(); + int cap = 0; (void)hipDeviceGetAttribute(&cap, hipDeviceAttributeSharedMemPerBlockOptin, 0); + int sh_sm = 0, thr_sm = 0, nsm = 0, regs_sm = 0; + (void)hipDeviceGetAttribute(&sh_sm, hipDeviceAttributeMaxSharedMemoryPerMultiprocessor, 0); + (void)hipDeviceGetAttribute(&thr_sm, hipDeviceAttributeMaxThreadsPerMultiProcessor, 0); + (void)hipDeviceGetAttribute(&nsm, hipDeviceAttributeMultiprocessorCount, 0); + (void)hipDeviceGetAttribute(®s_sm, hipDeviceAttributeMaxRegistersPerMultiprocessor, 0); + const bool og = hipFuncSetAttribute(reinterpret_cast(v41_vq_gateup_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)cbb) == hipSuccess && + hipFuncSetAttribute(reinterpret_cast(v41_vq_gateup_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)cbb) == hipSuccess; + const bool od = hipFuncSetAttribute(reinterpret_cast(v41_vq_down_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)cbb) == hipSuccess && + hipFuncSetAttribute(reinterpret_cast(v41_vq_down_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)cbb) == hipSuccess; + /* Occupancy must be asked AFTER the opt-in: before it the API uses the + * default 48 KB limit and reports "no block fits" (the engine printed + * 0% once and nearly drew the wrong conclusion). */ + hipFuncAttributes fa; memset(&fa, 0, sizeof fa); + (void)hipFuncGetAttributes(&fa, reinterpret_cast(v41_vq_gateup_kernel)); + int blocks_sm = 0; + const int thr_blk = (int)(V41_VQ_WARPS * 32u); + (void)hipOccupancyMaxActiveBlocksPerMultiprocessor(&blocks_sm, v41_vq_gateup_kernel, thr_blk, (size_t)cbb); + fprintf(stderr, "ds4: [ds41] device: %d SM / %d KB shared per SM / %d threads per SM / %d regs per SM\n" + "ds4: [ds41] VQ gateup kernel: %d threads/block, %d regs/thread, dynamic shared %u KB" + " => %d blocks/SM = %d threads, occupancy %.0f%%\n" + "ds4: [ds41] bound by: shared %d / registers %d / thread slots %d (min)\n", + nsm, sh_sm >> 10, thr_sm, regs_sm, thr_blk, fa.numRegs, cbb >> 10, + blocks_sm, blocks_sm * thr_blk, thr_sm ? 100.0 * blocks_sm * thr_blk / thr_sm : 0.0, + cbb ? sh_sm / (int)cbb : 0, + fa.numRegs ? regs_sm / (fa.numRegs * thr_blk) : 0, thr_sm / thr_blk); + (void)hipGetLastError(); + if (og) s_gu_optin = cbb; /* a failed grant leaves the layer on global gather */ + if (od) s_dn_optin = cbb; + fprintf(stderr, "ds4: [ds41] VQ codebook %u words (%u KB each, NBIT %d); per-block dynamic shared cap %d KB => gate+up %s / down %s\n", + nc, cbb >> 10, NBIT, cap >> 10, og ? "in shared" : "global gather", od ? "in shared" : "global gather"); + } + const bool shg = cbb <= s_gu_optin, shd = cbb <= s_dn_optin; + /* Threads: always 32 warps (shared path, one warp loops ITERS rows) or 8 + * warps (global gather, one warp one row) — never rows*32, which would ask + * for 8192 threads and exceed the 1024 limit. */ + const uint32_t rg = shg ? V41_VQ_GU_ROWS : 8u, rd = shd ? V41_VQ_DN_ROWS : 8u; + const uint32_t tg = shg ? V41_VQ_WARPS * 32u : 8u * 32u, td = shd ? V41_VQ_WARPS * 32u : 8u * 32u; + int32_t *ord = NULL; + if (n_tok >= 2u) { + ord = (int32_t *)v41_grow(&g_v41_vq_ord, np * 4, "v41 vq order"); + if (!ord) return 0; + v41_vq_order_kernel<<<1, (unsigned)np, 0, ds4_current_stream()>>>(ord, sel, (uint32_t)np); + if (!cuda_ok(hipGetLastError(), "v41 vq order")) return 0; + } + /* Drain upstream errors before launching: CUDA errors are sticky, so + * without this an unchecked launch elsewhere is reported as "vq gateup + * failed" and the real culprit is invisible (this cost the engine a full + * debugging round). */ + { + const hipError_t pre = hipGetLastError(); + if (pre != hipSuccess) + fprintf(stderr, "ds4: *[ds41] an unchecked CUDA error predates vq gateup: %s*\n", hipGetErrorString(pre)); + } + const bool pern = ord && V3 && shg && shd && g_ds4_v41_vq_group && + v41_vq_persist_n_launch(0, n_tok, h, part, blob, sel, ord, (const uint32_t *)xb, IN, MID, OUT, K, (uint32_t)np, clamp, cbb, gr); + int grp = 0; + if (!pern && ord && shg && shd && g_ds4_v41_vq_group) { + grp = v41_vq_grp_launch(0, n_tok, h, part, blob, sel, (const uint32_t *)xb, IN, MID, OUT, K, clamp, cbb, gr, ord, (uint32_t)np); + if (grp < 0) return 0; + } + /* Diagnostic switch, kept (P4-2): DS41_VQ_NO_PERSIST=1 forces the + * v2-shaped plain kernels so the persist path can be cmp'd against them — + * the engine's own equivalence gate ("gate = cmp against the v2-shaped + * kernels"). Verified byte-identical (mid + partial dumps) on the real + * layer-0 payload (sm_121, P4-2 evidence). Not usable on sm_89: the plain + * gateup kernel compiles to 92 registers there and cannot launch at 1024 + * threads ("too many resources"); the persist kernels are pinned by + * __launch_bounds__(1024, 1). */ + const bool per = n_tok == 1u && V3 && shg && shd && getenv("DS41_VQ_NO_PERSIST") == NULL; + if (pern) { /* already launched */ } + else if (per) { if (!v41_vq_persist_launch(0, h, part, blob, sel, (const uint32_t *)xb, IN, MID, OUT, (uint32_t)np, clamp, cbb, gr)) return 0; } + else if (ord) v41_vq_gateup_kernel<<>>( + h, blob, sel, (const uint32_t *)xb, IN, MID, K, clamp, shg ? cbb : 0u, ord, (uint32_t)np, grp > 0 ? 1u : 0u); + else v41_vq_gateup_kernel<<>>( + h, blob, sel, (const uint32_t *)xb, IN, MID, K, clamp, shg ? cbb : 0u, NULL, 0u, 0u); + if (!per && !pern && !cuda_ok(hipGetLastError(), "v41 vq gateup")) { + /* Log the launch parameters with the failure: "invalid argument" alone + * does not say which one, and the shared-memory grant is per instance + * while cbb varies per layer. */ + fprintf(stderr, "ds4: [ds41] gateup launch: grid(%u,%u) block %u dynamic shared %u B; " + "NBIT %d V3 %d EXT %d SORTED %d; n_tok %u K %u np %llu IN %u MID %u nc %u cbb %u B granted %u B\n", + (MID + rg - 1u) / rg, (unsigned)np, tg, shg ? cbb : 0u, + NBIT, V3, EXT, ord ? 1 : 0, n_tok, K, (unsigned long long)np, IN, MID, nc, cbb, s_gu_optin); + return 0; + } + { + extern int g_ds4_v41_prof; + if (g_ds4_v41_prof) v41_f16range_probe(NULL, h, np * MID, 1, "h"); + } + if (grp > 0 && v41_vq_grp_launch(1, n_tok, h, part, blob, sel, (const uint32_t *)xb, IN, MID, OUT, K, clamp, cbb, gr, ord, (uint32_t)np) < 0) return 0; + if (pern) { if (!v41_vq_persist_n_launch(1, n_tok, h, part, blob, sel, ord, (const uint32_t *)xb, IN, MID, OUT, K, (uint32_t)np, clamp, cbb, gr)) return 0; } + else if (per) { if (!v41_vq_persist_launch(1, h, part, blob, sel, (const uint32_t *)xb, IN, MID, OUT, (uint32_t)np, clamp, cbb, gr)) return 0; } + else if (ord) v41_vq_down_kernel<<>>( + part, blob, sel, (const uint32_t *)h, MID, OUT, K, shd ? cbb : 0u, gr, ord, (uint32_t)np, grp > 0 ? 1u : 0u); + else v41_vq_down_kernel<<>>( + part, blob, sel, (const uint32_t *)h, MID, OUT, K, shd ? cbb : 0u, gr, NULL, 0u, 0u); + if (!per && !pern && !cuda_ok(hipGetLastError(), "v41 vq down")) return 0; + if (!out) return 1; /* the caller folds the routed sum and the shared expert later via ds4_gpu_v41_moe_tail_tensor */ + v41_vq_reduce_kernel<<>>(out, part, w, K, OUT); + return cuda_ok(hipGetLastError(), "v41 vq reduce"); +} + +/* MoE tail, four launches folded into one: y = bf16(sum_k w*partial + so). The + * arithmetic is untouched (same k order, then + so, then bf16 round), so the + * output is bit-identical to the reduce -> copy -> add -> round chain it + * replaced. partial is what the down kernel left in the scratch. */ +__global__ static void v41_vq_tail_kernel(float *y, const float *partial, const float *w, const float *so, uint32_t K, uint32_t OUT) { + v41_pdl_wait(); + const uint32_t t = blockIdx.y, o = blockIdx.x * 256u + threadIdx.x; + if (o >= OUT) return; + float a = 0.f; + for (uint32_t k = 0; k < K; k++) a += w[(uint64_t)t * K + k] * partial[((uint64_t)t * K + k) * OUT + o]; + y[(uint64_t)t * OUT + o] = v41_bf16r(a + so[(uint64_t)t * OUT + o]); +} + +extern "C" int ds4_gpu_v41_moe_tail_tensor(ds4_gpu_tensor *y, const ds4_gpu_tensor *so, const ds4_gpu_tensor *weights, + uint32_t n_tok, uint32_t n_used, uint32_t out_dim) { + if (!y || !so || !weights || !g_v41_vq_part.p || g_v41_vq_part.cap < (uint64_t)n_tok * n_used * out_dim * 4) return 0; + v41_vq_tail_kernel<<>>( + (float *)y->ptr, (const float *)g_v41_vq_part.p, (const float *)weights->ptr, (const float *)so->ptr, n_used, out_dim); + return cuda_ok(hipGetLastError(), "v41 moe tail"); +} diff --git a/cuda/ds41_vq_group.cuh b/cuda/ds41_vq_group.cuh new file mode 100644 index 0000000000000000000000000000000000000000..112a1855be8d90ae4bd24c0cfd475f23d4c2aa1c --- /dev/null +++ b/cuda/ds41_vq_group.cuh @@ -0,0 +1,325 @@ +#include "hip/hip_runtime.h" +/* ds41_vq_group.cuh — the multi-token grouped VQ "decode is the multiply" + * kernels (verify batches and draft-tower small batches), ported from the C + * engine at /data/YoungAi (commit 3946dbc), file + * `src/cuda/cuda_vq_group.inc.cu`. + * + * The disease this treats: a verify batch of n rows through the per-pair + * kernels costs n=1 0.356 ms/layer and n=4 1.283 ms/layer = 3.6x — the weight + * bytes do not amortize at all, so "verifying k+1 positions costs about one" + * does not hold in that shape. ncu named the cost unit: L1TEX wavefronts — per + * round (32 indices) the codebook lookups are ~11.7 + activations 4 + bitstream + * ~1.5, and the per-pair path pays the codebook lookups m times when m tokens + * pick the same expert. Here one block owns one expert x the m tokens of this + * batch that chose it (m <= M = batch size): the bitstream is read once, each + * index's codeword decoded once, and each token does one 8-element multiply-add + * — 11.7 + 4m wavefronts per round versus 15.7m for per-pair. + * + * The "same track" property (speculative == pure decode, byte for byte) holds + * because each token's row goes through the same function the single-token + * kernel uses (v41_vq_dot8_cw): per-lane accumulation order, the cross-lane + * reduction tree and the gain multiply are untouched; only which block computes + * which pair when changes. Gate = cmp. + * A grouping table error (order sorted by expert) writes one token's output + * into another pair's slot — no error, only the temperature-0 byte gate catches + * it. + * + * Must follow ds41_vq_decode.cuh (its v41_vq_cb_to_shared / v41_vq_swiglu and + * the row family) and precede ds41_vq_launch.cuh (which instantiates + * v41_vq_fused_moe_n and needs v41_vq_grp_launch visible). + */ +#pragma once +#include +#include +#include + +#include "ds41_vq_decode.cuh" + +/* One block = 16 warps = 512 threads => 128 registers/thread, enough for M + * accumulators + M activation loads (the per-pair kernel allows only 64 at 1024 + * threads and sits at 56-64). At <= 64 registers two such blocks fit per SM + * (2x64 KB shared, 1024 threads — within the device budget), same occupancy as + * the per-pair kernel. Rows per warp: gateup 2 (32 rows/block), down 4 (64) — + * half the block size of the per-pair kernel; the load balance comes from more + * waves (measured: this kernel lives on multiple waves). */ +#define V41_VQG_WARPS 16u +#define V41_VQG_THREADS (V41_VQG_WARPS * 32u) +#define V41_VQG_ITERS_GU 2u +#define V41_VQG_ITERS_DN 4u + +/* Up to 8 rounds per block, M-token form: index and codeword decoded once, one + * v41_vq_dot8_cw per token. xs[j] = token j's activation row (packed bf16), + * boff = this block's word offset within the row (32 indices x 4 words per + * round); m is block-uniform, so the branch does not diverge. */ +template +__device__ __forceinline__ static void v41_vqg_blk_rounds(const v41_vq_blk &cur, uint32_t rounds, uint32_t a, uint32_t sh, uint32_t imsk, + const uint32_t *const *xs, uint32_t boff, uint32_t m, const uint8_t *cbs, float *acc) { + const uint32_t lane = threadIdx.x & 31u; + constexpr uint32_t MB = v41_vq_mbit(); + #pragma unroll + for (uint32_t k = 0; k < 8u; k++) { + if (k >= rounds) break; + const uint32_t f = k * MB; + const uint32_t lo_r = ((f & 31u) > 32u - MB) ? ((f + 31u - lane) >> 5) : (f >> 5); + const uint32_t hi_r = (((f + 1u) & 31u) > 32u - MB) ? ((f + 32u - lane) >> 5) : ((f + 1u) >> 5); + const uint32_t lo = __shfl_sync(0xffffffffu, v41_vq_sel3(lo_r, cur.w0, cur.w1, cur.w2), (int)(f + a)); + const uint32_t hi = __shfl_sync(0xffffffffu, v41_vq_sel3(hi_r, cur.w0, cur.w1, cur.w2), (int)(f + a + 1u)); + uint32_t v = __funnelshift_r(lo, hi, sh) & (EXT ? 0xFFFu : imsk); + if (EXT) { + const uint32_t exw = __shfl_sync(0xffffffffu, cur.ex, (int)k); + v |= ((exw >> lane) & 1u) << 12; + } + float c[8]; + v41_vq_cw(v, cbs, 1, c); /* codeword decoded once (codebook always in shared here) */ + const uint32_t xo = boff + (k * 32u + lane) * 4u; + #pragma unroll + for (int j = 0; j < M; j++) { + if ((uint32_t)j < m) { + const uint4 xa = *(const uint4 *)(xs[j] + xo); /* 8 bf16 = 16 B, L1 hit */ + acc[j] += v41_vq_dot8_cw(c, xa); /* the same multiply-add expression as the single-token kernel */ + } + } + } +} + +/* One warp dots one row against m tokens (same cross-block pipeline as + * v41_vq_row_dot: carry in = this row's first block, out = next row's first). + * out[j] = gain * sum (warp-reduced), the multiply order identical to the + * single-token kernel (acc * row gain * gov, no pre-multiply). */ +template +__device__ __forceinline__ static void v41_vqg_row_dot(const v41_vq_mat &mt, uint32_t r, const uint32_t *const *xs, uint32_t m, const uint8_t *cbs, + v41_vq_blk *carry, const uint32_t *next, const uint32_t *nextex, float *out) { + const uint32_t lane = threadIdx.x & 31u; + const uint32_t R = mt.nidx_row >> 5; + constexpr uint32_t MB = v41_vq_mbit(); + const uint32_t a = (lane * MB) >> 5, sh = (lane * MB) & 31u; + const uint32_t *row = v41_vq_row_ptr(mt, r); + const uint32_t *rex = EXT ? v41_vq_ext_ptr(mt, r) : NULL; + v41_vq_blk cur = *carry; + float acc[M]; + #pragma unroll + for (int j = 0; j < M; j++) acc[j] = 0.f; + for (uint32_t b = 0; b < R; b += 8u) { + v41_vq_blk nxt; + if (b + 8u < R) { + const uint32_t rem = R - b - 8u, nr = (rem < 8u ? rem : 8u); + nxt = v41_vq_blk_load(row + (size_t)(b + 8u) * MB, nr * MB, EXT ? rex + (b + 8u) : NULL, nr); /* plane advances per group (fixed 09-22) */ + } + else if (next) { const uint32_t nr = (R < 8u ? R : 8u); nxt = v41_vq_blk_load(next, nr * MB, nextex, nr); } + else { nxt.w0 = 0u; nxt.w1 = 0u; nxt.w2 = 0u; nxt.ex = 0u; } + v41_vqg_blk_rounds(cur, (R - b < 8u) ? R - b : 8u, a, sh, mt.imsk, xs, b * 32u * 4u, m, cbs, acc); + cur = nxt; + } + *carry = cur; + __half gh; memcpy(&gh, mt.gr + (size_t)r * 2u, 2); + #pragma unroll + for (int j = 0; j < M; j++) { + if ((uint32_t)j < m) { + float v = acc[j]; + for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o); + out[j] = v * __half2float(gh) * (mt.gov ? mt.gov[r] : 1.0f); + } + } +} + +/* Which expert and which tokens this block owns: order[] is sorted by (expert, + * pair) and same-expert pairs are adjacent; only the head block of a run works + * (the rest return immediately, block-uniform so no __syncthreads is harmed), + * run length m (a token picks an expert at most once). + * Mixed dispatch (2026-09-22): letting ALL groups go through the grouped kernel + * measured 94.0 ms versus 87.5 for per-pair on a k=3 verify batch — most groups + * hold one token, and they are slower in the 16-warp/high-register shape. So + * m=1 groups stay on the per-pair kernel (its SORTED instance skips m>=2 pairs) + * and the grouped kernel takes m in [lo, M]; M=2/4/6 instances (registers grow + * with M; large groups use the large instance). + * Empty slots j >= m point at the head pair: valid pointer, never read or + * written. Returns m; 0 = not a run head or the run length is outside this + * instance's band. */ +__device__ __forceinline__ static uint32_t v41_vqg_group(const int32_t *sel, const int32_t *order, uint32_t np, uint32_t q, int M, uint32_t lo, + int32_t *e_out, uint32_t *pairs) { + const uint32_t p0 = (uint32_t)order[q]; + const int32_t e = sel[p0]; + *e_out = e; + if (e < 0) return 0u; + if (q > 0u && sel[order[q - 1u]] == e) return 0u; + uint32_t m = 1u; + while (q + m < np && sel[order[q + m]] == e) m++; + if (m < lo || m > (uint32_t)M) return 0u; + for (int j = 0; j < M; j++) pairs[j] = ((uint32_t)j < m) ? (uint32_t)order[q + (uint32_t)j] : p0; + return m; +} + +/* gate/up in one kernel (same form as v41_vq_gateup_kernel: w1/w3 -> bf16 -> + * silu(g)*u -> bf16), one block per expert x m tokens. One shared buffer serves + * twice (gate's codebook then up's); the gate result parks as bf16 in each + * token's own h row. */ +template +__global__ static void __launch_bounds__(V41_VQG_THREADS) +v41_vqg_gateup_kernel(uint16_t *h, const uint8_t *blob, const int32_t *sel, const uint32_t *x, + uint32_t IN, uint32_t MID, uint32_t K, float clamp, uint32_t cb_bytes, const int32_t *order, uint32_t np, uint32_t mlo) { + extern __shared__ __align__(16) uint8_t vqsh[]; + constexpr uint32_t W = V41_VQG_WARPS, NIT = V41_VQG_ITERS_GU; + int32_t e; uint32_t pairs[M]; + const uint32_t m = v41_vqg_group(sel, order, np, blockIdx.y, M, mlo, &e, pairs); + if (!m) return; + const uint32_t *xs[M]; uint16_t *hp[M]; + #pragma unroll + for (int j = 0; j < M; j++) { xs[j] = x + (uint64_t)(pairs[j] / K) * (IN / 2u); hp[j] = h + (uint64_t)pairs[j] * MID; } + const v41_vq_mat mg = v41_vq_open(blob, e, 0, MID, IN, NULL), mu = v41_vq_open(blob, e, 1, MID, IN, NULL); + if (!mg.ok || !mu.ok) return; + constexpr uint32_t CW = V3 ? 8u : 16u; + if (mg.nc * CW != cb_bytes || mu.nc * CW != cb_bytes) return; + const uint32_t r0 = blockIdx.x * (W * NIT) + (threadIdx.x >> 5); + const bool lead = (threadIdx.x & 31u) == 0u; + v41_vq_blk carry; carry.w0 = 0u; carry.w1 = 0u; carry.w2 = 0u; carry.ex = 0u; + if (r0 < MID) carry = v41_vq_row_first_blk(mg, r0); + v41_vq_cb_to_shared(vqsh, mg.cb, cb_bytes); + __syncthreads(); + for (uint32_t i = 0; i < NIT; i++) { + const uint32_t r = r0 + i * W; + if (r >= MID) break; + const uint32_t rn = r + W; + const int own = (i + 1u < NIT && rn < MID); + const uint32_t *next = own ? v41_vq_row_ptr(mg, rn) : v41_vq_row_ptr(mu, r0); + const uint32_t *nextex = EXT ? (own ? v41_vq_ext_ptr(mg, rn) : v41_vq_ext_ptr(mu, r0)) : NULL; + float g[M]; + v41_vqg_row_dot(mg, r, xs, m, vqsh, &carry, next, nextex, g); + if (lead) { + #pragma unroll + for (int j = 0; j < M; j++) if ((uint32_t)j < m) hp[j][r] = (uint16_t)(__float_as_uint(v41_bf16r(g[j])) >> 16); + } + } + __syncthreads(); /* all warps done with gate's codebook before overwriting it */ + v41_vq_cb_to_shared(vqsh, mu.cb, cb_bytes); + __syncthreads(); + for (uint32_t i = 0; i < NIT; i++) { + const uint32_t r = r0 + i * W; + if (r >= MID) break; + const uint32_t rn = r + W; + const int own = (i + 1u < NIT && rn < MID); + const uint32_t *next = own ? v41_vq_row_ptr(mu, rn) : NULL; + float u[M]; + v41_vqg_row_dot(mu, r, xs, m, vqsh, &carry, next, EXT && own ? v41_vq_ext_ptr(mu, rn) : NULL, u); + if (lead) { + #pragma unroll + for (int j = 0; j < M; j++) + if ((uint32_t)j < m) hp[j][r] = v41_vq_swiglu(__uint_as_float((uint32_t)hp[j][r] << 16), v41_bf16r(u[j]), clamp); + } + } +} + +/* down: partial[pair][OUT] = bf16(W2 * h_pair), one block per expert x m tokens. */ +template +__global__ static void __launch_bounds__(V41_VQG_THREADS) +v41_vqg_down_kernel(float *partial, const uint8_t *blob, const int32_t *sel, const uint32_t *hh, + uint32_t MID, uint32_t OUT, uint32_t K, uint32_t cb_bytes, const float *gr, const int32_t *order, uint32_t np, uint32_t mlo) { + extern __shared__ __align__(16) uint8_t vqsh[]; + constexpr uint32_t W = V41_VQG_WARPS, NIT = V41_VQG_ITERS_DN; + int32_t e; uint32_t pairs[M]; + const uint32_t m = v41_vqg_group(sel, order, np, blockIdx.y, M, mlo, &e, pairs); + if (!m) return; + const uint32_t *hs[M]; + #pragma unroll + for (int j = 0; j < M; j++) hs[j] = hh + (uint64_t)pairs[j] * (MID / 2u); + const v41_vq_mat md = v41_vq_open(blob, e, 2, OUT, MID, gr ? gr + (size_t)e * OUT : NULL); + const uint32_t r0 = blockIdx.x * (W * NIT) + (threadIdx.x >> 5); + const bool lead = (threadIdx.x & 31u) == 0u; + if (!md.ok) { /* bad payload: this block's rows all write 0 (the reduce reads them) */ + for (uint32_t i = 0; i < NIT; i++) { + const uint32_t r = r0 + i * W; + if (r < OUT && lead) for (int j = 0; j < M; j++) if ((uint32_t)j < m) partial[(uint64_t)pairs[j] * OUT + r] = 0.f; + } + return; + } + if (md.nc * (V3 ? 8u : 16u) != cb_bytes) return; + v41_vq_blk carry; carry.w0 = 0u; carry.w1 = 0u; carry.w2 = 0u; carry.ex = 0u; + if (r0 < OUT) carry = v41_vq_row_first_blk(md, r0); + v41_vq_cb_to_shared(vqsh, md.cb, cb_bytes); + __syncthreads(); + for (uint32_t i = 0; i < NIT; i++) { + const uint32_t r = r0 + i * W; + if (r >= OUT) break; + const uint32_t rn = r + W; + const int own = (i + 1u < NIT && rn < OUT); + const uint32_t *next = own ? v41_vq_row_ptr(md, rn) : NULL; + float y[M]; + v41_vqg_row_dot(md, r, hs, m, vqsh, &carry, next, EXT && own ? v41_vq_ext_ptr(md, rn) : NULL, y); + if (lead) { + #pragma unroll + for (int j = 0; j < M; j++) if ((uint32_t)j < m) partial[(uint64_t)pairs[j] * OUT + r] = v41_bf16r(y[j]); + } + } + (void)K; +} + +/* Per-M-instance opt-in (template statics are per instantiation — the same rule + * as the per-pair kernel's 09-22 lesson). 1 = cbb granted; 0 = cannot (the whole + * grouped path falls back to per-pair). */ +template +static int v41_vqg_optin(uint32_t cbb) { + static uint32_t s_optin = 0u; + if (s_optin >= cbb) return 1; + (void)hipGetLastError(); /* the opt-in must not see a previously latched error (see ds41_vq_decode.cuh) */ + const bool ok = hipFuncSetAttribute(reinterpret_cast(v41_vqg_gateup_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)cbb) == hipSuccess && + hipFuncSetAttribute(reinterpret_cast(v41_vqg_down_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)cbb) == hipSuccess; + (void)hipGetLastError(); + if (!ok) return 0; + s_optin = cbb; + hipFuncAttributes fg, fd; memset(&fg, 0, sizeof fg); memset(&fd, 0, sizeof fd); + (void)hipFuncGetAttributes(&fg, reinterpret_cast(v41_vqg_gateup_kernel)); + (void)hipFuncGetAttributes(&fd, reinterpret_cast(v41_vqg_down_kernel)); + int bg = 0, bd = 0; + (void)hipOccupancyMaxActiveBlocksPerMultiprocessor(&bg, v41_vqg_gateup_kernel, (int)V41_VQG_THREADS, (size_t)cbb); + (void)hipOccupancyMaxActiveBlocksPerMultiprocessor(&bd, v41_vqg_down_kernel, (int)V41_VQG_THREADS, (size_t)cbb); + (void)hipGetLastError(); + fprintf(stderr, "ds4: [ds41] VQ grouped kernel M=%d (NBIT %d): %u threads/block, dynamic shared %u KB; gateup %d regs => %d blocks/SM, down %d regs => %d blocks/SM\n", + M, NBIT, V41_VQG_THREADS, cbb >> 10, fg.numRegs, bg, fd.numRegs, bd); + return 1; +} + +/* Launch one M-band's gateup or down (takes runs with m in [mlo, M]); grid.y = + * np, blocks outside the band return immediately. */ +template +static int v41_vqg_launch_gu(uint16_t *h, const uint8_t *blob, const int32_t *sel, const uint32_t *xb, uint32_t IN, uint32_t MID, uint32_t K, + float clamp, uint32_t cbb, const int32_t *ord, uint32_t np, uint32_t mlo) { + const uint32_t rg = V41_VQG_WARPS * V41_VQG_ITERS_GU; + v41_vqg_gateup_kernel<<>>( + h, blob, sel, xb, IN, MID, K, clamp, cbb, ord, np, mlo); + return cuda_ok(hipGetLastError(), "v41 vq gateup (grouped)"); +} + +template +static int v41_vqg_launch_dn(float *part, const uint8_t *blob, const int32_t *sel, const uint16_t *h, uint32_t MID, uint32_t OUT, uint32_t K, + uint32_t cbb, const float *gr, const int32_t *ord, uint32_t np, uint32_t mlo) { + const uint32_t rd = V41_VQG_WARPS * V41_VQG_ITERS_DN; + v41_vqg_down_kernel<<>>( + part, blob, sel, (const uint32_t *)h, MID, OUT, K, cbb, gr, ord, np, mlo); + return cuda_ok(hipGetLastError(), "v41 vq down (grouped)"); +} + +/* The two stages of the grouped path (gateup then down), called from + * v41_vq_fused_moe_n: the per-pair kernel's SORTED instance computes only m=1 + * pairs while the grouped kernel takes m>=2 runs by M band (2 / 4 / 6), each + * writing disjoint pair slots. Batch size picks the bands (m <= n_tok): n <= 2 + * only M=2; 3..4 adds M=4 (takes 3..4); 5..6 adds M=6 (takes 5..6). + * 1 = launched; 0 = some band could not opt in (the caller falls back to all + * per-pair); -1 = launch failure (hard error). */ +template +static int v41_vq_grp_launch(int stage, uint32_t n_tok, uint16_t *h, float *part, const uint8_t *blob, const int32_t *sel, const uint32_t *xb, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t K, float clamp, uint32_t cbb, const float *gr, + const int32_t *ord, uint32_t np) { + if (n_tok < 2u || n_tok > 6u) return 0; + if (!v41_vqg_optin(cbb)) return 0; + if (n_tok >= 3u && !v41_vqg_optin(cbb)) return 0; + if (n_tok >= 5u && !v41_vqg_optin(cbb)) return 0; + if (stage == 0) { + if (!v41_vqg_launch_gu(h, blob, sel, xb, IN, MID, K, clamp, cbb, ord, np, 2u)) return -1; + if (n_tok >= 3u && !v41_vqg_launch_gu(h, blob, sel, xb, IN, MID, K, clamp, cbb, ord, np, 3u)) return -1; + if (n_tok >= 5u && !v41_vqg_launch_gu(h, blob, sel, xb, IN, MID, K, clamp, cbb, ord, np, 5u)) return -1; + return 1; + } + if (!v41_vqg_launch_dn(part, blob, sel, h, MID, OUT, K, cbb, gr, ord, np, 2u)) return -1; + if (n_tok >= 3u && !v41_vqg_launch_dn(part, blob, sel, h, MID, OUT, K, cbb, gr, ord, np, 3u)) return -1; + if (n_tok >= 5u && !v41_vqg_launch_dn(part, blob, sel, h, MID, OUT, K, cbb, gr, ord, np, 5u)) return -1; + return 1; +} diff --git a/cuda/ds41_vq_launch.cuh b/cuda/ds41_vq_launch.cuh new file mode 100644 index 0000000000000000000000000000000000000000..e3c1ea0080053f89fa1f14d284220c514e9a7233 --- /dev/null +++ b/cuda/ds41_vq_launch.cuh @@ -0,0 +1,54 @@ +#include "hip/hip_runtime.h" +/* ds41_vq_launch.cuh — the VQ decode dispatcher, ported from the C engine at + * /data/YoungAi (commit 3946dbc), file `src/cuda/cuda_vq_decode_launch.inc.cu`. + * + * One job: pick the template instance by [on-disk version + codebook word count] + * and do the shape precheck. All numerics live in the template it calls. + * Width = ceil(log2(words)) decides words per round, and the kernel is + * instantiated on it. A round is 32 indices => the row's index count (cols/8) + * must be a multiple of 32 (IN/MID multiples of 256; V4.1 Flash: 5120 -> 20 + * rounds, 2304 -> 9); anything else is a hard error, not a slow path. + * + * The VERSION goes into the instance too (2026-09-21): a v2 payload carries its + * own f16 codebook and a bitstream packed at NBIT; v3 hoists one E4M3 codebook + * per layer and always packs a 12-bit main stream plus (13-bit layers) a bit + * plane. The layout differences live in ds41_vq_row.cuh; this file only picks + * the instance — picking wrong produces no error, just a whole set of fake + * weights, so the version is read from the blob header by the caller, never + * guessed from a filename. + * Instance table: v2 12-bit / v2 11-bit / v3 12-bit / v3 13-bit (with plane). + * + * Must follow ds41_vq_decode.cuh and ds41_vq_group.cuh (v41_vq_fused_moe_n is + * instantiated here; v41_vq_grp_launch must be visible by now). + */ +#pragma once +#include +#include + +#include "ds41_vq_group.cuh" + +static int v41_vq_fused_moe(float *out, const uint8_t *blob, uint32_t IN, uint32_t MID, uint32_t OUT, + const int32_t *sel, const float *w, uint32_t K, float clamp, const float *x, uint32_t n_tok, uint32_t nc, + const float *gr, uint32_t ver) { + uint32_t nbit = 0; while ((1u << nbit) < nc) nbit++; + if ((IN % 256u) || (MID % 256u)) { + fprintf(stderr, "ds4: [ds41] VQ decode kernels need IN/MID multiples of 256 (32 indices per round), got %u/%u\n", IN, MID); + return 0; + } + if (ver == 3u) { + /* v3: 10-12-bit layers carry the direct stream and no plane, 13-bit + * layers a 12-bit stream plus one plane. Anything wider would need + * another plane (the converter hard-stops too). */ + if (nbit == 10u) return v41_vq_fused_moe_n<10, 1, 0>(out, blob, IN, MID, OUT, sel, w, K, clamp, x, n_tok, nc, gr); + if (nbit == 11u) return v41_vq_fused_moe_n<11, 1, 0>(out, blob, IN, MID, OUT, sel, w, K, clamp, x, n_tok, nc, gr); + if (nbit == 12u) return v41_vq_fused_moe_n<12, 1, 0>(out, blob, IN, MID, OUT, sel, w, K, clamp, x, n_tok, nc, gr); + if (nbit == 13u) return v41_vq_fused_moe_n<13, 1, 1>(out, blob, IN, MID, OUT, sel, w, K, clamp, x, n_tok, nc, gr); + fprintf(stderr, "ds4: [ds41] DQVL v3 codebook with %u words (%u bits) has no decode instance (10-13 only)\n", nc, nbit); + return 0; + } + /* v2 instances: nc4096 = 12-bit, nc2048 = 11-bit */ + if (nbit == 12u) return v41_vq_fused_moe_n<12, 0, 0>(out, blob, IN, MID, OUT, sel, w, K, clamp, x, n_tok, nc, gr); + if (nbit == 11u) return v41_vq_fused_moe_n<11, 0, 0>(out, blob, IN, MID, OUT, sel, w, K, clamp, x, n_tok, nc, gr); + fprintf(stderr, "ds4: [ds41] VQ codebook with %u words (%u bits) has no decode instance\n", nc, nbit); + return 0; +} diff --git a/cuda/ds41_vq_persist.cuh b/cuda/ds41_vq_persist.cuh new file mode 100644 index 0000000000000000000000000000000000000000..5d5c90a695f02afb7386b3104ced2aa789a49680 --- /dev/null +++ b/cuda/ds41_vq_persist.cuh @@ -0,0 +1,782 @@ +#include "hip/hip_runtime.h" +/* ds41_vq_persist.cuh — the v3 persistent decode kernels, ported from the C + * engine at /data/YoungAi (commit 3946dbc), file `src/cuda/cuda_vq_persist.inc.cu`. + * + * Why: a v3 codebook is ONE PER LAYER, but the v2-shaped gateup/down kernels + * copy it into shared per block (and gateup twice, once per matrix) with three + * full-block barriers each time; at 1 block/SM the whole SM waits during every + * copy. Grid = SM count, one persistent block per SM: the codebook is copied + * once per layer, then each warp walks a contiguous span of (expert, row) work + * items with no further block-wide barriers. gateup computes gate and up for + * the same row in one warp and SwiGLUs in registers (no h round trip). + * + * Byte-identical: every row still goes through the same v41_vq_row_dot (same + * lane<->index map, same accumulation order, same reduction tree); the gate + * value used to park as bf16 in h and read back, and it is on the bf16 grid by + * construction (v41_bf16r exit), so the round trip was lossless and SwiGLU's + * input is bit-identical. Gate = cmp against the v2-shaped kernels. + * A payload whose codebook does not point at the layer's one codebook is a + * format violation => __trap(), never a silently different codebook. + * + * Must follow ds41_vq_decode.cuh (uses its v41_vq_cb_to_shared / v41_vq_swiglu + * / V41_VQ_WARPS) and precede ds41_vq_launch.cuh. + */ +#pragma once +#include +#include +#include + +#include "ds41_vq_decode.cuh" + +/* The first valid pair's codebook for a matrix = the layer codebook (same value + * across the whole block, so the early return is block-uniform — no warp can + * miss a barrier). */ +__device__ __forceinline__ static const uint8_t *v41_vq_layer_cb(const uint8_t *blob, const int32_t *sel, uint32_t np, int which, + uint32_t rows, uint32_t cols) { + for (uint32_t p = 0; p < np; p++) { + const int32_t e = sel[p]; + if (e < 0) continue; + const v41_vq_mat m = v41_vq_open<1>(blob, e, which, rows, cols, NULL); + if (m.ok) return m.cb; + } + return NULL; +} + +/* A run of consecutive rows read as ONE bitstream. The disease (ncu on the + * first persistent version): long_scoreboard ~14, and moving activations to + * shared changed nothing — the wait was the bitstream. v41_vq_row_dot's + * pipeline is "read the next block while computing this one" (8 rounds), but a + * gateup row is 20 rounds = 8+8+4 and a down row 9 = 8+1: the last block's + * 4 (1) rounds cannot hide the next row's first-block DRAM latency (~1.2-1.4 + * us loaded) => every row stalls once at its head, plus the row gain gr[r] + * (and the sidecar gov[r]) is a global read issued at the row's end and needed + * immediately. + * Fix: consecutive rows of the same matrix are contiguous in the bitstream (a + * row is R rounds x 12 words, the plane R words per row), so a warp's n + * consecutive rows are one continuous stream — cut into 8-round blocks by + * "global round number within the span", never restarting at row boundaries. + * A row that runs out of rounds reduces, applies the gain, and moves on; row + * gains are prefetched at the span head (lane i takes row i). + * Byte-identical: each row still starts at 0 and accumulates the same + * v41_vq_dot8 sequence, the reduction tree and gain multiply match + * v41_vq_row_dot; the lane<->word map within a round only depends on the + * round's position k in the block (exactly 12 whole words per round). */ +/* Archived negatives (engine, do not resurrect): f32 activations in shared + * (6.46 -> 6.67 ms, not instruction-count-bound); bitstream through a shared + * ring with cp.async (noise); M as a warp-count knob (fewer warps in flight + * loses more than the dedupe saves). */ +/* M = how many activations one span multiplies (2026-09-24): pure decode M=1 + * (xs[0]); in a verify batch, when one expert is chosen by nt <= M tokens, the + * bitstream is read once, the codeword decoded once (v41_vq_cw), and each + * token does its own v41_vq_dot8_cw — same expression tree as M=1's + * v41_vq_dot8, so each token's accumulation order / reduction / gain multiply + * is unchanged => verify row t == the pure-decode row. M=1 keeps the original + * single v41_vq_dot8 call, so pure decode's codegen is untouched. */ +/* CBH: the codebook sits in shared as f16 (v41_vq_cb_to_shared_h), so a lookup is one 16-byte read and + * no conversion — the same eight values v41_vq_cw<1> builds per lookup. XG (M=1): x / h stay in global. + * + * Round pipeline (2026-10-23, RDNA): a round is index (two bpermutes) -> codebook read -> eight FMAs, + * two dependent LDS round trips. Unrolled with the `k >= rounds` exit between rounds, the compiler + * could not hoist round k+1's loads above the branch and serialized the chain (s_waitcnt lgkmcnt(0) + * twice per round). Full blocks (the common case: a row is 9 or 20 rounds, a span up to 32 rows) + * take a branch-free body that issues round k+2's index and round k+1's codebook/activation reads + * before round k's FMAs; only a span's tail block runs the plain loop. Every round still computes + * the same v41_vq_dot8 expression in the same order => byte-identical. */ +/* Wave-level sum / broadcast without LDS (2026-10-23). __shfl_xor_sync compiles to ds_bpermute on AMD; + * inside the row-finish branch those LDS ops made the wait-count pass drain lgkmcnt at the branch join + * every round, which serialized the codebook lookahead. DPP row_xmask (lanes ^1..^8 within a 16-lane + * row) and permlanex16 (lane ^16) give the very same partner values in the same 16,8,4,2,1 order, so + * the sum is bit-identical to the shuffle tree. */ +#if defined(__HIP_PLATFORM_AMD__) +__device__ __forceinline__ static float v41_wave_sum32(float v) { + int b = __builtin_bit_cast(int, v); + b = __builtin_bit_cast(int, v + __builtin_bit_cast(float, __builtin_amdgcn_permlanex16(b, b, 0x76543210, 0xfedcba98, false, false))); + v = __builtin_bit_cast(float, b); + v += __builtin_bit_cast(float, __builtin_amdgcn_update_dpp(0, __builtin_bit_cast(int, v), 0x168, 0xf, 0xf, true)); /* row_xmask:8 */ + v += __builtin_bit_cast(float, __builtin_amdgcn_update_dpp(0, __builtin_bit_cast(int, v), 0x164, 0xf, 0xf, true)); /* row_xmask:4 */ + v += __builtin_bit_cast(float, __builtin_amdgcn_update_dpp(0, __builtin_bit_cast(int, v), 0x162, 0xf, 0xf, true)); /* row_xmask:2 */ + v += __builtin_bit_cast(float, __builtin_amdgcn_update_dpp(0, __builtin_bit_cast(int, v), 0x161, 0xf, 0xf, true)); /* row_xmask:1 */ + return v; +} +__device__ __forceinline__ static float v41_wave_bcast(float v, uint32_t lane) { /* lane is wave-uniform */ + return __builtin_bit_cast(float, __builtin_amdgcn_readlane(__builtin_bit_cast(int, v), (int)lane)); +} +#else +__device__ __forceinline__ static float v41_wave_sum32(float v) { for (int o = 16; o > 0; o >>= 1) v += __shfl_xor_sync(0xffffffffu, v, o); return v; } +__device__ __forceinline__ static float v41_wave_bcast(float v, uint32_t lane) { return __shfl_sync(0xffffffffu, v, (int)lane); } +#endif + +template struct v41_vq_cwraw { uint4 q; }; /* one codeword's shared-memory bytes, read early, expanded at use */ +template +__device__ __forceinline__ static v41_vq_cwraw v41_vq_cw_load(uint32_t v, const uint8_t *cbs) { + v41_vq_cwraw r; + if (FP8) { const uint2 w = v41_lds_ld2(cbs + (size_t)v * 8u); r.q.x = w.x; r.q.y = w.y; r.q.z = 0u; r.q.w = 0u; } + else r.q = v41_lds_ld4(cbs + (size_t)v * 16u); + return r; +} +template +__device__ __forceinline__ static void v41_vq_cw_expand(const v41_vq_cwraw &r, float *c) { /* the v41_vq_cw conversions, verbatim */ + if (FP8) { + const __half2 p0 = v41_e4m3x2_to_half2(r.q.x), p1 = v41_e4m3x2_to_half2(r.q.x >> 16), + p2 = v41_e4m3x2_to_half2(r.q.y), p3 = v41_e4m3x2_to_half2(r.q.y >> 16); + const float2 e0 = __half22float2(p0), e1 = __half22float2(p1), e2 = __half22float2(p2), e3 = __half22float2(p3); + c[0] = e0.x; c[1] = e0.y; c[2] = e1.x; c[3] = e1.y; c[4] = e2.x; c[5] = e2.y; c[6] = e3.x; c[7] = e3.y; + } else { +#if defined(__HIP_PLATFORM_AMD__) + /* Native f16x2 -> f32 (one v_cvt_f32_f16 per value, op_sel for the high half). Through HIP.s + * __half2 the same conversion lowered to a byte-level reassembly per value plus a second LDS + * read of the low half (~30 extra VALU per round in the ISA). The conversion is exact either way. */ + typedef _Float16 v41_f16x2 __attribute__((ext_vector_type(2))); + const v41_f16x2 h0 = __builtin_bit_cast(v41_f16x2, r.q.x), h1 = __builtin_bit_cast(v41_f16x2, r.q.y); + const v41_f16x2 h2 = __builtin_bit_cast(v41_f16x2, r.q.z), h3 = __builtin_bit_cast(v41_f16x2, r.q.w); + c[0] = (float)h0.x; c[1] = (float)h0.y; c[2] = (float)h1.x; c[3] = (float)h1.y; + c[4] = (float)h2.x; c[5] = (float)h2.y; c[6] = (float)h3.x; c[7] = (float)h3.y; +#else + __half2 h0, h1, h2, h3; memcpy(&h0, &r.q.x, 4); memcpy(&h1, &r.q.y, 4); memcpy(&h2, &r.q.z, 4); memcpy(&h3, &r.q.w, 4); + const float2 f0 = __half22float2(h0), f1 = __half22float2(h1), f2 = __half22float2(h2), f3 = __half22float2(h3); + c[0] = f0.x; c[1] = f0.y; c[2] = f1.x; c[3] = f1.y; c[4] = f2.x; c[5] = f2.y; c[6] = f3.x; c[7] = f3.y; +#endif + } +} + +/* ABL (knob 7, microbenchmark ablations; wrong numbers by design): bit0 no codebook read, bit1 no + * activation read, bit2 no bitstream load (words = lane), bit3 no multiply-add. */ +template +__device__ __forceinline__ static void v41_vq_stream(const v41_vq_mat &m, uint32_t r0, uint32_t n, const uint32_t *const *xs, uint32_t nt, + const uint8_t *cbs, v41_vq_blkd *carry, const uint32_t *next, const uint32_t *nextex, + float *res) { + constexpr int FP8 = CBH ? 0 : 1; + const uint32_t lane = threadIdx.x & 31u; + constexpr uint32_t MB = v41_vq_mbit(); /* v3 main stream: min(NBIT, 12) bits */ + const uint32_t R = m.nidx_row >> 5, G = n * R; /* rounds per row / rounds in the span */ + const uint32_t a = (lane * MB) >> 5, sh = (lane * MB) & 31u; + const uint32_t *base = v41_vq_row_ptr(m, r0); + const uint32_t *exb = EXT ? v41_vq_ext_ptr(m, r0) : NULL; + float g1 = 0.f, g2 = 1.f; /* lane i: row r0+i's payload gain / sidecar gain */ + if (lane < n) { __half gh; memcpy(&gh, m.gr + (size_t)(r0 + lane) * 2u, 2); g1 = __half2float(gh); if (m.gov) g2 = m.gov[r0 + lane]; } + v41_vq_blkd cur = *carry; + float acc[M]; + #pragma unroll + for (int j = 0; j < M; j++) { acc[j] = 0.f; res[j] = 0.f; } + uint32_t kin = 0, row = 0; + /* round k's codeword index: the lane's own two words (v41_vq_blkd), the same bits + * v41_vq_blk_rounds pulled through two shuffles. k must be a compile-time constant + * (register-array index), so every caller is an unrolled loop. */ + auto idx_of = [&](uint32_t k) -> uint32_t { + uint32_t v = __funnelshift_r(cur.w[2u * k], cur.w[2u * k + 1u], sh) & ((1u << MB) - 1u); /* MB bits: 12 for 12/13-bit layers, 10/11 for narrow ones */ + if (ABL & 1) v = lane * 7u; /* ablation: a lane-fixed index, the lookup hits the same 32 words every round */ + if (EXT) { const uint32_t exw = __shfl_sync(0xffffffffu, cur.ex, (int)k); v |= ((exw >> lane) & 1u) << 12; } + return v; + }; + auto cw_at = [&](uint32_t k) -> v41_vq_cwraw { /* round k.s codeword bytes (ABL bit4: no lookup at all) */ + if (ABL & 16) { v41_vq_cwraw z; z.q = make_uint4(0x3c003c00u ^ idx_of(k), 0x3c003c00u, 0x3c003c00u, 0x3c003c00u); return z; } + return v41_vq_cw_load(idx_of(k), cbs); + }; + /* the activation words of the span's round `kin` (M=1 only; M>1 reads per token at use) */ + auto x_of = [&](uint32_t kin_) -> uint4 { + if (ABL & 2) return make_uint4(0x3f803f80u, 0x3f803f80u, 0x3f803f80u, 0x3f803f80u); + if (XG) return v41_glb_ld4(xs[0] + (size_t)(kin_ * 32u + lane) * 4u); + return v41_lds_ld4(xs[0] + (size_t)(kin_ * 32u + lane) * 4u); + }; + /* one round's multiply-add and, when the row ends, its finish (same as v41_vq_row_dot) */ + auto use = [&](const v41_vq_cwraw &cw, const uint4 &xa0) { + float c[8]; + v41_vq_cw_expand(cw, c); + if (ABL & 8) { acc[0] += c[0] + __uint_as_float(xa0.x); if (++kin == R) { kin = 0u; if (lane == row) res[0] = acc[0]; row++; } return; } + if (M == 1) acc[0] += v41_vq_dot8_cw(c, xa0); + else { + #pragma unroll + for (int j = 0; j < M; j++) + if ((uint32_t)j < nt) acc[j] += v41_vq_dot8_cw(c, v41_glb_ld4(xs[j] + (size_t)(kin * 32u + lane) * 4u)); + } + if (++kin == R) { + #pragma unroll + for (int j = 0; j < M; j++) + if ((uint32_t)j < nt) acc[j] = v41_wave_sum32(acc[j]); + const float a1 = v41_wave_bcast(g1, row), a2 = v41_wave_bcast(g2, row); + #pragma unroll + for (int j = 0; j < M; j++) { + const float val = acc[j] * a1 * (m.gov ? a2 : 1.0f); + if (lane == row && (uint32_t)j < nt) res[j] = val; + acc[j] = 0.f; + } + kin = 0u; row++; + } + }; + for (uint32_t g0 = 0; g0 < G; g0 += 8u) { + v41_vq_blkd nxt; + if (ABL & 4) { for (int i = 0; i < 16; i++) nxt.w[i] = lane * 0x01010101u + (uint32_t)i; nxt.ex = 0u; } /* ablation: no bitstream traffic */ + else if (g0 + 8u < G) { const uint32_t rem = G - g0 - 8u, nr = rem < 8u ? rem : 8u; + nxt = v41_vq_blkd_load(base + (size_t)(g0 + 8u) * MB, nr * MB, EXT ? exb + (g0 + 8u) : NULL, nr, MB, a); } + else if (next) nxt = v41_vq_blkd_load(next, 8u * MB, nextex, 8u, MB, a); /* caller guarantees that span has >= 8 rounds */ + else { const v41_vq_blkd z = {}; nxt = z; } + if (G - g0 >= 8u) { /* full block: branch-free, two rounds of lookahead */ + /* Only the codebook lookup (random, bank-conflicted) runs ahead; the activation read is a + * contiguous LDS read issued at use — prefetching it too made the register allocator alias + * its destination with the pending load and the waitcnt pass then drained lgkmcnt every round. */ + v41_vq_cwraw c0 = cw_at(0), c1 = cw_at(1); + #pragma unroll + for (uint32_t k = 0; k < 8u; k++) { + uint4 xa0; if (M == 1) xa0 = x_of(kin); /* x first: the in-order lgkmcnt wait for it must not cover the lookahead */ + v41_vq_cwraw c2; + if (k + 2u < 8u) c2 = cw_at(k + 2u); +#if defined(__HIP_PLATFORM_AMD__) + __builtin_amdgcn_sched_barrier(0); /* keep the two-round lookahead ahead: the scheduler sank these loads to their use (ISA: lgkmcnt(0) + load + wait per round) */ +#endif + + use(c0, xa0); + c0 = c1; c1 = c2; + } + } else { + /* tail block: guards, never a break — a loop exit between rounds made the compiler keep k + * at run time and the word array (indexed by k) fell into scratch (private_seg_size 72) */ + const uint32_t rounds = G - g0; + #pragma unroll + for (uint32_t k = 0; k < 8u; k++) { + if (k < rounds) { + const v41_vq_cwraw c = cw_at(k); + uint4 xa0; if (M == 1) xa0 = x_of(kin); + use(c, xa0); + } + } + } + cur = nxt; + } + *carry = cur; +} + +/* The layer's E4M3 codebook into shared as f16 (twice the bytes): the per-lookup conversion was ~80 VALU + * per codeword on RDNA (no FP8 instruction), now paid once per block. Same converter as v41_vq_cw<1>, + * same element order (byte 2i -> low half of word i), so every codeword value is unchanged. */ +__device__ __forceinline__ static void v41_vq_cb_to_shared_h(uint8_t *dst, const uint8_t *src, uint32_t bytes) { + const uint32_t *s = (const uint32_t *)src; uint2 *d = (uint2 *)dst; + for (uint32_t i = threadIdx.x; i < bytes / 4u; i += blockDim.x) { + const uint32_t w = s[i]; + const __half2 lo = v41_e4m3x2_to_half2(w), hi = v41_e4m3x2_to_half2(w >> 16); + uint2 o; memcpy(&o.x, &lo, 4); memcpy(&o.y, &hi, 4); + d[i] = o; + } +} +template +__device__ __forceinline__ static void v41_vq_cb_stage(uint8_t *dst, const uint8_t *src, uint32_t bytes) { + if (CBH) v41_vq_cb_to_shared_h(dst, src, bytes); else v41_vq_cb_to_shared(dst, src, bytes); +} + +/* ==== two bitstreams per wave (2026-10-23) ==== + * S streams of the same width (gate + up of one row span; or the two halves of a + * down span, rows of the same pair) advance in lockstep and share every activation + * read: the kernel is LDS-bound (codebook lookups + activation reads, ~1 KB per + * wave-round), and the activation read is the half that can be shared. Each + * stream keeps its own words, lookahead and accumulator, so every row's + * multiply-add sequence is the single-stream one => byte-identical. Streams may + * differ in row count (an odd down span): stream s is active while row < ns[s]. */ +template +__device__ __forceinline__ static void v41_vq_stream2(const v41_vq_mat *ms, const uint32_t *r0s, const uint32_t *ns, const uint32_t *x, + const uint8_t *cbs, v41_vq_blkd *carry, float *res) { + constexpr int FP8 = CBH ? 0 : 1; + const uint32_t lane = threadIdx.x & 31u; + constexpr uint32_t MB = v41_vq_mbit(); + const uint32_t R = ms[0].nidx_row >> 5; /* rounds per row, the same for every stream */ + const uint32_t a = (lane * MB) >> 5, sh = (lane * MB) & 31u; + uint32_t G[S], Gmax = 0; const uint32_t *base[S], *exb[S]; float g1[S], g2[S], acc[S]; + v41_vq_blkd cur[S]; + #pragma unroll + for (int s = 0; s < S; s++) { + G[s] = ns[s] * R; if (G[s] > Gmax) Gmax = G[s]; + base[s] = v41_vq_row_ptr(ms[s], r0s[s]); + exb[s] = EXT ? v41_vq_ext_ptr(ms[s], r0s[s]) : NULL; + g1[s] = 0.f; g2[s] = 1.f; + if (lane < ns[s]) { __half gh; memcpy(&gh, ms[s].gr + (size_t)(r0s[s] + lane) * 2u, 2); g1[s] = __half2float(gh); if (ms[s].gov) g2[s] = ms[s].gov[r0s[s] + lane]; } + acc[s] = 0.f; res[s] = 0.f; + cur[s] = carry[s]; + } + uint32_t kin = 0, row = 0; + auto idx_of = [&](int s, uint32_t k) -> uint32_t { + uint32_t v = __funnelshift_r(cur[s].w[2u * k], cur[s].w[2u * k + 1u], sh) & ((1u << MB) - 1u); + if (ABL & 1) v = lane * 7u; + if (EXT) { const uint32_t exw = __shfl_sync(0xffffffffu, cur[s].ex, (int)k); v |= ((exw >> lane) & 1u) << 12; } + return v; + }; + auto cw_at = [&](int s, uint32_t k) -> v41_vq_cwraw { + if (ABL & 16) { v41_vq_cwraw z; z.q = make_uint4(0x3c003c00u ^ idx_of(s, k), 0x3c003c00u, 0x3c003c00u, 0x3c003c00u); return z; } + return v41_vq_cw_load(idx_of(s, k), cbs); + }; + auto x_of = [&](uint32_t kin_) -> uint4 { + if (ABL & 2) return make_uint4(0x3f803f80u, 0x3f803f80u, 0x3f803f80u, 0x3f803f80u); + if (XG) return v41_glb_ld4(x + (size_t)(kin_ * 32u + lane) * 4u); + return v41_lds_ld4(x + (size_t)(kin_ * 32u + lane) * 4u); + }; + /* one round for every active stream, then the shared row finish */ + auto use = [&](const v41_vq_cwraw *cw, const uint4 &xa0) { + #pragma unroll + for (int s = 0; s < S; s++) { + if (row < ns[s]) { + float c[8]; + v41_vq_cw_expand(cw[s], c); + if (ABL & 8) acc[s] += c[0] + __uint_as_float(xa0.x); + else acc[s] += v41_vq_dot8_cw(c, xa0); + } + } + if (++kin == R) { + #pragma unroll + for (int s = 0; s < S; s++) { + if (row < ns[s]) { + const float sum = v41_wave_sum32(acc[s]); + const float a1 = v41_wave_bcast(g1[s], row), a2 = v41_wave_bcast(g2[s], row); + const float val = sum * a1 * (ms[s].gov ? a2 : 1.0f); + if (lane == row) res[s] = val; + } + acc[s] = 0.f; + } + kin = 0u; row++; + } + }; + for (uint32_t g0 = 0; g0 < Gmax; g0 += 8u) { + v41_vq_blkd nxt[S]; + #pragma unroll + for (int s = 0; s < S; s++) { + if (ABL & 4) { for (int i = 0; i < 16; i++) nxt[s].w[i] = lane * 0x01010101u + (uint32_t)i; nxt[s].ex = 0u; } + else if (g0 + 8u < G[s]) { const uint32_t rem = G[s] - g0 - 8u, nr = rem < 8u ? rem : 8u; + nxt[s] = v41_vq_blkd_load(base[s] + (size_t)(g0 + 8u) * MB, nr * MB, EXT ? exb[s] + (g0 + 8u) : NULL, nr, MB, a); } + else { const v41_vq_blkd z = {}; nxt[s] = z; } + } + if (Gmax - g0 >= 8u) { + v41_vq_cwraw c0[S], c1[S]; + #pragma unroll + for (int s = 0; s < S; s++) { c0[s] = cw_at(s, 0); c1[s] = cw_at(s, 1); } + #pragma unroll + for (uint32_t k = 0; k < 8u; k++) { + const uint4 xa0 = x_of(kin); /* x first: its in-order wait must not cover the lookahead */ + v41_vq_cwraw c2[S]; + #pragma unroll + for (int s = 0; s < S; s++) if (k + 2u < 8u) c2[s] = cw_at(s, k + 2u); +#if defined(__HIP_PLATFORM_AMD__) + __builtin_amdgcn_sched_barrier(0); +#endif + use(c0, xa0); + #pragma unroll + for (int s = 0; s < S; s++) { c0[s] = c1[s]; c1[s] = c2[s]; } + } + } else { + const uint32_t rounds = Gmax - g0; + #pragma unroll + for (uint32_t k = 0; k < 8u; k++) { + if (k < rounds) { + const uint4 xa0 = x_of(kin); + v41_vq_cwraw c[S]; + #pragma unroll + for (int s = 0; s < S; s++) c[s] = cw_at(s, k); + use(c, xa0); + } + } + } + #pragma unroll + for (int s = 0; s < S; s++) cur[s] = nxt[s]; + } +} + +/* __launch_bounds__(1024, 1): 1024 threads/block caps 64 registers/thread; the + * 13-bit instance (plane) naturally exceeds it, so it must be pinned; a spill + * (cuobjdump LOCAL) disqualifies the build. Work split: all np*rows + * "(expert, row)" items cut into contiguous per-warp spans, a span at most 32 + * rows and never crossing experts (lane i receives row i's result). */ +#define V41_VQ_SEG(U, UEND, ROWS, P, R0, N) \ + const uint32_t P = (U) / (ROWS), R0 = (U) % (ROWS); \ + uint32_t N = (UEND) - (U); if (N > (ROWS) - R0) N = (ROWS) - R0; if (N > 32u) N = 32u + +/* ==== the pair kernels (any n <= 8; 2026-10-23) ==== + * Work items are (pair, row) for all np = n*K pairs, exactly the n=1 kernels' + * split; a verify batch no longer groups same-expert pairs (the grouped + * kernels below stay as the knob-6 A/B). Why: those kernels read the tokens' + * activations from global inside the round loop, and on RDNA the in-order + * vmcnt wait for that read also waits for the next bitstream block's prefetch + * — the whole software pipeline collapsed (n=2: 3x the n=1 time for 2x the + * bytes). Here every activation read is LDS: a block stages only the + * activation rows (tokens' x, or pairs' h) its own contiguous span of items + * touches — a span is total/grid items, within one or two units — so the + * staging stays small at any n (the host sizes it, v41_vq_pairs_stage_max). + * Byte-identical: each (pair, row) runs the same v41_vq_stream M=1 path as + * pure decode, so a verify row equals the pure-decode row (the engine's + * "same track" gate). */ +typedef struct { uint32_t u0, uend, unit0, nunit; } v41_vq_span; +/* This block's span of items and the unit (token / pair) range it touches; UNIT = items per unit. */ +__device__ __forceinline__ static v41_vq_span v41_vq_block_span(uint32_t total, uint32_t unit) { + v41_vq_span s; + const uint32_t nw = gridDim.x * V41_VQ_WARPS, per = (total + nw - 1u) / nw; + const uint32_t b0 = blockIdx.x * V41_VQ_WARPS * per, b1 = b0 + V41_VQ_WARPS * per; + s.u0 = b0 < total ? b0 : total; s.uend = b1 < total ? b1 : total; + s.unit0 = s.u0 / unit; + s.nunit = s.uend > s.u0 ? (s.uend - 1u) / unit - s.unit0 + 1u : 0u; + return s; +} +/* Host side: the most units any block's span touches (what the launcher must stage). */ +static uint32_t v41_vq_pairs_stage_max(uint32_t total, uint32_t grid, uint32_t unit) { + const uint32_t nw = grid * V41_VQ_WARPS, per = (total + nw - 1u) / nw; + uint32_t mx = 0; + for (uint32_t b = 0; b < grid; b++) { + const uint32_t b0 = b * V41_VQ_WARPS * per; if (b0 >= total) break; + uint32_t b1 = b0 + V41_VQ_WARPS * per; if (b1 > total) b1 = total; + const uint32_t n = (b1 - 1u) / unit - b0 / unit + 1u; + if (n > mx) mx = n; + } + return mx; +} + +/* HS = row halves per span: gateup runs gate + up of each half (S = 2*HS streams), down + * runs HS halves of the span (S = HS). The host picks HS from knob 8 (0 = the defaults). */ +template +__global__ static void __launch_bounds__(1024, 1) v41_vq_gu_pairs_kernel(uint16_t *h, const uint8_t *blob, const int32_t *sel, const uint32_t *x, + uint32_t IN, uint32_t MID, uint32_t np, uint32_t K, float clamp, uint32_t cbb) { + extern __shared__ __align__(16) uint8_t vqsh[]; + const uint32_t lane = threadIdx.x & 31u, cbsh = (CBH ? 2u : 1u) * cbb; + const v41_vq_span sp = v41_vq_block_span(np * MID, MID * K); /* unit = one token's K pairs */ + const uint32_t per = (np * MID + gridDim.x * V41_VQ_WARPS - 1u) / (gridDim.x * V41_VQ_WARPS); + uint32_t u = (blockIdx.x * V41_VQ_WARPS + (threadIdx.x >> 5)) * per; + const uint32_t uend = (u + per < np * MID) ? u + per : np * MID; + /* PDL: the codebook is a constant, located via expert 0's payload (v3 + * shares one per layer) and copied first; sel/x are upstream output and + * only touched after v41_pdl_wait. */ + const v41_vq_mat m0 = v41_vq_open<1>(blob, 0, 0, MID, IN, NULL); + const uint8_t *cb = m0.ok ? m0.cb : NULL; + if (cb) v41_vq_cb_stage(vqsh, cb, cbb); + v41_pdl_wait(); + if (!cb) { cb = v41_vq_layer_cb(blob, sel, np, 0, MID, IN); if (!cb) return; v41_vq_cb_stage(vqsh, cb, cbb); } + v41_vq_blkd carry[2 * HS] = {}; + if (u < uend) { /* the first span's first block issues behind the activation copy */ + const int32_t e = sel[u / MID]; + if (e >= 0) { const v41_vq_mat mg = v41_vq_open<1>(blob, e, 0, MID, IN, NULL); if (mg.ok) carry[0] = v41_vq_row_first_blkd(mg, u % MID); } + } + if (!XG) v41_vq_cb_to_shared(vqsh + cbsh, (const uint8_t *)(x + (size_t)sp.unit0 * (IN / 2u)), sp.nunit * IN * 2u); /* the span's tokens (bf16) */ + __syncthreads(); /* the whole kernel's only block-wide barrier */ + bool first = true; + while (u < uend) { + V41_VQ_SEG(u, uend, MID, p, r0, n); + u += n; + const int32_t e = sel[p]; + if (e < 0) { first = false; continue; } /* invalid expert: write nothing, as the v2 kernel does */ + const v41_vq_mat mg = v41_vq_open<1>(blob, e, 0, MID, IN, NULL), mu = v41_vq_open<1>(blob, e, 1, MID, IN, NULL); + if (!mg.ok || !mu.ok) { first = false; continue; } + if (mg.cb != cb || mu.cb != cb) __trap(); /* format violation: never compute with a foreign codebook */ + const uint32_t t = p / K; + const uint32_t *xp = XG ? x + (size_t)t * (IN / 2u) : (const uint32_t *)(vqsh + cbsh + (size_t)(t - sp.unit0) * IN * 2u); + /* streams 2i / 2i+1 = gate / up of half i (rows [hi, hi+ni)); a half may be empty on short spans */ + v41_vq_mat ms[2 * HS]; uint32_t r0s[2 * HS], ns[2 * HS]; float res[2 * HS]; + const uint32_t hn = (n + HS - 1u) / HS; + #pragma unroll + for (int i = 0; i < HS; i++) { + const uint32_t h0 = (uint32_t)i * hn, ni = h0 < n ? (n - h0 < hn ? n - h0 : hn) : 0u; + ms[2 * i] = mg; ms[2 * i + 1] = mu; r0s[2 * i] = r0s[2 * i + 1] = r0 + h0; ns[2 * i] = ns[2 * i + 1] = ni; + if (!(i == 0 && first)) carry[2 * i] = ni ? v41_vq_row_first_blkd(mg, r0 + h0) : v41_vq_blkd{}; + carry[2 * i + 1] = ni ? v41_vq_row_first_blkd(mu, r0 + h0) : v41_vq_blkd{}; + } + first = false; + v41_vq_stream2(ms, r0s, ns, xp, vqsh, carry, res); + #pragma unroll + for (int i = 0; i < HS; i++) + if (lane < ns[2 * i]) h[(uint64_t)p * MID + r0s[2 * i] + lane] = v41_vq_swiglu(v41_bf16r(res[2 * i]), v41_bf16r(res[2 * i + 1]), clamp); + } +} + +template +__global__ static void __launch_bounds__(1024, 1) v41_vq_dn_pairs_kernel(float *partial, const uint8_t *blob, const int32_t *sel, const uint32_t *h, + uint32_t MID, uint32_t OUT, uint32_t np, uint32_t cbb, const float *gr) { + extern __shared__ __align__(16) uint8_t vqsh[]; + const uint32_t lane = threadIdx.x & 31u, cbsh = (CBH ? 2u : 1u) * cbb; + const v41_vq_span sp = v41_vq_block_span(np * OUT, OUT); /* unit = one pair's h row */ + const uint32_t per = (np * OUT + gridDim.x * V41_VQ_WARPS - 1u) / (gridDim.x * V41_VQ_WARPS); + uint32_t u = (blockIdx.x * V41_VQ_WARPS + (threadIdx.x >> 5)) * per; + const uint32_t uend = (u + per < np * OUT) ? u + per : np * OUT; + const v41_vq_mat m0 = v41_vq_open<1>(blob, 0, 2, OUT, MID, NULL); /* PDL: codebook first, same as gateup */ + const uint8_t *cb = m0.ok ? m0.cb : NULL; + if (cb) v41_vq_cb_stage(vqsh, cb, cbb); + v41_pdl_wait(); + if (!cb) { cb = v41_vq_layer_cb(blob, sel, np, 2, OUT, MID); if (!cb) return; v41_vq_cb_stage(vqsh, cb, cbb); } + v41_vq_blkd carry[HS] = {}; + if (u < uend) { + const int32_t e = sel[u / OUT]; + if (e >= 0) { const v41_vq_mat md = v41_vq_open<1>(blob, e, 2, OUT, MID, NULL); if (md.ok) carry[0] = v41_vq_row_first_blkd(md, u % OUT); } + } + if (!XG) v41_vq_cb_to_shared(vqsh + cbsh, (const uint8_t *)(h + (size_t)sp.unit0 * (MID / 2u)), sp.nunit * MID * 2u); /* the span's pairs' h (bf16) */ + __syncthreads(); + bool first = true; + while (u < uend) { + V41_VQ_SEG(u, uend, OUT, p, r0, n); + u += n; + const int32_t e = sel[p]; + if (e < 0) { first = false; continue; } + const v41_vq_mat md = v41_vq_open<1>(blob, e, 2, OUT, MID, gr ? gr + (size_t)e * OUT : NULL); + if (!md.ok) { if (lane < n) partial[(uint64_t)p * OUT + r0 + lane] = 0.f; first = false; continue; } /* bad payload: zeros, as the v2 kernel */ + if (md.cb != cb) __trap(); + const uint32_t *hp = XG ? h + (size_t)p * (MID / 2u) : (const uint32_t *)(vqsh + cbsh + (size_t)(p - sp.unit0) * MID * 2u); + /* stream i = rows [r0 + i*hn, +ni) of this pair */ + v41_vq_mat ms[HS]; uint32_t r0s[HS], ns[HS]; float res[HS]; + const uint32_t hn = (n + HS - 1u) / HS; + #pragma unroll + for (int i = 0; i < HS; i++) { + const uint32_t h0 = (uint32_t)i * hn, ni = h0 < n ? (n - h0 < hn ? n - h0 : hn) : 0u; + ms[i] = md; r0s[i] = r0 + h0; ns[i] = ni; + if (!(i == 0 && first)) carry[i] = ni ? v41_vq_row_first_blkd(md, r0 + h0) : v41_vq_blkd{}; + } + first = false; + v41_vq_stream2(ms, r0s, ns, hp, vqsh, carry, res); + #pragma unroll + for (int i = 0; i < HS; i++) if (lane < ns[i]) partial[(uint64_t)p * OUT + r0s[i] + lane] = v41_bf16r(res[i]); + } +} + +/* ==== verify-batch (n_tok >= 2) persistent kernels (2026-09-24) ==== + * Work items change from "(pair, row)" to "(unique expert, row)": the block head + * groups same-expert pairs from `order` (pairs sorted by expert), the bitstream + * is read once, codewords decoded once, and the nt tokens of the group each + * multiply (v41_vq_stream). Codebook: one per layer, one persistent block + * per SM, copied once — same as the n=1 kernels. Activations/h stay in global + * (L1): a 13-bit codebook (64 KB) plus 4 rows of activations (40 KB) exceeds + * the ~99 KB opt-in cap. + * Registers: the n=1 kernel already sits at 64 (1024 threads); multi-token + * adds M-1 accumulators + results and reads activations on use, so M=2 fits in + * 32 warps. + * Byte-identical: see v41_vq_stream; the gate is the engine's "same track" + * test (speculative == pure decode, byte for byte). */ +#define V41_VQPN_MAXP 64u /* pair cap: V41_GEMV_MAX_TOK(8) * top-k(6) = 48 */ +/* One group holds at most M=2 tokens; larger groups split (measured: fewer + * warps in flight loses more than the byte savings — 32 warps, M=2, always). */ +#define V41_VQPN_M 2u + +/* Block head: group same-expert pairs from `order` (q = group start in order, + * m = pairs in the group, <= V41_VQPN_M); thread 0 does it, the rest wait at + * the barrier. (Moving this table into v41_vq_order_kernel was measured + * noise and reverted in the engine.) */ +__device__ __forceinline__ static void v41_vqpn_groups(const int32_t *sel, const int32_t *order, uint32_t np, + uint32_t *gq, uint32_t *gm, uint32_t *ng) { + if (threadIdx.x == 0) { + uint32_t g = 0, q = 0; + while (q < np) { + const int32_t e = sel[order[q]]; + uint32_t m = 1u; + while (q + m < np && sel[order[q + m]] == e && m < V41_VQPN_M) m++; + gq[g] = q; gm[g] = m; g++; q += m; + } + *ng = g; + } +} + +template +__global__ static void __launch_bounds__(NW * 32, 1) v41_vq_gu_persist_n_kernel(uint16_t *h, const uint8_t *blob, const int32_t *sel, + const int32_t *order, const uint32_t *x, uint32_t IN, uint32_t MID, uint32_t K, uint32_t np, float clamp, uint32_t cbb) { + extern __shared__ __align__(16) uint8_t vqsh[]; + __shared__ uint32_t gq[V41_VQPN_MAXP], gm[V41_VQPN_MAXP], ng; + const uint32_t lane = threadIdx.x & 31u; + const v41_vq_mat m0 = v41_vq_open<1>(blob, 0, 0, MID, IN, NULL); /* PDL: codebook first; sel/order/x after the wait */ + const uint8_t *cb = m0.ok ? m0.cb : NULL; + if (cb) v41_vq_cb_stage(vqsh, cb, cbb); + v41_pdl_wait(); + if (!cb) { cb = v41_vq_layer_cb(blob, sel, np, 0, MID, IN); if (!cb) return; v41_vq_cb_stage(vqsh, cb, cbb); } + v41_vqpn_groups(sel, order, np, gq, gm, &ng); + __syncthreads(); /* the kernel's only block-wide barrier (codebook + group table) */ + const uint32_t gw = blockIdx.x * NW + (threadIdx.x >> 5), nw = gridDim.x * NW; + const uint32_t total = ng * MID, per = (total + nw - 1u) / nw; + uint32_t u = gw * per; + const uint32_t uend = (u + per < total) ? u + per : total; + while (u < uend) { + V41_VQ_SEG(u, uend, MID, g, r0, n); + u += n; + const uint32_t q = gq[g], nt = gm[g]; + const int32_t e = sel[order[q]]; + if (e < 0) continue; + const v41_vq_mat mg = v41_vq_open<1>(blob, e, 0, MID, IN, NULL), mu = v41_vq_open<1>(blob, e, 1, MID, IN, NULL); + if (!mg.ok || !mu.ok) continue; + if (mg.cb != cb || mu.cb != cb) __trap(); + const uint32_t *xs[M]; uint32_t pr[M]; + #pragma unroll + for (int j = 0; j < M; j++) { + pr[j] = (uint32_t)order[q + ((uint32_t)j < nt ? (uint32_t)j : 0u)]; /* empty slots point at the group head: valid pointer, never read/written */ + xs[j] = x + (uint64_t)(pr[j] / K) * (IN / 2u); + } + v41_vq_blkd carry = v41_vq_row_first_blkd(mg, r0); + float gs[M], us[M]; + v41_vq_stream(mg, r0, n, xs, nt, vqsh, &carry, v41_vq_row_ptr(mu, r0), EXT ? v41_vq_ext_ptr(mu, r0) : NULL, gs); + v41_vq_stream(mu, r0, n, xs, nt, vqsh, &carry, NULL, NULL, us); + if (lane < n) { + #pragma unroll + for (int j = 0; j < M; j++) + if ((uint32_t)j < nt) h[(uint64_t)pr[j] * MID + r0 + lane] = v41_vq_swiglu(v41_bf16r(gs[j]), v41_bf16r(us[j]), clamp); + } + } +} + +template +__global__ static void __launch_bounds__(NW * 32, 1) v41_vq_dn_persist_n_kernel(float *partial, const uint8_t *blob, const int32_t *sel, + const int32_t *order, const uint32_t *h, uint32_t MID, uint32_t OUT, uint32_t np, uint32_t cbb, const float *gr) { + extern __shared__ __align__(16) uint8_t vqsh[]; + __shared__ uint32_t gq[V41_VQPN_MAXP], gm[V41_VQPN_MAXP], ng; + const uint32_t lane = threadIdx.x & 31u; + const v41_vq_mat m0 = v41_vq_open<1>(blob, 0, 2, OUT, MID, NULL); + const uint8_t *cb = m0.ok ? m0.cb : NULL; + if (cb) v41_vq_cb_stage(vqsh, cb, cbb); + v41_pdl_wait(); + if (!cb) { cb = v41_vq_layer_cb(blob, sel, np, 2, OUT, MID); if (!cb) return; v41_vq_cb_stage(vqsh, cb, cbb); } + v41_vqpn_groups(sel, order, np, gq, gm, &ng); + __syncthreads(); + const uint32_t gw = blockIdx.x * NW + (threadIdx.x >> 5), nw = gridDim.x * NW; + const uint32_t total = ng * OUT, per = (total + nw - 1u) / nw; + uint32_t u = gw * per; + const uint32_t uend = (u + per < total) ? u + per : total; + while (u < uend) { + V41_VQ_SEG(u, uend, OUT, g, r0, n); + u += n; + const uint32_t q = gq[g], nt = gm[g]; + const int32_t e = sel[order[q]]; + if (e < 0) continue; + uint32_t pr[M]; const uint32_t *hs[M]; + #pragma unroll + for (int j = 0; j < M; j++) { + pr[j] = (uint32_t)order[q + ((uint32_t)j < nt ? (uint32_t)j : 0u)]; + hs[j] = h + (uint64_t)pr[j] * (MID / 2u); + } + const v41_vq_mat md = v41_vq_open<1>(blob, e, 2, OUT, MID, gr ? gr + (size_t)e * OUT : NULL); + if (!md.ok) { /* bad payload: zeros (the tail reads them) */ + if (lane < n) for (uint32_t j = 0; j < nt; j++) partial[(uint64_t)pr[j] * OUT + r0 + lane] = 0.f; + continue; + } + if (md.cb != cb) __trap(); + v41_vq_blkd carry = v41_vq_row_first_blkd(md, r0); + float ys[M]; + v41_vq_stream(md, r0, n, hs, nt, vqsh, &carry, NULL, NULL, ys); + if (lane < n) { + #pragma unroll + for (int j = 0; j < M; j++) if ((uint32_t)j < nt) partial[(uint64_t)pr[j] * OUT + r0 + lane] = v41_bf16r(ys[j]); + } + } +} + +/* One kernel instance's opt-in + launch. The opt-in is recorded per instance + * (template statics are per instantiation; both codebook sizes coexist). */ +template +static int v41_vq_optin(uint32_t *granted, KF kern, uint32_t need, const char *what) { + if (*granted >= need) return 1; + (void)hipGetLastError(); /* the opt-in must not see a previously latched error (see ds41_vq_decode.cuh) */ + const hipError_t e = hipFuncSetAttribute(reinterpret_cast(kern), hipFuncAttributeMaxDynamicSharedMemorySize, (int)need); + if (e != hipSuccess) { + fprintf(stderr, "ds4: [ds41] %s cannot opt in %u B dynamic shared: %s\n", what, need, hipGetErrorString(e)); + (void)hipGetLastError(); + return 0; + } + *granted = need; + return 1; +} + +/* The grouped verify-batch kernels (knob 6 = 8 / 16 / 32 warps): the pre-pair A/B. */ +template +static int v41_vq_persist_n_go(int stage, uint16_t *h, float *part, const uint8_t *blob, const int32_t *sel, const int32_t *ord, + const uint32_t *xb, uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t K, uint32_t np, float clamp, + uint32_t cbb, const float *gr, int nsm) { + static uint32_t s_optin[2] = { 0u, 0u }; + const uint32_t need = CBH ? 2u * cbb : cbb; + if (stage == 0) { + if (!v41_vq_optin(&s_optin[0], v41_vq_gu_persist_n_kernel, need, "VQ verify-batch gateup kernel")) return 0; + v41_pdl_register((const void *)v41_vq_gu_persist_n_kernel); /* v41_pdl_wait before touching sel/order/x */ + v41_vq_gu_persist_n_kernel<<<(unsigned)nsm, NW * 32, need, ds4_current_stream()>>>(h, blob, sel, ord, xb, IN, MID, K, np, clamp, cbb); + } else { + if (!v41_vq_optin(&s_optin[1], v41_vq_dn_persist_n_kernel, need, "VQ verify-batch down kernel")) return 0; + v41_pdl_register((const void *)v41_vq_dn_persist_n_kernel); + v41_vq_dn_persist_n_kernel<<<(unsigned)nsm, NW * 32, need, ds4_current_stream()>>>(part, blob, sel, ord, (const uint32_t *)h, MID, OUT, np, cbb, gr); + } + return cuda_ok(hipGetLastError(), stage == 0 ? "v41 vq gateup persist (verify batch)" : "v41 vq down persist (verify batch)"); +} + +/* Shared budget per block (the device reports 64 KB per SM; one persistent + * block per SM). Knob 1 forces the E4M3 codebook lookup (the f16 A/B). */ +#define V41_VQ_SHARED_CAP 65536u +static inline int v41_vq_cbh_ok(uint32_t bytes) { return !v41_knob(1) && bytes <= V41_VQ_SHARED_CAP; } + +/* The pair kernels' launch: shared layout [codebook (f16 when CBH)][the span's + * activation rows unless XG]. Preference: f16 codebook + staged rows; f16 + + * global rows (12-bit layers: the 64 KB f16 codebook fills the block alone); + * E4M3 + staged; E4M3 + global. HS = row halves per span (see the kernels). */ +template +static int v41_vq_gu_go(uint16_t *h, const uint8_t *blob, const int32_t *sel, const uint32_t *xb, uint32_t IN, uint32_t MID, uint32_t K, + uint32_t np, float clamp, uint32_t cbb, uint32_t need, int grid) { + static uint32_t s_optin = 0u; + if (!v41_vq_optin(&s_optin, v41_vq_gu_pairs_kernel, need, "VQ pair gateup kernel")) return 0; + v41_pdl_register((const void *)v41_vq_gu_pairs_kernel); /* v41_pdl_wait before sel/x */ + v41_vq_gu_pairs_kernel<<<(unsigned)grid, V41_VQ_WARPS * 32u, need, ds4_current_stream()>>>(h, blob, sel, xb, IN, MID, np, K, clamp, cbb); + return cuda_ok(hipGetLastError(), "v41 vq gateup pairs"); +} +template +static int v41_vq_dn_go(float *part, const uint8_t *blob, const int32_t *sel, const uint16_t *h, uint32_t MID, uint32_t OUT, + uint32_t np, uint32_t cbb, const float *gr, uint32_t need, int grid) { + static uint32_t s_optin = 0u; + if (!v41_vq_optin(&s_optin, v41_vq_dn_pairs_kernel, need, "VQ pair down kernel")) return 0; + v41_pdl_register((const void *)v41_vq_dn_pairs_kernel); /* v41_pdl_wait before sel/h */ + v41_vq_dn_pairs_kernel<<<(unsigned)grid, V41_VQ_WARPS * 32u, need, ds4_current_stream()>>>(part, blob, sel, (const uint32_t *)h, MID, OUT, np, cbb, gr); + return cuda_ok(hipGetLastError(), "v41 vq down pairs"); +} +/* One (CBH, XG) layout: the stream count from knob 8 (0 = gate+up of one half, two + * down halves; 1 = single streams; 2 = two gate+up halves, three down halves; 3 = two + * and four), the ablation bits from knob 7 (at the default stream count only). */ +template +static int v41_vq_pairs_go(int stage, uint16_t *h, float *part, const uint8_t *blob, const int32_t *sel, const uint32_t *xb, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t K, uint32_t np, float clamp, uint32_t cbb, const float *gr, + uint32_t need, int grid) { + const int hs = v41_knob(8), abl = v41_knob(7); +#define V41_GU(ABL_, HS_) v41_vq_gu_go(h, blob, sel, xb, IN, MID, K, np, clamp, cbb, need, grid) +#define V41_DN(ABL_, HS_) v41_vq_dn_go(part, blob, sel, h, MID, OUT, np, cbb, gr, need, grid) + if (stage == 0) { + if (hs == 1) return V41_GU(0, 1); + if (hs == 2 || hs == 3) return V41_GU(0, 2); + switch (abl) { case 1: return V41_GU(1, 1); case 2: return V41_GU(2, 1); case 4: return V41_GU(4, 1); case 8: return V41_GU(8, 1); case 16: return V41_GU(16, 1); default: return V41_GU(0, 1); } + } + if (hs == 1) return V41_DN(0, 1); + if (hs == 2) return V41_DN(0, 3); + if (hs == 3) return V41_DN(0, 4); + switch (abl) { case 1: return V41_DN(1, 2); case 2: return V41_DN(2, 2); case 4: return V41_DN(4, 2); case 8: return V41_DN(8, 2); case 16: return V41_DN(16, 2); default: return V41_DN(0, 2); } +#undef V41_GU +#undef V41_DN +} +template +static int v41_vq_pairs_launch(int stage, uint16_t *h, float *part, const uint8_t *blob, const int32_t *sel, const uint32_t *xb, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t K, uint32_t np, float clamp, uint32_t cbb, const float *gr) { + static int s_nsm = 0; + if (!s_nsm && hipDeviceGetAttribute(&s_nsm, hipDeviceAttributeMultiprocessorCount, 0) != hipSuccess) { + fprintf(stderr, "ds4: [ds41] VQ pair kernel cannot read SM count: %s\n", hipGetErrorString(hipGetLastError())); + return 0; + } + const int grid = s_nsm * (v41_knob(0) > 0 ? v41_knob(0) : 1); + const uint32_t total = stage == 0 ? np * MID : np * OUT, unit = stage == 0 ? MID * K : OUT, ub = stage == 0 ? IN * 2u : MID * 2u; + const uint32_t st = v41_vq_pairs_stage_max(total, (uint32_t)grid, unit) * ub; + if (v41_vq_cbh_ok(2u * cbb + st)) return v41_vq_pairs_go(stage, h, part, blob, sel, xb, IN, MID, OUT, K, np, clamp, cbb, gr, 2u * cbb + st, grid); + if (v41_vq_cbh_ok(2u * cbb)) return v41_vq_pairs_go(stage, h, part, blob, sel, xb, IN, MID, OUT, K, np, clamp, cbb, gr, 2u * cbb, grid); + if (cbb + st <= V41_VQ_SHARED_CAP) return v41_vq_pairs_go(stage, h, part, blob, sel, xb, IN, MID, OUT, K, np, clamp, cbb, gr, cbb + st, grid); + return v41_vq_pairs_go(stage, h, part, blob, sel, xb, IN, MID, OUT, K, np, clamp, cbb, gr, cbb, grid); +} + +/* The two entries the worker calls (signatures kept). n >= 2: the pair kernels + * unless knob 6 picks a grouped warp count; 1 = launched, 0 = not launched + * (the worker falls back to the pair + group kernels). */ +template +static int v41_vq_persist_n_launch(int stage, uint32_t n_tok, uint16_t *h, float *part, const uint8_t *blob, const int32_t *sel, + const int32_t *ord, const uint32_t *xb, uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t K, + uint32_t np, float clamp, uint32_t cbb, const float *gr) { + static int s_nsm = 0; + if (np > V41_VQPN_MAXP || n_tok < 2u) return 0; + const int nw = v41_knob(6); + if (nw != 8 && nw != 16 && nw != 32) return v41_vq_pairs_launch(stage, h, part, blob, sel, xb, IN, MID, OUT, K, np, clamp, cbb, gr); + if (!s_nsm && hipDeviceGetAttribute(&s_nsm, hipDeviceAttributeMultiprocessorCount, 0) != hipSuccess) { (void)hipGetLastError(); return 0; } + const int g = s_nsm * (v41_knob(0) > 0 ? v41_knob(0) : 1); + const bool cbh = v41_vq_cbh_ok(2u * cbb + 1024u); /* + the static group tables */ +#define V41_VQPN_GO(NW_) (cbh ? v41_vq_persist_n_go(stage, h, part, blob, sel, ord, xb, IN, MID, OUT, K, np, clamp, cbb, gr, g) \ + : v41_vq_persist_n_go(stage, h, part, blob, sel, ord, xb, IN, MID, OUT, K, np, clamp, cbb, gr, g)) + if (nw == 8) return V41_VQPN_GO(8); + if (nw == 16) return V41_VQPN_GO(16); + return V41_VQPN_GO(32); +#undef V41_VQPN_GO +} +#undef V41_VQ_SEG + +/* n = 1: the pair kernels with K = np (one token). 1 = launched; 0 = launch + * failure (the caller treats it as an error, never falls back). */ +template +static int v41_vq_persist_launch(int stage, uint16_t *h, float *part, const uint8_t *blob, const int32_t *sel, const uint32_t *xb, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t np, float clamp, uint32_t cbb, const float *gr) { + return v41_vq_pairs_launch(stage, h, part, blob, sel, xb, IN, MID, OUT, np, np, clamp, cbb, gr); +} diff --git a/cuda/ds41_vq_prefill.cuh b/cuda/ds41_vq_prefill.cuh new file mode 100644 index 0000000000000000000000000000000000000000..3816167a147683f5669ed976ee1a28072a99717b --- /dev/null +++ b/cuda/ds41_vq_prefill.cuh @@ -0,0 +1,238 @@ +#include "hip/hip_runtime.h" +/* ds41_vq_prefill.cuh — DeepSeek V4.1 (ds41) routed-MoE prefill scheduler, + * ported from the C engine at /data/YoungAi (commit 3946dbc), file + * `src/cuda/cuda_vq_prefill.inc.cu` (the whole file minus the backward + * capture). + * + * This file does no multiply; it is what surrounds the multiply: + * 1. a per-layer slot-header cache (vqp_hdr_build, :79-136): copy the blob's + * n_expert*3 offset table and each slot's 16 B payload header to the host + * once per layer, validate magic/dims, cache dim/nc/nbit. The blob's + * host pages are dropped after the device copy, so every host touch of a + * header is a page fault back into the SSD (the engine's profile: 768 + * faults/layer, 8-35 ms of GPU idle between experts, pf1); + * 2. a stable host-side counting sort of the (token, pick) pairs by expert + * (:159-184) — one expert's weights are touched once, and the same input + * reproduces bit for bit (an atomicAdd reduce was deleted 2026-08-22 for + * exactly this: two runs of one prompt disagreed); + * 3. the fixed-pick-order weighted reduce (vqp_reduce_kernel, :60-73). + * The multiply itself is the run behind vqp_fused_run (declared here, defined + * in ds41_vq_prefill_fused.cuh -> ds41_vq_prefill_mma.cuh), which writes the + * sorted ys this file reduces. + * + * Scope: the inference scheduler. The engine's backward capture + * (g_vqp_last_*, ds4_gpu_v41_vq_capture_expert_out, :222-243) is not ported + * (the port has no backward pass). The gr-override store + * (ds4_gpu_v41_set_gr_override, :246-258) is ported below; g_v41_gr stays + * all-NULL until the zchain sidecar is applied, which is the state the BARE + * golden (NO_ZCHAIN=1) was captured in. + */ +#pragma once +#include +#include +#include +#include + +#include "ds41_primitives.cuh" +#include "ds41_vq_row.cuh" + +/* zchain per-expert gain override (cuda_vq_prefill.inc.cu:19-21): [layer] -> + * device s[n_expert][OUT] row gains for the down matrix, NULL = none for the + * layer. Only the down output consumes it (m.gov in ds41_vq_row.cuh); the + * store below mounts a table, the zchain sidecar's gr half. */ +static float *g_v41_gr[64]; + +/* Mount/unmount one layer's gain table (cuda_vq_prefill.inc.cu:246-258). + * host NULL unloads; n_expert x out_dim must match the layer's real shape + * (the caller checks, as the engine's loader does). A failed upload leaves + * the slot NULL and returns 0 — never a half-mounted table. */ +extern "C" int ds4_gpu_v41_set_gr_override(uint32_t layer, const float *host, uint32_t n_expert, uint32_t out_dim) { + if (layer >= 64u) return 0; + if (g_v41_gr[layer]) { (void)hipFree(g_v41_gr[layer]); g_v41_gr[layer] = NULL; } + if (!host) return 1; + const size_t nb = (size_t)n_expert * out_dim * 4; + if (hipMalloc((void **)&g_v41_gr[layer], nb) != hipSuccess) { (void)hipGetLastError(); g_v41_gr[layer] = NULL; return 0; } + if (hipMemcpy(g_v41_gr[layer], host, nb, hipMemcpyHostToDevice) != hipSuccess) { + (void)hipGetLastError(); (void)hipFree(g_v41_gr[layer]); g_v41_gr[layer] = NULL; return 0; + } + return 1; +} + +/* The NVFP4 prefill path's n-alignment pad (cuda_vq_prefill_nvfp4.inc.cu:24). + * That path is retired in the engine itself (2026-09-20: measured 0.013 Σmin + * quality loss, see the fused file's account) and is not ported; the +PAD + * allocation stays because the engine's buffers still carry it, and the pad + * rows are never written by any live path. */ +#define V41_VQN_PAD 2048u + +/* ---- per-layer slot header cache (cuda_vq_prefill.inc.cu:74-136) ---- + * off=0 -> slot absent. nbit = ceil(log2(nc)), floor 1. */ +typedef struct { uint64_t off; uint32_t dim, nc, nbit; } vqp_slot_hdr; +static vqp_slot_hdr *g_vqp_hdr[64]; /* [layer] -> [e*3+which] */ +static v41_scratch g_vqp_hdr_dev; /* [n_total*3][6] u32: off(2) magic dim|nc rows cols */ + +__global__ static void vqp_copy_hdr_kernel(uint32_t *dst, const uint8_t *blob, uint32_t n_total) { + const uint32_t e = blockIdx.x, w = threadIdx.x; + if (e >= n_total || w >= 3u) return; + uint64_t off; memcpy(&off, blob + 16 + ((size_t)e * 3 + w) * 8, 8); + uint32_t *d = dst + ((size_t)e * 3 + w) * 6; + memcpy(d, &off, 8); + if (off) memcpy(d + 2, blob + off, 16); + else d[2] = d[3] = d[4] = d[5] = 0; +} + +static int vqp_hdr_build(uint32_t layer, const uint8_t *blob, uint32_t n_total, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t ver) { + if (layer >= 64u) return 0; + if (g_vqp_hdr[layer]) return 1; + const uint64_t n = (uint64_t)n_total * 3u; + uint32_t *dst = (uint32_t *)v41_grow(&g_vqp_hdr_dev, n * 6u * sizeof(uint32_t), "vq prefill hdr"); + if (!dst) return 0; + vqp_copy_hdr_kernel<<>>(dst, blob, n_total); + if (!cuda_ok(hipGetLastError(), "vq prefill hdr copy launch")) return 0; + uint32_t *raw = (uint32_t *)malloc((size_t)n * 6u * sizeof(uint32_t)); + vqp_slot_hdr *tab = (vqp_slot_hdr *)calloc((size_t)n, sizeof(vqp_slot_hdr)); + if (!raw || !tab || + hipMemcpy(raw, dst, (size_t)n * 6u * sizeof(uint32_t), hipMemcpyDeviceToHost) != hipSuccess) { + (void)hipGetLastError(); free(raw); free(tab); return 0; + } + int bad = 0; + for (uint64_t k = 0; k < n; k++) { + const uint32_t *d = raw + k * 6u; + uint64_t off; memcpy(&off, d, 8); + tab[k].off = off; + if (!off) continue; + const uint32_t which = (uint32_t)(k % 3u), e = (uint32_t)(k / 3u); + const uint32_t exp_rows = which == 2u ? OUT : MID, exp_cols = which == 2u ? MID : IN; + const uint32_t d16 = d[3] & 0xFFFFu, n16 = d[3] >> 16; + /* The expected payload magic follows the on-disk version (:118-121): + * v3 payloads are 'DQV3' with a different layout. A wrong magic is a + * hard failure, never a decode under the wrong layout. */ + if (d[2] != (ver == 3u ? DS4VQ_MAT3_MAGIC : DS4VQ_MAT_MAGIC) || d[4] != exp_rows || d[5] != exp_cols) { + fprintf(stderr, "ds4: [ds41] vq-prefill L%u e=%u which=%u header bad (magic %08x %ux%u, want %ux%u) -- aborting\n", + layer, e, which, d[2], d[4], d[5], exp_rows, exp_cols); + bad = 1; break; + } + /* 48 KB is fused2(dim4)'s codebook-into-shared cap; only dim==4 is + * still held to it (the v3 dim8 codebook is 64 KB by design). */ + if (d16 == 4u && (size_t)n16 * d16 * 2u > 48u * 1024u) { + fprintf(stderr, "ds4: [ds41] vq-prefill L%u e=%u codebook %ux%u over the 48 KB shared cap -- aborting\n", layer, e, n16, d16); + bad = 1; break; + } + uint32_t nb = 0; while ((1u << nb) < n16) nb++; if (nb < 1u) nb = 1u; + tab[k].dim = d16; tab[k].nc = n16; tab[k].nbit = nb; + } + free(raw); + if (bad) { free(tab); return 0; } + g_vqp_hdr[layer] = tab; + return 1; +} + +/* ---- sorted-run scratch + the reduce (:29-38, :60-73) ---- */ +static struct { v41_scratch ys, perm, inv; } g_vqp; + +/* out[t][o] = sum_pk w[t][pk] * ys[inv[t*n_expert+pk]][o], pick order fixed + * -> reproducible. Never an atomicAdd (see the file header). */ +__global__ static void vqp_reduce_kernel(float *out, const float *ys, const int32_t *inv, const float *rw, + uint32_t n_expert, uint32_t OUT) { + const uint32_t t = blockIdx.y; + const uint32_t o = blockIdx.x * blockDim.x + threadIdx.x; + if (o >= OUT) return; + float s = 0.0f; + for (uint32_t pk = 0; pk < n_expert; pk++) { + const int32_t i = inv[(uint64_t)t * n_expert + pk]; + if (i < 0) continue; + s += rw[(uint64_t)t * n_expert + pk] * ys[(uint64_t)i * OUT + o]; + } + out[(uint64_t)t * OUT + o] = s; +} + +/* The multiply run (cuda_vq_prefill_fused.inc.cu:168-256). Returns 0 = the + * path is unavailable; the caller hard-fails rather than silently falling + * back (a fallback would hide which path ran). */ +static int vqp_fused_run(const uint8_t *blob, const uint32_t *cnt, const uint32_t *off_h, uint32_t n_total_expert, + uint32_t nvalid, uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t nc, float clamp, + const float *x, const int32_t *perm, uint32_t n_expert, uint32_t layer_index, uint32_t ver); + +/* ---- the prefill scheduler (cuda_vq_prefill.inc.cu:139-218) ---- + * Host-side counting sort, then the fused/tensor-core run, then the reduce. + * The sort's D2H of `selected` is per layer and the path is never captured, + * so plain synchronous copies are the engine's shape. */ +static int cuda_vq_moe_prefill_gemm( + ds4_gpu_tensor *out, const uint8_t *blob, + const void *model_map, uint64_t down_offset, uint64_t down_expert_bytes, + uint32_t IN, uint32_t MID, uint32_t OUT, + const ds4_gpu_tensor *selected, const ds4_gpu_tensor *weights, + uint32_t n_total_expert, uint32_t n_expert, float clamp, + const ds4_gpu_tensor *x, uint32_t layer_index, uint32_t n_tokens, uint32_t ver) { + if (!g_cublas_ready) { fprintf(stderr, "ds4: [ds41] vq-prefill cuBLAS not ready (L%u)\n", layer_index); return 0; } + if (!vqp_hdr_build(layer_index, blob, n_total_expert, IN, MID, OUT, ver)) return 0; + const vqp_slot_hdr *tab = g_vqp_hdr[layer_index]; + const uint64_t npair = (uint64_t)n_tokens * n_expert; + int32_t *sel_h = (int32_t *)malloc(npair * sizeof(int32_t)); + int32_t *perm_h = (int32_t *)malloc(npair * sizeof(int32_t)); + int32_t *inv_h = (int32_t *)malloc(npair * sizeof(int32_t)); + uint32_t *cnt = (uint32_t *)calloc((size_t)n_total_expert, sizeof(uint32_t)); + uint32_t *off = (uint32_t *)malloc(((size_t)n_total_expert + 1u) * sizeof(uint32_t)); + uint32_t *cur = (uint32_t *)malloc((size_t)n_total_expert * sizeof(uint32_t)); + int ok = 0; + do { + if (!sel_h || !perm_h || !inv_h || !cnt || !off || !cur) break; + if (hipMemcpy(sel_h, selected->ptr, npair * sizeof(int32_t), hipMemcpyDeviceToHost) != hipSuccess) { + (void)hipGetLastError(); break; + } + for (uint64_t k = 0; k < npair; k++) { + const int32_t e = sel_h[k]; + if (e >= 0 && (uint32_t)e < n_total_expert) cnt[e]++; + } + off[0] = 0; + uint32_t ne_max = 0; + for (uint32_t e = 0; e < n_total_expert; e++) { + off[e + 1] = off[e] + cnt[e]; + if (cnt[e] > ne_max) ne_max = cnt[e]; + } + const uint32_t nvalid = off[n_total_expert]; + if (nvalid == 0) { /* every pick empty: the routed output is 0 */ + ok = (hipMemsetAsync(out->ptr, 0, (size_t)n_tokens * OUT * sizeof(float), ds4_current_stream()) == hipSuccess); + break; + } + memcpy(cur, off, (size_t)n_total_expert * sizeof(uint32_t)); + for (uint64_t k = 0; k < npair; k++) { /* stable: same-expert pairs keep token order */ + const int32_t e = sel_h[k]; + if (e < 0 || (uint32_t)e >= n_total_expert) { inv_h[k] = -1; continue; } + const uint32_t pos = cur[e]++; + perm_h[pos] = (int32_t)k; + inv_h[k] = (int32_t)pos; + } + if (!v41_grow(&g_vqp.ys, ((uint64_t)nvalid + V41_VQN_PAD) * OUT * sizeof(float), "vq prefill ys") || + !v41_grow(&g_vqp.perm, (uint64_t)nvalid * sizeof(int32_t), "vq prefill perm") || + !v41_grow(&g_vqp.inv, npair * sizeof(int32_t), "vq prefill inv")) break; + (void)ne_max; /* the fused run accumulates per work item, not per max expert width */ + if (hipMemcpy(g_vqp.perm.p, perm_h, (size_t)nvalid * sizeof(int32_t), hipMemcpyHostToDevice) != hipSuccess || + hipMemcpy(g_vqp.inv.p, inv_h, (size_t)npair * sizeof(int32_t), hipMemcpyHostToDevice) != hipSuccess) { + (void)hipGetLastError(); break; + } + /* Walk the slot table on the host first: an expert with tokens but a + * missing w1/w3/w2 slot is a hard failure — never a kernel that + * writes zeros into the quality silently (:196-208). */ + int bad = 0; + for (uint32_t e = 0; e < n_total_expert && !bad; e++) { + if (!cnt[e]) continue; + const vqp_slot_hdr *h1 = &tab[(size_t)e * 3u]; + if (!h1->off || !(h1 + 1)->off || !(h1 + 2)->off) { + fprintf(stderr, "ds4: [ds41] vq-prefill L%u e=%u slot missing (w1/w3/w2) -- aborting (no silent quality downgrade)\n", + layer_index, e); + bad = 1; + } + } + if (bad) break; + (void)model_map; (void)down_offset; (void)down_expert_bytes; + if (!vqp_fused_run(blob, cnt, off, n_total_expert, nvalid, IN, MID, OUT, + tab[0].nc, clamp, (const float *)x->ptr, (const int32_t *)g_vqp.perm.p, n_expert, layer_index, ver)) break; + vqp_reduce_kernel<<>>( + (float *)out->ptr, (const float *)g_vqp.ys.p, (const int32_t *)g_vqp.inv.p, (const float *)weights->ptr, n_expert, OUT); + ok = cuda_ok(hipGetLastError(), "vq prefill reduce launch"); + } while (0); + free(sel_h); free(perm_h); free(inv_h); free(cnt); free(off); free(cur); + return ok; +} diff --git a/cuda/ds41_vq_prefill_fused.cuh b/cuda/ds41_vq_prefill_fused.cuh new file mode 100644 index 0000000000000000000000000000000000000000..6ff1cf58d3dbb4bb2f0dc4e4c518e7f4523ad934 --- /dev/null +++ b/cuda/ds41_vq_prefill_fused.cuh @@ -0,0 +1,295 @@ +#include "hip/hip_runtime.h" +/* ds41_vq_prefill_fused.cuh — DeepSeek V4.1 (ds41) VQ expert prefill, the + * v2 (f16-codebook) arm: decode is the multiply, no f16 staging, ported from + * the C engine at /data/YoungAi (commit 3946dbc), file + * `src/cuda/cuda_vq_prefill_fused.inc.cu` (whole file, the negative-result + * archives at its tail included as comments). + * + * Why this shape (the engine's 2026-09-15 measurement, kept): a 512-token + * block routes ~8 tokens per expert on average, while the expert holds 35.4M + * weights — dequantizing the whole expert to f16 for 8 tokens is arithmetic + * intensity 8 (225 ms/layer of traffic for 2.7 ms of math). Fused, one + * layer reads the blob once: ~11 ms. So a work item is (expert, token + * segment, count): a block loads one codebook and each of its weight rows is + * decoded once and dotted against the <= 8 activations the expert owns. + * + * The K-order numerics (kept from the engine): activation/row sums run f32 + * with the official bf16 boundary rounding — the same contract as the decode + * fused kernels — and the dot's K sum order is "32-lane groups + shuffle + * tree", which is not bit-equal to any GEMM by construction. The gate is the + * NLL/PPL scale (the engine's cuda_v41_hc.inc.cu:36 doctrine), not cmp. + * + * The v2 arm is what a DQVL v2 blob (V4 legacy) routes to; the artifact's + * real blob is DQVL v3 13-bit (measured, plan §6.7) and routes to the + * tensor-core run in ds41_vq_prefill_mma.cuh. Both arms are ported; only the + * v3 arm is gated (the v2 one has no artifact to gate against — named). + * + * Must follow ds41_vq_prefill.cuh (V41_VQN_PAD, g_v41_gr, g_vqp) and + * ds41_vq_decode.cuh (v41_vq_cb_to_shared, v41_vq_swiglu); the mma file + * follows this one (vqp_item, vqm_run's forward declaration). + */ +#pragma once + +#define V41_VQP_NT 8u /* tokens per work item (acc lives in local memory, see the NT archive below) */ +#define V41_VQP_ITERS 8u /* rows a warp loops: the codebook is copied once per 32*8 = 256 rows */ +#define V41_VQP_ROWS (32u * V41_VQP_ITERS) + +/* Sorted activations, f32 (the old path used f16; the fused kernel reads f32 + * directly — one less precision round trip). */ +__global__ static void vqp_gather32_kernel(float *xs, const float *x, const int32_t *perm, uint32_t K, uint32_t IN) { + const uint32_t i = blockIdx.x; + const uint32_t t = (uint32_t)perm[i] / K; + const float *src = x + (uint64_t)t * IN; + float *dst = xs + (uint64_t)i * IN; + for (uint32_t d = threadIdx.x; d < IN; d += blockDim.x) dst[d] = src[d]; +} + +/* One weight row x nt activation vectors. The decode part is + * v41_vq_row_dot's expression verbatim (bitstream -> 16 B codebook entry -> + * 8 halves); the one difference is that the weight row is decoded once and + * reused for nt activations — the whole reason this file exists. acc comes + * out warp-reduced and multiplied by the row gain (gov override included); + * only lane 0's value is valid. */ +template +__device__ __forceinline__ static void v41_vq_row_dotN(const v41_vq_mat &m, uint32_t r, const float *xs, + uint32_t xstride, uint32_t nt, const uint8_t *cbs, float *acc) { + const uint64_t i0 = (uint64_t)r * m.nidx_row; + const uint32_t MB = (V3 && m.nbit > 12u) ? 12u : m.nbit; /* main-stream bit width (v3: min(nbit, 12)) */ + const uint8_t *ex = EXT ? m.ex + (size_t)r * ((m.nidx_row + 7u) >> 3) : NULL; + for (uint32_t t = 0; t < V41_VQP_NT; t++) acc[t] = 0.f; + for (uint32_t j = threadIdx.x & 31u; j < m.nidx_row; j += 32u) { + const uint64_t bit = (V3 ? (uint64_t)j : (i0 + j)) * MB + (V3 ? (uint64_t)r * ((m.nidx_row * MB + 7u) / 8u) * 8u : 0u), by = bit >> 3; + uint32_t wv; memcpy(&wv, m.ix + by, 4); + uint32_t v = (wv >> (bit & 7)) & (EXT ? 0xFFFu : m.imsk); + if (EXT) v |= (uint32_t)((ex[j >> 3] >> (j & 7)) & 1u) << 12; + float f0x, f0y, f1x, f1y, f2x, f2y, f3x, f3y; + if (V3) { /* E4M3 via the hardware instruction, never a scalar table (the engine's v41_e4m3x2_to_half2 lesson) */ + const uint2 w = *(const uint2 *)(cbs + (size_t)v * 8u); + const float2 e0 = __half22float2(v41_e4m3x2_to_half2(w.x)), e1 = __half22float2(v41_e4m3x2_to_half2(w.x >> 16)), + e2 = __half22float2(v41_e4m3x2_to_half2(w.y)), e3 = __half22float2(v41_e4m3x2_to_half2(w.y >> 16)); + f0x = e0.x; f0y = e0.y; f1x = e1.x; f1y = e1.y; f2x = e2.x; f2y = e2.y; f3x = e3.x; f3y = e3.y; + } else { + const uint4 c = *(const uint4 *)(cbs + (size_t)v * 16u); + __half2 h0, h1, h2, h3; memcpy(&h0, &c.x, 4); memcpy(&h1, &c.y, 4); memcpy(&h2, &c.z, 4); memcpy(&h3, &c.w, 4); + const float2 g0 = __half22float2(h0), g1 = __half22float2(h1), g2 = __half22float2(h2), g3 = __half22float2(h3); + f0x = g0.x; f0y = g0.y; f1x = g1.x; f1y = g1.y; f2x = g2.x; f2y = g2.y; f3x = g3.x; f3y = g3.y; + } + const float2 f0 = make_float2(f0x, f0y), f1 = make_float2(f1x, f1y), f2 = make_float2(f2x, f2y), f3 = make_float2(f3x, f3y); + for (uint32_t t = 0; t < nt; t++) { + const float *xt = xs + (uint64_t)t * xstride + (size_t)j * 8u; + const float4 xa = *(const float4 *)xt, xb = *(const float4 *)(xt + 4u); + acc[t] += f0.x * xa.x + f0.y * xa.y + f1.x * xa.z + f1.y * xa.w + + f2.x * xb.x + f2.y * xb.y + f3.x * xb.z + f3.y * xb.w; + } + } + __half gh; memcpy(&gh, m.gr + (size_t)r * 2u, 2); + const float g = __half2float(gh) * (m.gov ? m.gov[r] : 1.0f); + for (uint32_t t = 0; t < nt; t++) { + for (int o = 16; o > 0; o >>= 1) acc[t] += __shfl_xor_sync(0xffffffffu, acc[t], o); + acc[t] *= g; + } +} + +/* A work item: the [t0, t0+nt) tokens of expert e in sorted order (= off[e]+t0 onwards). */ +typedef struct { int32_t e, t0, nt; } vqp_item; + +/* gate and up are two launches, not one (2026-09-15 measurement): keeping + * g[ITERS][NT] and u[ITERS][NT] live together is 128 f32 accumulators per + * thread, far over the register budget — everything spills to local memory + * and the fusion buys +5% only. Split, each thread holds acc[NT] = 8; the + * codebook is loaded twice (64 KB/block, L2-resident, negligible). + * which=0: gate -> g32; which=1: up, reads g32 back, clamp + SwiGLU -> h32 + * (the official Expert's order and rounding point). */ +template +__global__ static void vqp_fused_gu_kernel(float *dst, const float *g32, const uint8_t *blob, const vqp_item *items, + const float *xs, const uint32_t *off, uint32_t IN, uint32_t MID, + float clamp, uint32_t cb_bytes, int which) { + extern __shared__ __align__(16) uint8_t vqsh[]; + const vqp_item it = items[blockIdx.y]; + const uint32_t nt = (uint32_t)it.nt, base = off[it.e] + (uint32_t)it.t0; + const v41_vq_mat m = v41_vq_open(blob, it.e, which, MID, IN, NULL); + if (!m.ok || m.nc * (V3 ? 8u : 16u) != cb_bytes) return; + const uint32_t r0 = blockIdx.x * V41_VQP_ROWS + (threadIdx.x >> 5); + const float *xb = xs + (uint64_t)base * IN; + v41_vq_cb_to_shared(vqsh, m.cb, cb_bytes); + __syncthreads(); + for (uint32_t i = 0; i < V41_VQP_ITERS; i++) { + const uint32_t r = r0 + i * 32u; + if (r >= MID) break; + float acc[V41_VQP_NT]; + v41_vq_row_dotN(m, r, xb, IN, nt, vqsh, acc); + if ((threadIdx.x & 31u) != 0) continue; + for (uint32_t t = 0; t < nt; t++) { + const uint64_t o = (uint64_t)(base + t) * MID + r; + if (!which) { dst[o] = v41_bf16r(acc[t]); continue; } + float gi = g32[o], ui = v41_bf16r(acc[t]); + if (clamp > 0.f) { if (gi > clamp) gi = clamp; if (ui > clamp) ui = clamp; if (ui < -clamp) ui = -clamp; } + const float sg = gi / (1.0f + expf(-gi)); + dst[o] = v41_bf16r(sg * ui); + } + } +} + +/* down: ys[sorted pos][OUT] = bf16(W2*h) — the same buffer the old path + * wrote, so the downstream reduce is untouched. */ +template +__global__ static void vqp_fused_down_kernel(float *ys, const uint8_t *blob, const vqp_item *items, const float *h, + const uint32_t *off, uint32_t MID, uint32_t OUT, uint32_t cb_bytes, const float *gr) { + extern __shared__ __align__(16) uint8_t vqsh[]; + const vqp_item it = items[blockIdx.y]; + const uint32_t nt = (uint32_t)it.nt, base = off[it.e] + (uint32_t)it.t0; + const v41_vq_mat md = v41_vq_open(blob, it.e, 2, OUT, MID, gr ? gr + (size_t)it.e * OUT : NULL); + const uint32_t r0 = blockIdx.x * V41_VQP_ROWS + (threadIdx.x >> 5); + if (!md.ok || md.nc * (V3 ? 8u : 16u) != cb_bytes) { /* bad payload: zero the rows this block owns, no dirty values for the reduce */ + if ((threadIdx.x & 31u) == 0) + for (uint32_t i = 0; i < V41_VQP_ITERS; i++) { + const uint32_t r = r0 + i * 32u; + if (r < OUT) for (uint32_t t = 0; t < nt; t++) ys[(uint64_t)(base + t) * OUT + r] = 0.f; + } + return; + } + v41_vq_cb_to_shared(vqsh, md.cb, cb_bytes); + __syncthreads(); + const float *hb = h + (uint64_t)base * MID; + for (uint32_t i = 0; i < V41_VQP_ITERS; i++) { + const uint32_t r = r0 + i * 32u; + if (r >= OUT) break; + float acc[V41_VQP_NT]; + v41_vq_row_dotN(md, r, hb, MID, nt, vqsh, acc); + if ((threadIdx.x & 31u) == 0) + for (uint32_t t = 0; t < nt; t++) ys[(uint64_t)(base + t) * OUT + r] = v41_bf16r(acc[t]); + } +} + +static struct { v41_scratch d, doff, h32, g32, xs32; } g_vqpf; +static int g_vqpf_sh = 0; /* 0 undecided / 1 codebook in shared / -1 does not fit (then this path is unavailable, hard failure) */ +static uint32_t g_vqpf_cbb = 0; /* codebook bytes the opt-in was armed for; layers of one file can differ (10 vs 11 bit) */ + +/* The tensor-core run for DQVL v3 (ds41_vq_prefill_mma.cuh). */ +static int vqm_run(const uint8_t *blob, const uint32_t *cnt, const uint32_t *off_h, uint32_t n_total_expert, uint32_t nvalid, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t nc, float clamp, const float *x, const int32_t *perm, + uint32_t n_expert, uint32_t layer_index, float *ys, const float *gr); +#if defined(__HIP_PLATFORM_AMD__) +/* The rocWMMA run (ds41_vq_prefill_wmma.cuh): 10- and 12-bit v3 layers. */ +static int vqw_run(const uint8_t *blob, const uint32_t *cnt, const uint32_t *off_h, uint32_t n_total_expert, uint32_t nvalid, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t nc, float clamp, const float *x, const int32_t *perm, + uint32_t n_expert, uint32_t layer_index, float *ys, const float *gr); +#endif + +/* Returns 0 = this path is unavailable (the caller hard-fails; a silent + * fallback to another path would hide which path ran, cuda_vq_prefill_fused.inc.cu:162). */ +static int vqp_fused_run(const uint8_t *blob, const uint32_t *cnt, const uint32_t *off_h, uint32_t n_total_expert, + uint32_t nvalid, uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t nc, float clamp, + const float *x, const int32_t *perm, uint32_t n_expert, uint32_t layer_index, uint32_t ver) { + uint32_t nbit = 0; while ((1u << nbit) < nc) nbit++; + /* v3 (E4M3 codebook) takes the bf16 tensor cores (2026-09-24): same + * products, same rounding point, different K accumulation order, several + * times faster per expert block. v2's f16 codebook would lose 3 mantissa + * bits on the way to bf16, so v2 stays on the fused kernels below. The + * tensor-core arms unpack a 12-bit main stream, so narrower v3 layers + * (10/11 bit) take the fused kernels by width, not as a fallback. */ +#if defined(__HIP_PLATFORM_AMD__) + /* gfx11 has 64 KB of LDS per block; the tensor-core arms need 64-88 KB and their MMA is emulated with + * shuffles here, so every v3 width takes the fused kernels (32 KB codebook at 12 bit, 8 KB at 10). */ + const uint32_t vqm_min_nbit = 14u; +#else + const uint32_t vqm_min_nbit = 12u; +#endif +#if defined(__HIP_PLATFORM_AMD__) + /* AMD: the rocWMMA arm for 10/12-bit v3 layers (knob 11 = 1 keeps the fused kernels, the A/B). */ + if (ver == 3u && (nbit == 10u || nbit == 12u) && !v41_knob(11)) + return vqw_run(blob, cnt, off_h, n_total_expert, nvalid, IN, MID, OUT, nc, clamp, x, perm, n_expert, layer_index, + (float *)g_vqp.ys.p, g_v41_gr[layer_index < 64u ? layer_index : 0]); +#endif + if (ver == 3u && nbit >= vqm_min_nbit) + return vqm_run(blob, cnt, off_h, n_total_expert, nvalid, IN, MID, OUT, nc, clamp, x, perm, n_expert, layer_index, + (float *)g_vqp.ys.p, g_v41_gr[layer_index < 64u ? layer_index : 0]); + /* v3 codebooks are 8 E4M3 per word (nc8192 = 64 KB, same as today's + * nc4096 f16); v2 carries 8 f16 per word. */ + const uint32_t cbb = nc * (ver == 3u ? 8u : 16u); + if (g_vqpf_sh == 0 || (g_vqpf_sh == 1 && cbb > g_vqpf_cbb)) { + /* The opt-in shared size must be set on the instance pair that will + * actually launch; all four pairs are set here (two kernels each) so + * whichever fires later is already armed. One failure marks this + * path unavailable — never a silent fallback. */ + bool og = true, od = true; +#define VQP_ATTR(V3, EXT) do { \ + og = og && hipFuncSetAttribute(reinterpret_cast(vqp_fused_gu_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)cbb) == hipSuccess; \ + od = od && hipFuncSetAttribute(reinterpret_cast(vqp_fused_down_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)cbb) == hipSuccess; \ + } while (0) + if (ver == 3u) { if (nbit == 13u) VQP_ATTR(1, 1); else VQP_ATTR(1, 0); } + else VQP_ATTR(0, 0); +#undef VQP_ATTR + (void)hipGetLastError(); + g_vqpf_sh = (og && od) ? 1 : -1; + g_vqpf_cbb = cbb; + fprintf(stderr, "ds4: [ds41] vq-prefill fused codebook %u KB/copy (DQVL v%u, %u bit) -> %s\n", cbb >> 10, ver, nbit, + g_vqpf_sh == 1 ? "in shared" : "does not fit, path unavailable"); + } + if (g_vqpf_sh != 1) return 0; + /* Work items: each expert split into NT-token segments. */ + uint32_t nit = 0; + for (uint32_t e = 0; e < n_total_expert; e++) nit += (cnt[e] + V41_VQP_NT - 1u) / V41_VQP_NT; + if (!nit) return 0; + vqp_item *ih = (vqp_item *)malloc((size_t)nit * sizeof(vqp_item)); + if (!ih) return 0; + uint32_t k = 0; + for (uint32_t e = 0; e < n_total_expert; e++) + for (uint32_t t0 = 0; t0 < cnt[e]; t0 += V41_VQP_NT) { + const uint32_t nt = cnt[e] - t0 < V41_VQP_NT ? cnt[e] - t0 : V41_VQP_NT; + ih[k].e = (int32_t)e; ih[k].t0 = (int32_t)t0; ih[k].nt = (int32_t)nt; k++; + } + int ok = v41_grow(&g_vqpf.d, (uint64_t)nit * sizeof(vqp_item), "vq prefill fused items") && + v41_grow(&g_vqpf.doff, ((uint64_t)n_total_expert + 1u) * sizeof(uint32_t), "vq prefill fused off") && + v41_grow(&g_vqpf.xs32, ((uint64_t)nvalid + V41_VQN_PAD) * IN * sizeof(float), "vq prefill fused xs32") && + v41_grow(&g_vqpf.h32, ((uint64_t)nvalid + V41_VQN_PAD) * MID * sizeof(float), "vq prefill fused h32") && + v41_grow(&g_vqpf.g32, ((uint64_t)nvalid + V41_VQN_PAD) * MID * sizeof(float), "vq prefill fused g32"); + if (ok) ok = hipMemcpyAsync(g_vqpf.d.p, ih, (size_t)nit * sizeof(vqp_item), hipMemcpyHostToDevice, ds4_current_stream()) == hipSuccess && + hipMemcpyAsync(g_vqpf.doff.p, off_h, (size_t)(n_total_expert + 1u) * sizeof(uint32_t), hipMemcpyHostToDevice, ds4_current_stream()) == hipSuccess; + free(ih); + if (!ok) { (void)hipGetLastError(); fprintf(stderr, "ds4: [ds41] vq-prefill L%u fused scratch/copy failed\n", layer_index); return 0; } + vqp_gather32_kernel<<>>((float *)g_vqpf.xs32.p, x, perm, n_expert, IN); + if (!cuda_ok(hipGetLastError(), "vq prefill gather32")) return 0; + /* The NVFP4 tensor-core route was retired here on 2026-09-20 (the engine's + * own decision, measured): it requantized weights/activations/h to E2M1 + * three times per layer and cost 0.013 Σmin — more than the 12->11 bit + * width cost of the file itself. Not ported; the fused kernels below are + * the live v2 path. */ + const dim3 gm((MID + V41_VQP_ROWS - 1u) / V41_VQP_ROWS, nit); + const dim3 gd((OUT + V41_VQP_ROWS - 1u) / V41_VQP_ROWS, nit); + /* Version/width select the instance (2026-09-21): v2 and v3 layouts + * differ, a wrong pick decodes fake weights silently, so ver comes from + * the blob header (the caller) and the width from the codebook word count. + * One macro launches all three kernels with the same template pair — three + * separate blocks would eventually miss one. */ +#define VQP_LAUNCH(V3, EXT) do { \ + vqp_fused_gu_kernel<<>>((float *)g_vqpf.g32.p, NULL, blob, (const vqp_item *)g_vqpf.d.p, (const float *)g_vqpf.xs32.p, (const uint32_t *)g_vqpf.doff.p, IN, MID, clamp, cbb, 0); \ + if (!cuda_ok(hipGetLastError(), "vq prefill fused gate")) return 0; \ + vqp_fused_gu_kernel<<>>((float *)g_vqpf.h32.p, (const float *)g_vqpf.g32.p, blob, (const vqp_item *)g_vqpf.d.p, (const float *)g_vqpf.xs32.p, (const uint32_t *)g_vqpf.doff.p, IN, MID, clamp, cbb, 1); \ + if (!cuda_ok(hipGetLastError(), "vq prefill fused up")) return 0; \ + vqp_fused_down_kernel<<>>( \ + (float *)g_vqp.ys.p, blob, (const vqp_item *)g_vqpf.d.p, (const float *)g_vqpf.h32.p, (const uint32_t *)g_vqpf.doff.p, MID, OUT, cbb, g_v41_gr[layer_index < 64u ? layer_index : 0]); \ + return cuda_ok(hipGetLastError(), "vq prefill fused down"); \ + } while (0) + if (ver == 3u) { + if (nbit >= 10u && nbit <= 12u) VQP_LAUNCH(1, 0); /* the kernels read the width from the payload */ + if (nbit == 13u) VQP_LAUNCH(1, 1); + fprintf(stderr, "ds4: [ds41] vq-prefill L%u DQVL v3 codebook %u words has no instance (10-13 only)\n", layer_index, nc); + return 0; + } + VQP_LAUNCH(0, 0); +#undef VQP_LAUNCH +} + +/* Negative-result archive kept from the engine (2026-09-15), so nobody walks + * it again: "one warp owns R rows, activations read once and shared" lost in + * all three configurations. The account was right (each row reads nt x 20 KB + * of activations while its weights are 960 B, 170:1) but it hit two walls: + * 1. a 64 KB codebook fills shared => one block per SM => block size IS the + * occupancy. The R-row form's register need (acc[R][NT] + xv[NT][8]) + * forces 256-thread blocks, 8 warps/SM: 34.3 -> 38.1 s. + * 2. holding 1024 threads (<= 64 registers/thread, NT=2/R=16 = 48) still + * spills to local memory: 34.3 -> 78.1 s. + * Also all three R-row configs landed on PPL 15.53 vs this version's 15.17 — + * an unresolved numeric difference; restarting that road requires finding + * it first, never covering it with "faster". */ diff --git a/cuda/ds41_vq_prefill_mma.cuh b/cuda/ds41_vq_prefill_mma.cuh new file mode 100644 index 0000000000000000000000000000000000000000..76792a14bf585797f3e22208d3b1ef6758120e3c --- /dev/null +++ b/cuda/ds41_vq_prefill_mma.cuh @@ -0,0 +1,640 @@ +#include "hip/hip_runtime.h" +/* ds41_vq_prefill_mma.cuh — DeepSeek V4.1 (ds41) VQ expert prefill on bf16 + * tensor cores, the DQVL v3 arm, ported from the C engine at /data/YoungAi + * (commit 3946dbc), files `src/cuda/cuda_vq_prefill_mma.inc.cu` (vqm, the + * tile form) and `src/cuda/cuda_vq_reg_mma.inc.cu` (vqs, the register-direct + * form; bit-identical to vqm, the engine's own microbench). + * + * Why (the engine's 2026-09-24 profile): the fused scalar kernels were 76% of + * a 12k-prompt run's GPU time — 2.6 TFLOPS against a ~11 ms/layer read floor. + * A work item could only carry 8 tokens (64 KB codebook fills shared, 64 + * registers/thread), so the same expert's bitstream was re-read for hot + * experts. Tensor cores take the math out: one mma.m16n8k16 is 4096 MACs. + * + * Why bf16, and the same products as the fused path (the 2026-09-19 lesson: + * the NVFP4 route pressed activations to 4 bits and gave up 0.013 Σmin): + * - weights: the v3 codebook is E4M3 (3 mantissa bits); bf16 has 7 — E4M3 -> + * bf16 is bit-exact. + * - activations: rms_norm / swiglu exits are already rounded to bf16. + * - a bf16 x bf16 product is exact in f32; the tensor core accumulates in + * f32; the row gain (override included) multiplies after the sum, the + * same point as the fused path. + * => the only difference from the fused path is the K accumulation order + * (which is a 32-lane split + shuffle tree there and matches no GEMM + * bit-for-bit anyway). The criterion is PPL / the five metrics, not bytes. + * v2's f16 codebook would lose 3 mantissa bits to bf16, so v2 stays on the + * fused kernels. + * + * Shapes (2026-09-29, shaped by the engine's microbench on the real payload + * and real routing): vqm = one block is 16 warps / 512 threads = 128 weight + * rows x one work item (<= 128 tokens of one expert), stepping 64 columns of + * K per round; vqs = one block is (work item, 256 rows) x all of K with + * cp.async segments in flight and the decode landing straight in mma + * fragments. The engine's negative results are archived in the comments. + * + * Bit-exactness (kept from the engine): products, the per-k16 zero-start + + * FADD-back accumulation, and the exit rounding point are the same as the + * 09-24 shape — the engine's microbench compared g32/h16/H_u/ys byte for byte + * against vqm_kernel, 12/13-bit x both routings. vqm/vqs vs the fused path + * is the K-order difference above. + * + * Portability deviation, named: vqs's L2 bulk prefetch (vqs_pf_l2) is sm_90+; + * this TU also compiles for sm_89 (the local dev/test device). Below sm_90 + * the prefetch asm is compiled out — a pure performance hint, the segment + * cp.async loads are the real fetches — so those devices run vqs correctly + * and numerically identically; sm_121 gets the engine's verbatim code. + * + * Must follow ds41_vq_prefill_fused.cuh (vqp_item, g_vqp.ys, g_v41_gr). + */ +#pragma once +#include /* std::integral_constant for the vqs slice-count switch */ +#include "hip_ptx_emul.cuh" /* AMD: ldmatrix / mma / cp.async / cvt equivalents (empty on CUDA) */ + +#define VQM_BM 128u /* weight rows per block: 8 tiles of 16 */ +#define VQM_BN 128u /* tokens per work item: 16 n8 slices */ +#define VQM_BK 64u /* K columns per round: 8 code words per row */ +#define VQM_TW 2u /* token halves across warp groups => 16 warps, 512 threads */ +#define VQM_THREADS (256u * VQM_TW) +#define VQM_TILE_BYTES ((VQM_BM + VQM_BN) * VQM_BK * 2u) /* A 16 KB + B 16 KB, after the codebook */ + +/* Two E4M3 -> two bf16 (packed). Hardware cvt to f16 first (E4M3 is a + * subset of f16 normals, subnormals included), then shift the exponent bias: + * a f16 normal shifted right 3 lands its exponent in the bf16 exponent bits + * and its top 7 mantissa bits in bf16's (E4M3 only has 3 nonzero, the lost + * ones are all 0), then add (127-15)<<7 = 0x3800 to rebias. Zero must be + * preserved separately (else it becomes 2^-15). */ +__device__ __forceinline__ static uint32_t vqm_e4m3x2_to_bf16x2(uint32_t two) { + uint32_t h; +#if defined(__HIP_PLATFORM_AMD__) + h = ds4hip_e4m3x2_to_f16x2(two & 0xffffu); +#else + asm("cvt.rn.f16x2.e4m3x2 %0, %1;" : "=r"(h) : "h"((unsigned short)(two & 0xffffu))); +#endif + const uint32_t mag = h & 0x7fff7fffu, nz = __vcmpne2(mag, 0u); + return (h & 0x80008000u) | ((((mag >> 3) & 0x0fff0fffu) + 0x38003800u) & nz); +} +__device__ __forceinline__ static void vqm_ldsm4(uint32_t *r, const uint8_t *p) { +#if defined(__HIP_PLATFORM_AMD__) + ds4hip_ldmatrix<4>(r, p); +#else + const uint32_t a = (uint32_t)__cvta_generic_to_shared(p); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) : "r"(a)); +#endif +} +__device__ __forceinline__ static void vqm_mma(float *c, const uint32_t *a, uint32_t b0, uint32_t b1) { +#if defined(__HIP_PLATFORM_AMD__) + ds4hip_mma_m16n8k16(c, a[0], a[1], a[2], a[3], b0, b1); +#else + asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};" + : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1)); +#endif +} +/* A tile row is 64 bf16 = 8 chunks of 16 B; chunk ^ (row & 7) puts the 8 rows + * ldmatrix reads at once into 8 different bank groups. */ +__device__ __forceinline__ static uint32_t vqm_swz(uint32_t row, uint32_t chunk) { return row * 128u + ((chunk ^ (row & 7u)) << 4); } +/* One code word -> 8 bf16 (16 B): CB=0 converts the E4M3 codebook on the fly; + * CB=1's codebook was converted to bf16 when it entered shared. */ +template +__device__ __forceinline__ static uint4 vqm_cw(const uint8_t *cbs, uint32_t v) { + if (CB == 1) return *(const uint4 *)(cbs + (size_t)v * 16u); + const uint2 cw = *(const uint2 *)(cbs + (size_t)v * 8u); + uint4 o; + o.x = vqm_e4m3x2_to_bf16x2(cw.x); o.y = vqm_e4m3x2_to_bf16x2(cw.x >> 16); + o.z = vqm_e4m3x2_to_bf16x2(cw.y); o.w = vqm_e4m3x2_to_bf16x2(cw.y >> 16); + return o; +} + +/* Sorted activations, bf16. The values already sit on the bf16 grid; bf16r + * (round) not truncation, so a missed upstream round costs one correct + * rounding, not a bias. */ +__global__ static void vqm_gather16_kernel(uint16_t *xs, const float *x, const int32_t *perm, uint32_t K, uint32_t IN) { + const uint32_t i = blockIdx.x, t = (uint32_t)perm[i] / K; + const float *src = x + (uint64_t)t * IN; + uint16_t *dst = xs + (uint64_t)i * IN; + for (uint32_t d = threadIdx.x; d < IN; d += blockDim.x) dst[d] = (uint16_t)(__float_as_uint(v41_bf16r(src[d])) >> 16); +} + +/* MODE 0 = gate: g32 = bf16(W1*x*g) | 1 = up: read g32, clamp + SwiGLU -> h16 + * | 2 = down: ys = bf16(W2*h*g). MODE 1 with ys non-NULL also writes H_u + * (the up exit before SwiGLU) — the backward pass needs it; the inference + * path passes NULL. The exit rounding point is vqp_fused_gu/down's + * expression: sum -> row gain -> bf16r. */ +template +__global__ __launch_bounds__(VQM_THREADS, 1) static void vqm_kernel( + float *g32, uint16_t *h16, float *ys, const uint8_t *blob, const vqp_item *items, uint32_t nitems, + const uint16_t *act, const uint32_t *off, uint32_t M, uint32_t K, float clamp, uint32_t cb_bytes, const float *gr) { + constexpr uint32_t CPR = VQM_BK / 8u, TPR = VQM_THREADS / VQM_BM, CPT = CPR / TPR; /* 8 words per row-round, 4 threads x 2 */ + constexpr uint32_t NF = VQM_BN / 8u, NFW = NF / VQM_TW, CH = (VQM_BN * CPR) / VQM_THREADS; /* 16 n8 slices, 8 per warp; 2 x 16 B of B per thread */ + extern __shared__ __align__(16) uint8_t vqmsh[]; + uint8_t *cbs = vqmsh, *As = vqmsh + cb_bytes, *Bs = As + VQM_BM * VQM_BK * 2u; + const int which = MODE; /* payload slot: 0 w1(gate) / 1 w3(up) / 2 w2(down) */ + const uint32_t tid = threadIdx.x, lane = tid & 31u, warp = tid >> 5, wr = warp & 7u, th = warp >> 3; + const uint32_t ntile = (M + VQM_BM - 1u) / VQM_BM, nwork = nitems * ntile, nit = K / VQM_BK; + { /* the v3 codebook is one per layer (three matrices, all experts) => the first work item's is this layer's */ + const v41_vq_mat m0 = v41_vq_open<1>(blob, items[0].e, which, M, K, NULL); + if (!m0.ok || m0.nc * (CB ? 16u : 8u) != cb_bytes) return; /* one decision for the whole block, nobody stuck at a later barrier */ + if (CB == 0) v41_vq_cb_to_shared(cbs, m0.cb, cb_bytes); + else for (uint32_t v = tid; v < m0.nc; v += VQM_THREADS) *(uint4 *)(cbs + (size_t)v * 16u) = vqm_cw<0>(m0.cb, v); + } + __syncthreads(); + const uint32_t ar = tid / TPR, sub = tid % TPR; /* decode split: the thread owns row ar's code words sub*CPT.. */ + for (uint32_t w = blockIdx.x; w < nwork; w += gridDim.x) { + const vqp_item it = items[w / ntile]; + const uint32_t r0 = (w % ntile) * VQM_BM, nt = (uint32_t)it.nt, base = off[it.e] + (uint32_t)it.t0; + const v41_vq_mat m = v41_vq_open<1>(blob, it.e, which, M, K, (MODE == 2 && gr) ? gr + (size_t)it.e * M : NULL); + if (!m.ok) { /* the host walked the slot table already; reaching here means a bad payload; down writes 0, no dirty reduce input (same as the fused path) */ + if (MODE == 2) + for (uint32_t i = tid; i < VQM_BM * nt; i += VQM_THREADS) + if (r0 + i % VQM_BM < M) ys[(uint64_t)(base + i / VQM_BM) * M + r0 + i % VQM_BM] = 0.f; + continue; + } + const uint32_t grow = r0 + ar, mrow = m.nidx_row * 12u / 8u, erow = (m.nidx_row + 7u) >> 3; + const bool rv = grow < M; + const uint8_t *rowp = m.ix + (size_t)(rv ? grow : 0u) * mrow; + const uint8_t *ep = EXT ? m.ex + (size_t)(rv ? grow : 0u) * erow : NULL; + float acc[NFW][4]; + #pragma unroll + for (uint32_t f = 0; f < NFW; f++) { acc[f][0] = acc[f][1] = acc[f][2] = acc[f][3] = 0.f; } + uint32_t w0 = 0, w1 = 0, eb = 0; + uint4 bx[CH]; + #pragma unroll + for (uint32_t c = 0; c < CH; c++) bx[c] = make_uint4(0, 0, 0, 0); + /* This thread's code words this round sit at bit offset round*96 + sub*24: two aligned words are read (the bit offset mod 32 is in {0,24,16,8}, all 24 bits inside the 64-bit window) and shifted. */ + auto load_round = [&](uint32_t i) { + const uint32_t bitoff = i * 12u * CPR + sub * 12u * CPT, a = (bitoff >> 5) << 2; + if (rv) { w0 = __ldg((const unsigned int *)(rowp + a)); w1 = __ldg((const unsigned int *)(rowp + a + 4u)); + if (EXT) eb = __ldg(ep + i); } /* the bit plane: 8 code words per round = 1 byte */ + #pragma unroll + for (uint32_t c = 0; c < CH; c++) { + const uint32_t ch = tid + c * VQM_THREADS, bt = ch / CPR, bc = ch % CPR; + if (bt < nt) bx[c] = __ldg((const uint4 *)(act + (uint64_t)(base + bt) * K + (uint64_t)i * VQM_BK + bc * 8u)); + } + }; + load_round(0); + for (uint32_t i = 0; i < nit; i++) { + const uint32_t sh = (i * 12u * CPR + sub * 12u * CPT) & 31u; + const uint64_t u = (((uint64_t)w1 << 32) | w0) >> sh; + #pragma unroll + for (uint32_t q = 0; q < CPT; q++) { + uint32_t v = (uint32_t)(u >> (12u * q)) & 0xFFFu; + if (EXT) v |= ((eb >> (sub * CPT + q)) & 1u) << 12; /* bit 13: byte i of the plane, bit (j&7) */ + *(uint4 *)(As + vqm_swz(ar, sub * CPT + q)) = vqm_cw(cbs, v); + } + #pragma unroll + for (uint32_t c = 0; c < CH; c++) { const uint32_t ch = tid + c * VQM_THREADS; *(uint4 *)(Bs + vqm_swz(ch / CPR, ch % CPR)) = bx[c]; } + __syncthreads(); + if (i + 1u < nit) load_round(i + 1u); /* next round's bitstream/activations issue now, in flight during the mma */ + /* the warp's 16 rows x this half's n8 slices */ + #pragma unroll + for (uint32_t kk = 0; kk < VQM_BK / 16u; kk++) { + uint32_t a[4]; + { const uint32_t mt = lane >> 3, row = wr * 16u + (lane & 7u) + (mt & 1u) * 8u; + vqm_ldsm4(a, As + vqm_swz(row, kk * 2u + (mt >> 1))); } + #pragma unroll + for (uint32_t np = 0; np < NFW / 2u; np++) { + const uint32_t f0 = th * NFW + 2u * np; /* this warp's 2np-th slice among all */ + if (f0 * 8u >= nt) break; + uint32_t b[4]; + const uint32_t mt = lane >> 3, tok = (f0 + (mt >> 1)) * 8u + (lane & 7u); + vqm_ldsm4(b, Bs + vqm_swz(tok, kk * 2u + (mt & 1u))); + /* Every k16 starts from zero and adds back with a plain FADD (2026-09-24 measurement): accumulating K straight in the + * accumulator lets the tensor core's internal f32 addition drop low bits (truncation on alignment, not round-to-nearest) — + * 3.5% of outputs off by one bf16 ulp vs the fused path, relative L2 5e-4, and 2048-scale PPL 14.096 -> 14.192 (+0.68%, + * one-sided). Hoisting the add back cuts it to 0.5% / 1.7e-4. Same idea as the official FP8 GEMM's per-128 hoist. */ + float t0[4] = {0.f, 0.f, 0.f, 0.f}, t1[4] = {0.f, 0.f, 0.f, 0.f}; + vqm_mma(t0, a, b[0], b[1]); + #pragma unroll + for (int c = 0; c < 4; c++) acc[2 * np][c] += t0[c]; + if ((f0 + 1u) * 8u < nt) { vqm_mma(t1, a, b[2], b[3]); + #pragma unroll + for (int c = 0; c < 4; c++) acc[2 * np + 1][c] += t1[c]; } + } + } + __syncthreads(); + } + /* exit: c0,c1 = row lane/4, token (lane&3)*2+{0,1}; c2,c3 = row +8 */ + const uint32_t rr[2] = { r0 + wr * 16u + (lane >> 2), r0 + wr * 16u + (lane >> 2) + 8u }; + #pragma unroll + for (uint32_t h = 0; h < 2u; h++) { + const uint32_t r = rr[h]; + if (r >= M) continue; + __half gh; memcpy(&gh, m.gr + (size_t)r * 2u, 2); + const float g = __half2float(gh) * (m.gov ? m.gov[r] : 1.0f); + #pragma unroll + for (uint32_t f = 0; f < NFW; f++) { + const uint32_t gf = th * NFW + f; + if (gf * 8u >= nt) break; + #pragma unroll + for (uint32_t e = 0; e < 2u; e++) { + const uint32_t t = gf * 8u + (lane & 3u) * 2u + e; + if (t >= nt) continue; + const uint64_t o = (uint64_t)(base + t) * M + r; + const float val = v41_bf16r(acc[f][h * 2u + e] * g); + if (MODE == 0) g32[o] = val; + else if (MODE == 1) { h16[o] = v41_vq_swiglu(g32[o], val, clamp); if (ys) ys[o] = val; } + else ys[o] = val; + } + } + } + } +} + +static struct { v41_scratch d, doff, xs16, h16, g32; int ready, nsm; int occ[2][3]; } g_vqm; + +/* Per-instance grid: SM count x the occupancy API's blocks/SM (never a fixed + * tier; 96 KB shared gives 1 today). */ +template +static int vqm_grid_of(size_t shb, uint32_t nwork) { + int *occ = &g_vqm.occ[EXT][MODE]; + if (*occ == 0) { + if (hipFuncSetAttribute(reinterpret_cast(vqm_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)shb) != hipSuccess || + hipOccupancyMaxActiveBlocksPerMultiprocessor(occ, vqm_kernel, (int)VQM_THREADS, shb) != hipSuccess || *occ <= 0) { + (void)hipGetLastError(); *occ = -1; + } + } + if (*occ < 0) return 0; + const uint32_t g = (uint32_t)g_vqm.nsm * (uint32_t)*occ; + return (int)(g < nwork ? g : nwork); +} + +/* ---- vqs, the register-direct form (cuda_vq_reg_mma.inc.cu, whole file) ---- + * One block (16 warps) owns (work item, 256 rows) x all of K: + * 1. bitstream and activations move in RS-round segments via cp.async, two + * segments in flight, one barrier per segment; each row's bitstream is + * additionally bulk-prefetched into L2 in 384 B (3 whole lines) chunks + * two chunks ahead — the segment cp.async then reads only L2 and DRAM + * sees whole-line sequential requests (measured 2026-09-18: a load that + * cannot fill a 128 B line gets 142 GB/s; cold experts sat at ~130 GB/s, + * this cut 8.6 -> 6.2 ms); + * 2. lane (g = lane/4, q = lane%4) decodes rows 16w+g and 16w+8+g of the + * unit, and the codebook lookup (E4M3, 8 B/word) converts straight to + * bf16 — that IS the mma A fragment: no A/B tile writes, no ldmatrix. + * k16 logical k = 2q+{0,1} maps to physical column 16kk+4q+{0,1}, and + * 2q+8+{0,1} to 16kk+4q+{2,3} => this lane needs code word 2kk+q/2's + * elements 4(q&1)..+3 (half a word, one LDS.32); the activation columns + * are one 8 B read. + * Bit-exactness: the mma k16 groups are the same 16 columns, only who takes + * which column inside the group changes; the per-k16 zero-start + FADD-back, + * the row gain and the bf16r exit are unchanged — the engine's microbench + * compared g32/h16/H_u/ys byte for byte with vqm_kernel, 12/13-bit x both + * routings (a tensor core's intra-group sum is order-independent). + * Wrong writes here do not fault: the mma eats other columns and PPL explodes + * into the tens of thousands; the gate is the engine's warm-0 output cmp. */ +#define VQS_THREADS 512u +#define VQS_BM 256u /* rows per work unit: 16 warps x 16 rows (M must be a multiple: 2304 = 9*256, 5120 = 20*256) */ +#define VQS_NTM 32u /* tokens per work item (4 n8 slices); hot experts split by it, 64/128 measured no faster */ + +/* The kk-th k16 of a row's round needs code word c = 2kk + h (h = q/2), bit offset 12c; this round's 96 bits live in w0..w2. */ +__device__ __forceinline__ static uint32_t vqs_code(uint32_t w0, uint32_t w1, uint32_t w2, uint32_t kk, uint32_t h) { + if (kk == 0u) return (w0 >> (12u * h)) & 0xFFFu; + if (kk == 1u) return (h ? (w1 >> 4) : __funnelshift_r(w0, w1, 24u)) & 0xFFFu; + if (kk == 2u) return __funnelshift_r(w1, w2, 16u + 12u * h) & 0xFFFu; + return (w2 >> (8u + 12u * h)) & 0xFFFu; +} +template +__device__ __forceinline__ static void vqs_cp(uint8_t *dst, const void *src) { /* N = 16 takes .cg (L2 only), 8/4 take .ca */ +#if defined(__HIP_PLATFORM_AMD__) + ds4hip_copy(dst, src, (int)N, true); +#else + if (N == 16u) asm volatile("cp.async.cg.shared.global [%0], [%1], 16;" :: "r"((uint32_t)__cvta_generic_to_shared(dst)), "l"(src) : "memory"); + else asm volatile("cp.async.ca.shared.global [%0], [%1], %2;" :: "r"((uint32_t)__cvta_generic_to_shared(dst)), "l"(src), "n"(N) : "memory"); +#endif +} +#if defined(__HIP_PLATFORM_AMD__) /* copies complete synchronously (hip_ptx_emul.cuh) */ +__device__ __forceinline__ static void vqs_commit() {} +template __device__ __forceinline__ static void vqs_wait() {} +#else +__device__ __forceinline__ static void vqs_commit() { asm volatile("cp.async.commit_group;" ::: "memory"); } +template __device__ __forceinline__ static void vqs_wait() { asm volatile("cp.async.wait_group %0;" :: "n"(N) : "memory"); } +#endif +/* Portability deviation, named: the bulk prefetch is sm_90+ (ptxas rejects + * it below); compiled out there. It is a hint — the segment cp.async loads + * are the real fetches — so the kernel stays correct and numerically + * identical; sm_121 compiles the engine's asm verbatim. */ +__device__ __forceinline__ static void vqs_pf_l2(const void *p, uint32_t bytes) { +#if !defined(__HIP_PLATFORM_AMD__) && (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 900) + asm volatile("cp.async.bulk.prefetch.L2.global [%0], %1;" :: "l"(p), "r"(bytes) : "memory"); +#else + (void)p; (void)bytes; /* a hint only; AMD has no bulk L2 prefetch */ +#endif +} +__device__ __forceinline__ static uint32_t vqs_e4m3x2_bf16x2(uint32_t two) { + uint32_t hh, o; +#if defined(__HIP_PLATFORM_AMD__) + hh = ds4hip_e4m3x2_to_f16x2(two & 0xffffu); +#else + asm("cvt.rn.f16x2.e4m3x2 %0, %1;" : "=r"(hh) : "h"((unsigned short)(two & 0xffffu))); +#endif + const float lo = __half2float(__ushort_as_half((unsigned short)(hh & 0xffffu))), hi = __half2float(__ushort_as_half((unsigned short)(hh >> 16))); +#if defined(__HIP_PLATFORM_AMD__) + o = ds4hip_f32x2_to_bf16x2(hi, lo); +#else + asm("cvt.rn.bf16x2.f32 %0, %1, %2;" : "=r"(o) : "f"(hi), "f"(lo)); +#endif + return o; +} +/* D = A*B (C is always +0, the output gets its own register group). Not + * vqm_mma(t, ...) with t = {0}: that is a "+f" read-write constraint, ptxas + * schedules D into A's registers, and a reused A fragment costs 4 MOVs before + * every HMMA (10-03 SASS: 32 HMMAs per segment paired with 128 MOVs). Same + * numerics (C = +0). */ +__device__ __forceinline__ static void vqs_mma0(float *d, const uint32_t *a, uint32_t b0, uint32_t b1) { +#if defined(__HIP_PLATFORM_AMD__) + d[0] = d[1] = d[2] = d[3] = 0.f; + ds4hip_mma_m16n8k16(d, a[0], a[1], a[2], a[3], b0, b1); +#else + asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%10,%10,%10};" + : "=f"(d[0]), "=f"(d[1]), "=f"(d[2]), "=f"(d[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1), "f"(0.f)); +#endif +} +/* shared bytes: codebook (E4M3 nc*8) + S segments x (bitstream 256 rows x 12*RS B + plane 256 x 4 B + activations NTM x 128*RS B) */ +static constexpr uint32_t vqs_smem(uint32_t cbb, int ext, uint32_t rs, uint32_t s) { + return cbb + s * (VQS_BM * 12u * rs + (ext ? VQS_BM * 4u : 0u) + VQS_NTM * 128u * rs); +} + +/* MODE 0 = gate: g32 = bf16(W1*x*g) | 1 = up: read g32, clamp + SwiGLU -> h16 (ys non-NULL also writes H_u) | 2 = down: ys = bf16(W2*h*g). + * Parameters identical to vqm_kernel (same work items / offsets / scratch from vqm_run_impl); only the work-item cut (VQS_NTM) differs. */ +template +__global__ __launch_bounds__(VQS_THREADS, 1) static void vqs_kernel( + float *g32, uint16_t *h16, float *ys, const uint8_t *blob, const vqp_item *items, uint32_t nitems, + const uint16_t *act, const uint32_t *off, uint32_t M, uint32_t K, float clamp, uint32_t cb_bytes, const float *gr) { + constexpr uint32_t NTN = VQS_NTM / 8u, RB = 12u * RS, CP = RS == 4u ? 16u : 8u; /* per row per segment, moved in 3 chunks of CP bytes */ + constexpr uint32_t BSB = VQS_BM * RB, EXB = EXT ? VQS_BM * 4u : 0u, ATB = VQS_NTM * 128u * RS, STB = BSB + EXB + ATB; + constexpr uint32_t PCH = 384u, PD = 2u, SPC = PCH / RB; /* L2 prefetch: 384 B per row chunk, 2 chunks ahead; SPC = chunks per segment */ + static_assert(RS == 2u || RS == 4u, "a segment is 2 or 4 rounds"); + extern __shared__ __align__(16) uint8_t vqssh[]; + uint8_t *cbs = vqssh, *ring = vqssh + cb_bytes; + const int which = MODE; /* payload slot: 0 w1(gate) / 1 w3(up) / 2 w2(down) */ + const uint32_t tid = threadIdx.x, lane = tid & 31u, warp = tid >> 5, g = lane >> 2, q = lane & 3u, h = q >> 1, hf = q & 1u; + { /* the v3 codebook is one per layer (three matrices, all experts); enters shared as raw E4M3 */ + const v41_vq_mat m0 = v41_vq_open<1>(blob, items[0].e, which, M, K, NULL); + if (!m0.ok || m0.nc * 8u != cb_bytes) return; /* one decision for the whole block, nobody stuck at a later barrier */ + v41_vq_cb_to_shared(cbs, m0.cb, cb_bytes); + } + __syncthreads(); + const uint32_t ntile = M / VQS_BM, nwork = nitems * ntile, nst = K / (64u * RS); + const uint32_t la = warp * 16u + g, lb = la + 8u; + /* This thread moves the bitstream's two chunks (k = tid, tid+512; only k < 768 exist): chunk k%3 of row k/3 */ + const uint32_t k1 = tid + VQS_THREADS, rw0 = tid / 3u, pt0 = tid - rw0 * 3u, rw1 = k1 / 3u, pt1 = k1 - rw1 * 3u; + const uint32_t cofs[4] = { ((0u + h) ^ (2u * (g & 3u))) << 4, ((2u + h) ^ (2u * (g & 3u))) << 4, + ((4u + h) ^ (2u * (g & 3u))) << 4, ((6u + h) ^ (2u * (g & 3u))) << 4 }; + for (uint32_t w = blockIdx.x; w < nwork; w += gridDim.x) { + const vqp_item it = items[w / ntile]; + const uint32_t r0 = (w % ntile) * VQS_BM, nt = (uint32_t)it.nt, base = off[it.e] + (uint32_t)it.t0; + const v41_vq_mat m = v41_vq_open<1>(blob, it.e, which, M, K, (MODE == 2 && gr) ? gr + (size_t)it.e * M : NULL); + if (!m.ok) { /* host walked the slot table; a bad payload zeroes the rows (same as vqm_kernel) */ + if (MODE == 2) + for (uint32_t i = tid; i < VQS_BM * nt; i += VQS_THREADS) ys[(uint64_t)(base + i / VQS_BM) * M + r0 + i % VQS_BM] = 0.f; + continue; + } + const uint32_t mrow = m.nidx_row * 12u / 8u, erow = (m.nidx_row + 7u) >> 3, nach = nt * 8u * RS; + const uint8_t *prow = m.ix + (size_t)(r0 + (tid & 255u)) * mrow; + auto pf_chunk = [&](uint32_t c) { + if (tid < VQS_BM && c * PCH < mrow) vqs_pf_l2(prow + c * PCH, mrow - c * PCH < PCH ? mrow - c * PCH : PCH); + }; + #pragma unroll + for (uint32_t c = 0; c < PD; c++) pf_chunk(c); + if (EXT && tid == 0) vqs_pf_l2(m.ex + (size_t)r0 * erow, (VQS_BM * erow) & ~15u); /* the whole plane is 9-20 KB, one call */ + const uint8_t *bsrc0 = m.ix + (size_t)(r0 + rw0) * mrow + CP * pt0, *bsrc1 = m.ix + (size_t)(r0 + (k1 < 768u ? rw1 : 0u)) * mrow + CP * pt1; + const uint8_t *esrc = EXT ? m.ex + (size_t)(r0 + (tid & 255u)) * erow : NULL; + const uint16_t *xa = act + (uint64_t)base * K; + auto issue = [&](uint32_t s) { /* segment s (rounds RS*s ..) -> slot s % S */ + uint8_t *st = ring + (s % S) * STB; + vqs_cp(st + rw0 * RB + CP * pt0, bsrc0 + RB * s); + if (k1 < 768u) vqs_cp(st + rw1 * RB + CP * pt1, bsrc1 + RB * s); + if (EXT && tid < VQS_BM) vqs_cp<4>(st + BSB + tid * 4u, esrc + 4u * ((RS * s) >> 2)); /* plane: one word per 4 rounds, RS=2 shares it between two segments */ + uint8_t *ad = st + BSB + EXB; + for (uint32_t k = tid; k < nach; k += VQS_THREADS) { + const uint32_t t = k / (8u * RS), rem = k - t * 8u * RS, r = rem >> 3, c = rem & 7u; + vqs_cp<16>(ad + (r * VQS_NTM + t) * 128u + ((c ^ (2u * (t & 3u))) << 4), xa + (uint64_t)t * K + 64u * (RS * s + r) + 8u * c); + } + }; + /* Prologue: one group per segment 0..S-2. Then one group per segment, so segment s sits in group s and wait_group(S-2) reaches it. */ + #pragma unroll + for (uint32_t s = 0; s + 1u < S; s++) { if (s < nst) issue(s); vqs_commit(); } + const uint32_t nnt = (nt + 7u) >> 3; + float acc[NTN][4]; + #pragma unroll + for (uint32_t j = 0; j < NTN; j++) { acc[j][0] = acc[j][1] = acc[j][2] = acc[j][3] = 0.f; } + /* The segment loop is instantiated per n8-slice count (a compile-time constant): no per-slice branch in the slice loop */ + auto run = [&](auto nntc) { + constexpr uint32_t NNT = decltype(nntc)::value; + for (uint32_t s = 0; s < nst; s++) { + vqs_wait<(int)S - 2>(); + __syncthreads(); /* segment s is fully in; also proves everyone finished segment s-1's slot => segment s+S-1 can move in below */ + if (s + S - 1u < nst) issue(s + S - 1u); + vqs_commit(); + if (s % SPC == 0u) pf_chunk(s / SPC + PD); + const uint8_t *st = ring + (s % S) * STB; + uint32_t wa[3 * RS], wb[3 * RS]; + if (RS == 4u) { + #pragma unroll + for (uint32_t k = 0; k < 3u; k++) { + const uint4 u = *(const uint4 *)(st + la * RB + 16u * k), v = *(const uint4 *)(st + lb * RB + 16u * k); + wa[4 * k] = u.x; wa[4 * k + 1] = u.y; wa[4 * k + 2] = u.z; wa[4 * k + 3] = u.w; + wb[4 * k] = v.x; wb[4 * k + 1] = v.y; wb[4 * k + 2] = v.z; wb[4 * k + 3] = v.w; + } + } else { + #pragma unroll + for (uint32_t k = 0; k < 3u; k++) { + const uint2 u = *(const uint2 *)(st + la * RB + 8u * k), v = *(const uint2 *)(st + lb * RB + 8u * k); + wa[2 * k] = u.x; wa[2 * k + 1] = u.y; wb[2 * k] = v.x; wb[2 * k + 1] = v.y; + } + } + uint32_t xe = 0u, ye = 0u; + if (EXT) { xe = *(const uint32_t *)(st + BSB + la * 4u); ye = *(const uint32_t *)(st + BSB + lb * 4u); } + const uint8_t *as = st + BSB + EXB + 8u * hf + g * 128u; + #pragma unroll + for (uint32_t r = 0; r < RS; r++) { + const uint32_t eb = ((RS * s + r) & 3u) * 8u; /* this round's byte inside the plane word */ + #pragma unroll + for (uint32_t kk = 0; kk < 4u; kk++) { + uint32_t va = vqs_code(wa[3 * r], wa[3 * r + 1], wa[3 * r + 2], kk, h), vb = vqs_code(wb[3 * r], wb[3 * r + 1], wb[3 * r + 2], kk, h); + if (EXT) { const uint32_t c = eb + 2u * kk + h; va |= ((xe >> c) & 1u) << 12; vb |= ((ye >> c) & 1u) << 12; } + const uint32_t ea = *(const uint32_t *)(cbs + (size_t)va * 8u + hf * 4u), ebb = *(const uint32_t *)(cbs + (size_t)vb * 8u + hf * 4u); + const uint32_t a[4] = { vqs_e4m3x2_bf16x2(ea), vqs_e4m3x2_bf16x2(ebb), vqs_e4m3x2_bf16x2(ea >> 16), vqs_e4m3x2_bf16x2(ebb >> 16) }; + const uint8_t *ap = as + r * VQS_NTM * 128u + cofs[kk]; + #pragma unroll + for (uint32_t j = 0; j < NNT; j++) { + const uint2 x = *(const uint2 *)(ap + j * 8u * 128u); + /* Every k16 starts from zero and adds back with a plain FADD (same as vqm_kernel: a long in-core K accumulation truncates on alignment, a one-sided low-bit loss) */ + float t[4]; + vqs_mma0(t, a, x.x, x.y); + #pragma unroll + for (int c = 0; c < 4; c++) acc[j][c] += t[c]; + } + } + } + } + }; +#define VQS_NNT(n) std::integral_constant{} + switch (nnt) { + case 1: run(VQS_NNT(1)); break; + case 2: run(VQS_NNT(2)); break; + case 3: run(VQS_NNT(3)); break; + default: run(VQS_NNT(4)); break; + } +#undef VQS_NNT + __syncthreads(); /* the next unit's prologue overwrites these slots */ + /* exit: c0,c1 = row 16w+g, token 8j+2q+{0,1}; c2,c3 = row +8. Rounding point same as vqm_kernel (sum -> row gain -> bf16r) */ + const uint32_t ra = r0 + la, rb = r0 + lb; + float gna, gnb; + { __half gh; memcpy(&gh, m.gr + (size_t)ra * 2u, 2); gna = __half2float(gh) * (m.gov ? m.gov[ra] : 1.0f); + memcpy(&gh, m.gr + (size_t)rb * 2u, 2); gnb = __half2float(gh) * (m.gov ? m.gov[rb] : 1.0f); } + #pragma unroll + for (uint32_t j = 0; j < NTN; j++) { + if (j >= nnt) break; + #pragma unroll + for (uint32_t e = 0; e < 2u; e++) { + const uint32_t t = 8u * j + 2u * q + e; + if (t >= nt) continue; + const uint64_t o = (uint64_t)(base + t) * M; + const float va = v41_bf16r(acc[j][e] * gna), vb = v41_bf16r(acc[j][2u + e] * gnb); + if (MODE == 0) { g32[o + ra] = va; g32[o + rb] = vb; } + else if (MODE == 1) { h16[o + ra] = v41_vq_swiglu(g32[o + ra], va, clamp); h16[o + rb] = v41_vq_swiglu(g32[o + rb], vb, clamp); + if (ys) { ys[o + ra] = va; ys[o + rb] = vb; } } + else { ys[o + ra] = va; ys[o + rb] = vb; } + } + } + } +} + +static uint32_t vqs_item_tokens(void) { return VQS_NTM; } +/* vqs/vqst preconditions: bitstream 16 B aligned + device memory (cp.async's + * 16 B copies and the L2 bulk prefetch both need it). Only the boot-cached + * device copy (the vq-align relocation) satisfies them; a flat copy of the + * raw GGUF bytes is 8 B aligned. Checked once per (layer, blob pointer). */ +static struct { const uint8_t *blob; int ok; } g_vqs_fit[64]; +static int vqs_blob_fits(uint32_t layer, const uint8_t *blob, uint32_t n_total, uint32_t IN, uint32_t MID, uint32_t OUT) { + if (layer >= 64u) return 0; + if (g_vqs_fit[layer].blob == blob) return g_vqs_fit[layer].ok; + if (!g_vqp_hdr[layer] && !vqp_hdr_build(layer, blob, n_total, IN, MID, OUT, 3u)) return 0; + int ok = ((uintptr_t)blob & 15u) == 0u; + hipPointerAttribute_t at; + if (ok && (hipPointerGetAttributes(&at, blob) != hipSuccess || at.type != hipMemoryTypeDevice)) { (void)hipGetLastError(); ok = 0; } + for (uint32_t k = 0; ok && k < n_total * 3u; k++) { /* bitstream start = payload + 32 + row gains rows*2 */ + const uint64_t off = g_vqp_hdr[layer][k].off; + if (off && ((off + 32u + (uint64_t)((k % 3u) == 2u ? OUT : MID) * 2u) & 15u)) ok = 0; + } + if (!ok) { static int said; if (!said++) fprintf(stderr, "ds4: [ds41] vq-prefill L%u expert blob is not an aligned device copy (mapping/flat copy), those layers take the tile kernel\n", layer); } + g_vqs_fit[layer].blob = blob; g_vqs_fit[layer].ok = ok; + return ok; +} +/* What vqs accepts: M (MID for gate/up, OUT for down) a multiple of 256, K a multiple of one segment (64*RS columns) */ +static bool vqs_shape_ok(uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t nbit) { + const uint32_t seg = 64u * (nbit == 13u ? 2u : 4u); + return (nbit == 12u || nbit == 13u) && MID % VQS_BM == 0u && OUT % VQS_BM == 0u && IN % seg == 0u && MID % seg == 0u; +} +/* Three launches (gate -> up -> down). 13-bit layers: 64 KB codebook, shared fits one 2-round segment; 12-bit layers: 32 KB codebook, 4-round segments (half the barriers, 7% faster in the microbench). */ +static int vqs_launch3(float *g32, uint16_t *h16, float *hu, float *ys, const uint8_t *blob, const vqp_item *items, uint32_t nitems, + const uint16_t *xs, const uint32_t *doff, uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t nc, uint32_t nbit, + float clamp, const float *gr, uint32_t layer_index) { + static int occ[2][3]; + const uint32_t cbb = nc * 8u; +#define VQS_GO(E, RS_) do { \ + const uint32_t shb = vqs_smem(cbb, E, RS_, 2u); \ + int *oc = occ[E]; \ + if (!oc[0]) { \ + const bool ok = hipFuncSetAttribute(reinterpret_cast(vqs_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)shb) == hipSuccess && \ + hipFuncSetAttribute(reinterpret_cast(vqs_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)shb) == hipSuccess && \ + hipFuncSetAttribute(reinterpret_cast(vqs_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)shb) == hipSuccess && \ + hipOccupancyMaxActiveBlocksPerMultiprocessor(&oc[0], vqs_kernel, (int)VQS_THREADS, shb) == hipSuccess && oc[0] > 0; \ + if (!ok) { (void)hipGetLastError(); oc[0] = -1; } \ + } \ + if (oc[0] < 0) { fprintf(stderr, "ds4: [ds41] vq-prefill L%u register-direct instance cannot open %u KB shared\n", layer_index, shb >> 10); return 0; } \ + const uint32_t gcap = (uint32_t)g_vqm.nsm * (uint32_t)oc[0], ng = nitems * (MID / VQS_BM), nd = nitems * (OUT / VQS_BM); \ + vqs_kernel<<>>(g32, NULL, NULL, blob, items, nitems, xs, doff, MID, IN, clamp, cbb, NULL); \ + if (!cuda_ok(hipGetLastError(), "vq prefill reg gate")) return 0; \ + vqs_kernel<<>>(g32, h16, hu, blob, items, nitems, xs, doff, MID, IN, clamp, cbb, NULL); \ + if (!cuda_ok(hipGetLastError(), "vq prefill reg up")) return 0; \ + vqs_kernel<<>>(NULL, NULL, ys, blob, items, nitems, h16, doff, OUT, MID, clamp, cbb, gr); \ + return cuda_ok(hipGetLastError(), "vq prefill reg down"); \ + } while (0) + if (nbit == 13u) VQS_GO(1, 2u); + VQS_GO(0, 4u); +#undef VQS_GO +} + +/* ---- the run (cuda_vq_prefill_mma.inc.cu:233-302) ---- + * Returns 0 = failure (the caller hard-fails). ys = sorted down output + * [nvalid][OUT], the same buffer the fused path writes, so the reduce is + * untouched. hg/ha/hu non-NULL = per-pair intermediates into the caller's + * buffers (backward capture); NULL = this file's scratch, ys only (the + * inference path). */ +static int vqm_run_impl(const uint8_t *blob, const uint32_t *cnt, const uint32_t *off_h, uint32_t n_total_expert, uint32_t nvalid, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t nc, float clamp, const float *x, const int32_t *perm, + uint32_t n_expert, uint32_t layer_index, float *ys, const float *gr, float *hg, uint16_t *ha, float *hu) { + uint32_t nbit = 0; while ((1u << nbit) < nc) nbit++; + if ((nbit != 12u && nbit != 13u) || IN % VQM_BK || MID % VQM_BK) { + fprintf(stderr, "ds4: [ds41] vq-prefill L%u tensor-core path does not take this shape (codebook %u words, IN %u MID %u)\n", layer_index, nc, IN, MID); + return 0; + } + /* 12-bit layers convert the codebook to bf16 on the way into shared + * (nc*16 B = 64 KB); 13-bit layers' 8192 words would need 128 KB, so they + * keep E4M3 (64 KB) and convert on the fly. */ + const int cb = nbit == 12u ? 1 : 0; + const uint32_t cbb = nc * (cb ? 16u : 8u), shb = cbb + VQM_TILE_BYTES; + if (!g_vqm.ready) { + int nsm = 0; + const bool ok = hipDeviceGetAttribute(&nsm, hipDeviceAttributeMultiprocessorCount, 0) == hipSuccess && nsm > 0; + (void)hipGetLastError(); + g_vqm.ready = ok ? 1 : -1; g_vqm.nsm = nsm; + fprintf(stderr, "ds4: [ds41] vq-prefill tensor-core path (bf16 mma, <=%u tokens/item, %u threads) %s, %d SMs\n", + VQM_BN, VQM_THREADS, ok ? "ready" : "unavailable", nsm); + } + if (g_vqm.ready != 1) return 0; + const bool rs = vqs_shape_ok(IN, MID, OUT, nbit) && vqs_blob_fits(layer_index, blob, n_total_expert, IN, MID, OUT); + const uint32_t bn = rs ? vqs_item_tokens() : VQM_BN; /* tokens per work item: the two kernels cut differently */ + uint32_t nit = 0; + for (uint32_t e = 0; e < n_total_expert; e++) nit += (cnt[e] + bn - 1u) / bn; + if (!nit) return 0; + vqp_item *ih = (vqp_item *)malloc((size_t)nit * sizeof(vqp_item)); + if (!ih) return 0; + uint32_t k = 0; + for (uint32_t e = 0; e < n_total_expert; e++) + for (uint32_t t0 = 0; t0 < cnt[e]; t0 += bn) { + ih[k].e = (int32_t)e; ih[k].t0 = (int32_t)t0; ih[k].nt = (int32_t)(cnt[e] - t0 < bn ? cnt[e] - t0 : bn); k++; + } + int ok = v41_grow(&g_vqm.d, (uint64_t)nit * sizeof(vqp_item), "vq prefill mma items") && + v41_grow(&g_vqm.doff, ((uint64_t)n_total_expert + 1u) * sizeof(uint32_t), "vq prefill mma off") && + v41_grow(&g_vqm.xs16, (uint64_t)nvalid * IN * sizeof(uint16_t), "vq prefill mma xs16") && + (ha || v41_grow(&g_vqm.h16, (uint64_t)nvalid * MID * sizeof(uint16_t), "vq prefill mma h16")) && + (hg || v41_grow(&g_vqm.g32, (uint64_t)nvalid * MID * sizeof(float), "vq prefill mma g32")); + float *g32 = hg ? hg : (float *)g_vqm.g32.p; + uint16_t *h16 = ha ? ha : (uint16_t *)g_vqm.h16.p; + if (ok) ok = hipMemcpyAsync(g_vqm.d.p, ih, (size_t)nit * sizeof(vqp_item), hipMemcpyHostToDevice, ds4_current_stream()) == hipSuccess && + hipMemcpyAsync(g_vqm.doff.p, off_h, (size_t)(n_total_expert + 1u) * sizeof(uint32_t), hipMemcpyHostToDevice, ds4_current_stream()) == hipSuccess; + /* ih is the source of an async H2D from pageable memory => the runtime + * staged it before returning, freeing here is safe (same as the fused path) */ + free(ih); + if (!ok) { (void)hipGetLastError(); fprintf(stderr, "ds4: [ds41] vq-prefill L%u tensor-core scratch/copy failed\n", layer_index); return 0; } + vqm_gather16_kernel<<>>((uint16_t *)g_vqm.xs16.p, x, perm, n_expert, IN); + if (!cuda_ok(hipGetLastError(), "vq prefill mma gather16")) return 0; + if (rs) return vqs_launch3(g32, h16, hu, ys, blob, (const vqp_item *)g_vqm.d.p, nit, (const uint16_t *)g_vqm.xs16.p, (const uint32_t *)g_vqm.doff.p, IN, MID, OUT, nc, nbit, clamp, gr, layer_index); + const uint32_t tg = (MID + VQM_BM - 1u) / VQM_BM, td = (OUT + VQM_BM - 1u) / VQM_BM; +#define VQM_LAUNCH(E, CB) do { \ + const int bg = vqm_grid_of(shb, nit * tg), bu = vqm_grid_of(shb, nit * tg), bd = vqm_grid_of(shb, nit * td); \ + if (bg <= 0 || bu <= 0 || bd <= 0) { fprintf(stderr, "ds4: [ds41] vq-prefill L%u tensor-core instance cannot open %u KB shared\n", layer_index, shb >> 10); return 0; } \ + vqm_kernel<<>>(g32, NULL, NULL, blob, (const vqp_item *)g_vqm.d.p, nit, (const uint16_t *)g_vqm.xs16.p, (const uint32_t *)g_vqm.doff.p, MID, IN, clamp, cbb, NULL); \ + if (!cuda_ok(hipGetLastError(), "vq prefill mma gate")) return 0; \ + vqm_kernel<<>>(g32, h16, hu, blob, (const vqp_item *)g_vqm.d.p, nit, (const uint16_t *)g_vqm.xs16.p, (const uint32_t *)g_vqm.doff.p, MID, IN, clamp, cbb, NULL); \ + if (!cuda_ok(hipGetLastError(), "vq prefill mma up")) return 0; \ + vqm_kernel<<>>(NULL, NULL, ys, blob, (const vqp_item *)g_vqm.d.p, nit, h16, (const uint32_t *)g_vqm.doff.p, OUT, MID, clamp, cbb, gr); \ + return cuda_ok(hipGetLastError(), "vq prefill mma down"); \ + } while (0) + if (nbit == 13u) VQM_LAUNCH(1, 0); + VQM_LAUNCH(0, 1); +#undef VQM_LAUNCH +} + +/* Inference entry (prefill / scoring): ys only. The engine's backward + * capture (g_vqm_cap / vqm_run_parts / vqm_run's H_u hoist) is training-only + * and not ported — the port has no backward pass. */ +static int vqm_run(const uint8_t *blob, const uint32_t *cnt, const uint32_t *off_h, uint32_t n_total_expert, uint32_t nvalid, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t nc, float clamp, const float *x, const int32_t *perm, + uint32_t n_expert, uint32_t layer_index, float *ys, const float *gr) { + return vqm_run_impl(blob, cnt, off_h, n_total_expert, nvalid, IN, MID, OUT, nc, clamp, x, perm, n_expert, layer_index, ys, gr, NULL, NULL, NULL); +} diff --git a/cuda/ds41_vq_prefill_wmma.cuh b/cuda/ds41_vq_prefill_wmma.cuh new file mode 100644 index 0000000000000000000000000000000000000000..7ac8f7e9d875299979b7fc2f4ace6f47d3dfa8b5 --- /dev/null +++ b/cuda/ds41_vq_prefill_wmma.cuh @@ -0,0 +1,224 @@ +#include "hip/hip_runtime.h" +/* ds41_vq_prefill_wmma.cuh — DeepSeek V4.1 (ds41) VQ expert prefill on rocWMMA + * (AMD, 2026-10-23). The engine's tensor-core arm (ds41_vq_prefill_mma.cuh) + * needs 64-88 KB of shared and, on AMD, an mma emulated with shuffles; so every + * layer took the fused scalar kernels — 76% of a prefill's GPU time on the + * engine's own profile, ~1.9 s per layer for a 2048-token chunk here (~27 + * tokens/s). This arm keeps the engine's decomposition (one block decodes a + * BM x BK tile of one expert's code words into shared and multiplies it with the + * expert's token tile, the bitstream read once per tile pass) at a 64 KB + * budget, with wave32 rocWMMA bf16 16x16x16 instructions. + * + * Numerics follow the engine's arm: E4M3 -> bf16 is exact, activations are on + * the bf16 grid, products are exact in f32, every k16 step starts from zero and + * is added back with a plain FADD, the row gain multiplies after the sum, the + * exit rounds to bf16. The K accumulation order differs from the fused kernels + * (which differ from any GEMM anyway); the gate is the quality metrics, not + * bytes. DS41_VQW_CHECK=1 runs both arms and prints the difference per layer. + * + * Shared: codebook (10-bit: 1024 words as bf16 = 16 KB; 12-bit: 4096 words + * E4M3 = 32 KB, converted on the fly) + A tile 128 x 72 bf16 (18 KB) + B tile + * BN x 72 bf16 (BN = 128 tokens: 18 KB; 64: 9 KB) => 52 KB / 59 KB. + * + * Must follow ds41_vq_prefill_mma.cuh (vqm_cw, vqm_gather16_kernel, vqp_item). + */ +#pragma once +#if defined(__HIP_PLATFORM_AMD__) +#define VQW_BM 128u /* weight rows per block: 8 warps x 16 */ +#define VQW_BK 64u /* K columns per round: 8 code words per row */ +#define VQW_LD (VQW_BK + 8u) /* tile row stride (bf16): padding spreads the fragment reads over the banks */ +#define VQW_THREADS 512u /* 16 warps: 8 row tiles x 2 token halves */ + +template +__global__ __launch_bounds__(VQW_THREADS, 1) static void vqw_kernel( + float *g32, uint16_t *h16, float *ys, const uint8_t *blob, const vqp_item *items, uint32_t nitems, + const uint16_t *act, const uint32_t *off, uint32_t M, uint32_t K, float clamp, uint32_t cb_bytes, const float *gr, uint32_t MB) { + namespace wmma = nvcuda::wmma; + constexpr uint32_t CPR = VQW_BK / 8u, TPR = VQW_THREADS / VQW_BM, CPT = CPR / TPR; /* 8 words per row-round, 4 threads x 2 */ + constexpr uint32_t NFW = BN / 2u / 16u; /* 16-token fragments per warp (its token half) */ + constexpr uint32_t CH = (BN * CPR) / VQW_THREADS; /* 16 B chunks of B per thread per round */ + extern __shared__ __align__(16) uint8_t vqwsh[]; + uint8_t *cbs = vqwsh; + uint16_t *As = (uint16_t *)(vqwsh + cb_bytes); /* [VQW_BM][VQW_LD] bf16 */ + uint16_t *Bs = As + VQW_BM * VQW_LD; /* [BN][VQW_LD] bf16 */ + float *Os = (float *)As; /* exit scratch, per warp one 16x16 f32 tile (As is free after the K loop) */ + const int which = MODE; + const uint32_t tid = threadIdx.x, lane = tid & 31u, warp = tid >> 5, wr = warp & 7u, th = warp >> 3; + const uint32_t ntile = (M + VQW_BM - 1u) / VQW_BM, nwork = nitems * ntile, nit = K / VQW_BK; + { /* the v3 codebook is one per layer: the first work item's is this layer's */ + const v41_vq_mat m0 = v41_vq_open<1>(blob, items[0].e, which, M, K, NULL); + if (!m0.ok || m0.nc * (CB ? 16u : 8u) != cb_bytes) return; /* block-uniform: nobody is left at a barrier */ + if (CB == 0) v41_vq_cb_to_shared(cbs, m0.cb, cb_bytes); + else for (uint32_t v = tid; v < m0.nc; v += VQW_THREADS) *(uint4 *)(cbs + (size_t)v * 16u) = vqm_cw<0>(m0.cb, v); + } + __syncthreads(); + const uint32_t ar = tid / TPR, sub = tid % TPR; /* decode split: this thread owns row ar's code words sub*CPT.. of each round */ + for (uint32_t w = blockIdx.x; w < nwork; w += gridDim.x) { + const vqp_item it = items[w / ntile]; + const uint32_t r0 = (w % ntile) * VQW_BM, nt = (uint32_t)it.nt, base = off[it.e] + (uint32_t)it.t0; + const v41_vq_mat m = v41_vq_open<1>(blob, it.e, which, M, K, (MODE == 2 && gr) ? gr + (size_t)it.e * M : NULL); + if (!m.ok) { /* bad payload (the host walked the slot table): down writes 0, never a dirty reduce input */ + if (MODE == 2) + for (uint32_t i = tid; i < VQW_BM * nt; i += VQW_THREADS) + if (r0 + i % VQW_BM < M) ys[(uint64_t)(base + i / VQW_BM) * M + r0 + i % VQW_BM] = 0.f; + continue; + } + const uint32_t grow = r0 + ar, mrow = (m.nidx_row * MB + 7u) / 8u; + const bool rv = grow < M; + const uint8_t *rowp = m.ix + (size_t)(rv ? grow : 0u) * mrow; + wmma::fragment acc[NFW]; + #pragma unroll + for (uint32_t f = 0; f < NFW; f++) wmma::fill_fragment(acc[f], 0.0f); + uint32_t w0 = 0, w1 = 0; + uint4 bx[CH]; + #pragma unroll + for (uint32_t c = 0; c < CH; c++) bx[c] = make_uint4(0, 0, 0, 0); + /* this thread's CPT code words of round i start at bit (i*CPR + sub*CPT)*MB: two aligned words hold them (bit%32 + CPT*MB <= 55) */ + auto load_round = [&](uint32_t i) { + const uint32_t bitoff = (i * CPR + sub * CPT) * MB, a = (bitoff >> 5) << 2; + if (rv) { w0 = v41_glb_ld1((const uint32_t *)(rowp + a)); w1 = v41_glb_ld1((const uint32_t *)(rowp + a + 4u)); } + #pragma unroll + for (uint32_t c = 0; c < CH; c++) { + const uint32_t ch = tid + c * VQW_THREADS, bt = ch / CPR, bc = ch % CPR; + if (bt < nt) bx[c] = v41_glb_ld4(act + (uint64_t)(base + bt) * K + (uint64_t)i * VQW_BK + bc * 8u); + } + }; + load_round(0); + for (uint32_t i = 0; i < nit; i++) { + const uint32_t sh = ((i * CPR + sub * CPT) * MB) & 31u; + const uint64_t u = (((uint64_t)w1 << 32) | w0) >> sh; + #pragma unroll + for (uint32_t q = 0; q < CPT; q++) { + const uint32_t v = (uint32_t)(u >> (MB * q)) & ((1u << MB) - 1u); + *(uint4 *)(As + ar * VQW_LD + (sub * CPT + q) * 8u) = vqm_cw(cbs, v); + } + #pragma unroll + for (uint32_t c = 0; c < CH; c++) { const uint32_t ch = tid + c * VQW_THREADS; *(uint4 *)(Bs + (ch / CPR) * VQW_LD + (ch % CPR) * 8u) = bx[c]; } + __syncthreads(); + if (i + 1u < nit) load_round(i + 1u); /* the next round's bitstream / activations issue now, in flight during the mma */ + #pragma unroll + for (uint32_t kk = 0; kk < VQW_BK / 16u; kk++) { + wmma::fragment a; + wmma::load_matrix_sync(a, (const rocwmma::bfloat16_t *)(As + (wr * 16u) * VQW_LD + kk * 16u), VQW_LD); + #pragma unroll + for (uint32_t f = 0; f < NFW; f++) { + const uint32_t tok0 = (th * NFW + f) * 16u; + if (tok0 >= nt) break; /* warp-uniform */ + wmma::fragment b; /* B[k][tok] = Bs[tok][k] */ + wmma::load_matrix_sync(b, (const rocwmma::bfloat16_t *)(Bs + tok0 * VQW_LD + kk * 16u), VQW_LD); + wmma::fragment t; + wmma::fill_fragment(t, 0.0f); + wmma::mma_sync(t, a, b, t); /* zero-start per k16, FADD back (the engine's hoist) */ + #pragma unroll + for (int e = 0; e < (int)t.num_elements; e++) acc[f].x[e] += t.x[e]; + } + } + __syncthreads(); + } + /* exit: each warp parks one 16x16 tile at a time in its own scratch; lane l takes row l>>1, tokens (l&1)*8..+8 */ + float *os = Os + warp * 256u; + #pragma unroll + for (uint32_t f = 0; f < NFW; f++) { + const uint32_t tok0 = (th * NFW + f) * 16u; + if (tok0 < nt) wmma::store_matrix_sync(os, acc[f], 16, wmma::mem_row_major); + __syncthreads(); /* uniform: every warp runs every f */ + if (tok0 < nt) { + const uint32_t r = r0 + wr * 16u + (lane >> 1); + if (r < M) { + __half gh; memcpy(&gh, m.gr + (size_t)r * 2u, 2); + const float g = __half2float(gh) * (m.gov ? m.gov[r] : 1.0f); + #pragma unroll + for (uint32_t j = 0; j < 8u; j++) { + const uint32_t tl = (lane & 1u) * 8u + j, t = tok0 + tl; + if (t >= nt) break; + const uint64_t o = (uint64_t)(base + t) * M + r; + const float val = v41_bf16r(os[(lane >> 1) * 16u + tl] * g); + if (MODE == 0) g32[o] = val; + else if (MODE == 1) h16[o] = v41_vq_swiglu(g32[o], val, clamp); + else ys[o] = val; + } + } + } + __syncthreads(); + } + } +} + +static struct { v41_scratch d, doff, xs16, h16, g32; int ready, nsm; } g_vqw; + +template +static int vqw_launch(float *g32, uint16_t *h16, float *ys, const uint8_t *blob, const vqp_item *d, uint32_t nit, const uint16_t *act, + const uint32_t *doff, uint32_t M, uint32_t K, float clamp, uint32_t cbb, const float *gr, uint32_t MB, uint32_t shb, const char *what) { + static uint32_t s_optin = 0u; + if (s_optin < shb) { + (void)hipGetLastError(); + if (hipFuncSetAttribute(reinterpret_cast(vqw_kernel), hipFuncAttributeMaxDynamicSharedMemorySize, (int)shb) != hipSuccess) { + fprintf(stderr, "ds4: [ds41] vq-prefill wmma %s cannot open %u KB shared: %s\n", what, shb >> 10, hipGetErrorString(hipGetLastError())); + return 0; + } + s_optin = shb; + } + const uint32_t ntile = (M + VQW_BM - 1u) / VQW_BM, nwork = nit * ntile; + uint32_t grid = (uint32_t)g_vqw.nsm * 2u; /* 2 blocks per WGP would need 2 x 52 KB; the second queues behind the first, the grid keeps every WGP fed */ + if (grid > nwork) grid = nwork; + vqw_kernel<<>>(g32, h16, ys, blob, d, nit, act, doff, M, K, clamp, cbb, gr, MB); + return cuda_ok(hipGetLastError(), what); +} + +/* The run: same contract as vqm_run (ys = sorted down output [nvalid][OUT], the + * reduce is untouched). 0 = unavailable / failed (the caller hard-fails). */ +static int vqw_run(const uint8_t *blob, const uint32_t *cnt, const uint32_t *off_h, uint32_t n_total_expert, uint32_t nvalid, + uint32_t IN, uint32_t MID, uint32_t OUT, uint32_t nc, float clamp, const float *x, const int32_t *perm, + uint32_t n_expert, uint32_t layer_index, float *ys, const float *gr) { + uint32_t nbit = 0; while ((1u << nbit) < nc) nbit++; + if ((nbit != 10u && nbit != 12u) || IN % VQW_BK || MID % VQW_BK) { + fprintf(stderr, "ds4: [ds41] vq-prefill L%u wmma path does not take this shape (codebook %u words, IN %u MID %u)\n", layer_index, nc, IN, MID); + return 0; + } + const int cb = nbit == 10u ? 1 : 0; /* 10-bit: bf16 codebook 16 KB; 12-bit: E4M3 32 KB, convert on the fly */ + const uint32_t bn = cb ? 128u : 64u; + const uint32_t cbb = nc * (cb ? 16u : 8u), shb = cbb + (VQW_BM + bn) * VQW_LD * 2u; + if (!g_vqw.ready) { + int nsm = 0; + const bool ok = hipDeviceGetAttribute(&nsm, hipDeviceAttributeMultiprocessorCount, 0) == hipSuccess && nsm > 0; + (void)hipGetLastError(); + g_vqw.ready = ok ? 1 : -1; g_vqw.nsm = nsm; + fprintf(stderr, "ds4: [ds41] vq-prefill rocWMMA path (bf16 16x16x16, %u/%u tokens per item, %u threads, %u/%u KB shared) %s, %d SMs\n", + 128u, 64u, VQW_THREADS, (nc * 16u + (VQW_BM + 128u) * VQW_LD * 2u) >> 10, (nc * 8u + (VQW_BM + 64u) * VQW_LD * 2u) >> 10, + ok ? "ready" : "unavailable", nsm); + } + if (g_vqw.ready != 1) return 0; + uint32_t nit = 0; + for (uint32_t e = 0; e < n_total_expert; e++) nit += (cnt[e] + bn - 1u) / bn; + if (!nit) return 0; + vqp_item *ih = (vqp_item *)malloc((size_t)nit * sizeof(vqp_item)); + if (!ih) return 0; + uint32_t k = 0; + for (uint32_t e = 0; e < n_total_expert; e++) + for (uint32_t t0 = 0; t0 < cnt[e]; t0 += bn) { + ih[k].e = (int32_t)e; ih[k].t0 = (int32_t)t0; ih[k].nt = (int32_t)(cnt[e] - t0 < bn ? cnt[e] - t0 : bn); k++; + } + int ok = v41_grow(&g_vqw.d, (uint64_t)nit * sizeof(vqp_item), "vq prefill wmma items") && + v41_grow(&g_vqw.doff, ((uint64_t)n_total_expert + 1u) * sizeof(uint32_t), "vq prefill wmma off") && + v41_grow(&g_vqw.xs16, (uint64_t)nvalid * IN * sizeof(uint16_t), "vq prefill wmma xs16") && + v41_grow(&g_vqw.h16, (uint64_t)nvalid * MID * sizeof(uint16_t), "vq prefill wmma h16") && + v41_grow(&g_vqw.g32, (uint64_t)nvalid * MID * sizeof(float), "vq prefill wmma g32"); + if (ok) ok = hipMemcpyAsync(g_vqw.d.p, ih, (size_t)nit * sizeof(vqp_item), hipMemcpyHostToDevice, ds4_current_stream()) == hipSuccess && + hipMemcpyAsync(g_vqw.doff.p, off_h, (size_t)(n_total_expert + 1u) * sizeof(uint32_t), hipMemcpyHostToDevice, ds4_current_stream()) == hipSuccess; + free(ih); /* pageable source of an async H2D: staged before the call returned */ + if (!ok) { (void)hipGetLastError(); fprintf(stderr, "ds4: [ds41] vq-prefill L%u wmma scratch/copy failed\n", layer_index); return 0; } + vqm_gather16_kernel<<>>((uint16_t *)g_vqw.xs16.p, x, perm, n_expert, IN); + if (!cuda_ok(hipGetLastError(), "vq prefill wmma gather16")) return 0; + float *g32 = (float *)g_vqw.g32.p; uint16_t *h16 = (uint16_t *)g_vqw.h16.p; + const vqp_item *d = (const vqp_item *)g_vqw.d.p; const uint32_t *doff = (const uint32_t *)g_vqw.doff.p; + const uint16_t *xs = (const uint16_t *)g_vqw.xs16.p; +#define VQW_GO(CB_, BN_) do { \ + if (!vqw_launch<0, CB_, BN_>(g32, NULL, NULL, blob, d, nit, xs, doff, MID, IN, clamp, cbb, NULL, nbit, shb, "vq prefill wmma gate")) return 0; \ + if (!vqw_launch<1, CB_, BN_>(g32, h16, NULL, blob, d, nit, xs, doff, MID, IN, clamp, cbb, NULL, nbit, shb, "vq prefill wmma up")) return 0; \ + return vqw_launch<2, CB_, BN_>(NULL, NULL, ys, blob, d, nit, h16, doff, OUT, MID, clamp, cbb, gr, nbit, shb, "vq prefill wmma down"); \ + } while (0) + if (cb) VQW_GO(1, 128u); + VQW_GO(0, 64u); +#undef VQW_GO +} +#endif