engine/crates/core/src/serving.rs

//! Types shared by the HTTP server (`dllm-server`) and the request runtime (`dllm-runtime`).

/// One validated question, ready for the engine.
#[derive(Debug, Clone, Default)]
pub struct Item {
    pub key: String,
    pub noul: bool,
    /// free-form number question: the answer is generated greedily after the end-of-thought token (no options, `keys` / `label_ids` empty)
    pub number: bool,
    pub keys: Vec<String>,
    /// ids ending after `<|channel>thought\n` (the engine appends `<channel|>` for the no-think readout).
    /// `<|image|>` appears ONCE per image (unexpanded); the runtime expands each to `<|image>` + n_soft x `<|image|>` + `<image|>`
    /// once the image is preprocessed and n_soft is known.
    pub ids: Vec<u32>,
    /// token id of each option's label
    pub label_ids: Vec<u32>,
}

/// Engine result per question. `p0` = softcapped letter LOGITS of the no-think readout,
/// `think` = letter logits of the post-thought readout if the question thought.
#[derive(Debug, Clone, Default)]
pub struct ItemResult {
    pub p0: Vec<f32>,
    pub think: Option<Vec<f32>>,
    pub trace_tokens: usize,
    pub out_tokens: usize,
    /// generated thought token ids (only filled when `Policy::return_trace`; `trace_tokens` of them)
    pub trace_ids: Vec<u32>,
    /// number questions: the greedy answer tokens (without the closing end-of-turn) and the sum of their log-probabilities
    pub answer_ids: Vec<u32>,
    pub answer_logprob: f64,
}

thread_local! {
    static ARRIVAL: std::cell::Cell<Option<std::time::Instant>> = const { std::cell::Cell::new(None) };
}
/// the HTTP layer records the request arrival on the thread that calls `Backend::run` (budget accounting counts queueing from arrival)
pub fn set_arrival(t: std::time::Instant) {
    ARRIVAL.with(|a| a.set(Some(t)));
}
pub fn take_arrival() -> Option<std::time::Instant> {
    ARRIVAL.with(|a| a.take())
}