ctopd / kdflow /backend /sglang /monkey_patch.py
codemaivanngu's picture
chore(upstream): import Cross-Tokenizer-OPD source snapshot
0ca529a unverified
Raw History Blame Contribute Delete
10.7 kB
"""
Monkey patch for SGLang scheduler's process_batch_result_prefill method.
This allows using numpy() instead of tolist() for hidden_states, which is much faster.
"""
from __future__ import annotations
import logging
import os
import time
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
import torch
from kdflow.utils.logging_utils import init_logger
if TYPE_CHECKING:
from sglang.srt.managers.scheduler import (
EmbeddingBatchResult,
GenerationBatchResult,
ScheduleBatch,
Scheduler,
)
logger = init_logger(__name__)
# Flag to prevent multiple patch applications
_PATCH_APPLIED = False
def process_batch_result_prefill_patched(
self: "Scheduler",
batch: "ScheduleBatch",
result: Union["GenerationBatchResult", "EmbeddingBatchResult"],
):
"""
Patched version of process_batch_result_prefill.
Key change: Use .numpy() instead of .tolist() for hidden_states (much faster).
"""
from sglang.srt.environ import envs
from sglang.srt.managers.io_struct import AbortReq
from sglang.srt.mem_cache.common import release_kv_cache
skip_stream_req = None
if self.is_generation:
if result.copy_done is not None:
result.copy_done.synchronize()
(
logits_output,
next_token_ids,
extend_input_len_per_req,
extend_logprob_start_len_per_req,
) = (
result.logits_output,
result.next_token_ids,
result.extend_input_len_per_req,
result.extend_logprob_start_len_per_req,
)
# Move next_token_ids and logprobs to cpu
next_token_ids = next_token_ids.tolist()
if batch.return_logprob:
if logits_output.next_token_logprobs is not None:
logits_output.next_token_logprobs = (
logits_output.next_token_logprobs.tolist()
)
if logits_output.input_token_logprobs is not None:
logits_output.input_token_logprobs = tuple(
logits_output.input_token_logprobs.tolist()
)
hidden_state_offset = 0
# Check finish conditions
logprob_pt = 0
for i, (req, next_token_id) in enumerate(zip(batch.reqs, next_token_ids)):
if req.finished() or req.is_retracted:
# decode req in mixed batch or retracted req
continue
if req.is_chunked <= 0:
if hasattr(req.time_stats, "prefill_finished_ts"):
if req.time_stats.prefill_finished_ts == 0.0:
req.time_stats.prefill_finished_ts = time.time()
else:
req.time_stats.set_prefill_finished_time()
# req output_ids are set here
req.output_ids.append(next_token_id)
req.check_finished()
if req.finished():
self.maybe_collect_routed_experts(req)
release_kv_cache(req, self.tree_cache)
req.time_stats.completion_time = time.perf_counter()
elif not batch.decoding_reqs or req not in batch.decoding_reqs:
# This updates radix so others can match
self.tree_cache.cache_unfinished_req(req)
self.maybe_collect_customized_info(i, req, logits_output)
if batch.return_logprob:
assert extend_logprob_start_len_per_req is not None
assert extend_input_len_per_req is not None
extend_logprob_start_len = extend_logprob_start_len_per_req[i]
extend_input_len = extend_input_len_per_req[i]
num_input_logprobs = self._calculate_num_input_logprobs(
req, extend_input_len, extend_logprob_start_len
)
if req.return_logprob:
self.add_logprob_return_values(
i,
req,
logprob_pt,
next_token_ids,
num_input_logprobs,
logits_output,
)
logprob_pt += num_input_logprobs
if (
req.return_hidden_states
and logits_output.hidden_states is not None
):
req.hidden_states.append(
logits_output.hidden_states[
hidden_state_offset : (
hidden_state_offset := hidden_state_offset
+ len(req.origin_input_ids)
)
]
.half()
.cpu()
.numpy()
)
if req.grammar is not None:
# FIXME: this try-except block is for handling unexpected xgrammar issue.
try:
req.grammar.accept_token(next_token_id)
except ValueError as e:
# Grammar accept_token can raise ValueError if the token is not in the grammar.
# This can happen if the grammar is not set correctly or the token is invalid.
logger.error(
f"Grammar accept_token failed for req {req.rid} with token {next_token_id}: {e}"
)
self.abort_request(AbortReq(rid=req.rid))
req.grammar.finished = req.finished()
else:
# being chunked reqs' prefill is not finished
req.is_chunked -= 1
# There is only at most one request being currently chunked.
# Because this request does not finish prefill,
# we don't want to stream the request currently being chunked.
skip_stream_req = req
# Incrementally update input logprobs.
if batch.return_logprob:
extend_logprob_start_len = extend_logprob_start_len_per_req[i]
extend_input_len = extend_input_len_per_req[i]
if extend_logprob_start_len < extend_input_len:
# Update input logprobs.
num_input_logprobs = self._calculate_num_input_logprobs(
req, extend_input_len, extend_logprob_start_len
)
if req.return_logprob:
self.add_input_logprob_return_values(
i,
req,
logits_output,
logprob_pt,
num_input_logprobs,
last_prefill_chunk=False,
)
logprob_pt += num_input_logprobs
else: # embedding or reward model
if result.copy_done is not None:
result.copy_done.synchronize()
is_sparse = envs.SGLANG_EMBEDDINGS_SPARSE_HEAD.is_set()
embeddings = result.embeddings
if is_sparse:
batch_ids, token_ids = embeddings.indices()
values = embeddings.values()
embeddings = [{} for _ in range(embeddings.size(0))]
for i in range(batch_ids.shape[0]):
embeddings[batch_ids[i].item()][token_ids[i].item()] = values[
i
].item()
else:
if isinstance(embeddings, torch.Tensor):
embeddings = embeddings.tolist()
else:
embeddings = [tensor.tolist() for tensor in embeddings]
# Check finish conditions
for i, req in enumerate(batch.reqs):
if req.is_retracted:
continue
req.embedding = embeddings[i]
if req.is_chunked <= 0:
# Dummy output token for embedding models
req.output_ids.append(0)
req.check_finished()
if req.finished():
release_kv_cache(req, self.tree_cache)
else:
self.tree_cache.cache_unfinished_req(req)
else:
# being chunked reqs' prefill is not finished
req.is_chunked -= 1
self.stream_output(batch.reqs, batch.return_logprob, skip_stream_req)
if self.current_scheduler_metrics_enabled:
can_run_cuda_graph = getattr(result, "can_run_cuda_graph", False)
self.log_prefill_stats(
prefill_stats=batch.prefill_stats,
can_run_cuda_graph=can_run_cuda_graph,
dp_cooperation_info=batch.dp_cooperation_info,
)
def apply_patch():
"""
Apply the monkey patch to SGLang's SchedulerOutputProcessorMixin.
This function is idempotent - calling it multiple times is safe.
Returns True if patch was applied (or already applied), False otherwise.
"""
global _PATCH_APPLIED
if _PATCH_APPLIED:
return True
try:
from sglang.srt.managers.scheduler_output_processor_mixin import (
SchedulerOutputProcessorMixin,
)
# Check if already patched (by another mechanism like sitecustomize)
current_method = getattr(SchedulerOutputProcessorMixin, 'process_batch_result_prefill', None)
if current_method is not None and getattr(current_method, '_kdflow_patched', False):
_PATCH_APPLIED = True
print(f"[monkey_patch] Patch already applied, PID={os.getpid()}", flush=True)
return True
# Mark the patched function
process_batch_result_prefill_patched._kdflow_patched = True
# Apply patch
SchedulerOutputProcessorMixin.process_batch_result_prefill = process_batch_result_prefill_patched
_PATCH_APPLIED = True
print(f"[monkey_patch] SUCCESS: process_batch_result_prefill patched! PID={os.getpid()}", flush=True)
return True
except ImportError as e:
print(f"[monkey_patch] Cannot import SGLang module: {e}", flush=True)
return False
except Exception as e:
print(f"[monkey_patch] Error applying patch: {e}", flush=True)
import traceback
traceback.print_exc()
return False
def is_patch_applied():
"""Check if the patch has been applied in this process."""
return _PATCH_APPLIED