diff --git a/python/sglang/srt/entrypoints/grpc_server.py b/python/sglang/srt/entrypoints/grpc_server.py index f7edf7743..5e91210f4 100644 --- a/python/sglang/srt/entrypoints/grpc_server.py +++ b/python/sglang/srt/entrypoints/grpc_server.py @@ -438,6 +438,7 @@ class SGLangSchedulerServicer(sglang_scheduler_pb2_grpc.SglangSchedulerServicer) regex = None json_schema = None ebnf_grammar = None + structural_tag = None if grpc_params.HasField("regex"): regex = grpc_params.regex @@ -445,6 +446,8 @@ class SGLangSchedulerServicer(sglang_scheduler_pb2_grpc.SglangSchedulerServicer) json_schema = grpc_params.json_schema elif grpc_params.HasField("ebnf_grammar"): ebnf_grammar = grpc_params.ebnf_grammar + elif grpc_params.HasField("structural_tag"): + structural_tag = grpc_params.structural_tag return SGLSamplingParams( temperature=grpc_params.temperature or 1.0, @@ -465,6 +468,7 @@ class SGLangSchedulerServicer(sglang_scheduler_pb2_grpc.SglangSchedulerServicer) regex=regex, json_schema=json_schema, ebnf=ebnf_grammar, + structural_tag=structural_tag, n=grpc_params.n or 1, ignore_eos=grpc_params.ignore_eos, ) diff --git a/python/sglang/srt/grpc/sglang_scheduler.proto b/python/sglang/srt/grpc/sglang_scheduler.proto index e4c87925e..affa131db 100644 --- a/python/sglang/srt/grpc/sglang_scheduler.proto +++ b/python/sglang/srt/grpc/sglang_scheduler.proto @@ -47,24 +47,24 @@ message SamplingParams { string regex = 13; string json_schema = 14; string ebnf_grammar = 15; + string structural_tag = 16; } // LoRA adapter - string lora_path = 16; + string lora_path = 17; // Speculative decoding - int32 n = 17; // Number of samples + int32 n = 18; // Number of samples // Token healing - bool token_healing = 18; + bool token_healing = 19; // Additional parameters - int32 min_new_tokens = 19; - bool ignore_eos = 20; - bool no_stop_trim = 21; - int32 stream_interval = 22; - map logit_bias = 23; - string structural_tag = 24; + int32 min_new_tokens = 20; + bool ignore_eos = 21; + bool no_stop_trim = 22; + int32 stream_interval = 23; + map logit_bias = 24; // Custom parameters for extensibility google.protobuf.Struct custom_params = 25; diff --git a/python/sglang/srt/grpc/sglang_scheduler_pb2.py b/python/sglang/srt/grpc/sglang_scheduler_pb2.py index 4b288d768..2f614132a 100644 --- a/python/sglang/srt/grpc/sglang_scheduler_pb2.py +++ b/python/sglang/srt/grpc/sglang_scheduler_pb2.py @@ -1,3 +1,6 @@ +# This file is auto-generated. Do not edit manually. +# Regenerate with: python compile_proto.py + # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE @@ -26,7 +29,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16sglang_scheduler.proto\x12\x15sglang.grpc.scheduler\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xc7\x05\n\x0eSamplingParams\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_p\x18\x02 \x01(\x02\x12\r\n\x05top_k\x18\x03 \x01(\x05\x12\r\n\x05min_p\x18\x04 \x01(\x02\x12\x19\n\x11\x66requency_penalty\x18\x05 \x01(\x02\x12\x18\n\x10presence_penalty\x18\x06 \x01(\x02\x12\x1a\n\x12repetition_penalty\x18\x07 \x01(\x02\x12\x16\n\x0emax_new_tokens\x18\x08 \x01(\x05\x12\x0c\n\x04stop\x18\t \x03(\t\x12\x16\n\x0estop_token_ids\x18\n \x03(\x05\x12\x1b\n\x13skip_special_tokens\x18\x0b \x01(\x08\x12%\n\x1dspaces_between_special_tokens\x18\x0c \x01(\x08\x12\x0f\n\x05regex\x18\r \x01(\tH\x00\x12\x15\n\x0bjson_schema\x18\x0e \x01(\tH\x00\x12\x16\n\x0c\x65\x62nf_grammar\x18\x0f \x01(\tH\x00\x12\x11\n\tlora_path\x18\x10 \x01(\t\x12\t\n\x01n\x18\x11 \x01(\x05\x12\x15\n\rtoken_healing\x18\x12 \x01(\x08\x12\x16\n\x0emin_new_tokens\x18\x13 \x01(\x05\x12\x12\n\nignore_eos\x18\x14 \x01(\x08\x12\x14\n\x0cno_stop_trim\x18\x15 \x01(\x08\x12\x17\n\x0fstream_interval\x18\x16 \x01(\x05\x12H\n\nlogit_bias\x18\x17 \x03(\x0b\x32\x34.sglang.grpc.scheduler.SamplingParams.LogitBiasEntry\x12\x16\n\x0estructural_tag\x18\x18 \x01(\t\x12.\n\rcustom_params\x18\x19 \x01(\x0b\x32\x17.google.protobuf.Struct\x1a\x30\n\x0eLogitBiasEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\nconstraint\"]\n\x13\x44isaggregatedParams\x12\x16\n\x0e\x62ootstrap_host\x18\x01 \x01(\t\x12\x16\n\x0e\x62ootstrap_port\x18\x02 \x01(\x05\x12\x16\n\x0e\x62ootstrap_room\x18\x03 \x01(\x05\"\xe9\x04\n\x0fGenerateRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x04 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x16\n\x0ereturn_logprob\x18\x05 \x01(\x08\x12\x19\n\x11logprob_start_len\x18\x06 \x01(\x05\x12\x18\n\x10top_logprobs_num\x18\x07 \x01(\x05\x12\x19\n\x11token_ids_logprob\x18\x08 \x03(\x05\x12\x1c\n\x14return_hidden_states\x18\t \x01(\x08\x12H\n\x14\x64isaggregated_params\x18\n \x01(\x0b\x32*.sglang.grpc.scheduler.DisaggregatedParams\x12\x1e\n\x16\x63ustom_logit_processor\x18\x0b \x01(\t\x12-\n\ttimestamp\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x13\n\x0blog_metrics\x18\r \x01(\x08\x12\x14\n\x0cinput_embeds\x18\x0e \x03(\x02\x12\x0f\n\x07lora_id\x18\x0f \x01(\t\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x10 \x01(\x05\x12\x15\n\rdp_balance_id\x18\x11 \x01(\x05\":\n\x0eTokenizedInput\x12\x15\n\roriginal_text\x18\x01 \x01(\t\x12\x11\n\tinput_ids\x18\x02 \x03(\x05\"\xd3\x01\n\x10MultimodalInputs\x12\x12\n\nimage_urls\x18\x01 \x03(\t\x12\x12\n\nvideo_urls\x18\x02 \x03(\t\x12\x12\n\naudio_urls\x18\x03 \x03(\t\x12\x33\n\x12processed_features\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x12\n\nimage_data\x18\x05 \x03(\x0c\x12\x12\n\nvideo_data\x18\x06 \x03(\x0c\x12\x12\n\naudio_data\x18\x07 \x03(\x0c\x12\x12\n\nmodalities\x18\x08 \x03(\t\"\xe3\x01\n\x10GenerateResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12;\n\x05\x63hunk\x18\x02 \x01(\x0b\x32*.sglang.grpc.scheduler.GenerateStreamChunkH\x00\x12;\n\x08\x63omplete\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.GenerateCompleteH\x00\x12\x35\n\x05\x65rror\x18\x04 \x01(\x0b\x32$.sglang.grpc.scheduler.GenerateErrorH\x00\x42\n\n\x08response\"\xf5\x01\n\x13GenerateStreamChunk\x12\x10\n\x08token_id\x18\x01 \x01(\x05\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x15\n\rprompt_tokens\x18\x03 \x01(\x05\x12\x19\n\x11\x63ompletion_tokens\x18\x04 \x01(\x05\x12\x15\n\rcached_tokens\x18\x05 \x01(\x05\x12\x31\n\x08logprobs\x18\x06 \x01(\x0b\x32\x1f.sglang.grpc.scheduler.LogProbs\x12\x15\n\rhidden_states\x18\x07 \x03(\x02\x12\x17\n\x0fgeneration_time\x18\x08 \x01(\x02\x12\x12\n\nqueue_time\x18\t \x01(\x05\"\xcd\x02\n\x10GenerateComplete\x12\x12\n\noutput_ids\x18\x01 \x03(\x05\x12\x13\n\x0boutput_text\x18\x02 \x01(\t\x12K\n\rfinish_reason\x18\x03 \x01(\x0e\x32\x34.sglang.grpc.scheduler.GenerateComplete.FinishReason\x12\x35\n\x0c\x61ll_logprobs\x18\x0b \x03(\x0b\x32\x1f.sglang.grpc.scheduler.LogProbs\x12>\n\x11\x61ll_hidden_states\x18\x0c \x03(\x0b\x32#.sglang.grpc.scheduler.HiddenStates\"L\n\x0c\x46inishReason\x12\x08\n\x04STOP\x10\x00\x12\n\n\x06LENGTH\x10\x01\x12\r\n\tEOS_TOKEN\x10\x02\x12\x0c\n\x08STOP_STR\x10\x03\x12\t\n\x05\x41\x42ORT\x10\x04\"K\n\rGenerateError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x18\n\x10http_status_code\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"\x84\x01\n\x08LogProbs\x12\x16\n\x0etoken_logprobs\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x38\n\x0ctop_logprobs\x18\x03 \x03(\x0b\x32\".sglang.grpc.scheduler.TopLogProbs\x12\x13\n\x0btoken_texts\x18\x04 \x03(\t\"E\n\x0bTopLogProbs\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x13\n\x0btoken_texts\x18\x03 \x03(\t\"?\n\x0cHiddenStates\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05layer\x18\x02 \x01(\x05\x12\x10\n\x08position\x18\x03 \x01(\x05\"\xca\x02\n\x0c\x45mbedRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x04 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x05 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x13\n\x0blog_metrics\x18\x06 \x01(\x08\x12\x16\n\x0etoken_type_ids\x18\x07 \x03(\x05\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x08 \x01(\x05\x12\x18\n\x10is_cross_encoder\x18\t \x01(\x08\x12\r\n\x05texts\x18\n \x03(\t\"\x9d\x01\n\rEmbedResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\x08\x63omplete\x18\x02 \x01(\x0b\x32$.sglang.grpc.scheduler.EmbedCompleteH\x00\x12\x32\n\x05\x65rror\x18\x03 \x01(\x0b\x32!.sglang.grpc.scheduler.EmbedErrorH\x00\x42\n\n\x08response\"\xbc\x01\n\rEmbedComplete\x12\x11\n\tembedding\x18\x01 \x03(\x02\x12\x15\n\rprompt_tokens\x18\x02 \x01(\x05\x12\x15\n\rcached_tokens\x18\x03 \x01(\x05\x12\x15\n\rembedding_dim\x18\x04 \x01(\x05\x12\x17\n\x0fgeneration_time\x18\x05 \x01(\x02\x12:\n\x10\x62\x61tch_embeddings\x18\x06 \x03(\x0b\x32 .sglang.grpc.scheduler.Embedding\"*\n\tEmbedding\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05index\x18\x02 \x01(\x05\"<\n\nEmbedError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"N\n\x12HealthCheckRequest\x12\x38\n\ttokenized\x18\x01 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\"7\n\x13HealthCheckResponse\x12\x0f\n\x07healthy\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"2\n\x0c\x41\x62ortRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\"1\n\rAbortResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"I\n\x0fLoadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\x12\x14\n\x0c\x61\x64\x61pter_path\x18\x02 \x01(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\"H\n\x10LoadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x12\n\nadapter_id\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"\'\n\x11UnloadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\"6\n\x12UnloadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"w\n\x14UpdateWeightsRequest\x12\x13\n\tdisk_path\x18\x01 \x01(\tH\x00\x12\x15\n\x0btensor_data\x18\x02 \x01(\x0cH\x00\x12\x14\n\nremote_url\x18\x03 \x01(\tH\x00\x12\x13\n\x0bweight_name\x18\x04 \x01(\tB\x08\n\x06source\"9\n\x15UpdateWeightsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"-\n\x17GetInternalStateRequest\x12\x12\n\nstate_keys\x18\x01 \x03(\t\"B\n\x18GetInternalStateResponse\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"A\n\x17SetInternalStateRequest\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"<\n\x18SetInternalStateResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2\xfe\x02\n\x0fSglangScheduler\x12]\n\x08Generate\x12&.sglang.grpc.scheduler.GenerateRequest\x1a\'.sglang.grpc.scheduler.GenerateResponse0\x01\x12R\n\x05\x45mbed\x12#.sglang.grpc.scheduler.EmbedRequest\x1a$.sglang.grpc.scheduler.EmbedResponse\x12\x64\n\x0bHealthCheck\x12).sglang.grpc.scheduler.HealthCheckRequest\x1a*.sglang.grpc.scheduler.HealthCheckResponse\x12R\n\x05\x41\x62ort\x12#.sglang.grpc.scheduler.AbortRequest\x1a$.sglang.grpc.scheduler.AbortResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16sglang_scheduler.proto\x12\x15sglang.grpc.scheduler\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xc9\x05\n\x0eSamplingParams\x12\x13\n\x0btemperature\x18\x01 \x01(\x02\x12\r\n\x05top_p\x18\x02 \x01(\x02\x12\r\n\x05top_k\x18\x03 \x01(\x05\x12\r\n\x05min_p\x18\x04 \x01(\x02\x12\x19\n\x11\x66requency_penalty\x18\x05 \x01(\x02\x12\x18\n\x10presence_penalty\x18\x06 \x01(\x02\x12\x1a\n\x12repetition_penalty\x18\x07 \x01(\x02\x12\x16\n\x0emax_new_tokens\x18\x08 \x01(\x05\x12\x0c\n\x04stop\x18\t \x03(\t\x12\x16\n\x0estop_token_ids\x18\n \x03(\x05\x12\x1b\n\x13skip_special_tokens\x18\x0b \x01(\x08\x12%\n\x1dspaces_between_special_tokens\x18\x0c \x01(\x08\x12\x0f\n\x05regex\x18\r \x01(\tH\x00\x12\x15\n\x0bjson_schema\x18\x0e \x01(\tH\x00\x12\x16\n\x0c\x65\x62nf_grammar\x18\x0f \x01(\tH\x00\x12\x18\n\x0estructural_tag\x18\x10 \x01(\tH\x00\x12\x11\n\tlora_path\x18\x11 \x01(\t\x12\t\n\x01n\x18\x12 \x01(\x05\x12\x15\n\rtoken_healing\x18\x13 \x01(\x08\x12\x16\n\x0emin_new_tokens\x18\x14 \x01(\x05\x12\x12\n\nignore_eos\x18\x15 \x01(\x08\x12\x14\n\x0cno_stop_trim\x18\x16 \x01(\x08\x12\x17\n\x0fstream_interval\x18\x17 \x01(\x05\x12H\n\nlogit_bias\x18\x18 \x03(\x0b\x32\x34.sglang.grpc.scheduler.SamplingParams.LogitBiasEntry\x12.\n\rcustom_params\x18\x19 \x01(\x0b\x32\x17.google.protobuf.Struct\x1a\x30\n\x0eLogitBiasEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\nconstraint\"]\n\x13\x44isaggregatedParams\x12\x16\n\x0e\x62ootstrap_host\x18\x01 \x01(\t\x12\x16\n\x0e\x62ootstrap_port\x18\x02 \x01(\x05\x12\x16\n\x0e\x62ootstrap_room\x18\x03 \x01(\x05\"\xe9\x04\n\x0fGenerateRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x04 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x16\n\x0ereturn_logprob\x18\x05 \x01(\x08\x12\x19\n\x11logprob_start_len\x18\x06 \x01(\x05\x12\x18\n\x10top_logprobs_num\x18\x07 \x01(\x05\x12\x19\n\x11token_ids_logprob\x18\x08 \x03(\x05\x12\x1c\n\x14return_hidden_states\x18\t \x01(\x08\x12H\n\x14\x64isaggregated_params\x18\n \x01(\x0b\x32*.sglang.grpc.scheduler.DisaggregatedParams\x12\x1e\n\x16\x63ustom_logit_processor\x18\x0b \x01(\t\x12-\n\ttimestamp\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x13\n\x0blog_metrics\x18\r \x01(\x08\x12\x14\n\x0cinput_embeds\x18\x0e \x03(\x02\x12\x0f\n\x07lora_id\x18\x0f \x01(\t\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x10 \x01(\x05\x12\x15\n\rdp_balance_id\x18\x11 \x01(\x05\":\n\x0eTokenizedInput\x12\x15\n\roriginal_text\x18\x01 \x01(\t\x12\x11\n\tinput_ids\x18\x02 \x03(\x05\"\xd3\x01\n\x10MultimodalInputs\x12\x12\n\nimage_urls\x18\x01 \x03(\t\x12\x12\n\nvideo_urls\x18\x02 \x03(\t\x12\x12\n\naudio_urls\x18\x03 \x03(\t\x12\x33\n\x12processed_features\x18\x04 \x01(\x0b\x32\x17.google.protobuf.Struct\x12\x12\n\nimage_data\x18\x05 \x03(\x0c\x12\x12\n\nvideo_data\x18\x06 \x03(\x0c\x12\x12\n\naudio_data\x18\x07 \x03(\x0c\x12\x12\n\nmodalities\x18\x08 \x03(\t\"\xe3\x01\n\x10GenerateResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12;\n\x05\x63hunk\x18\x02 \x01(\x0b\x32*.sglang.grpc.scheduler.GenerateStreamChunkH\x00\x12;\n\x08\x63omplete\x18\x03 \x01(\x0b\x32\'.sglang.grpc.scheduler.GenerateCompleteH\x00\x12\x35\n\x05\x65rror\x18\x04 \x01(\x0b\x32$.sglang.grpc.scheduler.GenerateErrorH\x00\x42\n\n\x08response\"\xf5\x01\n\x13GenerateStreamChunk\x12\x10\n\x08token_id\x18\x01 \x01(\x05\x12\x0c\n\x04text\x18\x02 \x01(\t\x12\x15\n\rprompt_tokens\x18\x03 \x01(\x05\x12\x19\n\x11\x63ompletion_tokens\x18\x04 \x01(\x05\x12\x15\n\rcached_tokens\x18\x05 \x01(\x05\x12\x31\n\x08logprobs\x18\x06 \x01(\x0b\x32\x1f.sglang.grpc.scheduler.LogProbs\x12\x15\n\rhidden_states\x18\x07 \x03(\x02\x12\x17\n\x0fgeneration_time\x18\x08 \x01(\x02\x12\x12\n\nqueue_time\x18\t \x01(\x05\"\xcd\x02\n\x10GenerateComplete\x12\x12\n\noutput_ids\x18\x01 \x03(\x05\x12\x13\n\x0boutput_text\x18\x02 \x01(\t\x12K\n\rfinish_reason\x18\x03 \x01(\x0e\x32\x34.sglang.grpc.scheduler.GenerateComplete.FinishReason\x12\x35\n\x0c\x61ll_logprobs\x18\x0b \x03(\x0b\x32\x1f.sglang.grpc.scheduler.LogProbs\x12>\n\x11\x61ll_hidden_states\x18\x0c \x03(\x0b\x32#.sglang.grpc.scheduler.HiddenStates\"L\n\x0c\x46inishReason\x12\x08\n\x04STOP\x10\x00\x12\n\n\x06LENGTH\x10\x01\x12\r\n\tEOS_TOKEN\x10\x02\x12\x0c\n\x08STOP_STR\x10\x03\x12\t\n\x05\x41\x42ORT\x10\x04\"K\n\rGenerateError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x18\n\x10http_status_code\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"\x84\x01\n\x08LogProbs\x12\x16\n\x0etoken_logprobs\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x38\n\x0ctop_logprobs\x18\x03 \x03(\x0b\x32\".sglang.grpc.scheduler.TopLogProbs\x12\x13\n\x0btoken_texts\x18\x04 \x03(\t\"E\n\x0bTopLogProbs\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\x11\n\ttoken_ids\x18\x02 \x03(\x05\x12\x13\n\x0btoken_texts\x18\x03 \x03(\t\"?\n\x0cHiddenStates\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05layer\x18\x02 \x01(\x05\x12\x10\n\x08position\x18\x03 \x01(\x05\"\xca\x02\n\x0c\x45mbedRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\ttokenized\x18\x02 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\x12:\n\tmm_inputs\x18\x04 \x01(\x0b\x32\'.sglang.grpc.scheduler.MultimodalInputs\x12>\n\x0fsampling_params\x18\x05 \x01(\x0b\x32%.sglang.grpc.scheduler.SamplingParams\x12\x13\n\x0blog_metrics\x18\x06 \x01(\x08\x12\x16\n\x0etoken_type_ids\x18\x07 \x03(\x05\x12\x1a\n\x12\x64\x61ta_parallel_rank\x18\x08 \x01(\x05\x12\x18\n\x10is_cross_encoder\x18\t \x01(\x08\x12\r\n\x05texts\x18\n \x03(\t\"\x9d\x01\n\rEmbedResponse\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x38\n\x08\x63omplete\x18\x02 \x01(\x0b\x32$.sglang.grpc.scheduler.EmbedCompleteH\x00\x12\x32\n\x05\x65rror\x18\x03 \x01(\x0b\x32!.sglang.grpc.scheduler.EmbedErrorH\x00\x42\n\n\x08response\"\xbc\x01\n\rEmbedComplete\x12\x11\n\tembedding\x18\x01 \x03(\x02\x12\x15\n\rprompt_tokens\x18\x02 \x01(\x05\x12\x15\n\rcached_tokens\x18\x03 \x01(\x05\x12\x15\n\rembedding_dim\x18\x04 \x01(\x05\x12\x17\n\x0fgeneration_time\x18\x05 \x01(\x02\x12:\n\x10\x62\x61tch_embeddings\x18\x06 \x03(\x0b\x32 .sglang.grpc.scheduler.Embedding\"*\n\tEmbedding\x12\x0e\n\x06values\x18\x01 \x03(\x02\x12\r\n\x05index\x18\x02 \x01(\x05\"<\n\nEmbedError\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x0c\n\x04\x63ode\x18\x02 \x01(\t\x12\x0f\n\x07\x64\x65tails\x18\x03 \x01(\t\"N\n\x12HealthCheckRequest\x12\x38\n\ttokenized\x18\x01 \x01(\x0b\x32%.sglang.grpc.scheduler.TokenizedInput\"7\n\x13HealthCheckResponse\x12\x0f\n\x07healthy\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"2\n\x0c\x41\x62ortRequest\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\"1\n\rAbortResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"I\n\x0fLoadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\x12\x14\n\x0c\x61\x64\x61pter_path\x18\x02 \x01(\t\x12\x0c\n\x04rank\x18\x03 \x01(\x05\"H\n\x10LoadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x12\n\nadapter_id\x18\x02 \x01(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"\'\n\x11UnloadLoRARequest\x12\x12\n\nadapter_id\x18\x01 \x01(\t\"6\n\x12UnloadLoRAResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"w\n\x14UpdateWeightsRequest\x12\x13\n\tdisk_path\x18\x01 \x01(\tH\x00\x12\x15\n\x0btensor_data\x18\x02 \x01(\x0cH\x00\x12\x14\n\nremote_url\x18\x03 \x01(\tH\x00\x12\x13\n\x0bweight_name\x18\x04 \x01(\tB\x08\n\x06source\"9\n\x15UpdateWeightsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"-\n\x17GetInternalStateRequest\x12\x12\n\nstate_keys\x18\x01 \x03(\t\"B\n\x18GetInternalStateResponse\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"A\n\x17SetInternalStateRequest\x12&\n\x05state\x18\x01 \x01(\x0b\x32\x17.google.protobuf.Struct\"<\n\x18SetInternalStateResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t2\xfe\x02\n\x0fSglangScheduler\x12]\n\x08Generate\x12&.sglang.grpc.scheduler.GenerateRequest\x1a\'.sglang.grpc.scheduler.GenerateResponse0\x01\x12R\n\x05\x45mbed\x12#.sglang.grpc.scheduler.EmbedRequest\x1a$.sglang.grpc.scheduler.EmbedResponse\x12\x64\n\x0bHealthCheck\x12).sglang.grpc.scheduler.HealthCheckRequest\x1a*.sglang.grpc.scheduler.HealthCheckResponse\x12R\n\x05\x41\x62ort\x12#.sglang.grpc.scheduler.AbortRequest\x1a$.sglang.grpc.scheduler.AbortResponseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -36,71 +39,71 @@ if not _descriptor._USE_C_DESCRIPTORS: _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._loaded_options = None _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_options = b'8\001' _globals['_SAMPLINGPARAMS']._serialized_start=113 - _globals['_SAMPLINGPARAMS']._serialized_end=824 - _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_start=762 - _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_end=810 - _globals['_DISAGGREGATEDPARAMS']._serialized_start=826 - _globals['_DISAGGREGATEDPARAMS']._serialized_end=919 - _globals['_GENERATEREQUEST']._serialized_start=922 - _globals['_GENERATEREQUEST']._serialized_end=1539 - _globals['_TOKENIZEDINPUT']._serialized_start=1541 - _globals['_TOKENIZEDINPUT']._serialized_end=1599 - _globals['_MULTIMODALINPUTS']._serialized_start=1602 - _globals['_MULTIMODALINPUTS']._serialized_end=1813 - _globals['_GENERATERESPONSE']._serialized_start=1816 - _globals['_GENERATERESPONSE']._serialized_end=2043 - _globals['_GENERATESTREAMCHUNK']._serialized_start=2046 - _globals['_GENERATESTREAMCHUNK']._serialized_end=2291 - _globals['_GENERATECOMPLETE']._serialized_start=2294 - _globals['_GENERATECOMPLETE']._serialized_end=2627 - _globals['_GENERATECOMPLETE_FINISHREASON']._serialized_start=2551 - _globals['_GENERATECOMPLETE_FINISHREASON']._serialized_end=2627 - _globals['_GENERATEERROR']._serialized_start=2629 - _globals['_GENERATEERROR']._serialized_end=2704 - _globals['_LOGPROBS']._serialized_start=2707 - _globals['_LOGPROBS']._serialized_end=2839 - _globals['_TOPLOGPROBS']._serialized_start=2841 - _globals['_TOPLOGPROBS']._serialized_end=2910 - _globals['_HIDDENSTATES']._serialized_start=2912 - _globals['_HIDDENSTATES']._serialized_end=2975 - _globals['_EMBEDREQUEST']._serialized_start=2978 - _globals['_EMBEDREQUEST']._serialized_end=3308 - _globals['_EMBEDRESPONSE']._serialized_start=3311 - _globals['_EMBEDRESPONSE']._serialized_end=3468 - _globals['_EMBEDCOMPLETE']._serialized_start=3471 - _globals['_EMBEDCOMPLETE']._serialized_end=3659 - _globals['_EMBEDDING']._serialized_start=3661 - _globals['_EMBEDDING']._serialized_end=3703 - _globals['_EMBEDERROR']._serialized_start=3705 - _globals['_EMBEDERROR']._serialized_end=3765 - _globals['_HEALTHCHECKREQUEST']._serialized_start=3767 - _globals['_HEALTHCHECKREQUEST']._serialized_end=3845 - _globals['_HEALTHCHECKRESPONSE']._serialized_start=3847 - _globals['_HEALTHCHECKRESPONSE']._serialized_end=3902 - _globals['_ABORTREQUEST']._serialized_start=3904 - _globals['_ABORTREQUEST']._serialized_end=3954 - _globals['_ABORTRESPONSE']._serialized_start=3956 - _globals['_ABORTRESPONSE']._serialized_end=4005 - _globals['_LOADLORAREQUEST']._serialized_start=4007 - _globals['_LOADLORAREQUEST']._serialized_end=4080 - _globals['_LOADLORARESPONSE']._serialized_start=4082 - _globals['_LOADLORARESPONSE']._serialized_end=4154 - _globals['_UNLOADLORAREQUEST']._serialized_start=4156 - _globals['_UNLOADLORAREQUEST']._serialized_end=4195 - _globals['_UNLOADLORARESPONSE']._serialized_start=4197 - _globals['_UNLOADLORARESPONSE']._serialized_end=4251 - _globals['_UPDATEWEIGHTSREQUEST']._serialized_start=4253 - _globals['_UPDATEWEIGHTSREQUEST']._serialized_end=4372 - _globals['_UPDATEWEIGHTSRESPONSE']._serialized_start=4374 - _globals['_UPDATEWEIGHTSRESPONSE']._serialized_end=4431 - _globals['_GETINTERNALSTATEREQUEST']._serialized_start=4433 - _globals['_GETINTERNALSTATEREQUEST']._serialized_end=4478 - _globals['_GETINTERNALSTATERESPONSE']._serialized_start=4480 - _globals['_GETINTERNALSTATERESPONSE']._serialized_end=4546 - _globals['_SETINTERNALSTATEREQUEST']._serialized_start=4548 - _globals['_SETINTERNALSTATEREQUEST']._serialized_end=4613 - _globals['_SETINTERNALSTATERESPONSE']._serialized_start=4615 - _globals['_SETINTERNALSTATERESPONSE']._serialized_end=4675 - _globals['_SGLANGSCHEDULER']._serialized_start=4678 - _globals['_SGLANGSCHEDULER']._serialized_end=5060 + _globals['_SAMPLINGPARAMS']._serialized_end=826 + _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_start=764 + _globals['_SAMPLINGPARAMS_LOGITBIASENTRY']._serialized_end=812 + _globals['_DISAGGREGATEDPARAMS']._serialized_start=828 + _globals['_DISAGGREGATEDPARAMS']._serialized_end=921 + _globals['_GENERATEREQUEST']._serialized_start=924 + _globals['_GENERATEREQUEST']._serialized_end=1541 + _globals['_TOKENIZEDINPUT']._serialized_start=1543 + _globals['_TOKENIZEDINPUT']._serialized_end=1601 + _globals['_MULTIMODALINPUTS']._serialized_start=1604 + _globals['_MULTIMODALINPUTS']._serialized_end=1815 + _globals['_GENERATERESPONSE']._serialized_start=1818 + _globals['_GENERATERESPONSE']._serialized_end=2045 + _globals['_GENERATESTREAMCHUNK']._serialized_start=2048 + _globals['_GENERATESTREAMCHUNK']._serialized_end=2293 + _globals['_GENERATECOMPLETE']._serialized_start=2296 + _globals['_GENERATECOMPLETE']._serialized_end=2629 + _globals['_GENERATECOMPLETE_FINISHREASON']._serialized_start=2553 + _globals['_GENERATECOMPLETE_FINISHREASON']._serialized_end=2629 + _globals['_GENERATEERROR']._serialized_start=2631 + _globals['_GENERATEERROR']._serialized_end=2706 + _globals['_LOGPROBS']._serialized_start=2709 + _globals['_LOGPROBS']._serialized_end=2841 + _globals['_TOPLOGPROBS']._serialized_start=2843 + _globals['_TOPLOGPROBS']._serialized_end=2912 + _globals['_HIDDENSTATES']._serialized_start=2914 + _globals['_HIDDENSTATES']._serialized_end=2977 + _globals['_EMBEDREQUEST']._serialized_start=2980 + _globals['_EMBEDREQUEST']._serialized_end=3310 + _globals['_EMBEDRESPONSE']._serialized_start=3313 + _globals['_EMBEDRESPONSE']._serialized_end=3470 + _globals['_EMBEDCOMPLETE']._serialized_start=3473 + _globals['_EMBEDCOMPLETE']._serialized_end=3661 + _globals['_EMBEDDING']._serialized_start=3663 + _globals['_EMBEDDING']._serialized_end=3705 + _globals['_EMBEDERROR']._serialized_start=3707 + _globals['_EMBEDERROR']._serialized_end=3767 + _globals['_HEALTHCHECKREQUEST']._serialized_start=3769 + _globals['_HEALTHCHECKREQUEST']._serialized_end=3847 + _globals['_HEALTHCHECKRESPONSE']._serialized_start=3849 + _globals['_HEALTHCHECKRESPONSE']._serialized_end=3904 + _globals['_ABORTREQUEST']._serialized_start=3906 + _globals['_ABORTREQUEST']._serialized_end=3956 + _globals['_ABORTRESPONSE']._serialized_start=3958 + _globals['_ABORTRESPONSE']._serialized_end=4007 + _globals['_LOADLORAREQUEST']._serialized_start=4009 + _globals['_LOADLORAREQUEST']._serialized_end=4082 + _globals['_LOADLORARESPONSE']._serialized_start=4084 + _globals['_LOADLORARESPONSE']._serialized_end=4156 + _globals['_UNLOADLORAREQUEST']._serialized_start=4158 + _globals['_UNLOADLORAREQUEST']._serialized_end=4197 + _globals['_UNLOADLORARESPONSE']._serialized_start=4199 + _globals['_UNLOADLORARESPONSE']._serialized_end=4253 + _globals['_UPDATEWEIGHTSREQUEST']._serialized_start=4255 + _globals['_UPDATEWEIGHTSREQUEST']._serialized_end=4374 + _globals['_UPDATEWEIGHTSRESPONSE']._serialized_start=4376 + _globals['_UPDATEWEIGHTSRESPONSE']._serialized_end=4433 + _globals['_GETINTERNALSTATEREQUEST']._serialized_start=4435 + _globals['_GETINTERNALSTATEREQUEST']._serialized_end=4480 + _globals['_GETINTERNALSTATERESPONSE']._serialized_start=4482 + _globals['_GETINTERNALSTATERESPONSE']._serialized_end=4548 + _globals['_SETINTERNALSTATEREQUEST']._serialized_start=4550 + _globals['_SETINTERNALSTATEREQUEST']._serialized_end=4615 + _globals['_SETINTERNALSTATERESPONSE']._serialized_start=4617 + _globals['_SETINTERNALSTATERESPONSE']._serialized_end=4677 + _globals['_SGLANGSCHEDULER']._serialized_start=4680 + _globals['_SGLANGSCHEDULER']._serialized_end=5062 # @@protoc_insertion_point(module_scope) diff --git a/python/sglang/srt/grpc/sglang_scheduler_pb2.pyi b/python/sglang/srt/grpc/sglang_scheduler_pb2.pyi index d9388463d..34d9cd0fe 100644 --- a/python/sglang/srt/grpc/sglang_scheduler_pb2.pyi +++ b/python/sglang/srt/grpc/sglang_scheduler_pb2.pyi @@ -12,7 +12,7 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class SamplingParams(_message.Message): - __slots__ = ("temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "max_new_tokens", "stop", "stop_token_ids", "skip_special_tokens", "spaces_between_special_tokens", "regex", "json_schema", "ebnf_grammar", "lora_path", "n", "token_healing", "min_new_tokens", "ignore_eos", "no_stop_trim", "stream_interval", "logit_bias", "structural_tag", "custom_params") + __slots__ = ("temperature", "top_p", "top_k", "min_p", "frequency_penalty", "presence_penalty", "repetition_penalty", "max_new_tokens", "stop", "stop_token_ids", "skip_special_tokens", "spaces_between_special_tokens", "regex", "json_schema", "ebnf_grammar", "structural_tag", "lora_path", "n", "token_healing", "min_new_tokens", "ignore_eos", "no_stop_trim", "stream_interval", "logit_bias", "custom_params") class LogitBiasEntry(_message.Message): __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] @@ -35,6 +35,7 @@ class SamplingParams(_message.Message): REGEX_FIELD_NUMBER: _ClassVar[int] JSON_SCHEMA_FIELD_NUMBER: _ClassVar[int] EBNF_GRAMMAR_FIELD_NUMBER: _ClassVar[int] + STRUCTURAL_TAG_FIELD_NUMBER: _ClassVar[int] LORA_PATH_FIELD_NUMBER: _ClassVar[int] N_FIELD_NUMBER: _ClassVar[int] TOKEN_HEALING_FIELD_NUMBER: _ClassVar[int] @@ -43,7 +44,6 @@ class SamplingParams(_message.Message): NO_STOP_TRIM_FIELD_NUMBER: _ClassVar[int] STREAM_INTERVAL_FIELD_NUMBER: _ClassVar[int] LOGIT_BIAS_FIELD_NUMBER: _ClassVar[int] - STRUCTURAL_TAG_FIELD_NUMBER: _ClassVar[int] CUSTOM_PARAMS_FIELD_NUMBER: _ClassVar[int] temperature: float top_p: float @@ -60,6 +60,7 @@ class SamplingParams(_message.Message): regex: str json_schema: str ebnf_grammar: str + structural_tag: str lora_path: str n: int token_healing: bool @@ -68,9 +69,8 @@ class SamplingParams(_message.Message): no_stop_trim: bool stream_interval: int logit_bias: _containers.ScalarMap[str, float] - structural_tag: str custom_params: _struct_pb2.Struct - def __init__(self, temperature: _Optional[float] = ..., top_p: _Optional[float] = ..., top_k: _Optional[int] = ..., min_p: _Optional[float] = ..., frequency_penalty: _Optional[float] = ..., presence_penalty: _Optional[float] = ..., repetition_penalty: _Optional[float] = ..., max_new_tokens: _Optional[int] = ..., stop: _Optional[_Iterable[str]] = ..., stop_token_ids: _Optional[_Iterable[int]] = ..., skip_special_tokens: bool = ..., spaces_between_special_tokens: bool = ..., regex: _Optional[str] = ..., json_schema: _Optional[str] = ..., ebnf_grammar: _Optional[str] = ..., lora_path: _Optional[str] = ..., n: _Optional[int] = ..., token_healing: bool = ..., min_new_tokens: _Optional[int] = ..., ignore_eos: bool = ..., no_stop_trim: bool = ..., stream_interval: _Optional[int] = ..., logit_bias: _Optional[_Mapping[str, float]] = ..., structural_tag: _Optional[str] = ..., custom_params: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ... + def __init__(self, temperature: _Optional[float] = ..., top_p: _Optional[float] = ..., top_k: _Optional[int] = ..., min_p: _Optional[float] = ..., frequency_penalty: _Optional[float] = ..., presence_penalty: _Optional[float] = ..., repetition_penalty: _Optional[float] = ..., max_new_tokens: _Optional[int] = ..., stop: _Optional[_Iterable[str]] = ..., stop_token_ids: _Optional[_Iterable[int]] = ..., skip_special_tokens: bool = ..., spaces_between_special_tokens: bool = ..., regex: _Optional[str] = ..., json_schema: _Optional[str] = ..., ebnf_grammar: _Optional[str] = ..., structural_tag: _Optional[str] = ..., lora_path: _Optional[str] = ..., n: _Optional[int] = ..., token_healing: bool = ..., min_new_tokens: _Optional[int] = ..., ignore_eos: bool = ..., no_stop_trim: bool = ..., stream_interval: _Optional[int] = ..., logit_bias: _Optional[_Mapping[str, float]] = ..., custom_params: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ...) -> None: ... class DisaggregatedParams(_message.Message): __slots__ = ("bootstrap_host", "bootstrap_port", "bootstrap_room") diff --git a/python/sglang/srt/grpc/sglang_scheduler_pb2_grpc.py b/python/sglang/srt/grpc/sglang_scheduler_pb2_grpc.py index d9bdf0462..402f71725 100644 --- a/python/sglang/srt/grpc/sglang_scheduler_pb2_grpc.py +++ b/python/sglang/srt/grpc/sglang_scheduler_pb2_grpc.py @@ -1,3 +1,6 @@ +# This file is auto-generated. Do not edit manually. +# Regenerate with: python compile_proto.py + # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc diff --git a/sgl-router/src/proto/sglang_scheduler.proto b/sgl-router/src/proto/sglang_scheduler.proto index e2adc7863..55b3c77f8 100644 --- a/sgl-router/src/proto/sglang_scheduler.proto +++ b/sgl-router/src/proto/sglang_scheduler.proto @@ -47,24 +47,24 @@ message SamplingParams { string regex = 13; string json_schema = 14; string ebnf_grammar = 15; + string structural_tag = 16; } // LoRA adapter - string lora_path = 16; + string lora_path = 17; // Speculative decoding - int32 n = 17; // Number of samples + int32 n = 18; // Number of samples // Token healing - bool token_healing = 18; + bool token_healing = 19; // Additional parameters - int32 min_new_tokens = 19; - bool ignore_eos = 20; - bool no_stop_trim = 21; - int32 stream_interval = 22; - map logit_bias = 23; - string structural_tag = 24; + int32 min_new_tokens = 20; + bool ignore_eos = 21; + bool no_stop_trim = 22; + int32 stream_interval = 23; + map logit_bias = 24; // Custom parameters for extensibility google.protobuf.Struct custom_params = 25; diff --git a/sgl-router/src/routers/grpc/router.rs b/sgl-router/src/routers/grpc/router.rs index 5266bf913..c3a83389e 100644 --- a/sgl-router/src/routers/grpc/router.rs +++ b/sgl-router/src/routers/grpc/router.rs @@ -241,14 +241,14 @@ impl GrpcRouter { debug!("Tokenized {} tokens from input", token_ids.len()); // Step 5: Build tool constraints if needed - let structural_tag = if let Some(tools) = &body.tools { + let tool_call_constraint = if let Some(tools) = &body.tools { self.generate_tool_constraints(tools, &body.tool_choice, &body.model) } else { None }; // Step 6: Build SamplingParams for gRPC - let sampling_params = match self.build_grpc_sampling_params(body, structural_tag) { + let sampling_params = match self.build_grpc_sampling_params(body, tool_call_constraint) { Ok(params) => params, Err(e) => { error!("Failed to build sampling parameters: {}", e); @@ -286,6 +286,41 @@ impl GrpcRouter { } // ============ Helper Methods ============ + /// Select a worker for the request + fn select_worker_for_request( + &self, + model_id: Option<&str>, + text: Option<&str>, + ) -> Option> { + // Get workers for the specified model, filtered by connection mode + let workers = self.worker_registry.get_workers_filtered( + model_id, + Some(WorkerType::Regular), + Some(crate::core::ConnectionMode::Grpc { port: None }), + false, // get all workers, we'll filter by is_available() next + ); + + // Filter by availability (health + circuit breaker) + let available: Vec> = workers + .iter() + .filter(|w| w.is_available()) + .cloned() + .collect(); + + if available.is_empty() { + return None; + } + + // Get the appropriate policy for this model + let policy = match model_id { + Some(model) => self.policy_registry.get_policy_or_default(model), + None => self.policy_registry.get_default_policy(), + }; + + // Select worker using the policy + let idx = policy.select_worker(&available, text)?; + Some(available[idx].clone()) + } /// Process chat messages and apply template fn process_chat_messages( @@ -516,7 +551,7 @@ impl GrpcRouter { fn build_grpc_sampling_params( &self, request: &ChatCompletionRequest, - structural_tag: Option, + tool_call_constraint: Option<(String, String)>, ) -> Result { let stop_sequences = self.extract_stop_strings(request); @@ -555,8 +590,7 @@ impl GrpcRouter { stop_token_ids: request.stop_token_ids.clone().unwrap_or_default(), skip_special_tokens, n: request.n.unwrap_or(1) as i32, - structural_tag: structural_tag.unwrap_or_default(), - constraint: self.build_constraint(request)?, + constraint: self.build_constraint(request, tool_call_constraint)?, ..Default::default() }) } @@ -574,28 +608,48 @@ impl GrpcRouter { fn build_constraint( &self, request: &ChatCompletionRequest, + tool_call_constraint: Option<(String, String)>, ) -> Result, String> { + let mut constraints = Vec::new(); + if let Some(ResponseFormat::JsonSchema { json_schema }) = &request.response_format { let schema_str = serde_json::to_string(&json_schema.schema) .map_err(|e| format!("Failed to serialize JSON schema: {}", e))?; - return Ok(Some(proto::sampling_params::Constraint::JsonSchema( - schema_str, - ))); + constraints.push(proto::sampling_params::Constraint::JsonSchema(schema_str)); } if let Some(ebnf) = &request.ebnf { - return Ok(Some(proto::sampling_params::Constraint::EbnfGrammar( + constraints.push(proto::sampling_params::Constraint::EbnfGrammar( ebnf.clone(), - ))); + )); } if let Some(regex) = &request.regex { - return Ok(Some(proto::sampling_params::Constraint::Regex( - regex.clone(), - ))); + constraints.push(proto::sampling_params::Constraint::Regex(regex.clone())); } - Ok(None) + // Handle tool call constraint + if let Some((constraint_type, constraint_value)) = tool_call_constraint { + if !constraints.is_empty() { + return Err("Constrained decoding is not compatible with tool calls.".to_string()); + } + let tool_constraint = match constraint_type.as_str() { + "structural_tag" => { + proto::sampling_params::Constraint::StructuralTag(constraint_value) + } + "json_schema" => proto::sampling_params::Constraint::JsonSchema(constraint_value), + "ebnf" => proto::sampling_params::Constraint::EbnfGrammar(constraint_value), + "regex" => proto::sampling_params::Constraint::Regex(constraint_value), + _ => return Err(format!("Unknown constraint type: {}", constraint_type)), + }; + constraints.push(tool_constraint); + } + + match constraints.len() { + 0 => Ok(None), + 1 => Ok(constraints.pop()), + _ => Err("Multiple constraints are not allowed.".to_string()), + } } /// Generate tool constraints for structured generation @@ -604,52 +658,19 @@ impl GrpcRouter { _tools: &[crate::protocols::spec::Tool], _tool_choice: &Option, model: &str, - ) -> Option { + ) -> Option<(String, String)> { let _parser = self.tool_parser_registry.get_parser(model)?; + // TODO: Implement actual constraint generation logic + // For now, return None as this is placeholder implementation None } - /// Select a worker for the request - fn select_worker_for_request( - &self, - model_id: Option<&str>, - text: Option<&str>, - ) -> Option> { - // Get workers for the specified model, filtered by connection mode - let workers = self.worker_registry.get_workers_filtered( - model_id, - Some(WorkerType::Regular), - Some(crate::core::ConnectionMode::Grpc { port: None }), - false, // get all workers, we'll filter by is_available() next - ); - - // Filter by availability (health + circuit breaker) - let available: Vec> = workers - .iter() - .filter(|w| w.is_available()) - .cloned() - .collect(); - - if available.is_empty() { - return None; - } - - // Get the appropriate policy for this model - let policy = match model_id { - Some(model) => self.policy_registry.get_policy_or_default(model), - None => self.policy_registry.get_default_policy(), - }; - - // Select worker using the policy - let idx = policy.select_worker(&available, text)?; - Some(available[idx].clone()) - } - /// Get or create a gRPC client for the worker async fn get_or_create_grpc_client( &self, worker_url: &str, ) -> Result { + // TODO: move to worker debug!("Creating new gRPC client for worker: {}", worker_url); SglangSchedulerClient::connect(worker_url) .await