project_6_89d52222/upstream_ref/xllm/xllm/proto/completion.proto

syntax = "proto3";

option go_package = "jd.com/jd-infer/xllm;xllm";
package xllm.proto;

import "common.proto";
import "rec.proto";

// Next ID: 41
message CompletionRequest {
  // ID of the model to use. (required)
  // You can use the ListModels endpoint to list available models.
  string model = 1;

  // the prompt to generate completions for. (required)
  string prompt = 2;

  // the number of sequence to generate server-side and returns the "best". default = None
  // Results can't be streamed.
  // when used with n, best_of controls the number of candidate completions and n specifies
  // how many to return. best_of must be greater than or equal to n.
  optional uint32 best_of = 3;

  // number of tokens to generate
  // the prompt token count + max_tokens can't exceed the model's max context length.
  optional uint32 max_tokens = 4;

  // temperature of the sampling, between [0, 2]. default = 0.0
  // higher value will make the ouput more random.
  optional float temperature = 5;

  // top_p sampling cutoff, between [0, 1.0]. default = 1.0
  optional float top_p = 6;

  // number of completions to return for each prompt. default = 1
  optional uint32 n = 7;

  // whether to stream partial completions back as they are generated. default = false
  optional bool stream = 8;

  // include the log probabilities of the chosen tokens. the maximum value is 5.
  optional uint32 logprobs = 9;

  // whether to include the original prompt in the completion response. default = true
  optional bool echo = 10;

  // up to 4 sequences where the API will stop generating further tokens.
  repeated string stop = 11;

  // presence penalty to reduce the likelihood of generating words already in the prompt.
  // values between [-2.0, 2.0]. Positive values penalize new tokens based on their existing
  // in the prompt. default = 0.0
  optional float presence_penalty = 12;

  // frequency penalty to reduce the likelihood of generating the same word multiple times.
  // values between [0.0, 2.0]. 0.0 means no penalty. default = 0.0
  // Positive values penalize new tokens based on their existing frequency in the text.
  optional float frequency_penalty = 13;

  // whether to skip special tokens in the output. default = true
  optional bool skip_special_tokens = 14;

  // whether to ignore the end of sequence token. default = false.
  optional bool ignore_eos = 15;

  // A unique identifier representing your end-user, which can help system to monitor and detect abuse.
  string user = 16;

  // the list of token ids where the API will stop generating further tokens.
  repeated int32 stop_token_ids = 18;

  // top_k sampling cutoff, default = -1 (no cutoff)
  optional int64 top_k = 19;

  // repetition penalty to penalize new tokens based on their occurence in the
  // text. values > 1.0 encourage the model to use new tokens, while values
  // < 1.0 encourage the model to repeat tokens. default = 1.0
  optional float repetition_penalty = 20;

  // options for streaming response. Only set this when you set stream: true
  optional StreamOptions stream_options = 21;

  optional string request_id = 22;

  optional string service_request_id = 23;

  repeated int32 token_ids = 24;

  Routing routing = 25;

  optional bool offline = 26;

  optional int32 ttlt_slo_ms = 27;

  // request priority. default = DEFAULT
  optional Priority priority = 28;

  optional int32 beam_width = 29;

  // Final number of beam results to return. Defaults to beam_width when unset.
  optional int32 num_return_sequences = 41;

  optional bool add_special_tokens = 30;
  // tensor for rec embedding.
  repeated InferInputTensor input_tensors = 31;

  optional int32 ttft_slo_ms = 32;

  optional int32 tpot_slo_ms = 33;

  optional int32 tpot_priority_weight = 34;

  optional int32 ttft_priority_weight = 35;

  optional int32 ttlt_priority_weight = 36;

  optional int32 priority_weight = 37;

  // xllm_service rpc addr(ip:rpc_port) that forwards this request.
  optional string source_xservice_addr = 38;

  // request trace headers from client.
  optional string x_request_id = 39;
  optional string x_request_time = 40;
}

message LogProbs {
  repeated float token_logprobs = 1 [json_name="token_logprobs"];

  repeated string tokens = 2;

  repeated int32 token_ids = 3 [json_name="token_ids"];

  // rpc doesn't allow: repeated map<string, float> top_logprobs = 4;
}

message Choice {
  // the index of the generated completion
  optional uint32 index = 1;

  // the generated completion
  optional string text = 2;

  // the log probability of of output tokens.
  optional LogProbs logprobs = 3;

  // the reason of the model stoped generating tokens.
  // "stop" - the model hit a natural stop point or a provided stop sequence.
  // "length" - the maximum number of tokens specified in the request was reached.
  // "function_call" - the model called a function.
  optional string finish_reason = 4 [json_name="finish_reason"];
}

message CompletionResponse {
  // unique id for the completion request
  string id = 1;

  // the object type, which is always "text_completion".
  string object = 2;

  // the unix timestamp (in seconds) of when the completion was created.
  uint32 created = 3;

  // the model used for the completion
  string model = 4;

  // list of generated completion choices for the input prompt
  repeated Choice choices = 5;

  // usage statistics for the completion request.
  Usage usage = 6;

  // for rec output
  repeated InferOutputTensor output_tensors = 7;
}