Gemini Backend (#9)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
This commit is contained in:
shiyi.c_98
2024-01-16 22:29:37 -08:00
committed by GitHub
parent c4707f1bb5
commit fd7c479239
13 changed files with 311 additions and 2 deletions

View File

@@ -428,6 +428,7 @@ class StreamExecutor:
self.messages_.append(last_msg)
self.cur_images = []
else:
# OpenAI chat API format
self.messages_.append({"role": expr.role, "content": new_text})
self.cur_role = None

View File

@@ -49,6 +49,16 @@ class SglSamplingParams:
"presence_penalty": self.presence_penalty,
}
def to_gemini_kwargs(self):
return {
"candidate_count": 1,
"max_output_tokens": self.max_new_tokens,
"stop_sequences": self.stop,
"temperature": self.temperature,
"top_p": self.top_p,
"top_k": self.top_k if self.top_k > 0 else None,
}
def to_anthropic_kwargs(self):
# Anthropic does not support frequency_penalty or presence_penalty, so we drop it here
return {