Use Anthropic messages API (#304)

examples/quick_start/anthropic_example_chat.py

@@ -52,7 +52,7 @@ def batch():
 
 
 if __name__ == "__main__":
-    sgl.set_default_backend(sgl.Anthropic("claude-2"))
+    sgl.set_default_backend(sgl.Anthropic("claude-3-haiku-20240307"))
 
     # Run a single request
     print("\n========== single ==========\n")

examples/quick_start/anthropic_example_complete.py

@@ -19,7 +19,7 @@ def few_shot_qa(s, question):
 \n\nAssistant: Rome
 """)
     s += "\n\nHuman: " + question + "\n"
-    s += "\n\nAssistant:" + sgl.gen("answer", stop="\n", temperature=0)
+    s += "\n\nAssistant:" + sgl.gen("answer", stop="\\n", temperature=0)
 
 
 def single():
@@ -52,7 +52,7 @@ def batch():
 
 
 if __name__ == "__main__":
-    sgl.set_default_backend(sgl.Anthropic("claude-2"))
+    sgl.set_default_backend(sgl.Anthropic("claude-3-haiku-20240307"))
 
     # Run a single request
     print("\n========== single ==========\n")

python/pyproject.toml

@@ -22,7 +22,7 @@ srt = ["aiohttp", "fastapi", "psutil", "rpyc", "torch", "uvloop", "uvicorn",
        "zmq", "vllm>=0.3.3", "interegular", "lark", "numba",
        "pydantic", "referencing", "diskcache", "cloudpickle", "pillow", "outlines>=0.0.27"]
 openai = ["openai>=1.0", "numpy"]
-anthropic = ["anthropic", "numpy"]
+anthropic = ["anthropic>=0.20.0", "numpy"]
 all = ["sglang[srt]", "sglang[openai]", "sglang[anthropic]"]
 
 [project.urls]

python/sglang/backend/anthropic.py

@@ -30,13 +30,17 @@ class Anthropic(BaseBackend):
         s: StreamExecutor,
         sampling_params: SglSamplingParams,
     ):
-        prompt = s.text_
+        if s.messages_:
-        ret = anthropic.Anthropic().completions.create(
+            messages = s.messages_
+        else:
+            messages = [{"role": "user", "content": s.text_}]
+
+        ret = anthropic.Anthropic().messages.create(
             model=self.model_name,
-            prompt=prompt,
+            messages=messages,
             **sampling_params.to_anthropic_kwargs(),
         )
-        comp = ret.completion
+        comp = ret.content[0].text
 
         return comp, {}
 
@@ -45,13 +49,15 @@ class Anthropic(BaseBackend):
         s: StreamExecutor,
         sampling_params: SglSamplingParams,
     ):
-        prompt = s.text_
+        if s.messages_:
-        generator = anthropic.Anthropic().completions.create(
+            messages = s.messages_
-            model=self.model_name,
+        else:
-            prompt=prompt,
+            messages = [{"role": "user", "content": s.text_}]
-            stream=True,
-            **sampling_params.to_anthropic_kwargs(),
-        )
 
-        for ret in generator:
+        with anthropic.Anthropic().messages.stream(
-            yield ret.completion, {}
+            model=self.model_name,
+            messages=messages,
+            **sampling_params.to_anthropic_kwargs(),
+        ) as stream:
+            for text in stream.text_stream:
+                yield text, {}

python/sglang/lang/ir.py

@@ -73,7 +73,7 @@ class SglSamplingParams:
                 "Regular expression is not supported in the Anthropic backend."
             )
         return {
-            "max_tokens_to_sample": self.max_new_tokens,
+            "max_tokens": self.max_new_tokens,
             "stop_sequences": (
                 self.stop if isinstance(self.stop, (list, tuple)) else [self.stop]
             ),

test/lang/test_anthropic_backend.py

@@ -14,7 +14,7 @@ class TestAnthropicBackend(unittest.TestCase):
         cls = type(self)
 
         if cls.backend is None:
-            cls.backend = Anthropic("claude-2")
+            cls.backend = Anthropic("claude-3-haiku-20240307")
             set_default_backend(cls.backend)
 
     def test_mt_bench(self):