[router] add tokenizer chat template support (#9370)

Co-authored-by: Chang Su <chang.s.su@oracle.com>

sgl-router/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [features]
 default = ["huggingface", "grpc-client"]
-huggingface = ["tokenizers"]
+huggingface = ["tokenizers", "minijinja"]
 tiktoken = ["tiktoken-rs"]
 grpc-client = []
 grpc-server = []
@@ -52,7 +52,8 @@ url = "2.5.4"
 tokio-stream = { version = "0.1", features = ["sync"] }
 anyhow = "1.0"
 tokenizers = { version = "0.21.4", optional = true }
-tiktoken-rs = { version = "0.5", optional = true }
+tiktoken-rs = { version = "0.7.0", optional = true }
+minijinja = { version = "2.0", optional = true }
 
 # gRPC and Protobuf dependencies
 tonic = { version = "0.12", features = ["tls", "gzip", "transport"] }
@@ -71,6 +72,7 @@ criterion = { version = "0.5", features = ["html_reports"] }
 tower = { version = "0.5", features = ["util"] }
 http-body-util = "0.1"
 portpicker = "0.1"
+tempfile = "3.8"
 
 [[bench]]
 name = "request_processing"