EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[router] add tiktokenizer and sequence in router (#9354)

Browse Source 
Co-authored-by: Chang Su <chang.s.su@oracle.com>
This commit is contained in:
Simo Lin
2025-08-19 10:46:28 -07:00
committed by GitHub
parent a3b810ebdb
commit 0b95a01a8f
7 changed files with 578 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
sgl-router/src/tokenizer/mod.rs
View File

@@ -4,6 +4,7 @@ use std::sync::Arc;
pub mod factory;
pub mod mock;
pub mod sequence;
pub mod stop;
pub mod stream;
pub mod traits;
@@ -12,11 +13,15 @@ pub mod traits;
#[cfg(feature = "huggingface")]
pub mod huggingface;
#[cfg(feature = "tiktoken")]
pub mod tiktoken;
#[cfg(test)]
mod tests;
// Re-exports
pub use factory::{create_tokenizer, create_tokenizer_from_file, TokenizerType};
pub use sequence::Sequence;
pub use stop::{SequenceDecoderOutput, StopSequenceConfig, StopSequenceDecoder};
pub use stream::DecodeStream;
pub use traits::{Decoder, Encoder, Encoding, SpecialTokens, Tokenizer as TokenizerTrait};
@@ -24,6 +29,9 @@ pub use traits::{Decoder, Encoder, Encoding, SpecialTokens, Tokenizer as Tokeniz
#[cfg(feature = "huggingface")]
pub use huggingface::{ChatMessage, HuggingFaceTokenizer};
#[cfg(feature = "tiktoken")]
pub use tiktoken::{TiktokenModel, TiktokenTokenizer};
/// Main tokenizer wrapper that provides a unified interface for different tokenizer implementations
#[derive(Clone)]
pub struct Tokenizer(Arc<dyn traits::Tokenizer>);