[router] introducing tokenizer trait (#9287)

sgl-router/src/lib.rs

@@ -10,6 +10,7 @@ pub mod policies;
 pub mod routers;
 pub mod server;
 pub mod service_discovery;
+pub mod tokenizer;
 pub mod tree;
 use crate::metrics::PrometheusConfig;