[router] remove old/oudated/useless comments across code base (#10968)

sgl-router/src/tokenizer/factory.rs

@@ -279,11 +279,9 @@ mod tests {
 
     #[test]
     fn test_create_tiktoken_tokenizer() {
-        // Test creating tokenizer for GPT models
         let tokenizer = create_tokenizer("gpt-4").unwrap();
         assert!(tokenizer.vocab_size() > 0);
 
-        // Test encoding and decoding
         let text = "Hello, world!";
         let encoding = tokenizer.encode(text).unwrap();
         let decoded = tokenizer.decode(encoding.token_ids(), false).unwrap();
@@ -292,7 +290,6 @@ mod tests {
 
     #[tokio::test]
     async fn test_download_tokenizer_from_hf() {
-        // Test with a small model that should have tokenizer files
         // Skip this test if HF_TOKEN is not set and we're in CI
         if std::env::var("CI").is_ok() && std::env::var("HF_TOKEN").is_err() {
             println!("Skipping HF download test in CI without HF_TOKEN");

sgl-router/src/tokenizer/sequence.rs

@@ -206,7 +206,6 @@ mod tests {
         // The incremental text should be " world" (with the space that the mock tokenizer adds)
         assert_eq!(text2, " world");
 
-        // Verify the full text
         assert_eq!(seq.text().unwrap(), "Hello world");
     }
 

sgl-router/src/tokenizer/stop.rs

@@ -398,7 +398,6 @@ mod tests {
         // The fix ensures we only output NEW text, not accumulated text
         assert_eq!(outputs.len(), 3);
 
-        // Verify no text is repeated
         for i in 0..outputs.len() {
             for j in i + 1..outputs.len() {
                 // No output should contain another (no accumulation)

sgl-router/src/tokenizer/tests.rs

@@ -36,22 +36,17 @@ fn test_tokenizer_wrapper() {
     let mock_tokenizer = Arc::new(mock::MockTokenizer::new());
     let tokenizer = Tokenizer::from_arc(mock_tokenizer);
 
-    // Test encoding
     let encoding = tokenizer.encode("Hello world").unwrap();
     assert_eq!(encoding.token_ids(), &[1, 2]);
 
-    // Test decoding
     let text = tokenizer.decode(&[1, 2], false).unwrap();
     assert_eq!(text, "Hello world");
 
-    // Test vocab size
     assert_eq!(tokenizer.vocab_size(), 8);
 
-    // Test token to ID
     assert_eq!(tokenizer.token_to_id("Hello"), Some(1));
     assert_eq!(tokenizer.token_to_id("unknown"), None);
 
-    // Test ID to token
     assert_eq!(tokenizer.id_to_token(1), Some("Hello".to_string()));
     assert_eq!(tokenizer.id_to_token(9999), None);
 }

sgl-router/src/tokenizer/tiktoken.rs

@@ -246,7 +246,6 @@ mod tests {
 
     #[test]
     fn test_unrecognized_model_name_returns_error() {
-        // Test that unrecognized model names return an error
         let result = TiktokenTokenizer::from_model_name("distilgpt-2");
         assert!(result.is_err());
         if let Err(e) = result {
@@ -268,7 +267,6 @@ mod tests {
 
     #[test]
     fn test_recognized_model_names() {
-        // Test that recognized model names work correctly
         assert!(TiktokenTokenizer::from_model_name("gpt-4").is_ok());
         assert!(TiktokenTokenizer::from_model_name("gpt-3.5-turbo").is_ok());
         assert!(TiktokenTokenizer::from_model_name("text-davinci-003").is_ok());