[router] create worker removal step and clean up worker manager (#11921)

sgl-router/tests/streaming_tests.rs

@@ -8,7 +8,6 @@ use reqwest::Client;
 use serde_json::json;
 use sglang_router_rs::{
     config::{RouterConfig, RoutingMode},
-    core::WorkerManager,
     routers::{RouterFactory, RouterTrait},
 };
 
@@ -52,13 +51,6 @@ impl TestContext {
 
         let app_context = common::create_test_context(config.clone());
 
-        // Initialize workers in the registry before creating router
-        if !worker_urls.is_empty() {
-            WorkerManager::initialize_workers(&config, &app_context.worker_registry, None)
-                .await
-                .expect("Failed to initialize workers");
-        }
-
         let router = RouterFactory::create_router(&app_context).await.unwrap();
         let router = Arc::from(router);