[router] Add Configurable L0 and L1 Tokenizer Caching (#11688)

This commit is contained in:
Simo Lin
2025-10-18 18:33:53 -07:00
committed by GitHub
parent fda0cb2a30
commit a7ae61ed77
22 changed files with 2385 additions and 24 deletions

View File

@@ -42,6 +42,9 @@ impl ConfigValidator {
}
}
// Validate tokenizer cache configuration
Self::validate_tokenizer_cache(&config.tokenizer_cache)?;
Ok(())
}
@@ -446,6 +449,29 @@ impl ConfigValidator {
Ok(())
}
/// Validate tokenizer cache configuration
fn validate_tokenizer_cache(cache: &TokenizerCacheConfig) -> ConfigResult<()> {
// Validate L0 max entries when L0 is enabled
if cache.enable_l0 && cache.l0_max_entries == 0 {
return Err(ConfigError::InvalidValue {
field: "tokenizer_cache.l0_max_entries".to_string(),
value: cache.l0_max_entries.to_string(),
reason: "Must be > 0 when L0 cache is enabled".to_string(),
});
}
// Validate L1 max memory when L1 is enabled
if cache.enable_l1 && cache.l1_max_memory == 0 {
return Err(ConfigError::InvalidValue {
field: "tokenizer_cache.l1_max_memory".to_string(),
value: cache.l1_max_memory.to_string(),
reason: "Must be > 0 when L1 cache is enabled".to_string(),
});
}
Ok(())
}
/// Validate compatibility between different configuration sections
fn validate_compatibility(config: &RouterConfig) -> ConfigResult<()> {
// IGW mode is independent - skip other compatibility checks when enabled