[router] add tokenizer integration test with real mini tokenizer (#9413)

2025-08-20 17:56:23 -07:00
parent 3828db4309
commit e65231022f
2 changed files with 393 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -48,6 +48,9 @@ coverage.xml
 *.cover
 *.py,cover
 .hypothesis/
+
+# Tokenizer cache for tests
+.tokenizer_cache/
 .pytest_cache/
 cover/