Support MetaX and Vastai tokenizer compatibility images

tests/test_engine_compat.py

@@ -84,6 +84,47 @@ class EngineCompatTests(unittest.TestCase):
             )
             self.assertIsNone(prepare_model(source, work_root=root))
 
+    def test_explicit_tokenizer_directory_gets_qwen_template(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            model = self._model(root, {"eos_token": "<|im_end|>"})
+            tokenizer = root / "vendor-tokenizer"
+            tokenizer.mkdir()
+            (tokenizer / "tokenizer_config.json").write_text(
+                json.dumps(
+                    {
+                        "eos_token": "<|im_end|>",
+                        "extra_special_tokens": ["<|im_start|>"],
+                        "tokenizer_class": "TokenizersBackend",
+                    }
+                ),
+                encoding="utf-8",
+            )
+            (tokenizer / "tokenizer.json").write_text("{}", encoding="utf-8")
+            view = Path(prepare_model(model, tokenizer_dir=tokenizer, work_root=root))
+            fixed = json.loads((view / "tokenizer_config.json").read_text("utf-8"))
+            self.assertEqual(fixed["chat_template"], QWEN_CHAT_TEMPLATE)
+            self.assertEqual(fixed["additional_special_tokens"], ["<|im_start|>"])
+            self.assertEqual(fixed["tokenizer_class"], "PreTrainedTokenizerFast")
+            self.assertEqual((model / "model.safetensors").read_bytes(), b"sample-weights")
+            self.assertFalse((view / "model.safetensors").exists())
+            self.assertEqual((view / "tokenizer.json").read_text("utf-8"), "{}")
+
+    def test_explicit_tokenizer_directory_preserves_model_standalone_template(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            model = self._model(root, {"eos_token": "<|im_end|>"})
+            (model / "chat_template.jinja").write_text("{{ messages }}", encoding="utf-8")
+            tokenizer = root / "vendor-tokenizer"
+            tokenizer.mkdir()
+            (tokenizer / "tokenizer_config.json").write_text(
+                json.dumps({"eos_token": "<|im_end|>"}), encoding="utf-8"
+            )
+            view = Path(prepare_model(model, tokenizer_dir=tokenizer, work_root=root))
+            fixed = json.loads((view / "tokenizer_config.json").read_text("utf-8"))
+            self.assertNotIn("chat_template", fixed)
+            self.assertEqual((view / "chat_template.jinja").read_text("utf-8"), "{{ messages }}")
+
 
 if __name__ == "__main__":
     unittest.main()