chore: limit crawler pages to 20

app/scheduler/submitter.py

@@ -1,6 +1,9 @@
 from app.clients.modelhub import ModelHubClient
 from app.domain.vllm_configs import gen_vllm_config
 from app.storage.repositories import Repository, future_seconds
+import logging
+
+LOG = logging.getLogger(__name__)
 
 
 class Submitter:
@@ -13,6 +16,7 @@ class Submitter:
         for task in self.repo.due_tasks(("ready_to_submit",), limit=limit):
             config = gen_vllm_config(task["gpu_type"], task["model_id"], task["max_model_len"])
             result = self.client.create_contest_task(task["model_id"], task["gpu_type"], config)
+            LOG.info("submit %s / %s / %s -> %s (%s)", task["model_id"], task["gpu_type"], task["source"], result.result, result.message)
             self.repo.record_submission_attempt(
                 task["id"],
                 task["model_id"],

config.local.json

@@ -5,7 +5,7 @@
   "EMAIL": "i-hengyan@4paradigm.com",
   "USER_ID": "83",
   "CONTRIBUTORS": "i-hengyan",
-  "STRATEGY_ID": "280bc4a7-a460-4778-bf74-89ccd3162bf5",
+  "STRATEGY_ID": "a2514012-5ac8-4187-a3dd-8d74d8582827",
 
   "MODELHUB_URL": "https://modelhub.org.cn",
   "MODELHUB_API_BASE": "https://modelhub.org.cn/api",
@@ -32,11 +32,11 @@
   "TARGET_MACHINE_NAMES": "MTT S4000,Hygon K100,Kunlunxin P800",
   "TARGET_TASK_LEVEL": "文本生成",
   "MODELHUB_PAGE_SIZE": 100,
-  "CRAWLER_MAX_PAGES": 0,
+  "CRAWLER_MAX_PAGES": 20,
 
   "ENABLE_DOWNLOAD_SUCCESS_CRAWLER": true,
   "DOWNLOAD_SUCCESS_PAGE_SIZE": 50,
-  "DOWNLOAD_SUCCESS_MAX_PAGES": 0,
+  "DOWNLOAD_SUCCESS_MAX_PAGES": 20,
 
   "BOUNTY_ALLOW_DIRECT_SUBMIT": true,
   "AUTO_LOAD_SEED_PATH": ""