state: generation 4949 (cycle)

This commit is contained in:
2026-09-10 04:30:39 +00:00
parent 2eafacb35b
commit 96637a9c48
10 changed files with 2055 additions and 2034 deletions

View File

@@ -1325,7 +1325,7 @@
"taskType": "text-generation"
}
},
"generatedAt": "2026-09-10T04:26:57.745662+00:00",
"generatedAt": "2026-09-10T04:30:38.934819+00:00",
"summary": {
"activeBlockCount": 66,
"byGpuFramework": {

View File

@@ -75,7 +75,7 @@
"7": {
"complete": false,
"lastError": "ModelHubAPIError: 系统错误",
"listingErrors": 352,
"listingErrors": 353,
"nextPage": 1,
"recordsScanned": 0,
"uniqueRecords": 0
@@ -102,12 +102,12 @@
"cutoffAt": "2026-09-04T03:55:51.365685+00:00",
"failureLogsInspected": 0,
"mode": "incremental_decision_only",
"nextAccountIndex": 7,
"nextAccountIndex": 8,
"recordsScanned": 0,
"seenTaskIds": [],
"startedAt": "2026-09-04T03:55:51.365685+00:00",
"terminalRecords": 0,
"uniqueRecords": 0,
"updatedAt": "2026-09-10T04:26:57.718209+00:00",
"updatedAt": "2026-09-10T04:30:38.907475+00:00",
"version": 1
}

View File

@@ -416,7 +416,7 @@
}
},
"frameworkUpdatedAt": null,
"generatedAt": "2026-09-10T04:25:19.574564+00:00",
"generatedAt": "2026-09-10T04:28:11.208939+00:00",
"gpuStats": {
"Ascend_910-b3": {
"available": true,

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"generatedAt": "2026-09-10T04:17:40.484591+00:00",
"lastSyncTime": "2026-09-10T04:17:40.427917+00:00",
"generatedAt": "2026-09-10T04:27:59.073636+00:00",
"lastSyncTime": "2026-09-10T04:27:58.701482+00:00",
"recentLimit": 300,
"report": {
"architectureCompatibilityBlocks": {
@@ -1486,13 +1486,13 @@
"decisionSuccessRate": 0.0,
"decisionTotal": 37,
"failureBreakdown": {
"ambiguous_runtime": 22,
"ambiguous_runtime": 25,
"framework_architecture_unsupported": 33,
"memory_capacity": 1,
"repository_structure": 1,
"tokenizer_compatibility": 2
},
"failureCount": 59,
"failureCount": 62,
"failureRate": 1.0,
"framework": "vllm",
"pendingCount": 0,
@@ -1502,8 +1502,8 @@
"successRate": 0.0,
"targetGpu": "Ascend_910-b4",
"taskType": "text-generation",
"total": 59,
"unresolvedFailureCount": 22
"total": 62,
"unresolvedFailureCount": 25
},
"Biren_166m|unknown|text-generation": {
"attributableFailureCount": 24,
@@ -1919,19 +1919,19 @@
"unresolvedFailureCount": 56
},
"MetaX_c-500|vllm|text-generation": {
"attributableFailureCount": 38,
"attributableFailureCount": 39,
"decisionFailureRate": 1.0,
"decisionSuccessRate": 0.0,
"decisionTotal": 38,
"decisionTotal": 39,
"failureBreakdown": {
"ambiguous_runtime": 7,
"backend_operator": 19,
"backend_operator": 20,
"framework_architecture_unsupported": 14,
"memory_capacity": 1,
"model_load": 4,
"参数/模板问题": 9
},
"failureCount": 54,
"failureCount": 55,
"failureRate": 1.0,
"framework": "vllm",
"pendingCount": 0,
@@ -1941,7 +1941,7 @@
"successRate": 0.0,
"targetGpu": "MetaX_c-500",
"taskType": "text-generation",
"total": 54,
"total": 55,
"unresolvedFailureCount": 16
},
"Mthreads_s4000|llamacpp|text-generation": {
@@ -2328,13 +2328,13 @@
"unresolvedFailureCount": 830
},
"vllm": {
"attributableFailureCount": 229,
"attributableFailureCount": 230,
"decisionFailureRate": 1.0,
"decisionSuccessRate": 0.0,
"decisionTotal": 229,
"decisionTotal": 230,
"failureBreakdown": {
"ambiguous_runtime": 131,
"backend_operator": 23,
"ambiguous_runtime": 134,
"backend_operator": 24,
"framework_architecture_unsupported": 169,
"memory_capacity": 6,
"model_load": 18,
@@ -2344,15 +2344,15 @@
"tokenizer_compatibility": 3,
"参数/模板问题": 21
},
"failureCount": 382,
"failureCount": 386,
"failureRate": 1.0,
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 1,
"successCount": 0,
"successRate": 0.0,
"total": 382,
"unresolvedFailureCount": 152
"total": 386,
"unresolvedFailureCount": 155
},
"vllm-mlu": {
"attributableFailureCount": 7,
@@ -2455,7 +2455,7 @@
"unresolvedFailureCount": 1
}
},
"generatedAt": "2026-09-10T04:17:40.458056+00:00",
"generatedAt": "2026-09-10T04:27:59.069416+00:00",
"gpuSummaries": {
"Ascend_910-b3": {
"attributableFailureCount": 26,
@@ -2486,7 +2486,7 @@
"decisionSuccessRate": 0.05,
"decisionTotal": 40,
"failureBreakdown": {
"ambiguous_runtime": 23,
"ambiguous_runtime": 26,
"framework_architecture_unsupported": 33,
"memory_capacity": 1,
"model_load": 1,
@@ -2495,15 +2495,15 @@
"参数/模板问题": 11,
"验证失败": 175
},
"failureCount": 247,
"failureRate": 0.992,
"failureCount": 250,
"failureRate": 0.9921,
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
"successCount": 2,
"successRate": 0.008,
"total": 249,
"unresolvedFailureCount": 209
"successRate": 0.0079,
"total": 252,
"unresolvedFailureCount": 212
},
"Biren_166m": {
"attributableFailureCount": 25,
@@ -2662,27 +2662,27 @@
"unresolvedFailureCount": 78
},
"MetaX_c-500": {
"attributableFailureCount": 38,
"decisionFailureRate": 0.8085,
"decisionSuccessRate": 0.1915,
"decisionTotal": 47,
"attributableFailureCount": 39,
"decisionFailureRate": 0.8125,
"decisionSuccessRate": 0.1875,
"decisionTotal": 48,
"failureBreakdown": {
"ambiguous_runtime": 7,
"backend_operator": 19,
"backend_operator": 20,
"framework_architecture_unsupported": 14,
"memory_capacity": 1,
"model_load": 4,
"参数/模板问题": 17,
"验证失败": 48
},
"failureCount": 110,
"failureRate": 0.9244,
"failureCount": 111,
"failureRate": 0.925,
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
"successCount": 9,
"successRate": 0.0756,
"total": 119,
"successRate": 0.075,
"total": 120,
"unresolvedFailureCount": 72
},
"Mthreads_s4000": {
@@ -3489,14 +3489,14 @@
"unresolvedFailureCount": 0
},
"MetaX_c-500|vllm|text-generation|llama|compressed-tensors": {
"attributableFailureCount": 9,
"attributableFailureCount": 10,
"decisionFailureRate": 1.0,
"decisionSuccessRate": 0.0,
"decisionTotal": 9,
"decisionTotal": 10,
"failureBreakdown": {
"backend_operator": 9
"backend_operator": 10
},
"failureCount": 9,
"failureCount": 10,
"failureRate": 1.0,
"framework": "vllm",
"modelType": "llama",
@@ -3508,7 +3508,7 @@
"successRate": 0.0,
"targetGpu": "MetaX_c-500",
"taskType": "text-generation",
"total": 9,
"total": 10,
"unresolvedFailureCount": 0
},
"MetaX_c-500|vllm|text-generation|phi3small|compressed-tensors": {
@@ -4002,21 +4002,21 @@
"unresolvedFailureCount": 0
},
"Ascend_910-b4|vllm|text-generation": {
"attributableFailureCount": 11,
"consecutiveFailures": 11,
"attributableFailureCount": 9,
"consecutiveFailures": 9,
"consecutivePlatformFailures": 0,
"decisionFailureRate": 1.0,
"decisionSuccessRate": 0.0,
"decisionTotal": 11,
"decisionTotal": 9,
"failureBreakdown": {
"ambiguous_runtime": 9,
"framework_architecture_unsupported": 11
"ambiguous_runtime": 11,
"framework_architecture_unsupported": 9
},
"failureCount": 20,
"failureRate": 1.0,
"framework": "vllm",
"lastPlatformFailureAt": null,
"lastTerminalAt": "2026-09-10T01:08:07.916865+00:00",
"lastTerminalAt": "2026-09-10T04:27:58.701433+00:00",
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
@@ -4025,7 +4025,7 @@
"targetGpu": "Ascend_910-b4",
"taskType": "text-generation",
"total": 20,
"unresolvedFailureCount": 9
"unresolvedFailureCount": 11
},
"Biren_166m|unknown|text-generation": {
"attributableFailureCount": 8,
@@ -4346,7 +4346,7 @@
"consecutivePlatformFailures": 0,
"decisionFailureRate": 0.0,
"decisionSuccessRate": 1.0,
"decisionTotal": 3,
"decisionTotal": 2,
"failureBreakdown": {},
"failureCount": 0,
"failureRate": 0.0,
@@ -4356,11 +4356,11 @@
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
"successCount": 3,
"successCount": 2,
"successRate": 1.0,
"targetGpu": "MetaX_c-500",
"taskType": "text-generation",
"total": 3,
"total": 2,
"unresolvedFailureCount": 0
},
"MetaX_c-500|vllm|text-generation": {
@@ -4529,10 +4529,10 @@
"decisionSuccessRate": 1.0,
"decisionTotal": 2,
"failureBreakdown": {
"参数/模板问题": 3
"参数/模板问题": 2
},
"failureCount": 3,
"failureRate": 0.6,
"failureCount": 2,
"failureRate": 0.5,
"framework": "unknown",
"lastPlatformFailureAt": null,
"lastTerminalAt": "2026-09-07T22:32:11.428244+00:00",
@@ -4540,11 +4540,11 @@
"pendingRate": 0.0,
"platformFailureCount": 0,
"successCount": 2,
"successRate": 0.4,
"successRate": 0.5,
"targetGpu": "Vastai_va16",
"taskType": "text-generation",
"total": 5,
"unresolvedFailureCount": 3
"total": 4,
"unresolvedFailureCount": 2
},
"Vastai_va16|vllm_fix_tokenizer|text-generation": {
"attributableFailureCount": 1,
@@ -6233,6 +6233,30 @@
"total": 3,
"unresolvedFailureCount": 0
},
"MetaX_c-500|vllm|text-generation|llama|compressed-tensors|31": {
"attributableFailureCount": 1,
"decisionFailureRate": 1.0,
"decisionSuccessRate": 0.0,
"decisionTotal": 1,
"failureBreakdown": {
"backend_operator": 1
},
"failureCount": 1,
"failureRate": 1.0,
"framework": "vllm",
"loadSizeLog2Bucket": 31,
"modelType": "llama",
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
"quantizationMethod": "compressed-tensors",
"successCount": 0,
"successRate": 0.0,
"targetGpu": "MetaX_c-500",
"taskType": "text-generation",
"total": 1,
"unresolvedFailureCount": 0
},
"MetaX_c-500|vllm|text-generation|llama|compressed-tensors|32": {
"attributableFailureCount": 1,
"decisionFailureRate": 1.0,
@@ -6834,16 +6858,16 @@
"unresolvedFailureCount": 0
}
},
"terminalRecords": 1496,
"totalRecords": 1583,
"terminalRecords": 1500,
"totalRecords": 1587,
"totals": {
"attributableFailureCount": 398,
"decisionFailureRate": 0.8904,
"decisionSuccessRate": 0.1096,
"decisionTotal": 447,
"attributableFailureCount": 399,
"decisionFailureRate": 0.8906,
"decisionSuccessRate": 0.1094,
"decisionTotal": 448,
"failureBreakdown": {
"ambiguous_runtime": 279,
"backend_operator": 28,
"ambiguous_runtime": 282,
"backend_operator": 29,
"framework_architecture_unsupported": 270,
"memory_capacity": 11,
"model_load": 46,
@@ -6854,21 +6878,21 @@
"参数/模板问题": 95,
"验证失败": 673
},
"failureCount": 1447,
"failureRate": 0.9672,
"failureCount": 1451,
"failureRate": 0.9673,
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 2,
"successCount": 49,
"successRate": 0.0328,
"total": 1496,
"unresolvedFailureCount": 1047
"successRate": 0.0327,
"total": 1500,
"unresolvedFailureCount": 1050
},
"warnings": [
"GPU Cambricon_mlu-370-x8 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Ascend_910-b3 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Iluvatar_mrv-100 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Sunrise_pt-200-x1 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Iluvatar_bi-150 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Ascend_910-b4 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Vastai_va16 本地统计失败率偏高≥50%),建议重点关注。",
"GPU MetaX_c-500 本地统计失败率偏高≥50%),建议重点关注。",
@@ -6876,7 +6900,7 @@
"GPU hygon_k100-ai 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Cambricon_mlu-370-x4 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Mthreads_s4000 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Iluvatar_bi-150 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Sunrise_pt-200-x1 本地统计失败率偏高≥50%),建议重点关注。",
"组合 Iluvatar_mrv-100|unknown|text-generation 近期失败集中,建议降低该 GPU+框架的提交优先级。",
"组合 Cambricon_mlu-370-x8|vllm|text-generation 近期失败集中,建议降低该 GPU+框架的提交优先级。",
"组合 Biren_166m|unknown|text-generation 近期失败集中,建议降低该 GPU+框架的提交优先级。",
@@ -6897,6 +6921,6 @@
]
},
"storageMode": "decision_state_only",
"summarizedRecords": 1583,
"summarizedRecords": 1587,
"version": 1
}

View File

@@ -1,3 +1,6 @@
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "driver_error", "failureCode": "DRIVER_ERROR", "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "vllm", "lastSyncTime": "2026-09-10T04:27:58.701433+00:00", "modelId": "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-FP8-Channelwise-compressed-tensors", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-10T04:25:21+00:00", "targetGpu": "Ascend_910-b4", "taskId": "4471917", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "driver_error", "failureCode": "DRIVER_ERROR", "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "vllm", "lastSyncTime": "2026-09-10T04:27:58.701482+00:00", "modelId": "neuralmagic/SmolLM-1.7B-Instruct-quantized.w8a16", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-10T04:23:21+00:00", "targetGpu": "Ascend_910-b4", "taskId": "4471910", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "driver_error", "failureCode": "DRIVER_ERROR", "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "vllm", "lastSyncTime": "2026-09-10T04:27:58.701457+00:00", "modelId": "nm-testing/Meta-Llama-3-8B-Instruct-FP8-K-V", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-10T04:21:21+00:00", "targetGpu": "Ascend_910-b4", "taskId": "4471915", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "framework_architecture_unsupported", "failureAction": "block_gpu_framework_architecture", "failureCategory": "framework_architecture_unsupported", "failureClassificationReason": "explicit_framework_model_unsupported", "failureCode": "MODEL_NOT_SUPPORTED", "failureDetectedFramework": "vllm", "failureDeterministic": true, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": false, "failureScope": "model_gpu_framework", "failureUnsupportedModelTypes": ["rwkv7"], "framework": "vllm", "lastSyncTime": "2026-09-10T04:09:13.117076+00:00", "modelId": "RWKV/RWKV7-1.5B-20260805", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-10T04:01:22+00:00", "targetGpu": "Mthreads_s4000", "taskId": "4609102", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "backend_operator", "failureAction": "prefer_other_proven_framework_or_gpu", "failureCategory": "backend_operator", "failureClassificationReason": "backend_version_dependent", "failureCode": "MISSING_OPERATOR", "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "gpu_framework", "framework": "vllm", "lastSyncTime": "2026-09-10T04:00:09.998782+00:00", "modelId": "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t-alt", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-10T03:57:21+00:00", "targetGpu": "Mthreads_s4000", "taskId": "4532022", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "execute_empty_result", "failureCode": "EXECUTE_EMPTY_RESULT", "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "vllm", "lastSyncTime": "2026-09-10T02:50:12.317776+00:00", "modelId": "nm-testing/Meta-Llama-3-8B-Instruct-W4A16-compressed-tensors-test", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-10T02:43:21+00:00", "targetGpu": "hygon_k100-ai", "taskId": "4493607", "taskType": "text-generation", "verifyResult": -1}
@@ -295,6 +298,3 @@
{"failReason": "framework_architecture_unsupported", "failureAction": "block_gpu_framework_architecture", "failureCategory": "framework_architecture_unsupported", "failureClassificationReason": "explicit_framework_model_unsupported", "failureCode": "MODEL_NOT_SUPPORTED", "failureDetectedFramework": "vllm", "failureDeterministic": true, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": false, "failureScope": "model_gpu_framework", "failureUnsupportedArchitectures": ["GPTNeoForCausalLM"], "framework": "vllm", "lastSyncTime": "2026-09-07T09:53:28.210465+00:00", "modelId": "KoboldAI/GPT-Neo-2.7B-Janeway", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-07T09:45:21+00:00", "targetGpu": "Vastai_va16", "taskId": "4079146", "taskType": "text-generation", "verifyResult": -1}
{"failReason": null, "framework": "", "lastSyncTime": "2026-09-07T09:29:22.999691+00:00", "modelId": "dickeyli/llmrec-onereason-8b-sh2-grpo4-step160", "modelProfile": {}, "outcome": "success", "status": "success", "submitTime": "2026-09-07T09:17:22+00:00", "targetGpu": "Cambricon_mlu-370-x8", "taskId": "4471037", "taskType": "text-generation", "verifyResult": 1}
{"failReason": "framework_architecture_unsupported", "failureAction": "block_gpu_framework_architecture", "failureCategory": "framework_architecture_unsupported", "failureClassificationReason": "explicit_framework_model_unsupported", "failureCode": "MODEL_NOT_SUPPORTED", "failureDeterministic": true, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": false, "failureScope": "model_gpu_framework", "failureUnsupportedArchitectures": ["NemotronHForCausalLM"], "framework": "", "lastSyncTime": "2026-09-07T09:00:01.243299+00:00", "modelId": "nv-community/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-07T08:59:21+00:00", "targetGpu": "Cambricon_mlu-370-x8", "taskId": "4471038", "taskType": "text-generation", "verifyResult": -1}
{"failReason": null, "framework": "", "lastSyncTime": "2026-09-07T09:00:01.243313+00:00", "modelId": "dickeyli/llmrec-onereason-8b-sh2-grpo4-step160", "modelProfile": {}, "outcome": "success", "status": "success", "submitTime": "2026-09-07T08:55:21+00:00", "targetGpu": "MetaX_c-500", "taskId": "4460896", "taskType": "text-generation", "verifyResult": 1}
{"failReason": "参数/模板问题", "framework": "", "lastSyncTime": "2026-09-07T09:00:01.243276+00:00", "modelId": "nightmedia/Mistral-Nemo-Instruct-2407-12B-Thinking-HI-Claude-Opus-High-Reasoning-mxfp4-mlx", "modelProfile": {}, "outcome": "failed", "status": "failed", "submitTime": "2026-09-07T08:43:54+00:00", "targetGpu": "Vastai_va16", "taskId": "4079294", "taskType": "text-generation", "verifyResult": null}
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "missing_structured_error_code", "failureCode": null, "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "vllm", "lastSyncTime": "2026-09-07T08:32:22.399378+00:00", "modelId": "microsoft/Dayhoff-3b-GR-HM-c", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-07T08:31:21+00:00", "targetGpu": "Vastai_va16", "taskId": "4079222", "taskType": "text-generation", "verifyResult": -1}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff