state: generation 4940 (cycle)

This commit is contained in:
2026-09-10 04:01:55 +00:00
parent 7348025ffa
commit 48dbd92cde
10 changed files with 2154 additions and 2156 deletions

View File

@@ -1306,7 +1306,7 @@
"taskType": "text-generation"
}
},
"generatedAt": "2026-09-10T03:59:09.192440+00:00",
"generatedAt": "2026-09-10T04:01:54.724985+00:00",
"summary": {
"activeBlockCount": 65,
"byGpuFramework": {

View File

@@ -19,7 +19,7 @@
"10": {
"complete": false,
"lastError": "ModelHubAPIError: 系统错误",
"listingErrors": 351,
"listingErrors": 352,
"nextPage": 1,
"recordsScanned": 0,
"uniqueRecords": 0
@@ -102,12 +102,12 @@
"cutoffAt": "2026-09-04T03:55:51.365685+00:00",
"failureLogsInspected": 0,
"mode": "incremental_decision_only",
"nextAccountIndex": 10,
"nextAccountIndex": 11,
"recordsScanned": 0,
"seenTaskIds": [],
"startedAt": "2026-09-04T03:55:51.365685+00:00",
"terminalRecords": 0,
"uniqueRecords": 0,
"updatedAt": "2026-09-10T03:59:09.167128+00:00",
"updatedAt": "2026-09-10T04:01:54.694031+00:00",
"version": 1
}

View File

@@ -416,7 +416,7 @@
}
},
"frameworkUpdatedAt": null,
"generatedAt": "2026-09-10T03:57:31.899272+00:00",
"generatedAt": "2026-09-10T04:00:16.124696+00:00",
"gpuStats": {
"Ascend_910-b3": {
"available": true,

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"generatedAt": "2026-09-10T03:15:36.085648+00:00",
"lastSyncTime": "2026-09-10T03:15:35.806659+00:00",
"generatedAt": "2026-09-10T04:00:10.615681+00:00",
"lastSyncTime": "2026-09-10T04:00:09.998782+00:00",
"recentLimit": 300,
"report": {
"architectureCompatibilityBlocks": {
@@ -1738,26 +1738,26 @@
"unresolvedFailureCount": 31
},
"Iluvatar_bi-150|vllm_0_17_0_corex_4_4_0|text-generation": {
"attributableFailureCount": 3,
"decisionFailureRate": 0.75,
"decisionSuccessRate": 0.25,
"decisionTotal": 4,
"attributableFailureCount": 4,
"decisionFailureRate": 0.8,
"decisionSuccessRate": 0.2,
"decisionTotal": 5,
"failureBreakdown": {
"ambiguous_runtime": 2,
"backend_operator": 2,
"backend_operator": 3,
"framework_architecture_unsupported": 1
},
"failureCount": 5,
"failureRate": 0.8333,
"failureCount": 6,
"failureRate": 0.8571,
"framework": "vllm_0_17_0_corex_4_4_0",
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
"successCount": 1,
"successRate": 0.1667,
"successRate": 0.1429,
"targetGpu": "Iluvatar_bi-150",
"taskType": "text-generation",
"total": 6,
"total": 7,
"unresolvedFailureCount": 2
},
"Iluvatar_bi-150|vllm|text-generation": {
@@ -1990,20 +1990,20 @@
"unresolvedFailureCount": 1
},
"Mthreads_s4000|vllm|text-generation": {
"attributableFailureCount": 28,
"attributableFailureCount": 29,
"decisionFailureRate": 1.0,
"decisionSuccessRate": 0.0,
"decisionTotal": 28,
"decisionTotal": 29,
"failureBreakdown": {
"ambiguous_runtime": 23,
"backend_operator": 1,
"backend_operator": 2,
"framework_architecture_unsupported": 21,
"memory_capacity": 1,
"model_load": 2,
"repository_structure": 1,
"runtime_memory": 2
},
"failureCount": 51,
"failureCount": 52,
"failureRate": 1.0,
"framework": "vllm",
"pendingCount": 0,
@@ -2013,7 +2013,7 @@
"successRate": 0.0,
"targetGpu": "Mthreads_s4000",
"taskType": "text-generation",
"total": 51,
"total": 52,
"unresolvedFailureCount": 23
},
"Sunrise_pt-200-x1|unknown|text-generation": {
@@ -2309,13 +2309,13 @@
"unresolvedFailureCount": 830
},
"vllm": {
"attributableFailureCount": 227,
"attributableFailureCount": 228,
"decisionFailureRate": 1.0,
"decisionSuccessRate": 0.0,
"decisionTotal": 227,
"decisionTotal": 228,
"failureBreakdown": {
"ambiguous_runtime": 131,
"backend_operator": 22,
"backend_operator": 23,
"framework_architecture_unsupported": 168,
"memory_capacity": 6,
"model_load": 18,
@@ -2325,14 +2325,14 @@
"tokenizer_compatibility": 3,
"参数/模板问题": 21
},
"failureCount": 380,
"failureCount": 381,
"failureRate": 1.0,
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 1,
"successCount": 0,
"successRate": 0.0,
"total": 380,
"total": 381,
"unresolvedFailureCount": 152
},
"vllm-mlu": {
@@ -2375,23 +2375,23 @@
"unresolvedFailureCount": 8
},
"vllm_0_17_0_corex_4_4_0": {
"attributableFailureCount": 3,
"decisionFailureRate": 0.75,
"decisionSuccessRate": 0.25,
"decisionTotal": 4,
"attributableFailureCount": 4,
"decisionFailureRate": 0.8,
"decisionSuccessRate": 0.2,
"decisionTotal": 5,
"failureBreakdown": {
"ambiguous_runtime": 2,
"backend_operator": 2,
"backend_operator": 3,
"framework_architecture_unsupported": 1
},
"failureCount": 5,
"failureRate": 0.8333,
"failureCount": 6,
"failureRate": 0.8571,
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
"successCount": 1,
"successRate": 0.1667,
"total": 6,
"successRate": 0.1429,
"total": 7,
"unresolvedFailureCount": 2
},
"vllm_fix_tokenizer": {
@@ -2436,7 +2436,7 @@
"unresolvedFailureCount": 1
}
},
"generatedAt": "2026-09-10T03:15:36.079303+00:00",
"generatedAt": "2026-09-10T04:00:10.611135+00:00",
"gpuSummaries": {
"Ascend_910-b3": {
"attributableFailureCount": 26,
@@ -2575,27 +2575,27 @@
"unresolvedFailureCount": 26
},
"Iluvatar_bi-150": {
"attributableFailureCount": 18,
"decisionFailureRate": 0.75,
"decisionSuccessRate": 0.25,
"decisionTotal": 24,
"attributableFailureCount": 19,
"decisionFailureRate": 0.76,
"decisionSuccessRate": 0.24,
"decisionTotal": 25,
"failureBreakdown": {
"ambiguous_runtime": 10,
"backend_operator": 4,
"backend_operator": 5,
"framework_architecture_unsupported": 8,
"model_load": 4,
"runtime_memory": 2,
"参数/模板问题": 1,
"验证失败": 30
},
"failureCount": 59,
"failureRate": 0.9077,
"failureCount": 60,
"failureRate": 0.9091,
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
"successCount": 6,
"successRate": 0.0923,
"total": 65,
"successRate": 0.0909,
"total": 66,
"unresolvedFailureCount": 41
},
"Iluvatar_mrv-100": {
@@ -2667,13 +2667,13 @@
"unresolvedFailureCount": 72
},
"Mthreads_s4000": {
"attributableFailureCount": 29,
"decisionFailureRate": 0.9355,
"decisionSuccessRate": 0.0645,
"decisionTotal": 31,
"attributableFailureCount": 30,
"decisionFailureRate": 0.9375,
"decisionSuccessRate": 0.0625,
"decisionTotal": 32,
"failureBreakdown": {
"ambiguous_runtime": 23,
"backend_operator": 1,
"backend_operator": 2,
"framework_architecture_unsupported": 21,
"memory_capacity": 1,
"model_load": 3,
@@ -2682,14 +2682,14 @@
"参数/模板问题": 7,
"验证失败": 26
},
"failureCount": 85,
"failureRate": 0.977,
"failureCount": 86,
"failureRate": 0.9773,
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
"successCount": 2,
"successRate": 0.023,
"total": 87,
"successRate": 0.0227,
"total": 88,
"unresolvedFailureCount": 56
},
"Sunrise_pt-200-x1": {
@@ -3148,15 +3148,15 @@
"unresolvedFailureCount": 2
},
"Iluvatar_bi-150|vllm_0_17_0_corex_4_4_0|text-generation|llama|compressed-tensors": {
"attributableFailureCount": 2,
"decisionFailureRate": 0.6667,
"decisionSuccessRate": 0.3333,
"decisionTotal": 3,
"attributableFailureCount": 3,
"decisionFailureRate": 0.75,
"decisionSuccessRate": 0.25,
"decisionTotal": 4,
"failureBreakdown": {
"backend_operator": 2
"backend_operator": 3
},
"failureCount": 2,
"failureRate": 0.6667,
"failureCount": 3,
"failureRate": 0.75,
"framework": "vllm_0_17_0_corex_4_4_0",
"modelType": "llama",
"pendingCount": 0,
@@ -3164,10 +3164,10 @@
"platformFailureCount": 0,
"quantizationMethod": "compressed-tensors",
"successCount": 1,
"successRate": 0.3333,
"successRate": 0.25,
"targetGpu": "Iluvatar_bi-150",
"taskType": "text-generation",
"total": 3,
"total": 4,
"unresolvedFailureCount": 0
},
"Iluvatar_bi-150|vllm_0_17_0_corex_4_4_0|text-generation|rwkv7|none": {
@@ -4396,23 +4396,24 @@
"unresolvedFailureCount": 0
},
"Mthreads_s4000|vllm|text-generation": {
"attributableFailureCount": 4,
"consecutiveFailures": 4,
"attributableFailureCount": 5,
"consecutiveFailures": 5,
"consecutivePlatformFailures": 0,
"decisionFailureRate": 1.0,
"decisionSuccessRate": 0.0,
"decisionTotal": 4,
"decisionTotal": 5,
"failureBreakdown": {
"ambiguous_runtime": 6,
"backend_operator": 1,
"model_load": 2,
"repository_structure": 1,
"runtime_memory": 1
},
"failureCount": 10,
"failureCount": 11,
"failureRate": 1.0,
"framework": "vllm",
"lastPlatformFailureAt": null,
"lastTerminalAt": "2026-09-09T21:11:06.904677+00:00",
"lastTerminalAt": "2026-09-10T04:00:09.998782+00:00",
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 0,
@@ -4420,7 +4421,7 @@
"successRate": 0.0,
"targetGpu": "Mthreads_s4000",
"taskType": "text-generation",
"total": 10,
"total": 11,
"unresolvedFailureCount": 6
},
"Sunrise_pt-200-x1|unknown|text-generation": {
@@ -4633,12 +4634,12 @@
"decisionSuccessRate": 0.0,
"decisionTotal": 10,
"failureBreakdown": {
"ambiguous_runtime": 7,
"ambiguous_runtime": 6,
"framework_architecture_unsupported": 8,
"model_load": 1,
"repository_structure": 1
},
"failureCount": 17,
"failureCount": 16,
"failureRate": 1.0,
"framework": "vllm",
"lastPlatformFailureAt": null,
@@ -4650,8 +4651,8 @@
"successRate": 0.0,
"targetGpu": "hygon_k100-ai",
"taskType": "text-generation",
"total": 17,
"unresolvedFailureCount": 7
"total": 16,
"unresolvedFailureCount": 6
}
},
"recentProfileCombinationStats": {
@@ -5732,14 +5733,14 @@
"unresolvedFailureCount": 0
},
"Iluvatar_bi-150|vllm_0_17_0_corex_4_4_0|text-generation|llama|compressed-tensors|33": {
"attributableFailureCount": 1,
"attributableFailureCount": 2,
"decisionFailureRate": 1.0,
"decisionSuccessRate": 0.0,
"decisionTotal": 1,
"decisionTotal": 2,
"failureBreakdown": {
"backend_operator": 1
"backend_operator": 2
},
"failureCount": 1,
"failureCount": 2,
"failureRate": 1.0,
"framework": "vllm_0_17_0_corex_4_4_0",
"loadSizeLog2Bucket": 33,
@@ -5752,7 +5753,7 @@
"successRate": 0.0,
"targetGpu": "Iluvatar_bi-150",
"taskType": "text-generation",
"total": 1,
"total": 2,
"unresolvedFailureCount": 0
},
"Iluvatar_bi-150|vllm_0_17_0_corex_4_4_0|text-generation|rwkv7|none|31": {
@@ -6812,16 +6813,16 @@
"unresolvedFailureCount": 0
}
},
"terminalRecords": 1492,
"totalRecords": 1576,
"terminalRecords": 1494,
"totalRecords": 1578,
"totals": {
"attributableFailureCount": 395,
"decisionFailureRate": 0.8896,
"decisionSuccessRate": 0.1104,
"decisionTotal": 444,
"attributableFailureCount": 397,
"decisionFailureRate": 0.8901,
"decisionSuccessRate": 0.1099,
"decisionTotal": 446,
"failureBreakdown": {
"ambiguous_runtime": 279,
"backend_operator": 26,
"backend_operator": 28,
"framework_architecture_unsupported": 269,
"memory_capacity": 11,
"model_load": 46,
@@ -6832,21 +6833,21 @@
"参数/模板问题": 94,
"验证失败": 673
},
"failureCount": 1443,
"failureCount": 1445,
"failureRate": 0.9672,
"pendingCount": 0,
"pendingRate": 0.0,
"platformFailureCount": 2,
"successCount": 49,
"successRate": 0.0328,
"total": 1492,
"total": 1494,
"unresolvedFailureCount": 1046
},
"warnings": [
"GPU Cambricon_mlu-370-x8 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Ascend_910-b3 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Iluvatar_mrv-100 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Sunrise_pt-200-x1 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Iluvatar_bi-150 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Ascend_910-b4 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Vastai_va16 本地统计失败率偏高≥50%),建议重点关注。",
"GPU MetaX_c-500 本地统计失败率偏高≥50%),建议重点关注。",
@@ -6854,7 +6855,7 @@
"GPU hygon_k100-ai 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Cambricon_mlu-370-x4 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Mthreads_s4000 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Iluvatar_bi-150 本地统计失败率偏高≥50%),建议重点关注。",
"GPU Sunrise_pt-200-x1 本地统计失败率偏高≥50%),建议重点关注。",
"组合 Iluvatar_mrv-100|unknown|text-generation 近期失败集中,建议降低该 GPU+框架的提交优先级。",
"组合 Cambricon_mlu-370-x8|vllm|text-generation 近期失败集中,建议降低该 GPU+框架的提交优先级。",
"组合 Biren_166m|unknown|text-generation 近期失败集中,建议降低该 GPU+框架的提交优先级。",
@@ -6875,6 +6876,6 @@
]
},
"storageMode": "decision_state_only",
"summarizedRecords": 1576,
"summarizedRecords": 1578,
"version": 1
}

View File

@@ -1,3 +1,4 @@
{"failReason": "backend_operator", "failureAction": "prefer_other_proven_framework_or_gpu", "failureCategory": "backend_operator", "failureClassificationReason": "backend_version_dependent", "failureCode": "MISSING_OPERATOR", "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "gpu_framework", "framework": "vllm", "lastSyncTime": "2026-09-10T04:00:09.998782+00:00", "modelId": "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t-alt", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-10T03:57:21+00:00", "targetGpu": "Mthreads_s4000", "taskId": "4532022", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "execute_empty_result", "failureCode": "EXECUTE_EMPTY_RESULT", "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "vllm", "lastSyncTime": "2026-09-10T02:50:12.317776+00:00", "modelId": "nm-testing/Meta-Llama-3-8B-Instruct-W4A16-compressed-tensors-test", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-10T02:43:21+00:00", "targetGpu": "hygon_k100-ai", "taskId": "4493607", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "framework_architecture_unsupported", "failureAction": "block_gpu_framework_architecture", "failureCategory": "framework_architecture_unsupported", "failureClassificationReason": "explicit_framework_model_unsupported", "failureCode": "MODEL_NOT_SUPPORTED", "failureDeterministic": true, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": false, "failureScope": "model_gpu_framework", "failureUnsupportedModelTypes": ["nemotron_h"], "framework": "", "lastSyncTime": "2026-09-10T02:50:12.317748+00:00", "modelId": "nota-ai/Nemotron-3.5-Lightning-30B-A3B-NVFP4-Global-Pruned-15", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-10T02:41:21+00:00", "targetGpu": "Iluvatar_mrv-100", "taskId": "4271675", "taskType": "text-generation", "verifyResult": -1}
{"failReason": null, "framework": "", "lastSyncTime": "2026-09-10T02:40:44.943952+00:00", "modelId": "ysqlian/YZH_Model", "modelProfile": {}, "outcome": "success", "status": "success", "submitTime": "2026-09-10T02:37:23+00:00", "targetGpu": "Iluvatar_mrv-100", "taskId": "4483083", "taskType": "text-generation", "verifyResult": 1}
@@ -297,4 +298,3 @@
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "missing_structured_error_code", "failureCode": null, "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "vllm", "lastSyncTime": "2026-09-07T08:32:22.399378+00:00", "modelId": "microsoft/Dayhoff-3b-GR-HM-c", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-07T08:31:21+00:00", "targetGpu": "Vastai_va16", "taskId": "4079222", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "missing_structured_error_code", "failureCode": null, "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "", "lastSyncTime": "2026-09-07T08:32:22.399402+00:00", "modelId": "RedHatAI/starcoder2-3b-quantized.w8a8", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-07T08:09:21+00:00", "targetGpu": "Iluvatar_mrv-100", "taskId": "4083445", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "driver_error", "failureCode": "DRIVER_ERROR", "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "vllm", "lastSyncTime": "2026-09-07T08:04:59.604222+00:00", "modelId": "nv-community/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-07T07:55:21+00:00", "targetGpu": "Ascend_910-b4", "taskId": "4464045", "taskType": "text-generation", "verifyResult": -1}
{"failReason": "ambiguous_runtime", "failureAction": "send_compact_profile_and_root_exception_to_llm", "failureCategory": "ambiguous_runtime", "failureClassificationReason": "missing_structured_error_code", "failureCode": null, "failureDetectedFramework": "vllm", "failureDeterministic": false, "failureEnrichmentAttempts": 1, "failureEnrichmentError": null, "failureNeedsLlm": true, "failureScope": "unknown", "framework": "vllm", "lastSyncTime": "2026-09-07T07:41:41.205199+00:00", "modelId": "nv-community/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", "modelProfile": {}, "outcome": "failed", "status": "success", "submitTime": "2026-09-07T07:33:21+00:00", "targetGpu": "hygon_k100-ai", "taskId": "4459518", "taskType": "text-generation", "verifyResult": -1}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff