diff --git a/sherpa-onnx/csrc/hypothesis.cc b/sherpa-onnx/csrc/hypothesis.cc
index 3dbd33ba..9f5c680f 100644
--- a/sherpa-onnx/csrc/hypothesis.cc
+++ b/sherpa-onnx/csrc/hypothesis.cc
@@ -18,8 +18,10 @@ void Hypotheses::Add(Hypothesis hyp) {
   } else {
     it->second.log_prob = LogAdd<double>()(it->second.log_prob, hyp.log_prob);
 
-    it->second.lm_log_prob =
+    if (it->second.lm_log_prob != 0 && hyp.lm_log_prob != 0){
+      it->second.lm_log_prob =
         LogAdd<double>()(it->second.lm_log_prob, hyp.lm_log_prob);
+    }
   }
 }