fix the d2s bug in qwen2 modeling (#9603)

* fix the d2s bug in qwen2 modeling * update the code for the predictor
PaddlePaddle · Dec 11, 2024 · cd5468c · cd5468c
1 parent b7c6b4e
commit cd5468c
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/llm/predict/predictor.py b/llm/predict/predictor.py
@@ -27,7 +27,10 @@
 from paddle.base.framework import in_cinn_mode, in_pir_executor_mode, use_pir_api
 from paddle.distributed import fleet
 
-from paddlenlp.experimental.transformers import InferenceWithReferenceProposer
+try:
+    from paddlenlp.experimental.transformers import InferenceWithReferenceProposer
+except:
+    pass
 from paddlenlp.generation import GenerationConfig, TextIteratorStreamer
 from paddlenlp.peft import LoRAConfig, LoRAModel, PrefixConfig, PrefixModelForCausalLM
 from paddlenlp.taskflow.utils import static_mode_guard

diff --git a/paddlenlp/transformers/qwen2/modeling.py b/paddlenlp/transformers/qwen2/modeling.py
@@ -1026,7 +1026,7 @@ def _prepare_decoder_attention_mask(attention_mask, input_shape, past_key_values
             y = paddle.to_tensor(-1.7005809656952787e38, dtype="float32")
             expanded_attn_mask = paddle.where(expanded_attn_mask, x, y)
         else:
-            expanded_attn_mask = paddle.where(expanded_attn_mask.to("bool"), 0.0, paddle.finfo(dtype).min).astype(
+            expanded_attn_mask = paddle.where(expanded_attn_mask.cast("bool"), 0.0, paddle.finfo(dtype).min).astype(
                 dtype
             )
         return expanded_attn_mask