fix loraga dp (#9719)

PaddlePaddle · Dec 30, 2024 · a6d01f1 · a6d01f1
1 parent 0c58a21
commit a6d01f1
Showing 1 changed file with 5 additions and 5 deletions.
diff --git a/paddlenlp/peft/lora/loraga_utils.py b/paddlenlp/peft/lora/loraga_utils.py
@@ -185,12 +185,12 @@ def get_module_gradient(
 
     # dp
     if dp_degree > 1:
-        if data_parallel_group.nranks > 1:
-            if is_fleet_init:
-                dist.all_reduce(gradient, op=dist.ReduceOp.SUM, group=data_parallel_group)
-            else:
-                dist.all_reduce(gradient, op=dist.ReduceOp.SUM)
+        if is_fleet_init:
+            dist.all_reduce(gradient, op=dist.ReduceOp.SUM, group=data_parallel_group)
             gradient /= data_parallel_group.nranks
+        else:
+            dist.all_reduce(gradient, op=dist.ReduceOp.SUM)
+            gradient /= dist.get_world_size()
     return gradient