[LLM] fix lora target modules on llama

PaddlePaddle · May 7, 2024 · 252fdc4 · 252fdc4
1 parent fdcabf8
commit 252fdc4
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/llm/utils.py b/llm/utils.py
@@ -125,9 +125,10 @@ def get_lora_target_modules(model):
             ".*v_proj.*",
             ".*k_proj.*",
             ".*o_proj.*",
-            ".*gate_proj.*",
+            ".*qkv_proj.*" ".*gate_proj.*",
             ".*down_proj.*",
             ".*up_proj.*",
+            ".*gate_up_fused_proj.*",
         ]
     elif model.base_model_prefix == "opt":
         target_modules = [