【Hackathon 5th No.83】PaddleMIX ppdiffusers models模块功能升级同步HF (PaddlePa…

…ddle#322)

PaddleMIX ppdiffusers models模块功能升级同步HF


https://github.com/PaddlePaddle/community/blob/master/hackathon/hackathon_5th/%E3%80%90PaddlePaddle%20Hackathon%205th%E3%80%91%E5%BC%80%E6%BA%90%E8%B4%A1%E7%8C%AE%E4%B8%AA%E4%BA%BA%E6%8C%91%E6%88%98%E8%B5%9B%E5%A5%97%E4%BB%B6%E5%BC%80%E5%8F%91%E4%BB%BB%E5%8A%A1%E5%90%88%E9%9B%86.md#no83paddlemix-ppdiffusers-models%E6%A8%A1%E5%9D%97%E5%8A%9F%E8%83%BD%E5%8D%87%E7%BA%A7%E5%90%8C%E6%AD%A5hf

PaddlePaddle#259
PaddlePaddle#260
PaddlePaddle#262
PaddlePaddle#264
PaddlePaddle#265

升级 models
增加scale
增加get_processor
diffusers上LoRA类逐渐取消，改为使用lora_layer
部分测试误差增加，测试修改前代码同样有误差
增加tests/lora目录
flax模型paddle没有使用，没有迁移

test_attention_processor.py

![image](https://github.com/PaddlePaddle/PaddleMIX/assets/4617245/3a7cdfcf-6313-42f7-8bb3-9fd7d46b36cf)

test_models_unet_2d.py

![image](https://github.com/PaddlePaddle/PaddleMIX/assets/4617245/51f9ec37-e0f2-4010-b621-966d596bf451)

test_lora_layers.py

![image](https://github.com/PaddlePaddle/PaddleMIX/assets/4617245/d4761fde-1eda-4329-8f94-79d4ed6535b3)

test_models_unet_motion.py

![image](https://github.com/PaddlePaddle/PaddleMIX/assets/4617245/f3ff8f8c-89f2-46e2-a733-4b6c442e36b5)

ppdiffusers/ppdiffusers/__init__.py

@@ -63,6 +63,7 @@
         LVDMAutoencoderKL,
         LVDMUNet3DModel,
         ModelMixin,
+        MotionAdapter,
         MultiAdapter,
         PriorTransformer,
         T2IAdapter,
@@ -72,6 +73,7 @@
         UNet2DConditionModel,
         UNet2DModel,
         UNet3DConditionModel,
+        UNetMotionModel,
         VQModel,
     )
     from .optimization import (

ppdiffusers/ppdiffusers/models/__init__.py

@@ -37,6 +37,7 @@
     from .unet_2d import UNet2DModel
     from .unet_2d_condition import UNet2DConditionModel
     from .unet_3d_condition import UNet3DConditionModel
+    from .unet_motion_model import MotionAdapter, UNetMotionModel
     from .vq_model import VQModel
 
 try:

ppdiffusers/ppdiffusers/models/activations.py

@@ -13,14 +13,94 @@
 # limitations under the License.
 
 import paddle.nn as nn
+import paddle.nn.functional as F
 
+from ..utils import USE_PEFT_BACKEND
+from .lora import LoRACompatibleLinear
 
-def get_activation(act_fn):
-    if act_fn in ["swish", "silu"]:
-        return nn.Silu()
-    elif act_fn == "mish":
-        return nn.Mish()
-    elif act_fn == "gelu":
-        return nn.GELU()
+ACTIVATION_FUNCTIONS = {
+    "swish": nn.Silu(),
+    "silu": nn.Silu(),
+    "mish": nn.Mish(),
+    "gelu": nn.GELU(),
+    "relu": nn.ReLU(),
+}
+
+
+def get_activation(act_fn: str) -> nn.Layer:
+    """Helper function to get activation function from string.
+
+    Args:
+        act_fn (str): Name of activation function.
+
+    Returns:
+        nn.Layer: Activation function.
+    """
+
+    act_fn = act_fn.lower()
+    if act_fn in ACTIVATION_FUNCTIONS:
+        return ACTIVATION_FUNCTIONS[act_fn]
     else:
         raise ValueError(f"Unsupported activation function: {act_fn}")
+
+
+class GELU(nn.Layer):
+    r"""
+    GELU activation function with tanh approximation support with `approximate="tanh"`.
+
+    Parameters:
+        dim_in (`int`): The number of channels in the input.
+        dim_out (`int`): The number of channels in the output.
+        approximate (`str`, *optional*, defaults to `"none"`): If `"tanh"`, use tanh approximation.
+    """
+
+    def __init__(self, dim_in: int, dim_out: int, approximate: str = "none"):
+        super().__init__()
+        self.proj = LoRACompatibleLinear(dim_in, dim_out)
+        self.approximate = approximate
+        self.approximate_bool = approximate == "tanh"
+
+    def forward(self, hidden_states):
+        hidden_states = self.proj(hidden_states)
+        hidden_states = F.gelu(hidden_states, approximate=self.approximate_bool)
+        return hidden_states
+
+
+class GEGLU(nn.Layer):
+    r"""
+    A [variant](https://arxiv.org/abs/2002.05202) of the gated linear unit activation function.
+
+    Parameters:
+        dim_in (`int`): The number of channels in the input.
+        dim_out (`int`): The number of channels in the output.
+    """
+
+    def __init__(self, dim_in: int, dim_out: int):
+        super().__init__()
+        linear_cls = LoRACompatibleLinear if not USE_PEFT_BACKEND else nn.Linear
+
+        self.proj = linear_cls(dim_in, dim_out * 2)
+
+    def forward(self, hidden_states, scale: float = 1.0):
+        args = () if USE_PEFT_BACKEND else (scale,)
+        hidden_states, gate = self.proj(hidden_states, *args).chunk(2, axis=-1)
+        return hidden_states * F.gelu(gate)
+
+
+class ApproximateGELU(nn.Layer):
+    r"""
+    The approximate form of the Gaussian Error Linear Unit (GELU). For more details, see section 2 of this
+    [paper](https://arxiv.org/abs/1606.08415).
+
+    Parameters:
+        dim_in (`int`): The number of channels in the input.
+        dim_out (`int`): The number of channels in the output.
+    """
+
+    def __init__(self, dim_in: int, dim_out: int):
+        super().__init__()
+        self.proj = nn.Linear(dim_in, dim_out)
+
+    def forward(self, x):
+        x = self.proj(x)
+        return x * F.sigmoid(1.702 * x)