[Feature] Add MaskFormer

configs/_base_/ade20k.yml

@@ -26,7 +26,6 @@ val_dataset:
     - type: Normalize
   mode: val
 
-
 optimizer:
   type: SGD
   momentum: 0.9

configs/maskformer/README.md

@@ -0,0 +1,17 @@
+# Per-Pixel Classification is Not All You Need for Semantic Segmentation
+
+## Reference
+
+> Cheng, Bowen, Alex Schwing, and Alexander Kirillov. "Per-pixel classification is not all you need for semantic segmentation." Advances in Neural Information Processing Systems 34 (2021): 17864-17875.
+
+## Performance
+
+### ADE20k
+
+| Model | Backbone | Resolution | Training Iters | mIoU | mIoU (flip) | mIoU (ms+flip) | Links |
+|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
+|Maskformer|SwinTransformer|512x512|160000|47.6|-|-|[model](https://bj.bcebos.com/paddleseg/dygraph/ade20k/maskformer_ade20k_swin_tiny/model.pdparams) \| [log](https://bj.bcebos.com/paddleseg/dygraph/ade20k/maskformer_ade20k_swin_tiny/train.log) \| [vdl](https://www.paddlepaddle.org.cn/paddle/visualdl/service/app/scalar?id=e59773eaad87f677837add5ff110441e)|
+
+* Maskformer support different backbone including tiny, small, base and large. Due to long training time, the accuracy result is not provided.
+
+* Maskformer-Base and Maskformer-Large need to be evaled with multi-scale and flip by default.

configs/maskformer/maskformer_ade20k_swin_base.yml

@@ -0,0 +1,73 @@
+batch_size: 4
+iters: 160000
+
+train_dataset:
+  type: ADE20K
+  dataset_root: data/ADEChallengeData2016/
+  transforms: 
+    - type: ResizeByShort
+      short_size: [320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1088, 1152, 1216, 1280, 1344]
+      max_size: 2560
+    - type: RandomPaddingCrop
+      crop_size: [640, 640] 
+    - type: RandomDistort
+      brightness_range: 0.125
+      brightness_prob: 1.0
+      contrast_range: 0.5
+      contrast_prob: 1.0
+      saturation_range: 0.5
+      saturation_prob: 1.0
+      hue_range: 18
+      hue_prob: 1.0
+    - type: RandomHorizontalFlip
+  to_mask: True
+  size_divisibility: 640
+  normalize: True
+
+
+val_dataset:
+  type: ADE20K
+  dataset_root: data/ADEChallengeData2016/
+  transforms:
+    - type: ResizeByShort
+      short_size: 512
+  mode: val
+  to_mask: True
+  normalize: True
+
+model:
+  type: MaskFormer
+  num_classes: 150
+  backbone:
+    type: SwinTransformer_base_patch4_window7_384_maskformer
+  pretrained: https://bj.bcebos.com/paddleseg/paddleseg/dygraph/ade20k/maskformer_ade20k_swin_base/pretrain/model.pdparams
+
+optimizer:
+  type: AdamW
+  backbone_lr_mult: 1.0
+  weight_decay: 0.01
+
+gradient_clipper:
+  enabled: True
+  clip_value: 0.01
+
+lr_scheduler:
+  type: PolynomialDecay
+  warmup_iters: 1500
+  warmup_start_lr: 6.0e-11
+  learning_rate: 6.0e-05
+  end_lr: 0
+  power: 0.9
+
+loss:
+  types:
+    - type: MaskFormerLoss
+      num_classes: 150
+      eos_coef: 0.1
+  coef: [1]
+
+export:
+  transforms:
+    - type: Normalize
+      mean: [0.485, 0.456, 0.406]
+      std: [0.229, 0.224, 0.225]

configs/maskformer/maskformer_ade20k_swin_large.yml

@@ -0,0 +1,73 @@
+batch_size: 4
+iters: 160000
+
+train_dataset:
+  type: ADE20K
+  dataset_root: data/ADEChallengeData2016/
+  transforms: 
+    - type: ResizeByShort
+      short_size: [320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1088, 1152, 1216, 1280, 1344]
+      max_size: 2560
+    - type: RandomPaddingCrop
+      crop_size: [640, 640] 
+    - type: RandomDistort
+      brightness_range: 0.125
+      brightness_prob: 1.0
+      contrast_range: 0.5
+      contrast_prob: 1.0
+      saturation_range: 0.5
+      saturation_prob: 1.0
+      hue_range: 18
+      hue_prob: 1.0
+    - type: RandomHorizontalFlip
+  to_mask: True
+  size_divisibility: 640
+  normalize: True
+
+
+val_dataset:
+  type: ADE20K
+  dataset_root: data/ADEChallengeData2016/
+  transforms:
+    - type: ResizeByShort
+      short_size: 512
+  mode: val
+  to_mask: True
+  normalize: True
+
+model:
+  type: MaskFormer
+  num_classes: 150
+  backbone:
+    type: SwinTransformer_large_patch4_window7_384_maskformer
+  pretrained: https://bj.bcebos.com/paddleseg/paddleseg/dygraph/ade20k/maskformer_ade20k_swin_large/pretrain/model.pdparams
+
+optimizer:
+  type: AdamW
+  backbone_lr_mult: 1.0
+  weight_decay: 0.01
+
+gradient_clipper:
+  enabled: True
+  clip_value: 0.01
+
+lr_scheduler:
+  type: PolynomialDecay
+  warmup_iters: 1500
+  warmup_start_lr: 6.0e-11
+  learning_rate: 6.0e-05
+  end_lr: 0
+  power: 0.9
+
+loss:
+  types:
+    - type: MaskFormerLoss
+      num_classes: 150
+      eos_coef: 0.1
+  coef: [1]
+
+export:
+  transforms:
+    - type: Normalize
+      mean: [0.485, 0.456, 0.406]
+      std: [0.229, 0.224, 0.225]

configs/maskformer/maskformer_ade20k_swin_small.yml

@@ -0,0 +1,73 @@
+batch_size: 4
+iters: 160000
+
+train_dataset:
+  type: ADE20K
+  dataset_root: data/ADEChallengeData2016/
+  transforms: 
+    - type: ResizeByShort
+      short_size: [256, 307, 358, 409, 460, 512, 563, 614, 665, 716, 768, 819, 870, 921, 972, 1024]
+      max_size: 2048
+    - type: RandomPaddingCrop
+      crop_size: [512, 512] 
+    - type: RandomDistort
+      brightness_range: 0.125
+      brightness_prob: 1.0
+      contrast_range: 0.5
+      contrast_prob: 1.0
+      saturation_range: 0.5
+      saturation_prob: 1.0
+      hue_range: 18
+      hue_prob: 1.0
+    - type: RandomHorizontalFlip
+  to_mask: True
+  size_divisibility: 512
+  normalize: True
+
+
+val_dataset:
+  type: ADE20K
+  dataset_root: data/ADEChallengeData2016/
+  transforms:
+    - type: ResizeByShort
+      short_size: 512
+  mode: val
+  to_mask: True
+  normalize: True
+
+model:
+  type: MaskFormer
+  num_classes: 150
+  backbone:
+    type: SwinTransformer_small_patch4_window7_224_maskformer
+  pretrained: https://bj.bcebos.com/paddleseg/paddleseg/dygraph/ade20k/maskformer_ade20k_swin_small/pretrain/model.pdparams
+
+optimizer:
+  type: AdamW
+  backbone_lr_mult: 1.0
+  weight_decay: 0.01
+
+gradient_clipper:
+  enabled: True
+  clip_value: 0.01
+
+lr_scheduler:
+  type: PolynomialDecay
+  warmup_iters: 1500
+  warmup_start_lr: 6.0e-11
+  learning_rate: 6.0e-05
+  end_lr: 0
+  power: 0.9
+
+loss:
+  types:
+    - type: MaskFormerLoss
+      num_classes: 150
+      eos_coef: 0.1
+  coef: [1]
+
+export:
+  transforms:
+    - type: Normalize
+      mean: [0.485, 0.456, 0.406]
+      std: [0.229, 0.224, 0.225]

configs/maskformer/maskformer_ade20k_swin_tiny.yml

@@ -0,0 +1,73 @@
+batch_size: 4
+iters: 160000
+
+train_dataset:
+  type: ADE20K
+  dataset_root: data/ADEChallengeData2016/
+  transforms: 
+    - type: ResizeByShort
+      short_size: [256, 307, 358, 409, 460, 512, 563, 614, 665, 716, 768, 819, 870, 921, 972, 1024]
+      max_size: 2048
+    - type: RandomPaddingCrop
+      crop_size: [512, 512] 
+    - type: RandomDistort
+      brightness_range: 0.125
+      brightness_prob: 1.0
+      contrast_range: 0.5
+      contrast_prob: 1.0
+      saturation_range: 0.5
+      saturation_prob: 1.0
+      hue_range: 18
+      hue_prob: 1.0
+    - type: RandomHorizontalFlip
+  to_mask: True
+  size_divisibility: 512
+  normalize: True
+
+
+val_dataset:
+  type: ADE20K
+  dataset_root: data/ADEChallengeData2016/
+  transforms:
+    - type: ResizeByShort
+      short_size: 512
+  mode: val
+  to_mask: True
+  normalize: True
+
+model:
+  type: MaskFormer
+  num_classes: 150
+  backbone:
+    type: SwinTransformer_tiny_patch4_window7_224_maskformer
+  pretrained: https://bj.bcebos.com/paddleseg/paddleseg/dygraph/ade20k/maskformer_ade20k_swin_tiny/pretrain/model.pdparams
+
+optimizer:
+  type: AdamW
+  backbone_lr_mult: 1.0
+  weight_decay: 0.01
+
+gradient_clipper:
+  enabled: True
+  clip_value: 0.01
+
+lr_scheduler:
+  type: PolynomialDecay
+  warmup_iters: 1500
+  warmup_start_lr: 6.0e-11
+  learning_rate: 6.0e-05
+  end_lr: 0
+  power: 0.9
+
+loss:
+  types:
+    - type: MaskFormerLoss
+      num_classes: 150
+      eos_coef: 0.1
+  coef: [1]
+
+export:
+  transforms:
+    - type: Normalize
+      mean: [0.485, 0.456, 0.406]
+      std: [0.229, 0.224, 0.225]

paddleseg/core/train.py

@@ -34,7 +34,7 @@ def check_logits_losses(logits_list, losses):
             .format(len_logits, len_losses))
 
 
-def loss_computation(logits_list, labels, edges, losses):
+def loss_computation(logits_list, labels, edges, losses, targets=None):
     check_logits_losses(logits_list, losses)
     loss_list = []
     for i in range(len(logits_list)):
@@ -52,7 +52,10 @@ def loss_computation(logits_list, labels, edges, losses):
             loss_list.append(coef_i *
                              loss_i(logits_list[0], logits_list[1].detach()))
         else:
-            loss_list.append(coef_i * loss_i(logits, labels))
+            if targets is not None:
+                loss_list.append(coef_i * loss_i(logits, targets))
+            else:
+                loss_list.append(coef_i * loss_i(logits, labels))
     return loss_list
 
 
@@ -176,6 +179,26 @@ def train(model,
             edges = None
             if 'edge' in data.keys():
                 edges = data['edge'].astype('int64')
+
+            targets = None
+            if "instances" in data.keys():
+                targets = []
+                # split targets in the batch
+                for target_per_image_idx in range(batch_size):
+                    gt_masks = data['instances']['gt_masks'][
+                        target_per_image_idx, ...]
+                    padded_masks = paddle.zeros(
+                        (gt_masks.shape[0], gt_masks.shape[-2],
+                         gt_masks.shape[-1]),
+                        dtype=gt_masks.dtype)
+                    padded_masks[:, :gt_masks.shape[1], :gt_masks.shape[
+                        2]] = gt_masks
+                    targets.append({
+                        "labels": data['instances']['gt_classes'][
+                            target_per_image_idx, ...],
+                        "masks": padded_masks
+                    })
+
             if hasattr(model, 'data_format') and model.data_format == 'NHWC':
                 images = images.transpose((0, 2, 3, 1))
 
@@ -193,7 +216,8 @@ def train(model,
                         logits_list=logits_list,
                         labels=labels,
                         edges=edges,
-                        losses=losses)
+                        losses=losses,
+                        targets=targets)
                     loss = sum(loss_list)
 
                 scaled = scaler.scale(loss)  # scale the loss
@@ -208,7 +232,8 @@ def train(model,
                     logits_list=logits_list,
                     labels=labels,
                     edges=edges,
-                    losses=losses)
+                    losses=losses,
+                    targets=targets)
                 loss = sum(loss_list)
                 loss.backward()
                 optimizer.step()