fix missing modality

README.md

@@ -16,3 +16,4 @@ mamba activate ai4eo
 ---
 
 - [Model Fusion for Building Type Classification from Aerial and Street View Images](https://www.mdpi.com/2072-4292/11/11/1259#)
+- [Multi-modal fusion of satellite and street-view images for urban village classification based on a dual-branch deep neural network](https://www.sciencedirect.com/science/article/pii/S0303243422001209)

config/base.yaml

@@ -1,31 +1,36 @@
 ---
-seed: 1984
+seed: 2024
 
 num_workers: 4
-experiment_name: "2024-04-07"
+experiment_name: "2024-07-09-missing-modality"
 
 dataset:
-  val_split: 0.1
+  n_splits: 10
+  fold_th: 3
   train_dir: ~/publicWorkspace/data/building-age-dataset/train/data
   test_dir: ~/publicWorkspace/data/building-age-dataset/test/data
   train_csv: ~/publicWorkspace/data/building-age-dataset/train/train-set.csv
   test_csv: ~/publicWorkspace/data/building-age-dataset/test/test-set.csv
 
 model:
-  encoder_name: efficientnet_b2
+  encoder_name: tf_efficientnetv2_b3
   num_classes: 7
 
 optimizer:
   type: timm.optim.AdamP
-  lr: 0.0005
+  lr: 0.00025
+  weight_decay: 0.00001
 
 scheduler:
-  type: torch.optim.lr_scheduler.CosineAnnealingWarmRestarts
-  T_0: 10
-  T_mult: 2
+  type: torch.optim.lr_scheduler.ReduceLROnPlateau
+  mode: min
+  factor: 0.5
+  patience: 5
+  threshold: 0.00005
+  verbose: True
 
 trainer:
-  devices: 1
+  devices: [0]
   accelerator: "cuda"
   max_epochs: 50
   gradient_clip_val: 5.0
@@ -36,7 +41,7 @@ train_parameters:
   batch_size: 3
 
 val_parameters:
-  batch_size: 3
+  batch_size: 4
 
 output_root_dir: experiments
 image_size: 512

config/base_full_modality_1.yaml

@@ -17,23 +17,24 @@ model:
   encoder_name: tf_efficientnetv2_s
   num_classes: 7
 
+loss:
+  classification:
+    type: src.models.FocalLossLabelSmoothing
+
 optimizer:
   type: timm.optim.AdamW
-  lr: 0.0005
+  lr: 0.0002
   weight_decay: 0.001
 
 scheduler:
-  type: torch.optim.lr_scheduler.ReduceLROnPlateau
-  mode: min
-  factor: 0.5
-  patience: 10
-  threshold: 0.00005
-  verbose: True
+  type: torch.optim.lr_scheduler.CosineAnnealingLR
+  T_max: 30
+  eta_min: 0.00005
 
 trainer:
   devices: [0]
   accelerator: "cuda"
-  max_epochs: 50
+  max_epochs: 30
   gradient_clip_val: 5.0
   accumulate_grad_batches: 8
   resume_from_checkpoint:

config/base_full_modality_2.yaml

@@ -14,26 +14,27 @@ dataset:
 
 model:
   type: src.models.MultiModalNetFullModalityFeatureFusion
-  encoder_name: nextvit_base
+  encoder_name: mobilevitv2_150
   num_classes: 7
 
+loss:
+  classification:
+    type: src.models.FocalLossLabelSmoothing
+
 optimizer:
   type: timm.optim.AdamW
-  lr: 0.0005
+  lr: 0.0002
   weight_decay: 0.001
 
 scheduler:
-  type: torch.optim.lr_scheduler.ReduceLROnPlateau
-  mode: min
-  factor: 0.5
-  patience: 10
-  threshold: 0.00005
-  verbose: True
+  type: torch.optim.lr_scheduler.CosineAnnealingLR
+  T_max: 30
+  eta_min: 0.00005
 
 trainer:
   devices: [1]
   accelerator: "cuda"
-  max_epochs: 50
+  max_epochs: 30
   gradient_clip_val: 5.0
   accumulate_grad_batches: 8
   resume_from_checkpoint:

config/base_full_modality_3.yaml

@@ -17,23 +17,24 @@ model:
   encoder_name: tf_efficientnetv2_b3
   num_classes: 7
 
+loss:
+  classification:
+    type: src.models.FocalLossLabelSmoothing
+
 optimizer:
   type: timm.optim.AdamW
-  lr: 0.0005
+  lr: 0.0002
   weight_decay: 0.001
 
 scheduler:
-  type: torch.optim.lr_scheduler.ReduceLROnPlateau
-  mode: min
-  factor: 0.5
-  patience: 10
-  threshold: 0.00005
-  verbose: True
+  type: torch.optim.lr_scheduler.CosineAnnealingLR
+  T_max: 30
+  eta_min: 0.00005
 
 trainer:
   devices: [0]
   accelerator: "cuda"
-  max_epochs: 50
+  max_epochs: 30
   gradient_clip_val: 5.0
   accumulate_grad_batches: 8
   resume_from_checkpoint:

config/base_full_modality_4.yaml

@@ -0,0 +1,49 @@
+---
+seed: 2024
+
+num_workers: 4
+experiment_name: "2024-07-08-f2"
+
+dataset:
+  n_splits: 10
+  fold_th: 2
+  train_dir: ~/publicWorkspace/data/building-age-dataset/train/data
+  test_dir: ~/publicWorkspace/data/building-age-dataset/test/data
+  train_csv: ~/publicWorkspace/data/building-age-dataset/train/train-set.csv
+  test_csv: ~/publicWorkspace/data/building-age-dataset/test/test-set.csv
+
+model:
+  type: src.models.MultiModalNetFullModalityGeometricFusion
+  encoder_name: tf_efficientnetv2_b3
+  num_classes: 7
+
+loss:
+  classification:
+    type: src.models.FocalLossLabelSmoothing
+
+optimizer:
+  type: timm.optim.AdamW
+  lr: 0.0002
+  weight_decay: 0.001
+
+scheduler:
+  type: torch.optim.lr_scheduler.CosineAnnealingLR
+  T_max: 30
+  eta_min: 0.00005
+
+trainer:
+  devices: [0]
+  accelerator: "cuda"
+  max_epochs: 30
+  gradient_clip_val: 5.0
+  accumulate_grad_batches: 8
+  resume_from_checkpoint:
+
+train_parameters:
+  batch_size: 6
+
+val_parameters:
+  batch_size: 6
+
+output_root_dir: experiments
+image_size: 512

config/base_full_modality_input_dropout_1.yaml

@@ -0,0 +1,49 @@
+---
+seed: 2024
+
+num_workers: 4
+experiment_name: "2024-07-08-input-dropout-f5"
+
+dataset:
+  n_splits: 10
+  fold_th: 5
+  train_dir: ~/publicWorkspace/data/building-age-dataset/train/data
+  test_dir: ~/publicWorkspace/data/building-age-dataset/test/data
+  train_csv: ~/publicWorkspace/data/building-age-dataset/train/train-set.csv
+  test_csv: ~/publicWorkspace/data/building-age-dataset/test/test-set.csv
+
+model:
+  type: src.models.MultiModalNetFullModalityGeometricFusion
+  encoder_name: tf_efficientnetv2_b3
+  num_classes: 7
+
+loss:
+  classification:
+    type: src.models.FocalLossLabelSmoothing
+
+optimizer:
+  type: timm.optim.AdamW
+  lr: 0.0002
+  weight_decay: 0.001
+
+scheduler:
+  type: torch.optim.lr_scheduler.CosineAnnealingLR
+  T_max: 30
+  eta_min: 0.00005
+
+trainer:
+  devices: [0]
+  accelerator: "cuda"
+  max_epochs: 30
+  gradient_clip_val: 5.0
+  accumulate_grad_batches: 8
+  resume_from_checkpoint:
+
+train_parameters:
+  batch_size: 6
+
+val_parameters:
+  batch_size: 6
+
+output_root_dir: experiments
+image_size: 512

config/base_full_modality_input_dropout_2.yaml

@@ -0,0 +1,49 @@
+---
+seed: 2024
+
+num_workers: 4
+experiment_name: "2024-07-08-input-dropout-f6"
+
+dataset:
+  n_splits: 10
+  fold_th: 6
+  train_dir: ~/publicWorkspace/data/building-age-dataset/train/data
+  test_dir: ~/publicWorkspace/data/building-age-dataset/test/data
+  train_csv: ~/publicWorkspace/data/building-age-dataset/train/train-set.csv
+  test_csv: ~/publicWorkspace/data/building-age-dataset/test/test-set.csv
+
+model:
+  type: src.models.MultiModalNetFullModalityFeatureFusion
+  encoder_name: fastvit_sa24
+  num_classes: 7
+
+loss:
+  classification:
+    type: src.models.FocalLossLabelSmoothing
+
+optimizer:
+  type: timm.optim.AdamW
+  lr: 0.0002
+  weight_decay: 0.001
+
+scheduler:
+  type: torch.optim.lr_scheduler.CosineAnnealingLR
+  T_max: 30
+  eta_min: 0.00005
+
+trainer:
+  devices: [0]
+  accelerator: "cuda"
+  max_epochs: 30
+  gradient_clip_val: 5.0
+  accumulate_grad_batches: 8
+  resume_from_checkpoint:
+
+train_parameters:
+  batch_size: 4
+
+val_parameters:
+  batch_size: 4
+
+output_root_dir: experiments
+image_size: 512

config/base_full_modality_input_dropout_3.yaml

@@ -0,0 +1,49 @@
+---
+seed: 2024
+
+num_workers: 4
+experiment_name: "2024-07-08-input-dropout-f6"
+
+dataset:
+  n_splits: 10
+  fold_th: 6
+  train_dir: ~/publicWorkspace/data/building-age-dataset/train/data
+  test_dir: ~/publicWorkspace/data/building-age-dataset/test/data
+  train_csv: ~/publicWorkspace/data/building-age-dataset/train/train-set.csv
+  test_csv: ~/publicWorkspace/data/building-age-dataset/test/test-set.csv
+
+model:
+  type: src.models.MultiModalNetFullModalityFeatureFusion
+  encoder_name: mobilevitv2_150
+  num_classes: 7
+
+loss:
+  classification:
+    type: src.models.FocalLossLabelSmoothing
+
+optimizer:
+  type: timm.optim.AdamW
+  lr: 0.0002
+  weight_decay: 0.001
+
+scheduler:
+  type: torch.optim.lr_scheduler.CosineAnnealingLR
+  T_max: 30
+  eta_min: 0.00005
+
+trainer:
+  devices: [0]
+  accelerator: "cuda"
+  max_epochs: 30
+  gradient_clip_val: 5.0
+  accumulate_grad_batches: 10
+  resume_from_checkpoint:
+
+train_parameters:
+  batch_size: 4
+
+val_parameters:
+  batch_size: 4
+
+output_root_dir: experiments
+image_size: 512