Merge branch 'main' into contributors-readme-action-DsWKcGzj5w

.github/workflows/releasing.yml

@@ -38,7 +38,7 @@ jobs:
 
             - name: Publish distribution 📦 to PyPI
               if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
-              uses: pypa/gh-action-pypi-publish@v1.11.0
+              uses: pypa/gh-action-pypi-publish@v1.12.2
               with:
                   user: __token__
                   password: ${{ secrets.pypi_password }}

docs/apidocs_model.md

@@ -30,6 +30,9 @@
 ::: pytorch_tabular.models.TabTransformerConfig
     options:
             heading_level: 3
+::: pytorch_tabular.models.StackingModelConfig
+    options:
+            heading_level: 3
 ::: pytorch_tabular.config.ModelConfig
     options:
             heading_level: 3
@@ -66,7 +69,9 @@
 ::: pytorch_tabular.models.TabTransformerModel
     options:
             heading_level: 3
-
+::: pytorch_tabular.models.StackingModel
+    options:
+            heading_level: 3
 ## Base Model Class
 ::: pytorch_tabular.models.BaseModel
     options:

docs/models.md

@@ -253,6 +253,30 @@ All the parameters have beet set to recommended values from the paper. Let's loo
 **For a complete list of parameters refer to the API Docs**
 [pytorch_tabular.models.DANetConfig][]
 
+## Model Stacking
+
+Model stacking is an ensemble learning technique that combines multiple base models to create a more powerful predictive model. Each base model processes the input features independently, and their outputs are concatenated before making the final prediction. This allows the model to leverage different learning patterns captured by each backbone architecture. You can use it by choosing `StackingModelConfig`. 
+
+The following diagram shows the concept of model stacking in PyTorch Tabular.
+![Model Stacking](imgs/model_stacking_concept.png)
+
+The following model architectures are supported for stacking:
+- Category Embedding Model
+- TabNet Model
+- FTTransformer Model
+- Gated Additive Tree Ensemble Model
+- DANet Model
+- AutoInt Model
+- GANDALF Model
+- Node Model
+
+All the parameters have been set to provide flexibility while maintaining ease of use. Let's look at them:
+
+- `model_configs`: List[ModelConfig]: List of configurations for each base model. Each config should be a valid PyTorch Tabular model config (e.g., NodeConfig, GANDALFConfig)
+
+**For a complete list of parameters refer to the API Docs**
+[pytorch_tabular.models.StackingModelConfig][]
+
 ## Implementing New Architectures
 
 PyTorch Tabular is very easy to extend and infinitely customizable. All the models that have been implemented in PyTorch Tabular inherits an Abstract Class `BaseModel` which is in fact a PyTorchLightning Model.

mkdocs.yml

@@ -24,6 +24,7 @@ nav:
                 - SHAP, Deep LIFT and so on through Captum Integration: "tutorials/14-Explainability.ipynb"
           - Custom PyTorch Models:
                 - Implementing New Supervised Architectures: "tutorials/04-Implementing New Architectures.ipynb"
+          - Model Stacking: "tutorials/16-Model Stacking.ipynb"
           - Other Features:
                 - Using Neural Categorical Embeddings in Scikit-Learn Workflows: "tutorials/03-Neural Embedding in Scikit-Learn Workflows.ipynb"
                 - Self-Supervised Learning using Denoising Autoencoders: "tutorials/08-Self-Supervised Learning-DAE.ipynb"

requirements/base.txt

@@ -6,7 +6,7 @@ pytorch-lightning >=2.0.0, <2.5.0
 omegaconf >=2.3.0
 torchmetrics >=0.10.0, <1.6.0
 tensorboard >2.2.0, !=2.5.0
-protobuf >=3.20.0, <5.29.0
+protobuf >=3.20.0, <5.30.0
 pytorch-tabnet ==4.1
 PyYAML >=5.4, <6.1.0
 # importlib-metadata <1,>=0.12

src/pytorch_tabular/categorical_encoders.py

@@ -68,7 +68,7 @@ def transform(self, X):
             X_encoded[col] = X_encoded[col].fillna(NAN_CATEGORY).map(mapping["value"])
 
             if self.handle_unseen == "impute":
-                X_encoded[col].fillna(self._imputed, inplace=True)
+                X_encoded[col] = X_encoded[col].fillna(self._imputed)
             elif self.handle_unseen == "error":
                 if np.unique(X_encoded[col]).shape[0] > mapping.shape[0]:
                     raise ValueError(f"Unseen categories found in `{col}` column.")

src/pytorch_tabular/config/config.py

@@ -96,6 +96,8 @@ class DataConfig:
         handle_missing_values (bool): Whether to handle missing values in categorical columns as
                 unknown
 
+        pickle_protocol (int): pickle protocol version passed to `torch.save` for dataset caching to disk
+
         dataloader_kwargs (Dict[str, Any]): Additional kwargs to be passed to PyTorch DataLoader. See
                 https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader
 
@@ -179,6 +181,11 @@ class DataConfig:
         metadata={"help": "Whether or not to handle missing values in categorical columns as unknown"},
     )
 
+    pickle_protocol: int = field(
+        default=2,
+        metadata={"help": "pickle protocol version passed to `torch.save` for dataset caching to disk"},
+    )
+
     dataloader_kwargs: Dict[str, Any] = field(
         default_factory=dict,
         metadata={"help": "Additional kwargs to be passed to PyTorch DataLoader."},
@@ -351,8 +358,8 @@ class TrainerConfig:
 
         progress_bar (str): Progress bar type. Can be one of: `none`, `simple`, `rich`. Defaults to `rich`.
 
-        precision (int): Precision of the model. Can be one of: `32`, `16`, `64`. Defaults to `32`..
-                Choices are: [`32`,`16`,`64`].
+        precision (str): Precision of the model. Defaults to `32`. See
+                https://lightning.ai/docs/pytorch/stable/common/trainer.html#precision
 
         seed (int): Seed for random number generators. Defaults to 42
 
@@ -536,11 +543,10 @@ class TrainerConfig:
         default="rich",
         metadata={"help": "Progress bar type. Can be one of: `none`, `simple`, `rich`. Defaults to `rich`."},
     )
-    precision: int = field(
-        default=32,
+    precision: str = field(
+        default="32",
         metadata={
-            "help": "Precision of the model. Can be one of: `32`, `16`, `64`. Defaults to `32`.",
-            "choices": [32, 16, 64],
+            "help": "Precision of the model. Defaults to `32`.",
         },
     )
     seed: int = field(

src/pytorch_tabular/feature_extractor.py

@@ -79,15 +79,21 @@ def transform(self, X: pd.DataFrame, y=None) -> pd.DataFrame:
                 if k in ret_value.keys():
                     logits_predictions[k].append(ret_value[k].detach().cpu())
 
+        logits_dfs = []
         for k, v in logits_predictions.items():
             v = torch.cat(v, dim=0).numpy()
             if v.ndim == 1:
                 v = v.reshape(-1, 1)
-            for i in range(v.shape[-1]):
-                if v.shape[-1] > 1:
-                    X_encoded[f"{k}_{i}"] = v[:, i]
-                else:
-                    X_encoded[f"{k}"] = v[:, i]
+            if v.shape[-1] > 1:
+                temp_df = pd.DataFrame({f"{k}_{i}": v[:, i] for i in range(v.shape[-1])})
+            else:
+                temp_df = pd.DataFrame({f"{k}": v[:, 0]})
+
+            # Append the temp DataFrame to the list
+            logits_dfs.append(temp_df)
+
+        preds = pd.concat(logits_dfs, axis=1)
+        X_encoded = pd.concat([X_encoded, preds], axis=1)
 
         if self.drop_original:
             X_encoded.drop(columns=orig_features, inplace=True)

src/pytorch_tabular/models/__init__.py

@@ -19,6 +19,7 @@
 from .gate import GatedAdditiveTreeEnsembleConfig, GatedAdditiveTreeEnsembleModel
 from .mixture_density import MDNConfig, MDNModel
 from .node import NodeConfig, NODEModel
+from .stacking import StackingModel, StackingModelConfig
 from .tab_transformer import TabTransformerConfig, TabTransformerModel
 from .tabnet import TabNetModel, TabNetModelConfig
 
@@ -45,6 +46,8 @@
     "GANDALFBackbone",
     "DANetConfig",
     "DANetModel",
+    "StackingModel",
+    "StackingModelConfig",
     "category_embedding",
     "node",
     "mixture_density",
@@ -55,4 +58,5 @@
     "gate",
     "gandalf",
     "danet",
+    "stacking",
 ]

src/pytorch_tabular/models/base_model.py

@@ -244,13 +244,14 @@ def _setup_metrics(self):
         else:
             self.metrics = self.custom_metrics
 
-    def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tensor:
+    def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str, sync_dist: bool = False) -> torch.Tensor:
         """Calculates the loss for the model.
 
         Args:
             output (Dict): The output dictionary from the model
             y (torch.Tensor): The target tensor
             tag (str): The tag to use for logging
+            sync_dist (bool): enable distributed sync of logs
 
         Returns:
             torch.Tensor: The loss value
@@ -270,6 +271,7 @@ def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tenso
                     on_step=False,
                     logger=True,
                     prog_bar=False,
+                    sync_dist=sync_dist,
                 )
         if self.hparams.task == "regression":
             computed_loss = reg_loss
@@ -284,6 +286,7 @@ def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tenso
                         on_step=False,
                         logger=True,
                         prog_bar=False,
+                        sync_dist=sync_dist,
                     )
         else:
             # TODO loss fails with batch size of 1?
@@ -301,6 +304,7 @@ def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tenso
                         on_step=False,
                         logger=True,
                         prog_bar=False,
+                        sync_dist=sync_dist,
                     )
                 start_index = end_index
         self.log(
@@ -311,10 +315,13 @@ def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tenso
             # on_step=False,
             logger=True,
             prog_bar=True,
+            sync_dist=sync_dist,
         )
         return computed_loss
 
-    def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> List[torch.Tensor]:
+    def calculate_metrics(
+        self, y: torch.Tensor, y_hat: torch.Tensor, tag: str, sync_dist: bool = False
+    ) -> List[torch.Tensor]:
         """Calculates the metrics for the model.
 
         Args:
@@ -324,6 +331,8 @@ def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> L
 
             tag (str): The tag to use for logging
 
+            sync_dist (bool): enable distributed sync of logs
+
         Returns:
             List[torch.Tensor]: The list of metric values
 
@@ -356,6 +365,7 @@ def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> L
                             on_step=False,
                             logger=True,
                             prog_bar=False,
+                            sync_dist=sync_dist,
                         )
                     _metrics.append(_metric)
                 avg_metric = torch.stack(_metrics, dim=0).sum()
@@ -379,6 +389,7 @@ def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> L
                             on_step=False,
                             logger=True,
                             prog_bar=False,
+                            sync_dist=sync_dist,
                         )
                     _metrics.append(_metric)
                     start_index = end_index
@@ -391,6 +402,7 @@ def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> L
                 on_step=False,
                 logger=True,
                 prog_bar=True,
+                sync_dist=sync_dist,
             )
         return metrics
 
@@ -523,19 +535,19 @@ def validation_step(self, batch, batch_idx):
             # fetched from the batch
             y = batch["target"] if y is None else y
             y_hat = output["logits"]
-            self.calculate_loss(output, y, tag="valid")
-            self.calculate_metrics(y, y_hat, tag="valid")
+            self.calculate_loss(output, y, tag="valid", sync_dist=True)
+            self.calculate_metrics(y, y_hat, tag="valid", sync_dist=True)
         return y_hat, y
 
     def test_step(self, batch, batch_idx):
         with torch.no_grad():
             output, y = self.forward_pass(batch)
-            # y is not None for SSL task.Rest of the tasks target is
+            # y is not None for SSL task. Rest of the tasks target is
             # fetched from the batch
             y = batch["target"] if y is None else y
             y_hat = output["logits"]
-            self.calculate_loss(output, y, tag="test")
-            self.calculate_metrics(y, y_hat, tag="test")
+            self.calculate_loss(output, y, tag="test", sync_dist=True)
+            self.calculate_metrics(y, y_hat, tag="test", sync_dist=True)
         return y_hat, y
 
     def configure_optimizers(self):

src/pytorch_tabular/models/stacking/__init__.py

@@ -0,0 +1,4 @@
+from .config import StackingModelConfig
+from .stacking_model import StackingBackbone, StackingModel
+
+__all__ = ["StackingModel", "StackingModelConfig", "StackingBackbone"]

src/pytorch_tabular/models/stacking/config.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass, field
+
+from pytorch_tabular.config import ModelConfig
+
+
+@dataclass
+class StackingModelConfig(ModelConfig):
+    """StackingModelConfig is a configuration class for the StackingModel. It is used to stack multiple models
+    together. Now, CategoryEmbeddingModel, TabNetModel, FTTransformerModel, GatedAdditiveTreeEnsembleModel, DANetModel,
+    AutoIntModel, GANDALFModel, NodeModel are supported.
+
+    Args:
+        model_configs (list[ModelConfig]): List of model configs to stack.
+
+    """
+
+    model_configs: list = field(default_factory=list, metadata={"help": "List of model configs to stack"})
+    _module_src: str = field(default="models.stacking")
+    _model_name: str = field(default="StackingModel")
+    _backbone_name: str = field(default="StackingBackbone")
+    _config_name: str = field(default="StackingConfig")
+
+
+# if __name__ == "__main__":
+#    from pytorch_tabular.utils import generate_doc_dataclass
+#    print(generate_doc_dataclass(StackingModelConfig))