From f04a05c783c40a18a0277808cb54aca340cb4178 Mon Sep 17 00:00:00 2001 From: Soren Macbeth Date: Tue, 10 Dec 2024 02:53:06 +0100 Subject: [PATCH 1/4] Enable Support for Multi-GPU Training (#517) * Add `pickle_protocol` to `DataConfig` for passing it down to `torch.save()` when caching datasets to disk * Save the index of the original dataframe in `TabularDataset` so that it can be restored when accessing `TabularDataset.data` * Add `sync_dist=True` to all calls to self.log() in `validation_step()` and `test_step()` to distributed training * Fix `TrainerConfig.precision` to be a string and remove integer choices. Add a pointer to docs with possible options * Add `sync_dist` to SSL Models * Address `PerformanceWarning` related to `frame.insert()` * Only load best checkpoint on rank zero in distributed training * if logging with wandb, `unwatch` the model after training * address `FutureWarning` re inplace=True --- src/pytorch_tabular/categorical_encoders.py | 2 +- src/pytorch_tabular/config/config.py | 18 +++++++++----- src/pytorch_tabular/feature_extractor.py | 16 ++++++++---- src/pytorch_tabular/models/base_model.py | 26 ++++++++++++++------ src/pytorch_tabular/ssl_models/base_model.py | 12 ++++----- src/pytorch_tabular/ssl_models/dae/dae.py | 6 +++-- src/pytorch_tabular/tabular_datamodule.py | 13 +++++++--- src/pytorch_tabular/tabular_model.py | 4 +++ 8 files changed, 66 insertions(+), 31 deletions(-) diff --git a/src/pytorch_tabular/categorical_encoders.py b/src/pytorch_tabular/categorical_encoders.py index 35b771fe..b3d7a1ee 100644 --- a/src/pytorch_tabular/categorical_encoders.py +++ b/src/pytorch_tabular/categorical_encoders.py @@ -68,7 +68,7 @@ def transform(self, X): X_encoded[col] = X_encoded[col].fillna(NAN_CATEGORY).map(mapping["value"]) if self.handle_unseen == "impute": - X_encoded[col].fillna(self._imputed, inplace=True) + X_encoded[col] = X_encoded[col].fillna(self._imputed) elif self.handle_unseen == "error": if np.unique(X_encoded[col]).shape[0] > mapping.shape[0]: raise ValueError(f"Unseen categories found in `{col}` column.") diff --git a/src/pytorch_tabular/config/config.py b/src/pytorch_tabular/config/config.py index 2e410170..55aa500b 100644 --- a/src/pytorch_tabular/config/config.py +++ b/src/pytorch_tabular/config/config.py @@ -96,6 +96,8 @@ class DataConfig: handle_missing_values (bool): Whether to handle missing values in categorical columns as unknown + pickle_protocol (int): pickle protocol version passed to `torch.save` for dataset caching to disk + dataloader_kwargs (Dict[str, Any]): Additional kwargs to be passed to PyTorch DataLoader. See https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader @@ -179,6 +181,11 @@ class DataConfig: metadata={"help": "Whether or not to handle missing values in categorical columns as unknown"}, ) + pickle_protocol: int = field( + default=2, + metadata={"help": "pickle protocol version passed to `torch.save` for dataset caching to disk"}, + ) + dataloader_kwargs: Dict[str, Any] = field( default_factory=dict, metadata={"help": "Additional kwargs to be passed to PyTorch DataLoader."}, @@ -351,8 +358,8 @@ class TrainerConfig: progress_bar (str): Progress bar type. Can be one of: `none`, `simple`, `rich`. Defaults to `rich`. - precision (int): Precision of the model. Can be one of: `32`, `16`, `64`. Defaults to `32`.. - Choices are: [`32`,`16`,`64`]. + precision (str): Precision of the model. Defaults to `32`. See + https://lightning.ai/docs/pytorch/stable/common/trainer.html#precision seed (int): Seed for random number generators. Defaults to 42 @@ -536,11 +543,10 @@ class TrainerConfig: default="rich", metadata={"help": "Progress bar type. Can be one of: `none`, `simple`, `rich`. Defaults to `rich`."}, ) - precision: int = field( - default=32, + precision: str = field( + default="32", metadata={ - "help": "Precision of the model. Can be one of: `32`, `16`, `64`. Defaults to `32`.", - "choices": [32, 16, 64], + "help": "Precision of the model. Defaults to `32`.", }, ) seed: int = field( diff --git a/src/pytorch_tabular/feature_extractor.py b/src/pytorch_tabular/feature_extractor.py index 33f84f09..424a03e4 100644 --- a/src/pytorch_tabular/feature_extractor.py +++ b/src/pytorch_tabular/feature_extractor.py @@ -79,15 +79,21 @@ def transform(self, X: pd.DataFrame, y=None) -> pd.DataFrame: if k in ret_value.keys(): logits_predictions[k].append(ret_value[k].detach().cpu()) + logits_dfs = [] for k, v in logits_predictions.items(): v = torch.cat(v, dim=0).numpy() if v.ndim == 1: v = v.reshape(-1, 1) - for i in range(v.shape[-1]): - if v.shape[-1] > 1: - X_encoded[f"{k}_{i}"] = v[:, i] - else: - X_encoded[f"{k}"] = v[:, i] + if v.shape[-1] > 1: + temp_df = pd.DataFrame({f"{k}_{i}": v[:, i] for i in range(v.shape[-1])}) + else: + temp_df = pd.DataFrame({f"{k}": v[:, 0]}) + + # Append the temp DataFrame to the list + logits_dfs.append(temp_df) + + preds = pd.concat(logits_dfs, axis=1) + X_encoded = pd.concat([X_encoded, preds], axis=1) if self.drop_original: X_encoded.drop(columns=orig_features, inplace=True) diff --git a/src/pytorch_tabular/models/base_model.py b/src/pytorch_tabular/models/base_model.py index 12b5518c..824eb710 100644 --- a/src/pytorch_tabular/models/base_model.py +++ b/src/pytorch_tabular/models/base_model.py @@ -244,13 +244,14 @@ def _setup_metrics(self): else: self.metrics = self.custom_metrics - def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tensor: + def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str, sync_dist: bool = False) -> torch.Tensor: """Calculates the loss for the model. Args: output (Dict): The output dictionary from the model y (torch.Tensor): The target tensor tag (str): The tag to use for logging + sync_dist (bool): enable distributed sync of logs Returns: torch.Tensor: The loss value @@ -270,6 +271,7 @@ def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tenso on_step=False, logger=True, prog_bar=False, + sync_dist=sync_dist, ) if self.hparams.task == "regression": computed_loss = reg_loss @@ -284,6 +286,7 @@ def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tenso on_step=False, logger=True, prog_bar=False, + sync_dist=sync_dist, ) else: # TODO loss fails with batch size of 1? @@ -301,6 +304,7 @@ def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tenso on_step=False, logger=True, prog_bar=False, + sync_dist=sync_dist, ) start_index = end_index self.log( @@ -311,10 +315,13 @@ def calculate_loss(self, output: Dict, y: torch.Tensor, tag: str) -> torch.Tenso # on_step=False, logger=True, prog_bar=True, + sync_dist=sync_dist, ) return computed_loss - def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> List[torch.Tensor]: + def calculate_metrics( + self, y: torch.Tensor, y_hat: torch.Tensor, tag: str, sync_dist: bool = False + ) -> List[torch.Tensor]: """Calculates the metrics for the model. Args: @@ -324,6 +331,8 @@ def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> L tag (str): The tag to use for logging + sync_dist (bool): enable distributed sync of logs + Returns: List[torch.Tensor]: The list of metric values @@ -356,6 +365,7 @@ def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> L on_step=False, logger=True, prog_bar=False, + sync_dist=sync_dist, ) _metrics.append(_metric) avg_metric = torch.stack(_metrics, dim=0).sum() @@ -379,6 +389,7 @@ def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> L on_step=False, logger=True, prog_bar=False, + sync_dist=sync_dist, ) _metrics.append(_metric) start_index = end_index @@ -391,6 +402,7 @@ def calculate_metrics(self, y: torch.Tensor, y_hat: torch.Tensor, tag: str) -> L on_step=False, logger=True, prog_bar=True, + sync_dist=sync_dist, ) return metrics @@ -523,19 +535,19 @@ def validation_step(self, batch, batch_idx): # fetched from the batch y = batch["target"] if y is None else y y_hat = output["logits"] - self.calculate_loss(output, y, tag="valid") - self.calculate_metrics(y, y_hat, tag="valid") + self.calculate_loss(output, y, tag="valid", sync_dist=True) + self.calculate_metrics(y, y_hat, tag="valid", sync_dist=True) return y_hat, y def test_step(self, batch, batch_idx): with torch.no_grad(): output, y = self.forward_pass(batch) - # y is not None for SSL task.Rest of the tasks target is + # y is not None for SSL task. Rest of the tasks target is # fetched from the batch y = batch["target"] if y is None else y y_hat = output["logits"] - self.calculate_loss(output, y, tag="test") - self.calculate_metrics(y, y_hat, tag="test") + self.calculate_loss(output, y, tag="test", sync_dist=True) + self.calculate_metrics(y, y_hat, tag="test", sync_dist=True) return y_hat, y def configure_optimizers(self): diff --git a/src/pytorch_tabular/ssl_models/base_model.py b/src/pytorch_tabular/ssl_models/base_model.py index 7db2b226..03b31313 100644 --- a/src/pytorch_tabular/ssl_models/base_model.py +++ b/src/pytorch_tabular/ssl_models/base_model.py @@ -136,11 +136,11 @@ def _setup_metrics(self): pass @abstractmethod - def calculate_loss(self, output, tag): + def calculate_loss(self, output, tag, sync_dist): pass @abstractmethod - def calculate_metrics(self, output, tag): + def calculate_metrics(self, output, tag, sync_dist): pass @abstractmethod @@ -167,15 +167,15 @@ def training_step(self, batch, batch_idx): def validation_step(self, batch, batch_idx): with torch.no_grad(): output = self.forward(batch) - self.calculate_loss(output, tag="valid") - self.calculate_metrics(output, tag="valid") + self.calculate_loss(output, tag="valid", sync_dist=True) + self.calculate_metrics(output, tag="valid", sync_dist=True) return output def test_step(self, batch, batch_idx): with torch.no_grad(): output = self.forward(batch) - self.calculate_loss(output, tag="test") - self.calculate_metrics(output, tag="test") + self.calculate_loss(output, tag="test", sync_dist=True) + self.calculate_metrics(output, tag="test", sync_dist=True) return output def on_validation_epoch_end(self) -> None: diff --git a/src/pytorch_tabular/ssl_models/dae/dae.py b/src/pytorch_tabular/ssl_models/dae/dae.py index 172586c4..6550018f 100644 --- a/src/pytorch_tabular/ssl_models/dae/dae.py +++ b/src/pytorch_tabular/ssl_models/dae/dae.py @@ -200,7 +200,7 @@ def forward(self, x: Dict): else: return z.features - def calculate_loss(self, output, tag): + def calculate_loss(self, output, tag, sync_dist=False): total_loss = 0 for type_, out in output.items(): if type_ == "categorical": @@ -220,6 +220,7 @@ def calculate_loss(self, output, tag): on_step=False, logger=True, prog_bar=False, + sync_dist=sync_dist, ) total_loss += loss self.log( @@ -230,10 +231,11 @@ def calculate_loss(self, output, tag): # on_step=False, logger=True, prog_bar=True, + sync_dist=sync_dist, ) return total_loss - def calculate_metrics(self, output, tag): + def calculate_metrics(self, output, tag, sync_dist=False): pass def featurize(self, x: Dict): diff --git a/src/pytorch_tabular/tabular_datamodule.py b/src/pytorch_tabular/tabular_datamodule.py index d4e7fc60..3d09bb2e 100644 --- a/src/pytorch_tabular/tabular_datamodule.py +++ b/src/pytorch_tabular/tabular_datamodule.py @@ -61,6 +61,7 @@ def __init__( self.task = task self.n = data.shape[0] self.target = target + self.index = data.index if target: self.y = data[target].astype(np.float32).values if isinstance(target, str): @@ -87,11 +88,12 @@ def data(self): data = pd.DataFrame( np.concatenate([self.categorical_X, self.continuous_X], axis=1), columns=self.categorical_cols + self.continuous_cols, + index=self.index, ) elif self.continuous_cols: - data = pd.DataFrame(self.continuous_X, columns=self.continuous_cols) + data = pd.DataFrame(self.continuous_X, columns=self.continuous_cols, index=self.index) elif self.categorical_cols: - data = pd.DataFrame(self.categorical_X, columns=self.categorical_cols) + data = pd.DataFrame(self.categorical_X, columns=self.categorical_cols, index=self.index) else: data = pd.DataFrame() for i, t in enumerate(self.target): @@ -474,6 +476,7 @@ def _cache_dataset(self): target=self.target, ) self.train = None + validation_dataset = TabularDataset( task=self.config.task, data=self.validation, @@ -484,8 +487,10 @@ def _cache_dataset(self): self.validation = None if self.cache_mode is self.CACHE_MODES.DISK: - torch.save(train_dataset, self.cache_dir / "train_dataset") - torch.save(validation_dataset, self.cache_dir / "validation_dataset") + torch.save(train_dataset, self.cache_dir / "train_dataset", pickle_protocol=self.config.pickle_protocol) + torch.save( + validation_dataset, self.cache_dir / "validation_dataset", pickle_protocol=self.config.pickle_protocol + ) elif self.cache_mode is self.CACHE_MODES.MEMORY: self.train_dataset = train_dataset self.validation_dataset = validation_dataset diff --git a/src/pytorch_tabular/tabular_model.py b/src/pytorch_tabular/tabular_model.py index 4b3fd36a..0b34adf4 100644 --- a/src/pytorch_tabular/tabular_model.py +++ b/src/pytorch_tabular/tabular_model.py @@ -31,6 +31,7 @@ ) from pytorch_lightning.tuner.tuning import Tuner from pytorch_lightning.utilities.model_summary import summarize +from pytorch_lightning.utilities.rank_zero import rank_zero_only from rich import print as rich_print from rich.pretty import pprint from sklearn.base import TransformerMixin @@ -685,6 +686,8 @@ def train( "/n" + "Original Error: " + oom_handler.oom_msg ) self._is_fitted = True + if self.track_experiment and self.config.log_target == "wandb": + self.logger.experiment.unwatch(self.model) if self.verbose: logger.info("Training the model completed") if self.config.load_best: @@ -1522,6 +1525,7 @@ def add_noise(module, input, output): ) return pred_df + @rank_zero_only def load_best_model(self) -> None: """Loads the best model after training is done.""" if self.trainer.checkpoint_callback is not None: From e042201882252890a62feedc385aa6893690f3f1 Mon Sep 17 00:00:00 2001 From: taimo <132860814+taimo3810@users.noreply.github.com> Date: Tue, 17 Dec 2024 09:56:09 +0900 Subject: [PATCH 2/4] Add Built-in Support for Model Stacking (#520) * add stacking model & config * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * refactor: Add StackingEmbeddingLayer to delete "forward" from StackingModel * refactor: remove the use of eval for passing ruff format. * fix typo * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add Stacking Model Documentation and Tutorial - Updated API documentation to include `StackingModelConfig` and `StackingModel`. - Added a new tutorial notebook demonstrating model stacking in PyTorch Tabular, covering setup, configuration, training, and evaluation. - Enhanced existing documentation to explain the model stacking concept and its benefits. This commit improves the usability and understanding of the stacking functionality in the library. * Refactor: Remove GatedAdditiveTreeEnsembleConfig from model configuration This commit removes the GatedAdditiveTreeEnsembleConfig lambda function from the get_model_configs function in the test_model_stacking.py file, streamlining the model configuration process. This change enhances code clarity and focuses on the relevant model configurations for stacking. * Update mkdocs.yml to include new Model Stacking section in documentation - Added a new entry for "Model Stacking" in the navigation structure. - Included a link to the tutorial notebook "tutorials/16-Model Stacking.ipynb" for users to learn about model stacking. This change enhances the documentation by providing users with direct access to resources related to model stacking. * Refactor mkdocs.yml to streamline navigation structure - Removed unnecessary indentation for the "Model Stacking" entry in the navigation. - Maintained the link to the tutorial notebook "tutorials/16-Model Stacking.ipynb" for user access. This change improves the clarity of the documentation structure without altering the content. * Refactor StackingModelConfig to simplify model_configs type annotation - Changed the type annotation of model_configs from list[ModelConfig] to list * Refactor StackingBackbone forward method to remove type annotation * Refactor StackingEmbeddingLayer to remove type annotation from forward method * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add model stacking diagram and enhance documentation --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/apidocs_model.md | 7 +- docs/imgs/model_stacking_concept.png | Bin 0 -> 60611 bytes docs/models.md | 24 + docs/tutorials/16-Model Stacking.ipynb | 1486 +++++++++++++++++ mkdocs.yml | 1 + src/pytorch_tabular/models/__init__.py | 4 + .../models/stacking/__init__.py | 4 + src/pytorch_tabular/models/stacking/config.py | 26 + .../models/stacking/stacking_model.py | 140 ++ src/pytorch_tabular/models/tabnet/config.py | 1 + tests/test_model_stacking.py | 223 +++ 11 files changed, 1915 insertions(+), 1 deletion(-) create mode 100644 docs/imgs/model_stacking_concept.png create mode 100644 docs/tutorials/16-Model Stacking.ipynb create mode 100644 src/pytorch_tabular/models/stacking/__init__.py create mode 100644 src/pytorch_tabular/models/stacking/config.py create mode 100644 src/pytorch_tabular/models/stacking/stacking_model.py create mode 100644 tests/test_model_stacking.py diff --git a/docs/apidocs_model.md b/docs/apidocs_model.md index 01d8270f..d0312742 100644 --- a/docs/apidocs_model.md +++ b/docs/apidocs_model.md @@ -30,6 +30,9 @@ ::: pytorch_tabular.models.TabTransformerConfig options: heading_level: 3 +::: pytorch_tabular.models.StackingModelConfig + options: + heading_level: 3 ::: pytorch_tabular.config.ModelConfig options: heading_level: 3 @@ -66,7 +69,9 @@ ::: pytorch_tabular.models.TabTransformerModel options: heading_level: 3 - +::: pytorch_tabular.models.StackingModel + options: + heading_level: 3 ## Base Model Class ::: pytorch_tabular.models.BaseModel options: diff --git a/docs/imgs/model_stacking_concept.png b/docs/imgs/model_stacking_concept.png new file mode 100644 index 0000000000000000000000000000000000000000..6a0b36fd7fe473b300d7662ca884d6d96dbfb927 GIT binary patch literal 60611 zcmeGEcR1C5{6CH#$H6f>h=z3}D>E}Pj+qh4$jnHQWMuCVB@KiqTZ(8s!~QvmTGf^M1eG_x=7nC;FKF5qg@vG$<5` zURz7u5QU<^qEO`OSW0+j?Lf5}3Pq>yq^5RETTKmr%)`yW>HHZKN-O%7DV3Q~9czZA zAvXrAPSE+Khr7u{(7}q##o!NcQ{!~5Q>(m>5q;8U!N@2qR7P{RgN%`FtU&16DPNIm z6qjy!UpP5YpNb^zFu@x;jB& zL)l~GbXgSVQPsOX-cS4W%N^C=zPdQG!beD)l%~4huDrVG{FUkIAQ}~S_Daho4;9Xu zD6%R`jG?&_N?ma9v*VENt(SDCIQDqqe(vRMH122R)i>=Iijev9)aptmig03N_RJmB z**TRA6a2?Wd9U3f?(x|7Ojp+N3v~^t+>w5aH8Ni)8?{Zxid!n1>(){oR<7`6X3AYO zc}mCX5oRX(d8lbwx#dr#uJw~sehkiLZN;a#pVr^D>&Y1Wgf%+leWh2YnI{L|PKkQ| zzFH-a*TF$hmC1PF!3{#=B{}k=SNQ8Z2mI869>sCtD4%lI?1}&Jahf~zF5QK=X3F6m zk8vZjKe^i@UTW^&ah>g%9B(S)VTGyBbbFE|YkX(VL?{W&D+_rr9Xb$v^ssIXTXQWt zZSl_pe6A|{F}(iuNHg-ETK@W09dUZ+>{w+zIy7^yn0z=z_Vnm}>IKtL`=LG0vO2g| zyZvYSJDwVupRyjrRitFHaNVtESx~Ryn&bL#aXxx~TBIGv%^;5@cVX4zgscNTX~OaQ zYx4c93{s}J9`3!?rR|v$HE|*IxNA|oY{VgJYn-e?DV|GI|CW-q@aNbV)8%MWhgr28 zla0Prl|y|OQB`OfvzE7Xar8Bl%7JB5o5 z3p$;)XhW)>(C944RL;JRzXJGg0F5&m&LwC>p-gf9I{Pac8`6HfKgU zcz^GYn;BncJuB-gaHV!RB2V3(s&)8JNck{2zuikldpKI!&ZXl5FQ@qh14`2$)90T? zYzz|8zd8>$B$zlD{x%+I9hf`bmMoT%UKq6JmE=9o(?35w3Sy=Dy?Y<4!9I8PS`Qx& zA2PzL%?CxA1CjtJ|3=m zBI~F7&?QtQyVugfD7n2Y!=fxzAKj0zUDM>@_?_zEesOfUDpSRM^2y=mwwz{kS5*Yf z=-VT7idfE5N!yAY!tPa}JwzXZNv>jXC;xHjVIHjlRdAp5QWzdfoy|?B&UH{lm@%+s zj~$Pa7M>}lW?!Aw-uP%O9EHRV^NWf5oNwx!W;49TPKeo;ck`MSl?Rpol{tLqzNzDM zCP%!F@wC!##6CODu=nzZ9HXyCz4$Dyh-#SLWjQNdM&&{<$VvXfc9wsIeXo& zqfcb^_Ey3)x^E@K_mrZ9-qt@o_lVQ^a?|qO_UQRjQ||j3tIS`LTfOD>LyJ~9{-m8| z@#D3QknMB)mprWjn~Ti^U^(7K?1n8dN zNsUXdQyJX(EYeC{lz)mZ6K}(-%oCb*)=*e5BT?XLoJ7>KtIr~ukG?uOdN(NR-0{cx zJ*|G{WV{bPzi45gc%=MD@crCehuq<8)*QYQmiJC%m)cBT>{NU1JJvboGiEh!?0soP zXa3|^UhC9@%DZ3Bo{&Km_eptm4rh)G{Zjq>M(cE+bfuFnCwFzL zcYiN4DXaB9d}nvd)jIoO3Dbi+B-|u=V-N0qw})qManRBA+Yjw~e%)xBZDAh0^=q=_ zb@rll=6$XQnK_l3mB%+L$Xsn5arWX3w%6XuMf5zCw=e%d^C9WF;-Q6IYOP-y#oOJV z^!vP%c^ATCpJU6D|5YG zE{O(@h#~(5*%R$2v`;iV+WWZd@%$tI_Dk*F?ThEn%2dcmx)iuP>0)yEUMg3b>QdN| z?eoOP+{eTxdGdImY~a&%ht-%1kry}z_74=)c6%K2?K`ydP^+AR*J=67hgs!hEgXUIQ*N>^P%tfg%|Dyp1p%1*B-x%aVWI4na2Aq1q#od{o^+> zI(%o!|I6$SF{(oL!fxyCgh%|*e3Q55I=VikPL+Rt-4ZpFFg#mskXM*5*88G!@n+_$ z%r9$IHp8JcoAloj4rc})K+$3u1gOvMXk|ExeMA?8JI$K7$9Pxk zu8F-0vP;HT?)}22^*)MZFd9@n9GbsW7S*^zku8^#b=QeK3j#kxtyp{5!!;u{A6^!akKYqXb_GjbI!=F=p z9s32CESN{5^P`_fyTzm9+xfipIhFl426gKw>wXUA?ljanRo|GnAsBAdZZN9jttX;i zF4q!_MC&v17=)tE;)IDbz1$sfecuD1w@~8JBG{dEhlIbt}dfqLkEqAfvX`6SIh)UVK z^s@VQw@*X*TK+Xp)=jHH;s@+PWE)tF4y?e+s_8`ZnbKMlZuX zj+g%86Sm{r3)w60lA>fe?AQgJ2z!U6=A<2@ePnoLz1is%J3A-Zf(CC7JUuCrD`HaX z9a|``qWE;KrmC(wVLk6&UVwpPM#vBO^7-F)>KvbB-jM4G5uNyPi7O&R!85~iV~}Es zR%Dl=80UKQy3qjd3tf7bCoV@qeAbt>URwsfJG0oJXR7c-%J=Blp$lz`W24Sw-$LFr zz784H9J~>eN-gWwt5e?QZ|V1XRsVN-a^=ponf~8*10w@@7p13?mBoXOtv{SBT$S&Z zk5#7g>sS!@{k6OHVr^6A$IPG!iv^er-?{&RU218O@y7boqRE;q~lP zLXYyQ_&Cp~{d@i{gU^}olzQ1dvh9wWIkWcGhKx=iR%Q=s?G-J`LIOqS=Xy{7c=`@1 z+Pm^C4Ev@ti^k~W%zmIAL~YI;ET5hlW<{;~qHHPVNApqYEvPGts9!bFJl;#fvVOmf zS@$Wh1#!&_DvEKerQVfK2pL-Z#VTNOvg+4@&6;Ec^^2I`<4|B+J!7Wrpr?mA0MD@~ z4B1{31w12zKP57@f1YcS382V{@1s$uC?^z#v_>Buk$-XUhwRyUB#*z0qJmdU@aLC> z-d;_C%_84^POc80p;V02w6)>U$ll}38CTDhS@oWaY8ii?WxQ=-A+@d_TN9pnwwHA&mymEyj$US96pNOj`&(xzIoVw&*t$Ag=%4KKb{G|JiBw@6Lk~hYtOB>wiA_pIc2l&v>Z0xxh8Ol>U_$Y4d+S zByCg>LsgnjUdaxf`##D?vABU%Y(lSsficUXSP)cU3*^qr0Nx z-B+UYchRMMxNlV^kessrZo|Q|WjYB`f*&3m3uqjFwI0~MS=k=a6xdU|xcX%MSxA$4 zrsBYxCTnNKz|HoN#ZR}F8!ZAq`!7`%4r+v<$j}%p4o~>^2|f=3?#r+D@0;7p zvY2o<3+(^cfqWPsjK@!0m`#`e-_Apkk!4H&=lp+;szsx#Ic0nEng4tBtxw@1wg;LUpv|MCjTJw*G~R=;r}h#{Quid!Wq#vYd?1#VYibF|B9w%72X{j z$FuFuIj~slVZ!3AtpG-9s6G9zXKN%H!9B0T<{}|m!Q)XuA508fXjSHGF9u-D%B#O0Q4_QN@I7Gaj$zWe zQ8ZXb>gL+*QXrQbk58kFSKD%0;UVBisMbTL-%_<)6?$^0L;O^tn_GsYW8;;aq#hBx zRc8$=sg9`P;l<;F?wt-6-SRL+Kd^I(__apgiUiS$rJr--Hk_ zc8Pm`b&21fpMsWU!zs=uDot;14x3hti7D?Ky3FQ!duHj)L1n)|dN)6{;>{XngQFuE zfv&R^oC?cz2Kv6;-lpp^)L1qe@@;+}0W^;3tyClr+>lx|!-PX5?4Qx2+gjO0sm-cFYX+LF6(Pt7Eg;`K>@`&kHaqhpx5d*@s;TzKZD~8A8*~XTo0L}XarkuD<^LY+-Lv*JHpJMaun_-dtl(ku zYR_WD{h0J~?;cvu7Yw)mWSFI>v5-RBBf`CfB282 z3sS|-H;8Y#@(oB~Gf6Uhh!_9q#hgA}_n3^{8|jw}SpHs3E3U(B zy~!1G_uJf9hWRvjm!#rk99B;y^@%ZerI2dQNCY%IKxeXKC`X0~~NbMfWydd*vV z&e_{zFDMZdWl|9{Dp~9aH0|&EEZdv5&|x3zU{i5XJhZ(gZ_h}tP*8(qPc!>$1lJlq z_?H^jxgOg8k2+4^BClTk{aUR2w|Akl#huU{)6#|hdh<%kRf8oh`VT|w#ohesbgknv zHs?1Vil^3Qn+%q@ z`5e=|Pq0X}A0GaknrscZTORmknUnrQx8{e(3d7u~8QivQt{Buk%|xF|ljK-p5y!H* z@S@V>#kxM-BHyC#_~RE(Tq#0+Wk)r;zMf2iztXxW;rR{E%vGy^M$e3djrqdR``y01 zirp7#IdnNB{UgHMpIuqp=JPu<@R@PnVBc0&kz_Ha!{pX|L(X^IN3|hLCKUca}_DozE!|PTU zQ59{VSTX+Obpbzqb8SS~ZFrC>RoJ3LfWzY>_vc$tc^LucEJD}kQ|)#3LWU{l^!vvv zK^v3N?RL=>0X+VC2a7c0R??Y`(P6+k(#}kUZU)xxV5!+!hBcKn|0b4ylkL9+$p2I8DX2-WpS(Y| zkM>jgVObHgBD_cRIZ~DQKl;Mris9_I01?YRXPZ@vfS%b5)Abh%hrdt`*C}NY18u-~=D3at*6eCnM`ur?)31VyN&v)a?F)H=X*cFSVty z<0@uu6{dX+qWY)3>;);pL?RN7{p}0a$h1pM%4_7=7{W?&6$}-=5>#YB+hxCE* zw%@$%y4;g}k9gO3x4beokq3_#&NDr*{W_8fS#Lnjn+S_RLywWRy@n709K#~vHfZCC8~pkm*D zlYBcIVfbi{Rs^`W=*d3zvGHy*acVO$)eBwi>Me-6<#+dwy7V?c1S^2{N6&-nBxrGl zSrq}(S9oHynT9vt{6oo;z1eCfS z+)*QS@vgwf*IQ_BZ5Ja~cj9oY-(@tmMY$IkQL`h70f+yh2_=)}XLgbz5ZK}4H0uYE z+bRTr6@4rfOd-Ag0vG!?e@BJOz#Ky6-0=3re!|BCbE>@C#P0*V)NkC<30V_9MjHI?-!^Qs1hse+_pg1nApSCYn10R1Fy`-}(2n`x6_S*T>yW7f# zTr69BTmNxq5h>(2Fpv(%$ApE6>x&tylBg3fBoNF_ckSFB1D%D7{hNx05!iq>Hfhi8 zi(P<^F9c*WZA;-Mu(Gc!_w4rTzal3&m%k$4vX#G9zHQZit(@eO{yKJ2T>0zRw<6-- zSiT)z|KA$Ru?ME#i-jCKd*)z$+0Uy&1*)YTr>^a$pm==h?L`U|AtTl;C;G(~vXNfX zuvyaNOOs|WY&x8ZbGLsQIaXNL7CIQ(gqOk`U8K(06UgdL`c^HHQp$dcu`u_IwdHf$x>B)g_JCc--lFQhm|c4Igl^$-IlDw8zGZQ|GIjL zbnZ?#S1|pmd0v>fgN-q%2SW#I$Z2^<$yHS%BwK>!=hJfGz?5SQq~c2+`ba*SQ`bm5 z8^WIZUI!e~(bczukF#t6CyvA67=*8HtC>Sv3dB9({i?~1K~8SZuk zDE-Qb7*ba_-~bK=>d2&uVwY812n^fq*}wo~v?p*%&md$bp+%F;B{5_5$4IWri*A|6 z6)R(klUCPTSaBbIMI0bm;29`jlU?k0Y$s?;SN7sI`geqCajkaLHb4Jn{rZ#sP?7uB zyBbar^fG3{2YIIsjXlU|hEBXqdUaRt#zM(^R?Eo49AtJclLScSetSPyIp(Jb=}_Be zgqpDFjnyfGz{%&PW%KQp{Khzc} zaVy=O>RSccQMeaRAN?n3+nJc-PZ9hUzB}e2jds2KWtz8(FkEpepKsR<-CXT`*cD@- zygui!iFCtZ!t|&F+^I|#AH#1LHOxiwD{w!CK@_uW)5=nR6qml|@TWTqBQf&UghMVm z+ruqP5_l>rz8f4{CXWi=WWK_R8x|MaEVW&?@cb-o#6ZrkoHp2!x#6|)LAK&JSCYQs zd~3ni+p?S`Lfa6 z(Rf`^ZDiV0HV|{T!1A{5_cO^QFi{g>L2V3)h2~e+1qsc$L8GH&re4TYWf%sFD|4OS zDr6KFp2oVItB=qXE0-GToO%3ixp;O;(DhjzpVK>f*`s2i%MB09W*koj{dqt7x_A9I z({tr<^Cq3yktPG^`7bbKejj2p#rfnTG)D*J{iIO zc}d~tDdsLk7rbXzZ!cUke^n90c3~sE<^>EhbtjrU`9h9lm_}WN8DVBiYVX?cN=|_g zJxp;&@&H(0-A{GlT`(0`odbc%!9Y{lIi~5wapg%T1Krih7dGBXpJQUE(A6<)ed8po zk*>i_bp}#Bup){R!7D5eCM3U$`Qy_=?1UPopZa>_kKOD81Vr_-rTq^2!(b0Z=4q|5WR=1aS*Z!7(Y zHfT+4foaN#Zo{;9<*`A7JjlG{!Ga5P8{007r2AUS_eU}pxGc-ED9_$$x>u_(e6X$U z>fy-}qggpQ_TcXg)+fpr*&^w$Z4c^XEee zjFV+*L%}T_b@O}2ma?b}Q|uNhrRnhD^}8C^3@f_*JP1qcQ=!VA(+|Ip7u~$v51y<0 zlN>ZEjW$H8g2vr9ZoFHoR~f8{Dp;G#Q`YnDUhWrIohUXaTVLpE0W0`6w#2VEs@-OC zC3NW>R{=~u+h0_exHN4FTfG!7A2TT|K&E@F+w)t3Gq_V{d1jIXDzNIbSbpWVOLwn{ z7Gw|<(nDY%$1?hkYs201mnn2r6S+JLD}R?S_9TXm6ozoPSCO@;DTAiKJn1)+Dc#1X zL1;lt{zxlXe;_n?z2@6YO;cZdcS!a6$Q{)hB|JOj{q1D(%&kb!tA+q-Y%;I?;cY*g z4(O$)nv>9r-O|?AL9boc>rID;gy#4vXIdh}UiMvd@C=c^z|glb61tK1J*n_h;7{hv zjyFv^#PbUq8`ygP#Q5HWJ0k3(g|M>;F|(7E^UmgE{cxj!_4*S#C0Cfe>%#QXP4P^9 z<#N)%+Vm}UL;axgJ_<3dWjp?PU4ohJk}jzWcAEoIwuwV4JGW9TTRB$nc={E3ODu(c z8Y>;ON~OkB`f@{wiOz*K%XWNV(zQ$~-8H?l4Yhg|Cl4vt`8|AGdX5D*EJTS&TI*3*aAhtZlbLZ%YFSkB+b4 zNhJ8Q00|Bw(fyVonT2zfMYbvyc9&RYy1bzCYhpn4q-Sk0b43<4^=c#|(QWauO@iCH z;`&^R(MDlN+Y4HQH}Cz*OmP{0Kch!~cs4y&I-uNrsCREVmtVY16lcKM0BrIABH@;5R%DtF20cP$fkh?yMyHK+M+Hkj|+aVv&hDg=-LCed#0-6C<(a- z6#FP6WxKtMyvN;6+Ha&5mUX$mqi;dH!|>RffK6@w^u(FRWy`nRc5IB9R$5FIJn0Gi zdd)iVN#({0ud8+l+z65pqqOpC?4=BuT}AqG`c#nPtRf6ot?TXjAP9FwM);3Bt>sLW z={jFr0?IFIVA^OP)3`&lPC9s0*G6YKX#5-F5%!xilg-Q0e*oPzkG?yZ?r-W3JZz2f z-7qC|_|xtrHD<3v9vp*uM!J&%k2{F}bh56J>vpDu!q~0QZ*CBYoETn%*0gl>ER>G< z3^)wjQ>pkV)_eEOLZ`E7U?(Qh|0l$+uVaEW2}%|IsS$M$x1N;ET$pLOlN!NCKl9E2 z+)&=R@xdZVH-(L*ftEM^vwWY&em@Q?Ivm(bLhYyllp2csM@SA^krgtJ;iN&K@fnkl z`Ke{S^DRjDo^g307n!=YQmedi{N$^}?x_&pnI{LV8Y(wmz|>oIwc^hWw|ne;L#7pm zVm3F;DqSnr7fNOu4}WuO*;O%p3Aw?<;8ch}Nik?8d{re4k7?h6z%YGo{zWDh_?VBgx3z$#d{@;cs4pdDCGFu`F6?HzNzyzBZysE|dtl36@Nv*@KPE{dtU)ldZAsix zfB-%iQklO?C*3_I?g^&<>GP-Ekb+RLYLXn|QE=FwDjr-V#dydo=~}O&bs2H29a6l+ ziZ(n0ZVU=+ch4iau}@Z@R-TfQlt6m;CB`j(i!*{*MX^``sjfxC7INvFDnra)co3lf z!t;Nw_CU90Sn@M|Iuz|W(x?YFSXZf#q7D`+6uR%M8YGjuiSZ!(dVW4UHgs*kY2b#e zK7@+T&vraJLO4rL>b2T|RoYqRklHw5ggxkokDlq2!5vu5>x;cMnr{n8l_eHurb-~Z z#6-z%*&&#pB%>L4q87wM>3xTLU0NWhNhOTNY|n&oSa1MzUogC+YhiR)KAP_6@oZ8w zSzh*Cu+lI)$KOU8u+5o6lV#-hVc#wH5LUGt&0NU&NcqG6GAs6gCeF~D&V3RIFe^qD z(R#AjiS&zY=TXQ6RIc1mkn|dVy~@~hNBjF%SBsLSlS%dBvCH7Gt}ErolV0O=YzrPz zjiXO${3rVnP=hX(W~Q$3s z8}6%W)|2f{%$R@VXhPBBH*h1LVaG|0-47iLw#Sxq(57N&c^XS1gkVb@&W;*3WuP4Y zRufj#lN8$p0RlnR*llK;dIJ6dc+a!;iQD%i324|BgcT)rwr!J&kBblc^3Sjzw_6uH zAZzB9*AGL5H-gmDUl7Q)aeM}^&nz0uZ;ST7Us@ASf@mPDl(3Z{lcCe0J#g499KGE% zL)rxmzca_SKS!%Nlkmb5Y=yytn+5T0&#vVm!1(l_DYg}&o)vm&-O6p-pa18!`mic9 zaMw?lj&2W+z2KJCi8mB9r0NKpC<4Rf*P#WZo+reJ$0UgZgHJ(G?El=^&c=R01o%(7 zgA=d>$@**>8;~gdU%nz?kK9)JQ}PZ;B41vCs?4$qY%_JN6kbnkN@|@Xy=K+3-^V zsmN~ufEoXd$NzT;R}G8VOLw~;KaGhu-`imt4-H%|NGf0G6gsL$gXEL&bxZgvVSI8lxPv0;pCw~yhHFn!%ZEUKq`>q+x5x)~Ttwqzi3%(@DrF|w-FAFj}JM(oz zk?IN(<_WK8@wd{QtDP!Ob%C8<&WhY)ND^Mi=D zD{ik14ifKsGtGcdqE zQIQCTy{AT2zj(=fOdMAK(~Sss!mf$IuUBkt-g`DEy#13E6Y11HjO|y$+*T`e_jT@! zHaxCnp>-wx!Gn((IydjrvoLj*#GTDM+k4N@SZ6=)eGZoUhTna;HaCBa%%}M}r1+=$ zJ02dnGrksL;~3ajY_%Aqa5Cdea>0|EJ2}r__Ef66wACue@D~MOQfw{h0YUKQBLMk_k z47(JKy7)j_ftsSs2;E)jwCP*8l@~fffxD!H$6TfXumRw#rdMB1h*9Arb?!%>P zHMoy7Xl(T(tmKVI!Uh(Povpo5h=&7Dvyt&oewOR%GzcV~e*?~UW6Y_*X(___V|$Fy zm>^v^zw68K3&itziRV9er-=Y=P~k7B@z?>)4^+6K4A{AXYaw6LoE8T{en-3{ zCJ4~{wTf#-5y|H!l7IggO*oPKAZ7w?n2m!=waNmGRdrU=eTR=b1UrAy)sYd&#}UcT z{#?R{ke`W=&#pMe5Z;Od@-GX$Qmq19RfVdxcn~!2rG+s@45!VY1%Y-DAJe`t{820y zms*t$vg(~q4nB?@iima2+*5?q@`I98EaamGY!3?=y`x3G~1Y?^PzbN@%Z z?tu`*z0cR}L7>;$g1%GJk_f#PKyPpC#EfC$0RXRa*{D^W18g3?(;YxQ-+zOMP097T zE$3pW3fM3`I!!}n0}8q#A7spl&PSjZvGG7eZ%b7B?@2yi8Hl_9ubq3%J1N4~5Pi<% zuu-iFK%iGUN{&Fe{}S=|#G`c>BJ_p`^b#ef>BxFv=ZcJ%F*`b+AC6bF*Fl_)EjdxL zmu0?EBFD?Xp`Gj}J1N80K(fQJY&I%*A?%k6AFqnHXdw%ziE7N;7)OUNL%`q1{2IyI zDRxJ#DjBhDI<{Mv2?4M{Ylou_f;K2RIKkrHF12-{r1ir>Dkd3ogK_YKwHL)h9D>Ne z8xm@{omKVJi1e!?s-Q0*sf4&tRCpqDR%WJ0%}x|qFA%gMdfIq5dXxt?1V0iaDoZBy z?t^C=LtJUSiB5$c=y6m_bmGDM1o*GV?#kzmGr{~iblT*llQ42%5Rrk|bVe>!#Bd1~ z*e^{6v<@`kVTGWZ!7QQ&Xv622vIgR;dlD&u!>U4}cq>aI0@M&M#fqMrE#_@5VMad% ztUfeeO~ieaM|-h96CIS=jT~5kMF?81kBr?y$OS=&9tjSz2tw2>ESCr?)PRt(rw*dO zeMg80adi>W5dcmzldUX&-|5mx6W##?*$=26z@FwMlV>pBJhynJ%S-_7tzr^iPfef#uX=i*f9A~5QokH#hDAb@&J#KI1(14 zHr!z(fsmj~w6Aqz-!U45J~rwvO-&yjv^Zb~^+5S#xq4Lb4}sy2W792uhz=bjiokv& zE^;fnG6JNM7t>X_`fuLzgDXHej%?7cO@D;{cZ7^ zkbHbZ69WmW)Ez!zC=7cKk;;q6zTnmdTQ|7EqC+(^_Vi)!#HE|)v&xtmJox$@hu&O-P;ix)?U>xi zdrAx?wj2Oa`=vxm6}%k?Ba$(m3^Tz?sD0bh(JTSiS0Zu{WRtL$#DNIH0n3hacRaM5=C){{+BV1Sc0t>$ZSEinj8qyC7y47HQWISbYWo# zK$c8eDP-aMk))Q9&s?QS29}?{CV(WbGr9;sjc!UoBoR-efnn<_Xz#GF7mzJ%v0{K| zQNH`JJ#c{tB=U^Vow#%tNiIP1sF8$9TvZ}0%o_;W!pa|HwjasjQRodmq-t7yE#1mV zgy>m15^OgqzJy>WKuurDGH1Xcn{b6H17g|4fH1sAl*FwvoIW-3>$orW9#k{p(pGgW z=o7fC0b7mTP8i6oF+>3{J=;%9jGSSF50HK}?GB*19Oek2P?2r#0m|@C%#eO9?>+_q zRk0Bxr(5e)3!>Ek1V=ys8!!K`V9q0DqFh-CJ(?QH*YE@-$X*^1t3JyX@x5Ecy16zi zKXEx-Qw?tb8lmuKnhE_~4`RPhZDu=Q{)&jt4_PbV9pPaOj<47}rFnxp4zl)YV`jck z?9y!%2Nf4pYyqs-Ce~&A(w-8;q8QSzI{=UrdEO!HQW&IPR4R9`;y&&~(yx66uRx;T zh`ECGL#}xw9R?ZHfJTHUlxWe0PvD^q5Ob(S2RnETtTM=MfCo5MMIb7^q<2r7h-sV# zsD$J3BBukGQ~M#KW!zbZkGlZ^J8j?UBnin*G-3kmg7i&96GNdxxCzR=VmI$*;Iz2F zW>(CMm@q+*u0>3#YF^?-h3_P0YevR&-$^RntO^8Kh#O%sK^jm$KXgBCj=h%*1kKIl zN&!Kyi6WS4PfQ_Uw4fIjvQe=I?wkCMR2MFcdF8k(5wPKq?M*27ic%1Y&U@QCu@f0K zGXO~ExpTvDS5iSITu%p~!Y7cNKk$^EE)evFXaoLv!y;QoU)2R12Od^xWkBCS5W;xZ zM}`@IADY(AdcX%t|Um#!_#9+-KK`Uh&N5JWxPbZ@UMaL=r{fFab{uHm@C~ z#fmGs1>{)r=)Q(@){iKFKOqJITSgg{3`V(VDD{9A^F{?U;>Mw4v{=7*g#OEybb!Y_ zc!Yk(ESjCvq|me%uAr@^t@)oO#{Nvuh(SqSNU5tbi7FIYdeMUD=1>?9xWWhw*0GUL zL|ta?2WR6Balrk!JIL#_95H`yxbP*2NYJz#oUXJ3-xfun0VijKx+TmFWwAU%sd~H* zxP@pYq_x8QId+QVN-gg;?UkPY1fCNt+?^?(JVpaLeqUrpcz_;wKGt9Xo;n4JT7 zA7$YR%bK-OxS}M$zw=A8ICj520J_y2DvBU#^pS^WDTwHw7J}v9_xg9zhyMaVzkZ*l zM4KZ8Mf)ZO1x&3SK@@_L6!of_K%ZCT0$MrH`4BUX(wll*3S$BFF=Nz(Zy#d!6RVH3 zNZ~p(QQPow2SGh0FK1lj?gRBvZMuB)_Ot zU4tmdQo#URVnkgc2p3_)er#M5VB$DkH<1J% z#OZeb$(slL7bW7qNN>7zJQ=wA%&E%T$L+EAQeb^sWE~e;i4USA#K|#OZ%9-i?ayhS z5Z#1pIPrl1K354cpq0Ra=Tzbz?88WC06@7PjYOg3l_vVQZ(~P-NgMv)K%aMrG^^qd zgM^ED8cUea%lp7b*mNvGs!|k36o4JG6VsLe7(oE$JI|-kgzpB1?J*h>*wZ|q5#M)v zcS6;wz7J8*BRcQ4S~y{0Hi$;+-h050X+8oLtkj-}frbKv$=7km?I28Q5W~Pu7PV3m zq9s63ChMo2Xfhk5?{L7%m=!$=i2--{8$o&y0TH6_uchxhkTwKEajKw1M%d&>+oHy= zB@zRrn|lA?7(ezj*qC_nnji-{15u)u{%j9WqP;{VauQCtOVSB#r0+1QRfj`YAbp2m zEc|M^A<}ms_ImMsVnijXJZIoSGW%?>v#6Oq1~d+*f}0TnrtU&wfMcN-11BfUL-bkZ zvreFZprvL=-hPJcz8p>q4PlZAA5Vcfk7R!BACV>}-kKlg}hoKdX&e#td66~@KfqqJ&k?nba*4R!J;R+$m$Ks)(jiicviBcGJ9nz;*K3WHu zPxBM4IP-BazoZq)dwsY0K;nuD={63no*(XV*CjG6M*b zebM5_sKG~*6Mgg!sUd=GFNR%ZY0-C8UyBB?55+LZTjL^^)oQ^5g>53XYxC*BL5v|% zD4nYw+(E_#bo34M9rqr`?pE>Bi5_czegl&J7EHzRsS4~x@fN!*h#h#ylSc9wWi(W( z_yJNS>O_$?D56p zfgs}-KraLQjrL;JAnW)QC17rR1e2}t_MTt{v^M9gwBjwep=*C3eh)?! zT!NarJ2}KoGT6k5fszid540W%zo~E#9&48VNMmO9fn8sV+3e-Uu5+nb^qmAjOntbF zq;>xWCeqx4tDQsT*SoCaXL>=|SNM#V3j{8T6P@uv;O9eRv^q#e13+b^NXRANwc(jI zT3hBR^5KVQW86n%W)<>ad4vs<9ePSbZynt3FJcHG;Qt}$JVfV`g9V(f%oo4Bdz*cZ zWaqQz0eWIkQiWrw`v+i1Q5A&t=Q zKXL+Jfv2=}!Rr5Qp)vzKG8$dvE*iK1Jl6Q_)(~i1icaOMZ;;bT2RKZ!A+s6s(K!%7 zZl)ZsYSlPg(>gZp%)A=Tg7sL+fFK-faEWL1mI{uKLvW9371)K>R?0#R$0uP@98~cP z2<)~ur7j?_?};bJ+yzWDdF*RFG1(j!?y>xvlHlWDXj0w!l-Civv^C9|bt-3!RHA2Z zTc7}y6RS>~(%FZdy%DnmNl&3yi+JK0J=^NE=gS*OJB*IlexEs`Kg=RzeDoyuV453R zw)H^zl-n{6y^OeUF}aF{e}-zOK={|G6Wo}&vB7Ne2Y!c#LpzPxpW$5_xjNgpo?`@Df(_^?>fU1TxVw-$-nVM^Bb^yA=BB`k;`Tmj5tHq8F55GI>KAk)f7{ea(OZ-Rv-~l%1;lCKM#|^@; z$YI&2tQ)9A2;fC(lvfIMB1*7p$?vAT3Or)SX5^p~0#YWAp(xy9#}CO}i?g#&u>u_y zIQ0}5v4XqG&dtKYQH)vB0!YbEEGb~hv(5(paoAWSZq0d<4`T~Q#w!$00K2v-03 zv(Nh?JW`_vJ+XsO4cw+;r?uuLvNH{oW;?2>y%z>LO7K^txLshSlbW6iBQVTE=s!LN zk4CjgX-NEAl|_VDQ6gFli|z~=jQW6`5ecvYDhFrwzgU9R0|Odk7$Afh^Kk_CF# zaftPeDv_OkvSD*%ys3h7QtgBtEtgX+?gNH3!>V&y!PIhNMC8=q^F=pAU;ks;lSC~7 z0$u?Dbwk>&e-LL?d##lk@gGR097hG5bGi%rT_Upc7NInQo}Q%`sq8P}nEWg+8-s8X`Lvw_6|%;KZ{LjdeoKXFEr86&T)4MiXm@B(L(NgLH`MSCaJq4r1%V zp*k<1x+W4Zi13rl|LQ%g-pwUu$PXjQ+emwazzGcbS3T`xA+m#Mf`tLlru>8kXMkvI zGg8mXwC>y)4#1;#a^C-E&Izg?O8(aX{~92%9Q|v6e+>|BD)}1$|3<*S5%B-J6jY41 zTAtpbAv1KHg?VTsBZ$!H{Y!9jW7X~GsPtF4GdlHesW|pGm_CYLPDQ=~=Ah5HAAT}j z>r=0vVE>@g)W?Q*{PiO?LA&xKI?nM5(6d9RA`L%RgB%K9IR|vNTP%`5d0=zr~7FUp3DR4m=*Lw@7~3^iqqIc(P$g1pqzF1|WkUe%G~q3t<%F6{x&dy4*Z! z`Zw6-8QZN~x#LbCS?*8$K)Vpe$j853gcoP)^7U-DnFg4QyKUIP*oTrdIw3&-_@cd++#r|1(Wq6}0}l zUoKZ}u2x=Yl0IoYRXtnRR6e}9$`xAQAH}hE-ZTA~qI2WEk@fY*DLPY}J2doG;pb0y zls9J{vinYaJ>H@ocyp{Y#p8Jl6{la=s~gK{Pga~R;UW)4U5fl?7!vm$k}`AlpdC_B zafjni8?oG_uHIaqjit*JKkgvvXOI-MlD{@H?q=AxGyr$AJ~MnWXYSE3|4lm^P4SG{ zq`p%=w2o=?;92)i%ZsBwwCfWUc!uYm?zbxQxTSD#nu33nx|UOjCnJzAsVP3ip>AW&_30axtzL`5^ zIKJ1}0b6aRy)2n-gXu30m!{-87wBvi$FG+zBrDT!%XZxKXtLzh%p^z7!#=;0jC8Gk z=SE9WywF|TH8e0JH_qx4u+$i4yQZO8zL9%9U0k<4SVB)H>Cnjy?8mZ+dIlb%!N?W@NgcK-S1a4q|D;HxyVvQ?R(<{KH4+zL5^ zithRJgeMbn_wUV6jOauJBe_l~Qxnfl@3jlHTCvt@ zryE~$eO%>wg5&t^7doEXeD64iQDR_wfD?x@L)xv+-yK76m&kQwsv^+jp zsm(do+Vx7zBY=Cf{ndRsPI>DO`e{QYj)M&bubhnh!|_LXPtG(VEhuClyWxkFOW7Ko zr=gDGOBGli(#FT*K~V~aemBNO>*Sn`s;~Y-zN)=k+!T{^IdvCWn)Hn zb?w6k3^UqrA|@u?-nDar@!eqcZ2kK*rKHH>9!r56oj<1bw{?VU{7y4d(mj{*WSvT@ z*N!Jm!P&H2{R9247fov;!T~m}XVYJccEk!L$@N}OlQQpAxH+5LA$fJ@%`-;)lahtT z`=MOASl~OOj=TI}AFGPlYirs&ho{s=-sRpXy!oxiujR8kh27f({kU__-4bqPy0~XX z+0X?)O_g%9X!2M1b?ZX&%Ga zJQlA-3om4|`~LVU5z^enXSw)e%{B;+|nI#@VP?P)E{MS8i-r zik4;H45~PzdvyL-zOU1L+EY4g8Na%BC?&2emU?hH>gZhhCdj;qt!5v?Rv%qk5s+j= zgA+6T_GGV_P@zsu?wD4|zC4!B@skCbILd2Jr&UdGqMN60{0Iu9TD(P5P^`n^dH-(3 z7e~wXMxSO3t?KN}m>(@Sj#d14)26`0!LhVns?jK*dh4Y0_>a+w*V!Q=f@Xb~J<<+k zk``azbiAC=`~Jp1buPGH+I(rlWAWslyO)L@NnWetc+r|X{^YS>|Fh%0nF00Mfi8U# zzAFulSgp^YHtZKEvh!n0Rq{lS$d`O}h|H<@R&rSObAlR!oLT#$Rl~-|Ucnz7p;s&7-&XD_Ym5Gd#>GPCB|*jEWPtX`%UtMm2qb-C1t= zB>(85@@C>D?NPC-RD>KSL=~+|7V)G56nD| z&x#kDQP@q(e$~WnEUf-tRJ~gpF86-+alHRe>(hLgwXW-oU!3Pb<<->z4LKf-)2E)iE?xla zl#pBY+@O&BLGg#`DW2@Y7e=t(r~H4GX-RBZC%jV5K!gfBk0{p?j7=QxxQ(-g4N z7n?{{xkhFB$aj3^H++=Oh|~<(Rx4LCaC6+ekGkyd)el#Ro?frw&s1z6fZCT~qNlM7 zue>t7X8NJ?g3ANlN&Jp-{5z-1xdcY=@gRaud!m-BXeT+69XtY5gj1e2m%%StRN<+Zg;W*m#ZT`3lrs0<^x<3K!yU!9N7#PR4@P>nUa(wWN?|e0Q%=e)))v~OhkQCSv~=IUspC(28Z9!SY{>~p=h1pSlEQH! zZnzc?&X;v|+l&p~m=30m^}!(++vTphmXPjMhI-GdSiaw=cv*)klX zs)#ZQtGp|`CthzaG?kz1TLt3FqY^ppv%eNLXZ6pH7F8Fp9j2;*S^WAue$#=s06UdJ z?xt|H+Gl7Lev+iEZ)pI+LAs)Nj5JQR_?dGF;l|0vw=|Y74^-YkXhCnO_|3zV(odqU z&re2SS~ZGDpoS8*mSvUWK;0k*acsHhG35nJ`v(n*M8b#f^MBY&7JNSrM92=#hm#Ba zRNi%WT(m|u>L(p&mC`i9MXcX&Ub-fCjsCvAd{5=f)n-$EDgr-<&Btqb^}6L&+^#Ll zw`8t@Vdt4C&1}&`G^+Wol@|jJbZ!~sbeyxuleLz#2s>K9QVc4Vx2?vPI;i)T2IF_H z-s>-wLmne@+;bI$5Am5!+P_`~g*8etZOUhP~x7(tdh|ce@oL(Hm>H>US6|MmH#lhrD*&UvK~NdteG? zJ~zPhG*@Z+$%wFLdK0n)Ty&8joExCV!JjJBGVD=!+pf zXt`ryj=VLQR^ubCrUqk2sXbaDZ+dARHTyuFAE`6k&&+j5iY6z{>&56|DzN* z4PRw^AfYEZ66+?TfN){TwX1+Fw=^ix?#(FXDfuA!^@U2n;?7)>l;UFZlZ;cAxF$u9 z;(Y$9>Ma~ktvs56n{o7n(TnFVA}Al%d?rt?OX6L>rct$ba9Qo$ES=98 ztyo2`ZxZf^e3l7Z)`Bl=hlpa${{*UBqzE~M#Eb1J31wSw$W`LDxNth#ih)fJGTYy@ ziU>P>?}wry+(D}O_n|wA{vSp+T{mVkc+MqQ%rdt1WTNn;9X)xx!q`|PAedx6yQ}xV z8-JWaPj`3w1X>V;24(g9!DB2B2p%4@?1gZ8A2{?rCY|- zYy&wPU!i1HMkyMdWAiXOn+7DD`n5_s3l4RunbUGiI)_=6DoiGa1*#{I2S00-qm3mC zu$Soy6^H8`BG9B>1C$a00)D_{z)3h?9r^sESHCVk7~R}8Y%z>7`ncF+@L|Znhpv$- zinha=3NphHBC%XPy&9}>wB?ZiEy(ROi;62eurU$oJ1vDaFgHkljR=CyaVjl=EAZhORhI$ z$2}pa&hW%gjm^H8*f<%udX&qDCjpJyJw?$l#EZpaj0+27lU0y_5hMYwk!02EzqzYR zU1=jo|Gt=QU?7Cr<6Y1h!_C64Md4_~U_cp{HsYR^OHUepLPkPly=u)mD23XtXBqt7 zJ_$CK)Ggh)I7UY>yjEJe$cW{XfUy^W(HZ$rI_>B&dP?u24lX)-UXhFlH*7w60?yAwY* z*aElQ0TYO;>HKGbMq&A*joil=paQsLH~g{9>-gcj_d&D=c!WjkE0^S#)=(z_MphR2QoH(hhjYm-89D;FH$pFbT0UC3&Cw)pM? z;*>yK(=b6Se?uHdrJ#1~678z@bIzd%`bk^@a*ntwIyaYc=ktYV9108E4Ds;SpaZLu zfX7s^j7f9elWYGzh4{i_>AP|HHA`%{;hhKC=AV+a zsJ}Q|+>uaY-o73!tdXAICZd{B>(O@sxpTGNVpGrk{Gg#sW-vz04;oGVI$4U6RHo6}WLmv9x zYA>=~@PPsk;|G49^c2yzbW?7j=oF{r9IT$+bIrGQ z{A}9bLw;Cs&SP5Tccxo|F!}DV5x&_KmzISac`HrSP7846HmFHgvYHfCQ*LS(PQIXx zW}MxRLr@tGdDAIluKM8g#1f5mXZNb80Y1iCC;X!b2Cf6GvwguO`ccvAe|gg@pvE7H z%6zPd7(vb-R+DumU3>^w(|gZ+AW=d)ACqrKnmqPzsmtm2oS+1jMKy_6lLY=bD7J=d zr|Hr3eI*%0;qc`hq2rbl(&C={mE8N7rKxST#uM|=vxUW1%V+vRA7sX|NXyqes6)9kOg?Zkx}Kgm(0tQR{&MRqwJhL775r3mQx z9WL-NA=^y{QrFD)`ZCb6Ea{yz7*RL=V3mZm_fgv4fiXff+YY zu#w|s=Jf&u6{R4L_(y~AHLftqv-{|>l<$>V`&7D%ix)J4~-80+yUmD`` z^<$)_7WjElZw&N4!Gih{Jx90#YBBOfzvDh# zqUm7fLEZt($l40tYm4@(U*YzvxL~ANU>On)vso8`Hti~J_wuJKtVO+=j14?!5qJNa zJ;}7%fTAC?J4TH z&2q<8)N7HbQP9$<$Dk)WlKrT~yJy~xpSt5#jmM^Dwh9eXtrjwhVLmwsR$7isf?G00 zE$#X&1oZ{UBGcV5V~CNX?&b!x@b}BBLYS$k9u}E2A_9bby(-zF<+P^P<`u5f0mKp~ zY0cL?x~La7UDp<-$)8KM;Xcj3!NJ_kCB)28apO!1eZyFZH@usph>Z~^3=0>s$wFWXO5xj8!DQQ3wWXw>&`zRp} zA#Dpyz?)y#G2A8Ua4v%)qck6rM*-za8eAfVt-J|QiYU-&RE(2#PP+nDHtKK^>?Jna zAHh%%IjNNN_z4D(j^|s5Cdorp`PSXqhsWw|y$hE_FZlHmOLcb_D8z1_xt-r`f~1m& zC`c-j-Z$c|{{EG1wzp;C;Jdd_njg>RBRRAzI^9}hbNSLzc6^azzCV}W8g;S%G`co! zRD1K1Rhqt$e&9uv-`C{rd>mwCQzmNhbEfpg#o<6q!n zMlY7K?i@WN_L?^%K z0lVuY%aA*4u}e8JypqeAMC#N>Gq=8d&sh%pTyYoq-rPCh?gLz3IW{$7SkO*!$D zVLd)#j^Ys-cxagc)!?tgNbcjgNh|d8SW7Z4^qx1(wQa>7ub;iX@kxw@FQLT{DLWZv zHvU^kGiLQ{)7jAPZ8MP(beR)nm{4U}Xz>Nt^RDjus$FH&B)M${p{Z#RN>;HR z_ijRou(;{j$#sD`jn!OyP7HGAZ)}H{NA%z&`!l?j&nv`dYEGrqW+~Z0aCSZRV46zo zoKn*A_yX8NE&o>rb&R`hMqhGKo`$>g*K(oG2PP#=lqe=p6CvdruOK3L*dKlpXO0h! zxOF1m=_l2q=a-DWd>29K^hCu4yNm9MYe-Uxu8n$;BBk;@iJ#vzihQ0>5+PE#{gTu`YYGf7wWAGdf(|J&-?RtH?E5s zth;6VmfXw+{-l2K`?MTL!0$!wy>%H7z8MuSeLJ5ep6>UFX;TYh5Y#ttc)9(MD}32& zYF16SSquPu^a0T44yavb89@0;l#Il+cIaDURW@}cyT_nOZDJ=_ro%5N5FG$7s?7f+ zBaDBeLu1#TjNk@(kCHQS2-R^cpSf-EdLF)?n4%2jmj8f2HWiOWs-JJfF}aqu0g=M; zg}}#zAi)N=(qxRbvG1w$;?+vH^!>`M6@OwPs_xv*Px-W|renRni{3b?rayp>`9i-g zRw#T4P8+P+H5JU7F?0#D(6>F0A74@PcVUO82t-}WW(qH}Pn`1RTrV{|+- zUh9@po4|ozK6Zx}EI4MPZ;HFR7Z{$i>Sa63{4<+-R02zl6?X;8dHukbq-<_AKfhyk ziu{mXY5TW>iKVe@19fIB&dWDl!R{`o6`g>7Q|CFK73%dO0OPShsw+I`w%6vt%Yil_ z+{eiL9IJI}sqaMvJWWp}Fs|0y=bsAfjV@*%S~(4NRj!_Jpy9bAZ(erc;Y}+NiV9|5 z4UM#QWUm)N3?(;52hs=5Fw->Zr+aLRp5g|=dJ36(Lk)<1ROX@Oa|k^5Y?&7q2~6%!khnz4XbvvAuCt@YiuYPjRJGNR5v3w@yV zfc>lSA@Xa28TvG7ajnSECET;#Dc4crUmZCsH~2`~LFfF7W|r`>gPHuhH{0W4@KD@1 zlW>t21CFXlz|-9^ikIEMS~**Nk1jLV8I0PWPqUo*3Sl93 zeoSMcTYXOD5IG$V$6ZZSHtH*HyWX3&8qENYH#>}slUyFQ;?X6rl?|n1vPTs%H8{;I zBE{zsS%WWu?Xccg3blwU*VZe5j7uvV9UE{%c8mM^+&S3~yvx?KO1^9IvW@My1}{%CM)_r>c0$GQHH9iWz^489*0$V*FFvEV#V5@r z4exXsAMcNW?dn-kTyBlpiyBR10G; zZcZK$qgE5Kn}N-O+d~)a?QN9A_}5%ijG!y0oi}j4W|MMd1wVwt^##HRS;AK4gy7)Q zNc6whc3QkJg(P$l4U;xBO7;inRVgLp_DT~E)Yn-k?3Oj8xA*%)Vp1r;TRF4!ZsBP@ zix@k1Mo?F#geM~XXPdm3_uV>i!&~}le|W7*AP@<2gi|J|^Ol<0?jVgAx;GXyvR={j zeDRg*d*+f`D9|+qv0TYh;nr8TjDBSENqrz$Aua4|tFMtuz&4>~w>5ql=l*`q{oX;A z{IplBud;8H;a$EIl1jy=X}l$Pp=_cPUGRMP}5pDV7XJa zo+kCUwAtTLrFo>^`q{i;rL*WKYViK@-_htQwDYyzOsb+Oyf~YEjM4ZTW70j*RIj0< zX|IcQyk2%I#K&I>3M9;?_1-$B$@kGk*2fWJ?H!ST9CgalE*9z4G(y6Zj>D)E8D(y4+_nzE``Dn^xTlZ5EhZwK<+R13QAHt>&5oui)=X zf49Ednu~357u}r1{=WYpS7tZZeBs9`&&hD;@kF%5fVs|OY+hYW3=b+Xsqm>$(so?$ zfE*oXd>noTyAGm7ZLWHk9q-}rv+;=5#Rl*t#$7#;9kOz5Hw2ZOFmhA&%4a|6O;;{| z8reqMf=3jWJi#pFyi{RV<9Rx(9mkFauzSE`7V`l}D6cwS6?;7eK!ry6fPvbT^?1Jr z>1qZBbq63)+Z2n<3I>=%;IXeW-f}#7w6lZ6M5N5A2$*qXZ=slLc#F3sqs=UIvME0o zA{>l<;(6PE3R96FTzV#LRBJb_seqfut@_x;DU0#`Ft>@ZPoc9s8I{O7M?;h2NL$R1 zNP@P-1i^{Ys4R)ya*g6hQ34|^ra>imI@X?b#c6UKWdM__*m}s+s3u?H_vwUyHmd;Z zzBSU)NMLtJ$_escmT9zGT5;dT>Y{Q$Ql;V3EI`l0p@T4IC#Sq9_{{ZVeX@f~0W*-m zCCNHXckUApjgMEBe=v7ABgJ+BJOmn2oXPITm z1sH)~{MJ#yjWToVjN*ShO!{;k*;X}z7Afw)1v4*%&NwV5Eg}g<@qP9etZ#yw1!SD~ zmey&|jvL0m1AK#1YVTGM3#92+YyWEefR_0*q^83Jvt@>*Bh@B0vG2x5MR_gBLb&h* zMtOnDgl)O=oR-RXHe>#4${~lv) zF5z+hNhn{F1?6_0;>Qr`Z|C=5b-;Ny_v?D9|B`H*5)f0zoy6qQ_1nsEBGR$H*cTY%Y;-IRybup6E6+n zE#lH8`3kvI&eFg?V8XX-ZW^M(BaR`LYu6Z51Y8GnixA zM1FkHQ*(?-R2~sb@yTuf(gQ<|!t>NrX|XAylbbGr5?($o$*^N2#S5vOg|n>Q9OV@s zcYp}R0w_qlbUpo~P70K-gF=3*^eJnIliZBJ8*E)`LPhO z^7;`409yh*dIw+q+20vK%P%)iE@htt%)O%FCKm@L&uhu4Zi579wkbT7@FRdt^cAFX z6%iU)r+tfX`Tfm?yZi39V=AzDLc%mvk7hpI3^$g#r^dKht=F6L&|K@bdm~%zE3i}j z{GKImQ~ww%W7bE`)j!J;@)-x6a7}O8IeT>w1>`dhSUbUREvRzRgsMnwWbU+LL{HRlSOC$iDVr2gWT49AH=SN;i@GP@t*a^iexu|8|-+1iTkPKXq^ z>aDAL_3fbU`qFI!gn{v&nzuB1MXA5ivHjDRkqbW#8y%HEAwwArLnz z>$B&#E@<@NrIcKV`21~Fj9LM}Rx6@ctoFNlRAcZOvG&JPm)js+M;ib~8fiWL>c~c? zoI!6%KB!l&M7QG}e|a@dTEZ$`UZ^IWGF`lD^YgRp)2|-R&vJ#Phca03GVOnA*00l^ zgjD^{apL-S3YH^7mJ~VZVG;fH)j?a%=WVI7PwrOdxP{JIBktci);{GE6~CKcG$aw1 z*wSj)N!c}r8^1Ey6?vgpp_^7?92hTpvKRAlQ;wwQw2|vAO|5RII?Qesf8^O5Go8nu zex1P08LN}~);V-9Me2MQDU!H$67#dyo#G7vL_o0!jE>k5+4|SteDzw0!TKc;^>NTd zMs!j$SLDu={RvN{8eMf`RV;6*emS(cut9V}5Otv4^$;6}2E!f-89K=uPjo)2;tzd; zVnv2Gz5JB_p4jI`+0y+O5F|0!qOD#!S}McGcjy$gy*Px;-x-nwInFd+oWzg@(9j+* zz-v4|IoWmz^Rou^wQ|Ry6p-PZ35PBi+ILcAO{*4ew!am}bEVM_zdyBLcgp*Ece+jt z!*U>+MAURz>r*0<`IvvwmuO%g7P`zPsGpzyRME5ZUu!lFb4Y9dn4t+I&YZPZRbHea zuR+6b)`n@HhW=y~w>5{}(1FAWRWWm}ezLUND#91YXn=ALL&e9MMhOozhq+HXKY%|J z5V{a~I|=R%PkidRcd?2$TVYkTbrJD0Ut~hjnXt8)Xm~K^5gn%P#$U#lKZ}Uq!VtNJ zShh*RqzVXOK5=XARd=F=!x%!>>mMx*zjM85eplJ2tDfM~_;7`*?S?g1G2R|Q5uwR| zEY)z(inkg{$W)n)a=AW28CANk%U&FzGLg+yii1bJ)K57IuMACdE{Ye}!9QI?Hk_a4 zu1X;UF52D;3FB+et5qD!*cDoMNu2;X{f{$lvLtg;hItWE+6t&r^b4Aj)Ieijhh()4y*Z?!$Tpqa5`1 zv2>DOWHHH}wLVo-{c;D+rCm2TUcWg~#o^O46a42x+$#wPE@oih0S zkugO+`0u#4Xu#Ov{)$AiOW{6*ECAm?uCC=;3Ysi>Tho#kmty?kX#hCG8)Dbck}F@E zWaII+!W1@GOS6D=CoKoM2S+xE<)`0+mpp4>4H2&9@BdxO#=2Y50>7dwzj$r-Gy%AK z4cYq<^w%7RbbHoXLcLyg$9$HH=hFH2cVDw^0_PuvD`#g%35(COa(=05NiCu=;{rt(Jnfryk_f&{ZNJJTr zpp!Y1=8fQTiikgZqUXb>am4!#qGnQIBgw8sHTi83RbI>i5S9XgX?ln+~Kh z#4g%z#y6)e7)HN)Jr)0~c2e$OVxD|F9^TIxc0K-{UR|^cQ8W2<%R;EpO63p$2=h#q z`ThR1{uqCG@b~30lTZE@|NHWU{VJ5hU!$G$q#=kHHhXnR%Z>EEyj3Rp<~!fl?@cAK zBS9=W#~P z-n|K?O%eI}R`&;$WWh3Y_B3!y(*g{CJ2%#X3Lmf68+q~pAV$&j3uRNKcI1PIgLu=Z z{`8Jqhc{a1_E@i`GjQ${PP!DX$txnaN1YH=iP!#@61y92-BTE|KNUxIw;lWZeidiA z$TR)uu^alm&)269)WzS1@#vDY3VSD337F}QdoBVdxxmQi?|&h`S21Q!U)(dm%~H{5 zE#H00^SUAxR=wVEMvON-Z5jwYP4{(zKL{ z5QfifeTrfHVl{2d7wFj+cV8$jQ#it(OsD&5@g*}-X+#XCz+bD136|+hkoh(`LJxeq z(pkD1^NbHqXFoj1VmX;lpHHvBm(b-?T_TEj_?SGd;{^;jt89rruL#Rpf19E9s7tIF zQ%tAw2R9n$8Wguq=|uH+CY%1lFvb8z#yrYpkE_VzGXsru*6posZESGVHJ|g;iJBY@ z;}xC#`w)6huK|&fJob+5V~@!H7z7m`0vDR+&3Cz>zvnNn!@BKU%7R1pjnTmjr^L;5 zcs!^?Pq*lc8jHuYo>l5nvGtLdgl7)5>&iXY*OfLGh@)O)%rK`CSm-Lez0RXGuFw%B zEaKwRWEq_X8<$tF)K#;riYFMzc=3xH)ES~%;ui8BC-iTB01l*mjIRaE?9c(Y%${e$ zftWaFkwumw;zf9>h?igr{Unr}^J;;HP|(#_)eT)ZVz-bttoKrwOz=De4xATm?X?a6 zMfPFPJFkk1!BL>58+BP`dI@q4L!Tq$-nk`6v{g1SRz;odPOTrcSMNdWgsj^xXJ=nA zrO;m=^&aGQCO=p_?P!wP?ecEPh`$Bofn7G}C(RFmqu5UppeoAyK83GM&K+;=^mAL< z-8&!I!f_t+GdzH%70P~|#HqJYa`0sbyY;l?gSpKiND4PAt84ZE_*>r5|BmiJQ|V8s zyo;=b{u)K@0jcHxb=tXVk*NHqgnbPUt3a<5Gag&0WWJF$a$h1Gu#yQA+TGmtUr|;# zg&<8@cA?+FO)djlr2d###}6q_in+csfY{PCW)|P)>JPIaW96C#=cd@wN#Vj_W7HP7 zrsteJzm&MAYekn<#p4pTs*M(t&k;sGq-4Ga!yfZ)6zr|6)hu z(BU1`^s$spnL~HqZIS?s{r5=tusJIA>pKxj1i_x&_3(7L`DtGVxS{YFEjD}K&RwLE;&9=B2@prf-voC=V1%Ua{xsg-#FWv@N&ZI{ciazHVl=J z=#H3-mtxMOlvu%SEkG2MZbhxTH~*=tN`stl6g1pi&L8~T9_HRbqW=ojHWM9#KxR$! zlQk2v(Xo5dJQAXGM`bdsS)U{$LQ^M%EfX$oX6#h7z`x9sC zZl0t+F1YPi-p|(BrdHd`su$^$XAG>RHB)ftzruO=I415s@4dNdD@G>8L?yRm2pf>_ z4~vJ59!~Fzv%NNaQG0{F7ViR3-stQ{*=C^5-nqI&ZbqXh?dq#$MQkf)piMX4I&y($>_IzL;`$YH zhIgUCUZZNBeE*aDCaj@Z(ED0~PQ5CNek~m^vJ<-8+9GJKbi%fBojFdHWM>B zD3YHAbfl^-xN11MT8f#bwceFC! zMbmjy)>U;^B_dmh8bcdJwd(TS0}h(4_;$2+ka zi$PUbA49{gu1>jU9^3sU?DJQVsY|P#<*7g)V1@^`6-TZ5_JKlh>1x`TryK9>G9K)8!Rj>a(}jnueUiVuwE_ZFPP!u$kCRV|>Nl40Z`N?HM&;805gp$M2IC!X82{o7;o_MXi#fNY~Hdgqb|{p{zX(qSIK3q(*?&}kbw zT`eahO{dYxm?LTj2E)&<4M-}*suH`)YnSxvwV#*C66Q`bHTW=u0Er7NwsFE?kfvh` zZIJ=!aEBZ-(UX6Gyo2W#)5%H?as*&HbB&Fm?=sR~irLvd<)mEM?96z>hP4{7isBF^ zuZI!e6N&};f=27!Wj`*~5;Ni~(p1@*F68W`@u2QdL{ELY^DLS7Kw5A)2vbhN!Rs1!)Jw1&V+9>fszY%m5+{{8 z<;fa>n~FpyYu!Gu2)cB#yP0pix^SKSny-@5pptI~P{}_r__UHdfTM;=)W%Q4JAq(O zad;Gn6?!``r(^z_)ZV!nE20mJeuHMU-}NX0Q$e@7N@~DoBu5_q<({n z#j}gM&rFM|?N1E1p$K>`mu?~IvZpKeY{~Ov7{FF@9V{AwCbcwx z3nTp9zH$H{dM0)EOnn0!$)C6v;+*$q=|O@vbL3yL(y1VyS`{-2mKrR1GQ1W;|7-jjzp1Ac7+DH<`? zL2x>Fr>V_-G{Kv-(XI{afCZlihPA<$pReWJSzK`zA}S);@A%+?eJfC>^e*%89^uq* zpm@|+PGd$aUw^yO130&h;Uk<4VEgN1(ts31T~n*$L^6T0gsq{E&~-(c^G~N|M#V0R z=6xaZulpgEWZ~t}uG&$eFXi?gyPKy~d%v|L%=_!a$XS8VJ)nKOHKnG%wAkl{0q&bZ zoQQDWxja-(fweoFJaKjc0vjQHdj7YU%lFNmYX$7`r!qeZa^!&MCiWzRz4W2qTSc7A zLJLL^_H}voqa@|XTZ-0}wvT&{{N8eVLvEi-ZXvf`K?HG4%IGI!eS7KP)i{`CpF7pW za$QF2X|HXmX{@l{Ct@8r2`)l!ZzRS z^f4}sxo^%oBc|g+eTP52d?c0djbiG{*Y65}uv{dt0ao;~fbo@Q+!H~yfbiB)adum@ z?OL-N2D3eWozc6Xsh@u`kioJmXa|6XVBy-BUYxrLMqQE=Vy%xw?;mewxp}qO#`3yb z1Ew~Z&aU0-21jic$~4FUqWlLrk3RPX4&4$kmX4CIXzvAZ@s|U_!rcE(6u^<}TLbQA z)w}cLFYW^PosV#?6hmJMi9l`uTCLLgiik>r_id!5EP4czsg(J$TQ$5{Tibgr$HyeH zwpBY1cXhPwmNl3b7DDZ`;_D#DQhpX-dDiB0-`bqU{{8uSNj-q=(pda`g+pxPtW5%? z$=gv6C)K`DXtwmZErz@MVtiDi0C8Chxd3M@c(9j{6fXS7Hc=_vHnJ zTZJjtU&BZ$g3d$SCGQ^I!DrD63@SY~LYPU9IKk?#0KktK$=*)O394y57@>ZNO83ZP zH~6Lf73^5Y#)w(WD5n^Z@bxJpV*$^T`3f&$OZ&M7xaH_rd~nS+l6Sfj zqckA(^m}vL5Vw*pF_p&NOn1pf32fNa@HC-gdSRFF&FmyNpQBB5#X;_kO7d+P-6F7lC({pa5Bu;o%lirM+@I7HD|2cp}Pm4Rs_Cb0*qh_X0KytbE_C1AGYy@l)nrouYH|kiA09vEpI=` z0$LJ)#H+Oy0)$7L2?GgaVAEfFdX_0>)0x2BmS9RCQSP6SS%KrAQpQf1QHfWpy_x3g z6h=asTTWoGgsNN@hyHbFwGTTSF|7?3OuTIo#uX58Q%qR|WW#HKC3Mk8+jg!}*$4V9KOR(#VBF!Fn!!VVw5YN^%9jl|(O20~Y3T%C2sa?J2(4rMq~KHuHU( zi=%v_92?|AbX69YOWdaNZUeRo)ykz~v^>y~N&91kY-#Hqaz@!u@ z;)hJ(MFI3fKaD-I3PGJ|Z??8NnnX}4Wb?;MqC#g|Y1=S1L}^knx?aQk0#g^lpy4Lq zxJ9DlxgF9#9xaZJP>48}Tv@H|@XyglDva!Y^gA3ZZ%ZtSgt`oBGF+go8Ih;$Qga=C zpE8R*A4{mP=zd!AL^82BxQ~34_mcQ}{Hy^`RIdUOHZpHk*+|hTXQCQ*$AS$5tT{Sf z@}&9yFnQiOo-e1n<5`wbx=wTt!Ai8Jev{2xo&PeQL0$smp-O8X|E+SCQ%?Qj-(;MO zzMuiunfBTxi#z(B(wrAvF>aj6)HFe z^H{LVlh=&!Jk5e=Q<`#_T6rR|`H8G9u-oLF9`c*LXAfMo^2(F+_k zJDU-2os2#-)B~0?Dv?Ud__|w<2>MfCiq|h6AP3w$W&4`)-*L(Gsn)>wlFcp4fA?qK zfSk=%9Pawtd~g`wI^nGvALSls#V1R{0S~CSB|_dyJ4=a5Xyx0DjNEzhdg3Lb{~M=} zpTlYO_BbGS-%N1S4y9zL#CQYZ@{!Xedr93k7mmWQoMfA&#^D>6kv>w`85L97m9ZqH z`o|gN^qnd5p_7*M{1<}I*pu=m@mGO%zX2B*-2zQ42H8zEFgRo)o#wRa_lfRn3ju|w zP(YHnktmwb^7JRH1kf5@_K7(K{G8S77B%GdDtabTTG3E*7V&pbno1W*K!e}gan2t6 z?Qyo0LBIxT?+Vu2eUi-W@b4bzV|h_cN^nV5{v*vFH>W4%=pGuthH%&C$g`5mtwP)A zvN9gnz*w9fY~r)0NCGiBfKlCjMf>~)c_Ll|RIMjQ zxv!vSg;19-%gz|qchoT9mFZfY;N-ngMj}~|&C1W^2pAe(;8^oKRAm&fVoY}q_!*^H zjRcxZV2{VD3~*1lWCMlW{*Iw4j_r?fXZSmWF~iCnNK4Ht#s3(HIR=TiAC|Y>C4Qc= zZiGRdmX~14kvl5n0MEkoLxBe1XHJ0FGa z793yclDPOB-b?0&wz5NRLk))Oxyx6Cs#2k`BaD|YA2CF^{dJ+0yq_l^d$ZCv!W#L!iJH#Vxttd~H{3o> zfSZ-;dsD6lDA5QycXXCG2`ISWAyf?r%l0>J^^SDd-mKKv$%bQfp|c;^8=LbYeKU`$ z(%`ZrLHT#6eCpML&@)eLb~6=a;g_g~2r#D{wYP5HGt=~GL%{OE(DYB6m~(}pc6Q~7 z9zaa;-=gdJu@VK%Jf1(Rzod#>x}T)z9S=u=M@}IcWp?d9U;O&P=-GI<+m)FRMEJFr z1Ej_Ga8RD(>hS&5OPXM6Sd0ipc-*5+O#1(iyO28%^8uA;9%IZRk`YuTa%%+@^RtI; z2v?$O#9PvwA8MWdN7S8TO=$ODQMZ~f7QGwxzUt&x**tti3sJxApFeT?RF|#7bad$$ z5TK%(p`OE2QRf8Y0q9_WfsZK$W4_1qcKB7u1HLoB-PL%2@IDGOoS}#33|3fXBtF!c zPShyYbVeGkTkrI(a8kp`l0Q%;Ipk{f*lcZrA6U#T438uVVXH>FTAFw*N77CJo%2d6FYk@d^lvgJ z2d2fkIx%`f7i47e=ZpTXqpo1OTam{Go;RV|BoHz5^u45$CBCx=l1z7MBj!wT);;HT zhuJGiTyfsp9-BZoMhpaPx=pL-lR61@BiGMAu81xlxDDh%GsB%Mgci93Q zlG4*KPjwi}+Gc52FJBxCjgTIgQ0fr$D2A+eNe`6M)Xu(2b32YjMz7-aW{^j5-e`Dt z4+c^|>|672I=M#q4Stx^k_{{d5R3{5*!W(+bxCjcAAva(xX&4DI+0?s)iXHvJL)M!n zjkgJkUv9POMa$7)i{D;R)7{zs#x8jX?o}$^85!QJZ8&zZSGl+}3#qi&gCBtibSiTm zvePRSrL{;TXHk+sya&oVn;kJXis!(B_BYL^JH@Mw^9La2OdrWy<6hbdqqD=vdWN!> zqT9`}CX+MP4-KOXtPI^Xh?>#O*rkf4s3uJpG0**(y;k1@Lgytue9$1V0{ZothwlYr1wJK;Nyz8x_Qj+i846l*Z?7IEljxi1((1bl z?yEpAmu2^#-`)y@+}lVg=Ab|~U)|d86W;h}cQX_9G86ae7(SrV5ll2ng*IyII_H{q z;t)p@J4~5ekV!Y4ze1iDd4KFT=H!mbS|gvQd1s6+*u+mVbdavUf%|Y~zHagq zf%K#gD8B!pz%-mq@gPX*{rP#okx6L+InA@X2;O086){UE-OJ(|9|qn69Of#uYwYB8 zl~*O~w5D!}B8aR0Bhqm{wQk9sk)7W&abPp`;L`es$I=B`W{Xuhuz#+l)g8)CPj9nl z3LUvG%}rl|eYW1GPxHN7(Pllj*5SeG86FLF&w|s*KNsirG0B!GzhKZ7*n7%(+m<`u zgxV=$kwTjiw~06O5h%B<6mFc1Cx$d68BV`fP3Dya%T6ip{7pv^w987!MPbp4U7NdQ zV#ZpgEn1-ldo`-$-+Q9gCX&mAb|Zj0+UhzAHkngByX#f5eP4gOQyh4qV^qbHDBt3? z>+8Whm2-IYARv&-qzl+XjQ21gvSMHrZ1a?hAu8#K>W&soNxIh(8ICsNH0=xj9uK>= zH$(iW61o>8J!D}7Tvh$wJm`AJf+tk2`tKFrv%cc=i9&@G6RjVdhb0O24*Fzzp>ttz zq29f3Pcr7Y9(M%~r|+kxeLGChSq62H8{C=tBW4^tcVM2EFa@rACJIx0Nf2N|wTfYE zead0pvINgjlzN$tIwNH9KVKBRb89$2@>ET1Ep0j$>Z9(Hz_DTE!gbyFD9FL3X4 z+8-FvJ}QRRDCep*U%Q~7P}vdyYr>dg#z;Yyn2ID zxCe2p$Vs}!G+1-b#E{grr#neAO2|Gby)&x`Q>q*Kq8O)h7@;SUrqzgNa-nq<6XP0s z)MH&Lv;tcZtDR-N{<7>f5NwttI$Eu?+)}^RL(^8H_wF{vKa~hnceRZr)6v5jpW z_j;sr`}NMeFM%dkP(y`V1}PprHLu~pMXyx)69>f`kJEV+Ek;JJ-+0Hx{BYpZD+P`g z>Uxr!e-GL=CCecGKKl}L!>9t15LsQ=Q8tMtHAvLH9G0L5v6tV56LPif^i-_d$tgdJ z?@n+{inwZKpZ||`(81RXfd>GRmC_n>Epxct2+a?FI~*1?BA!saYj}%{*O*SXguJ-ZXv`hg z%sl@9S#++UyJwFo6{F}b zug(u^IBH9tok#0H$b&|CN56eLJY0kmTpd)K&KM_=u0~1R7LK3Y{?aiw`MmInhGbe? z6B)3a_99C3*PSmT_eO*X>g)3C>ji8dqn?jL2^{S1aJCX(zk23zODV8WBk_%WS+(Nq z$+~+f3w-Gp{%J;`Ri}?x!98=3O0Ae=oG?G!fKIK?Bhn6YTd- zwK=>dC=bY2P-Tn4vZYz|jPi>Li+f!hM2k~pcplHbwX`BL^~67pF#3_T${kmAGKE+g z8adJ5##W5iujTe=sHXSaeVcLzfvjabU&my&e|ijpUf&pXHlJYba=j)aOlHRkhV*^K zOg9o%cdaqYK`7s=Tgnk}OwuaU$Lo#g#kuLM7(o))B%xxGXE2a|dWnAMZ5N8Iw?~lM zRrCbW8@#;ipm;tgjSYm~U8v|9XUQnq3O+Wlt+PH~8}dSZeMSNi-dV6m?F(vMf}vf* zU6pKQqAO3wVWbWcOks4^@PD+>%#8hDJqyOM(abUw>HN zP}n)R=#>|COBJd-fN7X#J-HJYFow+k5|BbF(IQsO(L?*Y8W=Z>tyPr0&fFspU!^i) z)g4zH1kE>cc9Ru$Oa7%a z03EVSbFvdnCYSH~pd=-*{qrdPEAWY)zf~-~(2J(nzvQZme$|JQbvjRR}dKdW0ni$_XG0qvh)d<_b8hA7LusU)KAxP0k3K^y`gjtruAw zIg&h=CL)0&RXG!{J&?p#2DEFFE!LusttESxh47t3wiR0tcb0X!!f}!d*4ymCm1Rj* zTN_%pH}YLYLCOvqEs&4t=pKE4@Li8Td+ceN&7NQ?dQdD`ltE1E8KZdNf|zrngroC& zkjpdHtd-B5xJAlU5UCW4vzoIdu^^iT-Mg&_;`fy8hHMp#>}^TaG3|POeD-5?Cs*K_ zUG&PEX6jkI?|G~eETfcp9c^}yr~+l?Q(%!-%tb}T26mZTO;2^GyV{qk3O;8q<2u6l zME5CmZ}%uS>NUJ%NY$uXcUDi#rT|HnU0#8DNM-L7s{`KW+CEWJF4t%k@^)!uR)70s z4fO(gGc^B1OHcQ|H(z3+2^?Xg_LR_Y+nzqdT-*m96HHJnccot2<7P<97c>SM6wiO` z?Kp*c+RAV-YFZ|tkAma!%zO$&ehG5Mm1p06yROa%+;4?`4!)?M#nKc+EMt0<)xxUI zO6F5z83DKAP~J(6wu5rUTDj3HpCOQ8Uja;uZ@8+2Y=`(Q)ukB51XOjnjSHsHij-4=1=V-=T{ zTKg6)QiFOmLO&gLwnj`!|7;GOj$@~4i-OEVHICAzPYBZbI`&k9zhnGdG5u2fWuV_2%5U>*lo-FV9WNflU zoG$EkUNK7bp;1r=&;5MbTk4~k5Q)g1 zd8;UUyb^5bDKcy7o*-(Grzm|E-=Bp_OBn6v{4_M4I6W6(tRTp8|B2ZUK5USB0e_3K zqj9e8WN8aPPa53kt>bl)2tQ<1Moe*2^>GW+DN4dh1M6wJP|4jpe83*6YP_)8^gN7J z*46^uzc{o?9wQFQn)$#@C*Z=jW30f^*>)*x94z#3mp_eB8U>zzxS4;&*Sn=v2;LPi zZss9yc<<9$AD4#DHPj2)uY!-TLr7H1coeS5VA9SfGGcK&e05=^g!7GOCPvpXc40G= zpt`<8zBvOfGBUF!2?By>Q-WR!hMO>bz%%wv3S$#tmGckEt>W*wXkML(@Y+s>v`C2Z zJljQ{{%R4>GAUKxE*?GBk76KqI>?kY?{odG5#h$Y|7}$ohEdfL)$jGAyE)d8J8 zV8u|Y4{a`iQ>bg_Z!L^i%|H0OotPCTC^2KWIy=2rt80j)nG8FyV0rD>gzB;8*5nd- zqDEDdJSAmEy2bX|IPRo?DvS$)uPjbvBBBUO^{%X&t{7j70D?I9DLoP0;jyLm3yvZ6 z8f5V2@;>$IeVu)oO|Ky=XJ>1J_}Si7h2t70Dg%f7q%ovc;lcuZ+dzzSWU}ojISSO^;5L(A8J$z%g z_pS5>(Yp&Cx+Gb0z7cSfsP7b!Z&ksN-cAiAA`Ep-XQ*i33eD_m&BVxXf(B-(4R+RR z7%ezs+xhK8oC3Y)4QeF@tWR@=eol}$eew41Daw-yA~wU$^dyDbafJ5>L9G6{inkmj zJ{1_ z3QMY5Inxz8``X}=8ucV9IV2VN(4y%0-Q2#N+MExio9-QumBBd_#`!-HqYpe!KK@Kk zTicL=)^G^iDxa88I}2^=Wipx(69_UYid;?eQD!<#1r@xnFfnv;-b}XOLFBOx ziGm#gda)KD+S=RW`3qPy@#JpTlD+dWi}YS+pExXfZV*ClX?Me z)1QdAIef0Ep@ck>Sg;{o1MTA(_LUTf13slbV~V#h$(@78kFaN3HsZ9%UQ1h8&4OGR zW9njjR7^;WfBXhSJQT5XBCcZkSOHSCqxbwQNG*ww8`X*#AEKu&6g+$re;-0CxHoz? z6G4&&K1Sj1J|;Jj+^K)YS`RR3*P_wd2t}w<@EN(JDhUZ}%&~D?O)aqK>Ju@%8P*}b z`fF53>#! z=`O&+G%B6rbNUCTr=)W?x;p*b4j2c!I7AkC@-%Dub>0}sn&ZEg$C|iIfjijD?^7z# z_Ktb|+TMUily}WszE-S=T50};#rGn%G~cm#Ho`T|)FLiAd0vbjJmkbq3%OK*Hs>$4 zm6`JDufaur=7n?4OIwXqv`ph417$?(%{`f&x6!oZN74(&a6gzXMRi>A&Qpa2&ooA!61zZ8*hgR{D z9RU{5c<4AxVq9)hU80G7?X+R_M`yX_{cfvQ@omh8iwb=~1))niJIZXrA_u>Z`Dn52 zYZgszM-S3xf*LNNWZ~9RBhv$M+<%$?Id?VYLI@R5@eoa(G5BdhpPp_@l=r zb(oz4u}IbGp&wZ*gAdfwETc)`3#{8uG56Gs9()VZMy3cbY4$IC)@ zf0+6H81rhj{`5%}HG zFEBSF2$UIz||psE%^5veX25=~w=_}uly(6fR)yQwS zGh@+dz%q=z3RTPNTq3CL_`(;_hULXVO-7?RF=lKcZZmI00!yOt?L8yACHI>~-x|B& z?33n}G50`HB8Rc$XOcz`+Z@oc;+I``x5b zs95qA)r}CAQaW6%!#nIw>W^yxI!_>Pq-ffErWXo(bbQrv3Wvk)by|v<&nHhhQ34HC zi;NLQYuOBvI_ik|dG5oV8$Yt`tL`G4xesHqy8p~(gjD)@cSZI(u4OZny|`QCEjiib zk?DH|y-wDXzlG9UFi`edGD8RNYx*FZ|IMV~1$t~(NWxlBy(7)t#`F#gi*fym@=j6Z zl04CkN5&p1%EQkuyl4^`QhvRPhvSjP)}mm~3xzQaFvbtGxLu9#;3Up0!q^&ySsp$_ zKY{mg>qcTC=A1EG*%ur1^0S{*6}kZKlcMhtp1c*R>(M%pdpnU_KH-0*P8kYS25%Us zQ4>RdJ;+zejt8-Rq}Ss2nr9Lq?`W=<{En0Uxbx{SOt#H&;U4)t8Q{7e(%fJ9%1%U+ie<2jVp820C@qo%L#g3O)fNdP%d#ZdF$Z{%vVo9jU z$jGP<{euj0stJ=9ZkEouD3}Bp3L~HlaMY z5J}yU9=5h!J_&)&UZ74Cy2%I$WUF&ygPcJ51ANDB5H-9cu9zoDfwrW~WjULnx-N%Z zznuhPEm_POzykiSARB}m|1VB%cEC8L>+<=alitAMz_n@7#KC1ED5`%e6dKhz*xd$J zC*KEI)&Tm>CKU|a2kA%CvAoC|Kv)@~>GUWGv;}bzqex(7w|w3b>kfuembZs20qlIO zz)}|kFxA&e?TS!n$AOoHXi$&eJ$?_+uSxFO<-b_)h0adS4Cs4wqOg1vw9yoEq5lc! z;)pFy>AW`3N#flaBZ!~ak$ow-K#U)JVUr;Q%x|qA4a~#pm&wDCpkAa)@I?Zshkg?a zM&KZ@0+;pWT3@d9-L<~E)}7(HGhB~d>ri7IYOEu}|7XY$JSFEpJkTPM`le>z-sUY? zJz<@XZ{7aJkpx-Qj#se<0XkPYXL9v0U>|D*JwF0OEm!O$4U%gb|HueflBrGnbYnvj zA25mDiQbX{No@b`sDZ}-cx!AtVOoC~AZ&^bx9Z3iz}a42{C@MKZMM^HaPTigOky)& z3cB9jt`n62c2DS-Bw|4)vMQZl!c_pAcPe7&0$A>N`htDpnslX(i@Xda5;LZAyQ=03 zQYONvDsrZ6-vP0QpqZ#seM{HZXn*+pb$ArSj~8@aNr)P8v_ZDO}uO>h$ zt-U!{)zZw~5_1~lIiKKtYC@|w9er){QQ5$~!f?XaYOy)yy<%kt+`9nx$! z$xvmW=l?}6mjHjA4f`VZH=#cilXMVRi`+@mWZnO0=|4BO(*ehW*-`(Z!GC|Z>lx7G z?Q?t2|FzeD$*p(xccrYW@y{LjhyAP@$hv_**FDJJ-Rim{UPBgthOl)5`HL>r9O$}% e{O=mbigA$MStm@SOVbAMckGDMVdBBlzx@-MdI2024-12-12 00:02:35,338 - {pytorch_tabular.tabular_model:147} - INFO - Experiment Tracking is turned off \n", + "\n" + ], + "text/plain": [ + "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:02:35\u001b[0m,\u001b[1;36m338\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m147\u001b[0m\u001b[1m}\u001b[0m - INFO - Experiment Tracking is turned off \n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Seed set to 42\n" + ] + }, + { + "data": { + "text/html": [ + "
2024-12-12 00:02:35,388 - {pytorch_tabular.tabular_model:549} - INFO - Preparing the DataLoaders                   \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:02:35\u001b[0m,\u001b[1;36m388\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m549\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the DataLoaders \n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
2024-12-12 00:02:35,394 - {pytorch_tabular.tabular_datamodule:527} - INFO - Setting up the datamodule for          \n",
+       "classification task                                                                                                \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:02:35\u001b[0m,\u001b[1;36m394\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_datamodul\u001b[1;92me:527\u001b[0m\u001b[1m}\u001b[0m - INFO - Setting up the datamodule for \n", + "classification task \n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
2024-12-12 00:02:35,462 - {pytorch_tabular.tabular_model:600} - INFO - Preparing the Model: StackingModel          \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:02:35\u001b[0m,\u001b[1;36m462\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m600\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the Model: StackingModel \n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
2024-12-12 00:02:35,516 - {pytorch_tabular.tabular_model:343} - INFO - Preparing the Trainer                       \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:02:35\u001b[0m,\u001b[1;36m516\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m343\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the Trainer \n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "HPU available: False, using: 0 HPUs\n" + ] + }, + { + "data": { + "text/html": [ + "
2024-12-12 00:02:35,813 - {pytorch_tabular.tabular_model:679} - INFO - Training Started                            \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:02:35\u001b[0m,\u001b[1;36m813\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m679\u001b[0m\u001b[1m}\u001b[0m - INFO - Training Started \n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n" + ] + }, + { + "data": { + "text/html": [ + "
┏━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓\n",
+       "┃   ┃ Name             ┃ Type                   ┃ Params ┃ Mode  ┃\n",
+       "┡━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩\n",
+       "│ 0 │ _backbone        │ StackingBackbone       │ 77.2 K │ train │\n",
+       "│ 1 │ _embedding_layer │ StackingEmbeddingLayer │    917 │ train │\n",
+       "│ 2 │ _head            │ LinearHead             │ 12.5 K │ train │\n",
+       "│ 3 │ loss             │ CrossEntropyLoss       │      0 │ train │\n",
+       "└───┴──────────────────┴────────────────────────┴────────┴───────┘\n",
+       "
\n" + ], + "text/plain": [ + "┏━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓\n", + "┃\u001b[1;35m \u001b[0m\u001b[1;35m \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mName \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mType \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mParams\u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mMode \u001b[0m\u001b[1;35m \u001b[0m┃\n", + "┡━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩\n", + "│\u001b[2m \u001b[0m\u001b[2m0\u001b[0m\u001b[2m \u001b[0m│ _backbone │ StackingBackbone │ 77.2 K │ train │\n", + "│\u001b[2m \u001b[0m\u001b[2m1\u001b[0m\u001b[2m \u001b[0m│ _embedding_layer │ StackingEmbeddingLayer │ 917 │ train │\n", + "│\u001b[2m \u001b[0m\u001b[2m2\u001b[0m\u001b[2m \u001b[0m│ _head │ LinearHead │ 12.5 K │ train │\n", + "│\u001b[2m \u001b[0m\u001b[2m3\u001b[0m\u001b[2m \u001b[0m│ loss │ CrossEntropyLoss │ 0 │ train │\n", + "└───┴──────────────────┴────────────────────────┴────────┴───────┘\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
Trainable params: 90.6 K                                                                                           \n",
+       "Non-trainable params: 0                                                                                            \n",
+       "Total params: 90.6 K                                                                                               \n",
+       "Total estimated model params size (MB): 0                                                                          \n",
+       "Modules in train mode: 188                                                                                         \n",
+       "Modules in eval mode: 0                                                                                            \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[1mTrainable params\u001b[0m: 90.6 K \n", + "\u001b[1mNon-trainable params\u001b[0m: 0 \n", + "\u001b[1mTotal params\u001b[0m: 90.6 K \n", + "\u001b[1mTotal estimated model params size (MB)\u001b[0m: 0 \n", + "\u001b[1mModules in train mode\u001b[0m: 188 \n", + "\u001b[1mModules in eval mode\u001b[0m: 0 \n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3cd6f3938b1f419c8b07eb89ffa13bf4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:02:39,304 - {pytorch_tabular.tabular_model:692} - INFO - Training the model completed                \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:02:39\u001b[0m,\u001b[1;36m304\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m692\u001b[0m\u001b[1m}\u001b[0m - INFO - Training the model completed                \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:02:39,307 - {pytorch_tabular.tabular_model:1533} - INFO - Loading the best model                     \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:02:39\u001b[0m,\u001b[1;36m307\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m1533\u001b[0m\u001b[1m}\u001b[0m - INFO - Loading the best model                     \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stacking_model = TabularModel(\n",
+    "    data_config=data_config,\n",
+    "    model_config=stacking_config,\n",
+    "    optimizer_config=optimizer_config,\n",
+    "    trainer_config=trainer_config,\n",
+    ")\n",
+    "stacking_model.fit(\n",
+    "    train=train,\n",
+    "    validation=valid\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Evaluate Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b1616690de674da8bbc8cc985f19686a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃        Test metric        ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       test_accuracy       │    0.5960000157356262     │\n",
+       "│         test_loss         │    0.7419928312301636     │\n",
+       "│        test_loss_0        │    0.7419928312301636     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n"
+      ],
+      "text/plain": [
+       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│\u001b[36m \u001b[0m\u001b[36m      test_accuracy      \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.5960000157356262    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "│\u001b[36m \u001b[0m\u001b[36m        test_loss        \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.7419928312301636    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "│\u001b[36m \u001b[0m\u001b[36m       test_loss_0       \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.7419928312301636    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "└───────────────────────────┴───────────────────────────┘\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "predictions = stacking_model.predict(test)\n",
+    "stacking_metrics = stacking_model.evaluate(test)[0]\n",
+    "stacking_acc = stacking_metrics[\"test_accuracy\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Compare with individual models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def train_and_evaluate_model(model_config, name):\n",
+    "    model = TabularModel(\n",
+    "    data_config=data_config,\n",
+    "    model_config=model_config,\n",
+    "    optimizer_config=optimizer_config,\n",
+    "    trainer_config=trainer_config,\n",
+    "    )\n",
+    "    model.fit(train=train, validation=valid)\n",
+    "    metrics = model.evaluate(test)\n",
+    "    print(f\"\\n{name} Metrics:\")\n",
+    "    print(metrics)\n",
+    "    return metrics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:01,257 - {pytorch_tabular.tabular_model:147} - INFO - Experiment Tracking is turned off           \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:01\u001b[0m,\u001b[1;36m257\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m147\u001b[0m\u001b[1m}\u001b[0m - INFO - Experiment Tracking is turned off           \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Seed set to 42\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:01,320 - {pytorch_tabular.tabular_model:549} - INFO - Preparing the DataLoaders                   \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:01\u001b[0m,\u001b[1;36m320\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m549\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the DataLoaders                   \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:01,340 - {pytorch_tabular.tabular_datamodule:527} - INFO - Setting up the datamodule for          \n",
+       "classification task                                                                                                \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:01\u001b[0m,\u001b[1;36m340\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_datamodul\u001b[1;92me:527\u001b[0m\u001b[1m}\u001b[0m - INFO - Setting up the datamodule for          \n",
+       "classification task                                                                                                \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:01,376 - {pytorch_tabular.tabular_model:600} - INFO - Preparing the Model: CategoryEmbeddingModel \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:01\u001b[0m,\u001b[1;36m376\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m600\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the Model: CategoryEmbeddingModel \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:01,411 - {pytorch_tabular.tabular_model:343} - INFO - Preparing the Trainer                       \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:01\u001b[0m,\u001b[1;36m411\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m343\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the Trainer                       \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "GPU available: True (cuda), used: True\n",
+      "TPU available: False, using: 0 TPU cores\n",
+      "HPU available: False, using: 0 HPUs\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:01,638 - {pytorch_tabular.tabular_model:679} - INFO - Training Started                            \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:01\u001b[0m,\u001b[1;36m638\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m679\u001b[0m\u001b[1m}\u001b[0m - INFO - Training Started                            \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
┏━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓\n",
+       "┃   ┃ Name             ┃ Type                      ┃ Params ┃ Mode  ┃\n",
+       "┡━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩\n",
+       "│ 0 │ _backbone        │ CategoryEmbeddingBackbone │ 12.1 K │ train │\n",
+       "│ 1 │ _embedding_layer │ Embedding1dLayer          │     53 │ train │\n",
+       "│ 2 │ head             │ LinearHead                │     66 │ train │\n",
+       "│ 3 │ loss             │ CrossEntropyLoss          │      0 │ train │\n",
+       "└───┴──────────────────┴───────────────────────────┴────────┴───────┘\n",
+       "
\n"
+      ],
+      "text/plain": [
+       "┏━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓\n",
+       "┃\u001b[1;35m \u001b[0m\u001b[1;35m \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mName            \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mType                     \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mParams\u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mMode \u001b[0m\u001b[1;35m \u001b[0m┃\n",
+       "┡━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩\n",
+       "│\u001b[2m \u001b[0m\u001b[2m0\u001b[0m\u001b[2m \u001b[0m│ _backbone        │ CategoryEmbeddingBackbone │ 12.1 K │ train │\n",
+       "│\u001b[2m \u001b[0m\u001b[2m1\u001b[0m\u001b[2m \u001b[0m│ _embedding_layer │ Embedding1dLayer          │     53 │ train │\n",
+       "│\u001b[2m \u001b[0m\u001b[2m2\u001b[0m\u001b[2m \u001b[0m│ head             │ LinearHead                │     66 │ train │\n",
+       "│\u001b[2m \u001b[0m\u001b[2m3\u001b[0m\u001b[2m \u001b[0m│ loss             │ CrossEntropyLoss          │      0 │ train │\n",
+       "└───┴──────────────────┴───────────────────────────┴────────┴───────┘\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
Trainable params: 12.2 K                                                                                           \n",
+       "Non-trainable params: 0                                                                                            \n",
+       "Total params: 12.2 K                                                                                               \n",
+       "Total estimated model params size (MB): 0                                                                          \n",
+       "Modules in train mode: 19                                                                                          \n",
+       "Modules in eval mode: 0                                                                                            \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1mTrainable params\u001b[0m: 12.2 K                                                                                           \n",
+       "\u001b[1mNon-trainable params\u001b[0m: 0                                                                                            \n",
+       "\u001b[1mTotal params\u001b[0m: 12.2 K                                                                                               \n",
+       "\u001b[1mTotal estimated model params size (MB)\u001b[0m: 0                                                                          \n",
+       "\u001b[1mModules in train mode\u001b[0m: 19                                                                                          \n",
+       "\u001b[1mModules in eval mode\u001b[0m: 0                                                                                            \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "03ed36b48da24bb19f036d1db4422cb7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "`Trainer.fit` stopped: `max_epochs=20` reached.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:04,935 - {pytorch_tabular.tabular_model:692} - INFO - Training the model completed                \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:04\u001b[0m,\u001b[1;36m935\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m692\u001b[0m\u001b[1m}\u001b[0m - INFO - Training the model completed                \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:04,938 - {pytorch_tabular.tabular_model:1533} - INFO - Loading the best model                     \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:04\u001b[0m,\u001b[1;36m938\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m1533\u001b[0m\u001b[1m}\u001b[0m - INFO - Loading the best model                     \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "bdcb7befb3b340a895a5399394780d7e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃        Test metric        ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       test_accuracy       │    0.4586666524410248     │\n",
+       "│         test_loss         │    0.8828091025352478     │\n",
+       "│        test_loss_0        │    0.8828091025352478     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n"
+      ],
+      "text/plain": [
+       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│\u001b[36m \u001b[0m\u001b[36m      test_accuracy      \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.4586666524410248    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "│\u001b[36m \u001b[0m\u001b[36m        test_loss        \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.8828091025352478    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "│\u001b[36m \u001b[0m\u001b[36m       test_loss_0       \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.8828091025352478    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "└───────────────────────────┴───────────────────────────┘\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Category Embedding Metrics:\n",
+      "[{'test_loss_0': 0.8828091025352478, 'test_loss': 0.8828091025352478, 'test_accuracy': 0.4586666524410248}]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:05,183 - {pytorch_tabular.tabular_model:147} - INFO - Experiment Tracking is turned off           \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:05\u001b[0m,\u001b[1;36m183\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m147\u001b[0m\u001b[1m}\u001b[0m - INFO - Experiment Tracking is turned off           \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Seed set to 42\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:05,263 - {pytorch_tabular.tabular_model:549} - INFO - Preparing the DataLoaders                   \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:05\u001b[0m,\u001b[1;36m263\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m549\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the DataLoaders                   \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:05,272 - {pytorch_tabular.tabular_datamodule:527} - INFO - Setting up the datamodule for          \n",
+       "classification task                                                                                                \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:05\u001b[0m,\u001b[1;36m272\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_datamodul\u001b[1;92me:527\u001b[0m\u001b[1m}\u001b[0m - INFO - Setting up the datamodule for          \n",
+       "classification task                                                                                                \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:05,294 - {pytorch_tabular.tabular_model:600} - INFO - Preparing the Model: FTTransformerModel     \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:05\u001b[0m,\u001b[1;36m294\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m600\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the Model: FTTransformerModel     \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:05,323 - {pytorch_tabular.tabular_model:343} - INFO - Preparing the Trainer                       \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:05\u001b[0m,\u001b[1;36m323\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m343\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the Trainer                       \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "GPU available: True (cuda), used: True\n",
+      "TPU available: False, using: 0 TPU cores\n",
+      "HPU available: False, using: 0 HPUs\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:05,623 - {pytorch_tabular.tabular_model:679} - INFO - Training Started                            \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:05\u001b[0m,\u001b[1;36m623\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m679\u001b[0m\u001b[1m}\u001b[0m - INFO - Training Started                            \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
┏━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓\n",
+       "┃   ┃ Name             ┃ Type                  ┃ Params ┃ Mode  ┃\n",
+       "┡━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩\n",
+       "│ 0 │ _backbone        │ FTTransformerBackbone │ 57.7 K │ train │\n",
+       "│ 1 │ _embedding_layer │ Embedding2dLayer      │    864 │ train │\n",
+       "│ 2 │ _head            │ LinearHead            │     66 │ train │\n",
+       "│ 3 │ loss             │ CrossEntropyLoss      │      0 │ train │\n",
+       "└───┴──────────────────┴───────────────────────┴────────┴───────┘\n",
+       "
\n"
+      ],
+      "text/plain": [
+       "┏━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓\n",
+       "┃\u001b[1;35m \u001b[0m\u001b[1;35m \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mName            \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mType                 \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mParams\u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mMode \u001b[0m\u001b[1;35m \u001b[0m┃\n",
+       "┡━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩\n",
+       "│\u001b[2m \u001b[0m\u001b[2m0\u001b[0m\u001b[2m \u001b[0m│ _backbone        │ FTTransformerBackbone │ 57.7 K │ train │\n",
+       "│\u001b[2m \u001b[0m\u001b[2m1\u001b[0m\u001b[2m \u001b[0m│ _embedding_layer │ Embedding2dLayer      │    864 │ train │\n",
+       "│\u001b[2m \u001b[0m\u001b[2m2\u001b[0m\u001b[2m \u001b[0m│ _head            │ LinearHead            │     66 │ train │\n",
+       "│\u001b[2m \u001b[0m\u001b[2m3\u001b[0m\u001b[2m \u001b[0m│ loss             │ CrossEntropyLoss      │      0 │ train │\n",
+       "└───┴──────────────────┴───────────────────────┴────────┴───────┘\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
Trainable params: 58.6 K                                                                                           \n",
+       "Non-trainable params: 0                                                                                            \n",
+       "Total params: 58.6 K                                                                                               \n",
+       "Total estimated model params size (MB): 0                                                                          \n",
+       "Modules in train mode: 56                                                                                          \n",
+       "Modules in eval mode: 0                                                                                            \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1mTrainable params\u001b[0m: 58.6 K                                                                                           \n",
+       "\u001b[1mNon-trainable params\u001b[0m: 0                                                                                            \n",
+       "\u001b[1mTotal params\u001b[0m: 58.6 K                                                                                               \n",
+       "\u001b[1mTotal estimated model params size (MB)\u001b[0m: 0                                                                          \n",
+       "\u001b[1mModules in train mode\u001b[0m: 56                                                                                          \n",
+       "\u001b[1mModules in eval mode\u001b[0m: 0                                                                                            \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "62184d0ac93049058c153f2e93518d0f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:07,482 - {pytorch_tabular.tabular_model:692} - INFO - Training the model completed                \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:07\u001b[0m,\u001b[1;36m482\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m692\u001b[0m\u001b[1m}\u001b[0m - INFO - Training the model completed                \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:07,488 - {pytorch_tabular.tabular_model:1533} - INFO - Loading the best model                     \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:07\u001b[0m,\u001b[1;36m488\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m1533\u001b[0m\u001b[1m}\u001b[0m - INFO - Loading the best model                     \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5a482fb9cd5045e3ada1beac9c114d97",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃        Test metric        ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       test_accuracy       │    0.5546666383743286     │\n",
+       "│         test_loss         │    0.6846821904182434     │\n",
+       "│        test_loss_0        │    0.6846821904182434     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n"
+      ],
+      "text/plain": [
+       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│\u001b[36m \u001b[0m\u001b[36m      test_accuracy      \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.5546666383743286    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "│\u001b[36m \u001b[0m\u001b[36m        test_loss        \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.6846821904182434    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "│\u001b[36m \u001b[0m\u001b[36m       test_loss_0       \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.6846821904182434    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "└───────────────────────────┴───────────────────────────┘\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "FT Transformer Metrics:\n",
+      "[{'test_loss_0': 0.6846821904182434, 'test_loss': 0.6846821904182434, 'test_accuracy': 0.5546666383743286}]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:07,824 - {pytorch_tabular.tabular_model:147} - INFO - Experiment Tracking is turned off           \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:07\u001b[0m,\u001b[1;36m824\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m147\u001b[0m\u001b[1m}\u001b[0m - INFO - Experiment Tracking is turned off           \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Seed set to 42\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:07,863 - {pytorch_tabular.tabular_model:549} - INFO - Preparing the DataLoaders                   \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:07\u001b[0m,\u001b[1;36m863\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m549\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the DataLoaders                   \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:07,870 - {pytorch_tabular.tabular_datamodule:527} - INFO - Setting up the datamodule for          \n",
+       "classification task                                                                                                \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:07\u001b[0m,\u001b[1;36m870\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_datamodul\u001b[1;92me:527\u001b[0m\u001b[1m}\u001b[0m - INFO - Setting up the datamodule for          \n",
+       "classification task                                                                                                \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:07,900 - {pytorch_tabular.tabular_model:600} - INFO - Preparing the Model: TabNetModel            \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:07\u001b[0m,\u001b[1;36m900\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m600\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the Model: TabNetModel            \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:07,965 - {pytorch_tabular.tabular_model:343} - INFO - Preparing the Trainer                       \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:07\u001b[0m,\u001b[1;36m965\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m343\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the Trainer                       \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "GPU available: True (cuda), used: True\n",
+      "TPU available: False, using: 0 TPU cores\n",
+      "HPU available: False, using: 0 HPUs\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:08,200 - {pytorch_tabular.tabular_model:679} - INFO - Training Started                            \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:08\u001b[0m,\u001b[1;36m200\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m679\u001b[0m\u001b[1m}\u001b[0m - INFO - Training Started                            \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "
┏━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓\n",
+       "┃   ┃ Name             ┃ Type             ┃ Params ┃ Mode  ┃\n",
+       "┡━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩\n",
+       "│ 0 │ _embedding_layer │ Identity         │      0 │ train │\n",
+       "│ 1 │ _backbone        │ TabNetBackbone   │  6.4 K │ train │\n",
+       "│ 2 │ _head            │ Identity         │      0 │ train │\n",
+       "│ 3 │ loss             │ CrossEntropyLoss │      0 │ train │\n",
+       "└───┴──────────────────┴──────────────────┴────────┴───────┘\n",
+       "
\n"
+      ],
+      "text/plain": [
+       "┏━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓\n",
+       "┃\u001b[1;35m \u001b[0m\u001b[1;35m \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mName            \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mType            \u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mParams\u001b[0m\u001b[1;35m \u001b[0m┃\u001b[1;35m \u001b[0m\u001b[1;35mMode \u001b[0m\u001b[1;35m \u001b[0m┃\n",
+       "┡━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩\n",
+       "│\u001b[2m \u001b[0m\u001b[2m0\u001b[0m\u001b[2m \u001b[0m│ _embedding_layer │ Identity         │      0 │ train │\n",
+       "│\u001b[2m \u001b[0m\u001b[2m1\u001b[0m\u001b[2m \u001b[0m│ _backbone        │ TabNetBackbone   │  6.4 K │ train │\n",
+       "│\u001b[2m \u001b[0m\u001b[2m2\u001b[0m\u001b[2m \u001b[0m│ _head            │ Identity         │      0 │ train │\n",
+       "│\u001b[2m \u001b[0m\u001b[2m3\u001b[0m\u001b[2m \u001b[0m│ loss             │ CrossEntropyLoss │      0 │ train │\n",
+       "└───┴──────────────────┴──────────────────┴────────┴───────┘\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
Trainable params: 6.4 K                                                                                            \n",
+       "Non-trainable params: 0                                                                                            \n",
+       "Total params: 6.4 K                                                                                                \n",
+       "Total estimated model params size (MB): 0                                                                          \n",
+       "Modules in train mode: 111                                                                                         \n",
+       "Modules in eval mode: 0                                                                                            \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1mTrainable params\u001b[0m: 6.4 K                                                                                            \n",
+       "\u001b[1mNon-trainable params\u001b[0m: 0                                                                                            \n",
+       "\u001b[1mTotal params\u001b[0m: 6.4 K                                                                                                \n",
+       "\u001b[1mTotal estimated model params size (MB)\u001b[0m: 0                                                                          \n",
+       "\u001b[1mModules in train mode\u001b[0m: 111                                                                                         \n",
+       "\u001b[1mModules in eval mode\u001b[0m: 0                                                                                            \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2e27939fc57d4c9585a3252b035e74f8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:09,766 - {pytorch_tabular.tabular_model:692} - INFO - Training the model completed                \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:09\u001b[0m,\u001b[1;36m766\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m692\u001b[0m\u001b[1m}\u001b[0m - INFO - Training the model completed                \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:09:09,767 - {pytorch_tabular.tabular_model:1533} - INFO - Loading the best model                     \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:09:09\u001b[0m,\u001b[1;36m767\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m1533\u001b[0m\u001b[1m}\u001b[0m - INFO - Loading the best model                     \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8a98c3a2c4ce4bcaac279982ec86bd8f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Output()"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃        Test metric        ┃       DataLoader 0        ┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       test_accuracy       │    0.4346666634082794     │\n",
+       "│         test_loss         │    1.1570961475372314     │\n",
+       "│        test_loss_0        │    1.1570961475372314     │\n",
+       "└───────────────────────────┴───────────────────────────┘\n",
+       "
\n"
+      ],
+      "text/plain": [
+       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
+       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│\u001b[36m \u001b[0m\u001b[36m      test_accuracy      \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.4346666634082794    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "│\u001b[36m \u001b[0m\u001b[36m        test_loss        \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   1.1570961475372314    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "│\u001b[36m \u001b[0m\u001b[36m       test_loss_0       \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   1.1570961475372314    \u001b[0m\u001b[35m \u001b[0m│\n",
+       "└───────────────────────────┴───────────────────────────┘\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
\n"
+      ],
+      "text/plain": []
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "TabNet Metrics:\n",
+      "[{'test_loss_0': 1.1570961475372314, 'test_loss': 1.1570961475372314, 'test_accuracy': 0.4346666634082794}]\n"
+     ]
+    }
+   ],
+   "source": [
+    "ce_metrics = train_and_evaluate_model(model_config_1, \"Category Embedding\")[0]\n",
+    "ft_metrics = train_and_evaluate_model(model_config_2, \"FT Transformer\")[0]\n",
+    "tab_metrics = train_and_evaluate_model(model_config_3, \"TabNet\")[0]\n",
+    "ce_acc = ce_metrics[\"test_accuracy\"]\n",
+    "ft_acc = ft_metrics[\"test_accuracy\"]\n",
+    "tab_acc = tab_metrics[\"test_accuracy\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Stacking Model Test Accuracy: 0.5960000157356262\n",
+      "Category Embedding Model Test Accucacy: 0.4586666524410248\n",
+      "FT Transformer Model Test Accuracy: 0.5546666383743286\n",
+      "TabNet Model Test Accuracy: 0.4346666634082794\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Stacking Model Test Accuracy: {}\".format(stacking_acc))\n",
+    "print(\"Category Embedding Model Test Accucacy: {}\".format(ce_acc))\n",
+    "print(\"FT Transformer Model Test Accuracy: {}\".format(ft_acc))\n",
+    "print(\"TabNet Model Test Accuracy: {}\".format(tab_acc))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Save the stacking model & load it"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:00:31,524 - {pytorch_tabular.tabular_model:1579} - WARNING - Directory is not empty. Overwriting the \n",
+       "contents.                                                                                                          \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:00:31\u001b[0m,\u001b[1;36m524\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m1579\u001b[0m\u001b[1m}\u001b[0m - WARNING - Directory is not empty. Overwriting the \n",
+       "contents.                                                                                                          \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "stacking_model.save_model(\"stacking_model\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:00:32,437 - {pytorch_tabular.tabular_model:172} - INFO - Experiment Tracking is turned off           \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:00:32\u001b[0m,\u001b[1;36m437\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m172\u001b[0m\u001b[1m}\u001b[0m - INFO - Experiment Tracking is turned off           \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "
2024-12-12 00:00:32,452 - {pytorch_tabular.tabular_model:343} - INFO - Preparing the Trainer                       \n",
+       "
\n"
+      ],
+      "text/plain": [
+       "\u001b[1;36m2024\u001b[0m-\u001b[1;36m12\u001b[0m-\u001b[1;36m12\u001b[0m \u001b[1;92m00:00:32\u001b[0m,\u001b[1;36m452\u001b[0m - \u001b[1m{\u001b[0mpytorch_tabular.tabular_model:\u001b[1;36m343\u001b[0m\u001b[1m}\u001b[0m - INFO - Preparing the Trainer                       \n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Trainer already configured with model summary callbacks: []. Skipping setting a default `ModelSummary` callback.\n",
+      "GPU available: True (cuda), used: True\n",
+      "TPU available: False, using: 0 TPU cores\n",
+      "HPU available: False, using: 0 HPUs\n"
+     ]
+    }
+   ],
+   "source": [
+    "loaded_model = TabularModel.load_model(\"stacking_model\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "\n",
+    "## Key Points About Stacking\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "\n",
+    "1. The stacking model combines predictions from multiple base models into a final prediction\n",
+    "2. Each base model can have its own architecture and hyperparameters\n",
+    "3. The head layer combines the outputs from all base models\n",
+    "4. Base models are trained simultaneously\n",
+    "5. The stacking model can often achieve better performance than individual models\n",
+    "\n",
+    "## Tips for Better Stacking Results\n",
+    "\n",
+    "1. Use diverse base models that capture different aspects of the data\n",
+    "2. Experiment with different head architectures\n",
+    "3. Consider using cross-validation for more robust stacking\n",
+    "4. Balance model complexity with training time\n",
+    "5. Monitor individual model performances to ensure they contribute meaningfully\n",
+    "\n",
+    "This example demonstrates basic stacking functionality. For production use cases, you may want to:\n",
+    "- Use cross-validation\n",
+    "- Implement more sophisticated ensemble techniques\n",
+    "- Add custom metrics\n",
+    "- Tune hyperparameters for both base models and stacking head"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/mkdocs.yml b/mkdocs.yml
index 59a38a83..9f3aeca7 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -24,6 +24,7 @@ nav:
                 - SHAP, Deep LIFT and so on through Captum Integration: "tutorials/14-Explainability.ipynb"
           - Custom PyTorch Models:
                 - Implementing New Supervised Architectures: "tutorials/04-Implementing New Architectures.ipynb"
+          - Model Stacking: "tutorials/16-Model Stacking.ipynb"
           - Other Features:
                 - Using Neural Categorical Embeddings in Scikit-Learn Workflows: "tutorials/03-Neural Embedding in Scikit-Learn Workflows.ipynb"
                 - Self-Supervised Learning using Denoising Autoencoders: "tutorials/08-Self-Supervised Learning-DAE.ipynb"
diff --git a/src/pytorch_tabular/models/__init__.py b/src/pytorch_tabular/models/__init__.py
index 0ae80b8e..e4d3353b 100644
--- a/src/pytorch_tabular/models/__init__.py
+++ b/src/pytorch_tabular/models/__init__.py
@@ -19,6 +19,7 @@
 from .gate import GatedAdditiveTreeEnsembleConfig, GatedAdditiveTreeEnsembleModel
 from .mixture_density import MDNConfig, MDNModel
 from .node import NodeConfig, NODEModel
+from .stacking import StackingModel, StackingModelConfig
 from .tab_transformer import TabTransformerConfig, TabTransformerModel
 from .tabnet import TabNetModel, TabNetModelConfig
 
@@ -45,6 +46,8 @@
     "GANDALFBackbone",
     "DANetConfig",
     "DANetModel",
+    "StackingModel",
+    "StackingModelConfig",
     "category_embedding",
     "node",
     "mixture_density",
@@ -55,4 +58,5 @@
     "gate",
     "gandalf",
     "danet",
+    "stacking",
 ]
diff --git a/src/pytorch_tabular/models/stacking/__init__.py b/src/pytorch_tabular/models/stacking/__init__.py
new file mode 100644
index 00000000..ca69b8ae
--- /dev/null
+++ b/src/pytorch_tabular/models/stacking/__init__.py
@@ -0,0 +1,4 @@
+from .config import StackingModelConfig
+from .stacking_model import StackingBackbone, StackingModel
+
+__all__ = ["StackingModel", "StackingModelConfig", "StackingBackbone"]
diff --git a/src/pytorch_tabular/models/stacking/config.py b/src/pytorch_tabular/models/stacking/config.py
new file mode 100644
index 00000000..586cddab
--- /dev/null
+++ b/src/pytorch_tabular/models/stacking/config.py
@@ -0,0 +1,26 @@
+from dataclasses import dataclass, field
+
+from pytorch_tabular.config import ModelConfig
+
+
+@dataclass
+class StackingModelConfig(ModelConfig):
+    """StackingModelConfig is a configuration class for the StackingModel. It is used to stack multiple models
+    together. Now, CategoryEmbeddingModel, TabNetModel, FTTransformerModel, GatedAdditiveTreeEnsembleModel, DANetModel,
+    AutoIntModel, GANDALFModel, NodeModel are supported.
+
+    Args:
+        model_configs (list[ModelConfig]): List of model configs to stack.
+
+    """
+
+    model_configs: list = field(default_factory=list, metadata={"help": "List of model configs to stack"})
+    _module_src: str = field(default="models.stacking")
+    _model_name: str = field(default="StackingModel")
+    _backbone_name: str = field(default="StackingBackbone")
+    _config_name: str = field(default="StackingConfig")
+
+
+# if __name__ == "__main__":
+#    from pytorch_tabular.utils import generate_doc_dataclass
+#    print(generate_doc_dataclass(StackingModelConfig))
diff --git a/src/pytorch_tabular/models/stacking/stacking_model.py b/src/pytorch_tabular/models/stacking/stacking_model.py
new file mode 100644
index 00000000..a1090089
--- /dev/null
+++ b/src/pytorch_tabular/models/stacking/stacking_model.py
@@ -0,0 +1,140 @@
+import inspect
+
+import torch
+import torch.nn as nn
+from omegaconf import DictConfig
+
+import pytorch_tabular.models as models
+from pytorch_tabular.models import BaseModel
+from pytorch_tabular.models.common.heads import blocks
+from pytorch_tabular.models.gate import GatedAdditiveTreesBackbone
+from pytorch_tabular.models.node import NODEBackbone
+
+
+def instantiate_backbone(hparams, backbone_name):
+    backbone_class = getattr(getattr(models, hparams._module_src.split(".")[-1]), backbone_name)
+    class_args = list(inspect.signature(backbone_class).parameters.keys())
+    if "config" in class_args:
+        return backbone_class(config=hparams)
+    else:
+        return backbone_class(
+            **{
+                arg: getattr(hparams, arg) if arg != "block_activation" else getattr(nn, getattr(hparams, arg))()
+                for arg in class_args
+            }
+        )
+
+
+class StackingEmbeddingLayer(nn.Module):
+    def __init__(self, embedding_layers: nn.ModuleList):
+        super().__init__()
+        self.embedding_layers = embedding_layers
+
+    def forward(self, x):
+        outputs = []
+        for embedding_layer in self.embedding_layers:
+            em_output = embedding_layer(x)
+            outputs.append(em_output)
+        return outputs
+
+
+class StackingBackbone(nn.Module):
+    def __init__(self, config: DictConfig):
+        super().__init__()
+        self.hparams = config
+        self._build_network()
+
+    def _build_network(self):
+        self._backbones = nn.ModuleList()
+        self._heads = nn.ModuleList()
+        self._backbone_output_dims = []
+        assert len(self.hparams.model_configs) > 0, "Stacking requires more than 0 model"
+        for model_i in range(len(self.hparams.model_configs)):
+            # move necessary params to each model config
+            self.hparams.model_configs[model_i].embedded_cat_dim = self.hparams.embedded_cat_dim
+            self.hparams.model_configs[model_i].continuous_dim = self.hparams.continuous_dim
+            self.hparams.model_configs[model_i].n_continuous_features = self.hparams.continuous_dim
+
+            self.hparams.model_configs[model_i].embedding_dims = self.hparams.embedding_dims
+            self.hparams.model_configs[model_i].categorical_cardinality = self.hparams.categorical_cardinality
+            self.hparams.model_configs[model_i].categorical_dim = self.hparams.categorical_dim
+            self.hparams.model_configs[model_i].cat_embedding_dims = self.hparams.embedding_dims
+
+            # if output_dim is not set, set it to 128
+            if getattr(self.hparams.model_configs[model_i], "output_dim", None) is None:
+                self.hparams.model_configs[model_i].output_dim = 128
+
+            # if inferred_config is not set, set it to None.
+            if getattr(self.hparams, "inferred_config", None) is not None:
+                self.hparams.model_configs[model_i].inferred_config = self.hparams.inferred_config
+
+            # instantiate backbone
+            _backbone = instantiate_backbone(
+                self.hparams.model_configs[model_i], self.hparams.model_configs[model_i]._backbone_name
+            )
+            # set continuous_dim
+            _backbone.continuous_dim = self.hparams.continuous_dim
+            # if output_dim is not set, set it to the output_dim in model_config
+            if getattr(_backbone, "output_dim", None) is None:
+                setattr(
+                    _backbone,
+                    "output_dim",
+                    self.hparams.model_configs[model_i].output_dim,
+                )
+            self._backbones.append(_backbone)
+            self._backbone_output_dims.append(_backbone.output_dim)
+
+        self.output_dim = sum(self._backbone_output_dims)
+
+    def _build_embedding_layer(self):
+        assert getattr(self, "_backbones", None) is not None, "Backbones are not built"
+        embedding_layers = nn.ModuleList()
+        for backbone in self._backbones:
+            if getattr(backbone, "_build_embedding_layer", None) is None:
+                embedding_layers.append(nn.Identity())
+            else:
+                embedding_layers.append(backbone._build_embedding_layer())
+        return StackingEmbeddingLayer(embedding_layers)
+
+    def forward(self, x_list):
+        outputs = []
+        for i, backbone in enumerate(self._backbones):
+            bb_output = backbone(x_list[i])
+            if len(bb_output.shape) == 3 and isinstance(backbone, GatedAdditiveTreesBackbone):
+                bb_output = bb_output.mean(dim=-1)
+            elif len(bb_output.shape) == 3 and isinstance(backbone, NODEBackbone):
+                bb_output = bb_output.mean(dim=1)
+            outputs.append(bb_output)
+        x = torch.cat(outputs, dim=1)
+        return x
+
+
+class StackingModel(BaseModel):
+    def __init__(self, config: DictConfig, **kwargs):
+        super().__init__(config, **kwargs)
+
+    def _build_network(self):
+        self._backbone = StackingBackbone(self.hparams)
+        self._embedding_layer = self._backbone._build_embedding_layer()
+        self.output_dim = self._backbone.output_dim
+        self._head = self._get_head_from_config()
+
+    def _get_head_from_config(self):
+        _head_callable = getattr(blocks, self.hparams.head)
+        return _head_callable(
+            in_units=self.output_dim,
+            output_dim=self.hparams.output_dim,
+            config=_head_callable._config_template(**self.hparams.head_config),
+        )
+
+    @property
+    def backbone(self):
+        return self._backbone
+
+    @property
+    def embedding_layer(self):
+        return self._embedding_layer
+
+    @property
+    def head(self):
+        return self._head
diff --git a/src/pytorch_tabular/models/tabnet/config.py b/src/pytorch_tabular/models/tabnet/config.py
index ade0c6a0..c1142273 100644
--- a/src/pytorch_tabular/models/tabnet/config.py
+++ b/src/pytorch_tabular/models/tabnet/config.py
@@ -129,6 +129,7 @@ class TabNetModelConfig(ModelConfig):
     _module_src: str = field(default="models.tabnet")
     _model_name: str = field(default="TabNetModel")
     _config_name: str = field(default="TabNetModelConfig")
+    _backbone_name: str = field(default="TabNetBackbone")
 
 
 # if __name__ == "__main__":
diff --git a/tests/test_model_stacking.py b/tests/test_model_stacking.py
new file mode 100644
index 00000000..629c19a4
--- /dev/null
+++ b/tests/test_model_stacking.py
@@ -0,0 +1,223 @@
+import numpy as np
+import pytest
+import torch
+from sklearn.preprocessing import PowerTransformer
+
+from pytorch_tabular import TabularModel
+from pytorch_tabular.config import DataConfig, OptimizerConfig, TrainerConfig
+from pytorch_tabular.models.autoint import AutoIntConfig
+from pytorch_tabular.models.category_embedding import CategoryEmbeddingModelConfig
+from pytorch_tabular.models.danet import DANetConfig
+from pytorch_tabular.models.ft_transformer import FTTransformerConfig
+from pytorch_tabular.models.gandalf import GANDALFConfig
+from pytorch_tabular.models.gate import GatedAdditiveTreeEnsembleConfig
+from pytorch_tabular.models.node import NodeConfig
+from pytorch_tabular.models.stacking import StackingModelConfig
+from pytorch_tabular.models.tabnet import TabNetModelConfig
+
+
+def fake_metric(y_hat, y):
+    return (y_hat - y).mean()
+
+
+def get_model_configs(task):
+    all_model_configs = [
+        lambda task: CategoryEmbeddingModelConfig(
+            task=task,
+        ),
+        lambda task: TabNetModelConfig(
+            task=task,
+        ),
+        lambda task: FTTransformerConfig(
+            task=task,
+        ),
+        lambda task: GatedAdditiveTreeEnsembleConfig(
+            task=task,
+        ),
+        lambda task: DANetConfig(
+            task=task,
+        ),
+        lambda task: AutoIntConfig(
+            task=task,
+        ),
+        lambda task: GANDALFConfig(
+            task=task,
+        ),
+        lambda task: NodeConfig(
+            task=task,
+        ),
+    ]
+    return [model_config(task) for model_config in all_model_configs]
+
+
+@pytest.mark.parametrize("multi_target", [True, False])
+@pytest.mark.parametrize(
+    "continuous_cols",
+    [
+        [
+            "AveRooms",
+            "AveBedrms",
+            "Population",
+            "AveOccup",
+            "Latitude",
+            "Longitude",
+        ],
+        [],
+    ],
+)
+@pytest.mark.parametrize("categorical_cols", [["HouseAgeBin"], []])
+@pytest.mark.parametrize("continuous_feature_transform", [None, "yeo-johnson"])
+@pytest.mark.parametrize("normalize_continuous_features", [True, False])
+@pytest.mark.parametrize("target_range", [True, False])
+@pytest.mark.parametrize(
+    "target_transform",
+    [None, PowerTransformer(), (lambda x: np.power(x, 2), lambda x: np.sqrt(x))],
+)
+@pytest.mark.parametrize("virtual_bz", [None, 32])
+# @pytest.mark.parametrize("custom_loss", [None, torch.nn.L1Loss()])
+# @pytest.mark.parametrize("custom_optimizer", [None, torch.optim.Adagrad])
+@pytest.mark.parametrize(
+    "custom_args", [(None, None, None, None), ([fake_metric], [False], torch.nn.L1Loss(), torch.optim.Adagrad)]
+)
+@pytest.mark.parametrize("custom_head_config", [None, "", "32", "32-32"])
+@pytest.mark.parametrize("model_configs", [get_model_configs("regression")])
+def test_regression(
+    regression_data,
+    multi_target,
+    continuous_cols,
+    categorical_cols,
+    continuous_feature_transform,
+    normalize_continuous_features,
+    target_range,
+    target_transform,
+    virtual_bz,
+    # custom_metrics,
+    # custom_loss,
+    # custom_optimizer,
+    custom_args,
+    custom_head_config,
+    model_configs,
+):
+    (train, test, target) = regression_data
+    (custom_metrics, custom_metrics_prob_input, custom_loss, custom_optimizer) = custom_args
+    if len(continuous_cols) + len(categorical_cols) == 0:
+        return
+
+    data_config = DataConfig(
+        target=target + ["MedInc"] if multi_target else target,
+        continuous_cols=continuous_cols,
+        categorical_cols=categorical_cols,
+        continuous_feature_transform=continuous_feature_transform,
+        normalize_continuous_features=normalize_continuous_features,
+    )
+    model_config_params = {"task": "regression", "virtual_batch_size": virtual_bz}
+
+    if target_range:
+        _target_range = []
+        for target in data_config.target:
+            _target_range.append(
+                (
+                    float(train[target].min()),
+                    float(train[target].max()),
+                )
+            )
+        model_config_params["target_range"] = _target_range
+    if custom_head_config is not None:
+        model_config_params["head"] = "LinearHead"
+        model_config_params["head_config"] = {"layers": custom_head_config}
+
+    model_config_params["model_configs"] = model_configs
+    model_config = StackingModelConfig(**model_config_params)
+    trainer_config = TrainerConfig(
+        max_epochs=3,
+        checkpoints=None,
+        early_stopping=None,
+        accelerator="cpu",
+        fast_dev_run=True,
+    )
+    optimizer_config = OptimizerConfig()
+
+    tabular_model = TabularModel(
+        data_config=data_config,
+        model_config=model_config,
+        optimizer_config=optimizer_config,
+        trainer_config=trainer_config,
+    )
+    tabular_model.fit(
+        train=train,
+        metrics=custom_metrics,
+        metrics_prob_inputs=custom_metrics_prob_input,
+        target_transform=target_transform,
+        loss=custom_loss,
+        optimizer=custom_optimizer,
+        optimizer_params={},
+    )
+
+    result = tabular_model.evaluate(test)
+    # print(result[0]["valid_loss"])
+    if custom_metrics is None:
+        assert "test_mean_squared_error" in result[0].keys()
+    else:
+        assert "test_fake_metric" in result[0].keys()
+    pred_df = tabular_model.predict(test)
+    assert pred_df.shape[0] == test.shape[0]
+
+
+@pytest.mark.parametrize("multi_target", [False, True])
+@pytest.mark.parametrize(
+    "continuous_cols",
+    [
+        [f"feature_{i}" for i in range(54)],
+        [],
+    ],
+)
+@pytest.mark.parametrize("categorical_cols", [["feature_0_cat"], []])
+@pytest.mark.parametrize("continuous_feature_transform", [None])
+@pytest.mark.parametrize("normalize_continuous_features", [True])
+@pytest.mark.parametrize("model_configs", [get_model_configs("classification")])
+def test_classification(
+    classification_data,
+    multi_target,
+    continuous_cols,
+    categorical_cols,
+    continuous_feature_transform,
+    normalize_continuous_features,
+    model_configs,
+):
+    (train, test, target) = classification_data
+    if len(continuous_cols) + len(categorical_cols) == 0:
+        return
+
+    data_config = DataConfig(
+        target=target + ["feature_53"] if multi_target else target,
+        continuous_cols=continuous_cols,
+        categorical_cols=categorical_cols,
+        continuous_feature_transform=continuous_feature_transform,
+        normalize_continuous_features=normalize_continuous_features,
+    )
+    model_config_params = {"task": "classification"}
+
+    model_config_params["model_configs"] = model_configs
+    model_config = StackingModelConfig(**model_config_params)
+    trainer_config = TrainerConfig(
+        max_epochs=3,
+        checkpoints=None,
+        early_stopping=None,
+        accelerator="cpu",
+        fast_dev_run=True,
+    )
+    optimizer_config = OptimizerConfig()
+
+    tabular_model = TabularModel(
+        data_config=data_config,
+        model_config=model_config,
+        optimizer_config=optimizer_config,
+        trainer_config=trainer_config,
+    )
+    tabular_model.fit(train=train)
+
+    result = tabular_model.evaluate(test)
+    # print(result[0]["valid_loss"])
+    assert "test_accuracy" in result[0].keys()
+    pred_df = tabular_model.predict(test)
+    assert pred_df.shape[0] == test.shape[0]

From 07305841ad35698a971ce834236831dcda3e2ef7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 17 Dec 2024 21:35:46 +0900
Subject: [PATCH 3/4] Bump pypa/gh-action-pypi-publish from 1.11.0 to 1.12.2
 (#513)

Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.11.0 to 1.12.2.
- [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases)
- [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.11.0...v1.12.2)

---
updated-dependencies:
- dependency-name: pypa/gh-action-pypi-publish
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/releasing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/releasing.yml b/.github/workflows/releasing.yml
index 02820918..8c04a794 100644
--- a/.github/workflows/releasing.yml
+++ b/.github/workflows/releasing.yml
@@ -38,7 +38,7 @@ jobs:
 
             - name: Publish distribution 📦 to PyPI
               if: startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release'
-              uses: pypa/gh-action-pypi-publish@v1.11.0
+              uses: pypa/gh-action-pypi-publish@v1.12.2
               with:
                   user: __token__
                   password: ${{ secrets.pypi_password }}

From de60f5f91a393933e49535d8eff842e7362912f1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 17 Dec 2024 21:36:19 +0900
Subject: [PATCH 4/4] Update protobuf requirement from <5.29.0,>=3.20.0 to
 >=3.20.0,<5.30.0 (#516)

Updates the requirements on [protobuf](https://github.com/protocolbuffers/protobuf) to permit the latest version.
- [Release notes](https://github.com/protocolbuffers/protobuf/releases)
- [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/protobuf_release.bzl)
- [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.20.0...v5.29.0)

---
updated-dependencies:
- dependency-name: protobuf
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] 
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 requirements/base.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/base.txt b/requirements/base.txt
index 6f23a333..db660230 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -6,7 +6,7 @@ pytorch-lightning >=2.0.0, <2.5.0
 omegaconf >=2.3.0
 torchmetrics >=0.10.0, <1.6.0
 tensorboard >2.2.0, !=2.5.0
-protobuf >=3.20.0, <5.29.0
+protobuf >=3.20.0, <5.30.0
 pytorch-tabnet ==4.1
 PyYAML >=5.4, <6.1.0
 # importlib-metadata <1,>=0.12