From c216ea799e5dfd06cbfc89b37408829d8c213db0 Mon Sep 17 00:00:00 2001 From: Wenjie Du Date: Mon, 29 Apr 2024 20:50:20 +0800 Subject: [PATCH 1/5] Fix failed CI testing on macOS with Python 3.7 (#373) * test: stick to macOS-13 to avoid python 3.7 missing for m1 chip; https://github.com/actions/setup-python/issues/850 --- .github/workflows/testing_ci.yml | 2 +- .github/workflows/testing_daily.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing_ci.yml b/.github/workflows/testing_ci.yml index d9380be9..a59122d0 100644 --- a/.github/workflows/testing_ci.yml +++ b/.github/workflows/testing_ci.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-latest, windows-latest, macOS-13] python-version: ["3.7", "3.11"] steps: diff --git a/.github/workflows/testing_daily.yml b/.github/workflows/testing_daily.yml index 81d686f7..66f61d55 100644 --- a/.github/workflows/testing_daily.yml +++ b/.github/workflows/testing_daily.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-latest, windows-latest, macOS-13] python-version: ["3.7", "3.11"] steps: From 4734710d5d33ce4c16e0299e7cfed1b2e26369b9 Mon Sep 17 00:00:00 2001 From: Wenjie Du Date: Tue, 30 Apr 2024 00:50:17 +0800 Subject: [PATCH 2/5] feat: add SaitsEmbedding; --- pypots/nn/modules/saits/__init__.py | 2 + pypots/nn/modules/saits/backbone.py | 42 ++++++++++-------- pypots/nn/modules/saits/embedding.py | 66 ++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 18 deletions(-) create mode 100644 pypots/nn/modules/saits/embedding.py diff --git a/pypots/nn/modules/saits/__init__.py b/pypots/nn/modules/saits/__init__.py index 8819692f..84fd811e 100644 --- a/pypots/nn/modules/saits/__init__.py +++ b/pypots/nn/modules/saits/__init__.py @@ -13,9 +13,11 @@ # License: BSD-3-Clause from .backbone import BackboneSAITS +from .embedding import SaitsEmbedding from .loss import SaitsLoss __all__ = [ "BackboneSAITS", + "SaitsEmbedding", "SaitsLoss", ] diff --git a/pypots/nn/modules/saits/backbone.py b/pypots/nn/modules/saits/backbone.py index 9435272f..b0aee630 100644 --- a/pypots/nn/modules/saits/backbone.py +++ b/pypots/nn/modules/saits/backbone.py @@ -17,8 +17,8 @@ import torch.nn as nn import torch.nn.functional as F +from .embedding import SaitsEmbedding from ..transformer import ( - PositionalEncoding, TransformerEncoderLayer, ScaledDotProductAttention, ) @@ -43,6 +43,14 @@ def __init__( # concatenate the feature vector and missing mask, hence double the number of features actual_n_features = n_features * 2 + # for the 1st block + self.embedding_1 = SaitsEmbedding( + actual_n_features, + d_model, + with_pos=True, + n_max_steps=n_steps, + dropout=dropout, + ) self.layer_stack_for_first_block = nn.ModuleList( [ TransformerEncoderLayer( @@ -57,6 +65,16 @@ def __init__( for _ in range(n_layers) ] ) + self.reduce_dim_z = nn.Linear(d_model, n_features) + + # for the 2nd block + self.embedding_2 = SaitsEmbedding( + actual_n_features, + d_model, + with_pos=True, + n_max_steps=n_steps, + dropout=dropout, + ) self.layer_stack_for_second_block = nn.ModuleList( [ TransformerEncoderLayer( @@ -72,15 +90,9 @@ def __init__( ] ) - self.dropout = nn.Dropout(p=dropout) - self.position_enc = PositionalEncoding(d_model, n_positions=n_steps) - # for the 1st block - self.embedding_1 = nn.Linear(actual_n_features, d_model) - self.reduce_dim_z = nn.Linear(d_model, n_features) - # for the 2nd block - self.embedding_2 = nn.Linear(actual_n_features, d_model) self.reduce_dim_beta = nn.Linear(d_model, n_features) self.reduce_dim_gamma = nn.Linear(n_features, n_features) + # for delta decay factor self.weight_combine = nn.Linear(n_features + n_steps, n_features) @@ -89,28 +101,22 @@ def forward( ) -> Tuple[torch.Tensor, ...]: # first DMSA block - input_X_for_first = torch.cat([X, missing_mask], dim=2) - input_X_for_first = self.embedding_1(input_X_for_first) - enc_output = self.dropout( - self.position_enc(input_X_for_first) + enc_output = self.embedding_1( + X, missing_mask ) # namely, term e in the math equation first_DMSA_attn_weights = None for encoder_layer in self.layer_stack_for_first_block: enc_output, first_DMSA_attn_weights = encoder_layer(enc_output, attn_mask) - X_tilde_1 = self.reduce_dim_z(enc_output) X_prime = missing_mask * X + (1 - missing_mask) * X_tilde_1 # second DMSA block - input_X_for_second = torch.cat([X_prime, missing_mask], dim=2) - input_X_for_second = self.embedding_2(input_X_for_second) - enc_output = self.position_enc( - input_X_for_second + enc_output = self.embedding_2( + X_prime, missing_mask ) # namely term alpha in math algo second_DMSA_attn_weights = None for encoder_layer in self.layer_stack_for_second_block: enc_output, second_DMSA_attn_weights = encoder_layer(enc_output, attn_mask) - X_tilde_2 = self.reduce_dim_gamma(F.relu(self.reduce_dim_beta(enc_output))) # attention-weighted combine diff --git a/pypots/nn/modules/saits/embedding.py b/pypots/nn/modules/saits/embedding.py new file mode 100644 index 00000000..323c954a --- /dev/null +++ b/pypots/nn/modules/saits/embedding.py @@ -0,0 +1,66 @@ +""" + +""" + +# Created by Wenjie Du +# License: BSD-3-Clause + +import torch +import torch.nn as nn + +from ..transformer import PositionalEncoding + + +class SaitsEmbedding(nn.Module): + """The embedding method from the SAITS paper :cite:`du2023saits`. + + Parameters + ---------- + d_in : + The input dimension. + + d_out : + The output dimension. + + with_pos : + Whether to add positional encoding. + + n_max_steps : + The maximum number of steps. + It only works when ``with_pos`` is True. + + dropout : + The dropout rate. + + """ + + def __init__( + self, + d_in: int, + d_out: int, + with_pos: bool, + n_max_steps: int = 1000, + dropout: float = 0, + ): + super().__init__() + self.with_pos = with_pos + self.dropout_rate = dropout + + self.embedding_layer = nn.Linear(d_in, d_out) + self.position_enc = ( + PositionalEncoding(d_out, n_positions=n_max_steps) if with_pos else None + ) + self.dropout = nn.Dropout(p=dropout) if dropout > 0 else None + + def forward(self, X, missing_mask=None, concat_dim=2, embedding_dim=2): + if missing_mask is not None: + X = torch.cat([X, missing_mask], dim=concat_dim) + + X_embedding = self.embedding_layer(X) + + if self.with_pos: + X_embedding = self.position_enc(X_embedding) + if self.dropout_rate > 0: + X_embedding = self.dropout(X_embedding) + + return X_embedding From 54873769008378c8e0828f3c2d7fb92e42183b86 Mon Sep 17 00:00:00 2001 From: Wenjie Du Date: Tue, 30 Apr 2024 13:50:26 +0800 Subject: [PATCH 3/5] feat: directly apply SaitsEmbedding to new models; --- pypots/imputation/autoformer/core.py | 17 ++++++----------- pypots/imputation/crossformer/core.py | 16 ++++++++++------ pypots/imputation/dlinear/core.py | 21 +++++++++++---------- pypots/imputation/etsformer/core.py | 17 +++++++---------- pypots/imputation/fedformer/core.py | 17 +++++++---------- pypots/imputation/film/core.py | 13 ++++--------- pypots/imputation/frets/core.py | 16 +++++----------- pypots/imputation/informer/core.py | 17 +++++++---------- pypots/imputation/itransformer/core.py | 16 ++++++++++------ pypots/imputation/patchtst/core.py | 13 ++++++------- pypots/imputation/transformer/core.py | 19 ++++++++++--------- pypots/nn/modules/saits/embedding.py | 4 ++-- 12 files changed, 85 insertions(+), 101 deletions(-) diff --git a/pypots/imputation/autoformer/core.py b/pypots/imputation/autoformer/core.py index 0b4b198c..7d1fa1f0 100644 --- a/pypots/imputation/autoformer/core.py +++ b/pypots/imputation/autoformer/core.py @@ -5,7 +5,6 @@ # Created by Wenjie Du # License: BSD-3-Clause -import torch import torch.nn as nn from ...nn.modules.autoformer import ( @@ -15,8 +14,7 @@ AutoCorrelationLayer, ) from ...nn.modules.informer import InformerEncoder -from ...nn.modules.saits import SaitsLoss -from ...nn.modules.transformer.embedding import DataEmbedding +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding class _Autoformer(nn.Module): @@ -39,11 +37,11 @@ def __init__( self.n_steps = n_steps - self.enc_embedding = DataEmbedding( + self.saits_embedding = SaitsEmbedding( n_features * 2, d_model, - dropout=dropout, with_pos=False, + dropout=dropout, ) self.encoder = InformerEncoder( [ @@ -73,13 +71,10 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # WDU: the original Autoformer paper isn't proposed for imputation task. Hence the model doesn't take # the missing mask into account, which means, in the process, the model doesn't know which part of - # the input data is missing, and this may hurt the model's imputation performance. Therefore, I add the - # embedding layers to project the concatenation of features and masks into a hidden space, as well as + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as # the output layers to project back from the hidden space to the original space. - - # the same as SAITS, concatenate the time series data and the missing mask for embedding - input_X = torch.cat([X, missing_mask], dim=2) - enc_out = self.enc_embedding(input_X) + enc_out = self.saits_embedding(X, missing_mask) # Autoformer encoder processing enc_out, attns = self.encoder(enc_out) diff --git a/pypots/imputation/crossformer/core.py b/pypots/imputation/crossformer/core.py index b45d8da1..a2d28cb4 100644 --- a/pypots/imputation/crossformer/core.py +++ b/pypots/imputation/crossformer/core.py @@ -13,7 +13,7 @@ from ...nn.modules.crossformer import CrossformerEncoder, ScaleBlock from ...nn.modules.patchtst import PredictionHead, PatchEmbedding -from ...nn.modules.saits import SaitsLoss +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding class _Crossformer(nn.Module): @@ -72,7 +72,11 @@ def __init__( ) self.head = PredictionHead(d_model, out_seg_num, n_steps, dropout) - self.embedding = nn.Linear(n_features * 2, d_model) + self.saits_embedding = SaitsEmbedding( + n_features * 2, + d_model, + with_pos=False, + ) self.output_projection = nn.Linear(d_model, n_features) # apply SAITS loss function to Crossformer on the imputation task @@ -83,11 +87,11 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # WDU: the original Crossformer paper isn't proposed for imputation task. Hence the model doesn't take # the missing mask into account, which means, in the process, the model doesn't know which part of - # the input data is missing, and this may hurt the model's imputation performance. Therefore, I add the - # embedding layers to project the concatenation of features and masks into a hidden space, as well as + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as # the output layers to project back from the hidden space to the original space. - # embedding - input_X = self.embedding(torch.cat([X, missing_mask], dim=2)) + input_X = self.saits_embedding(X, missing_mask) + x_enc = self.enc_value_embedding(input_X.permute(0, 2, 1)) x_enc = rearrange( x_enc, "(b d) seg_num d_model -> b d seg_num d_model", d=self.d_model diff --git a/pypots/imputation/dlinear/core.py b/pypots/imputation/dlinear/core.py index 1a8e448f..8adfbe88 100644 --- a/pypots/imputation/dlinear/core.py +++ b/pypots/imputation/dlinear/core.py @@ -7,12 +7,11 @@ from typing import Optional -import torch import torch.nn as nn from ...nn.modules.autoformer import SeriesDecompositionBlock from ...nn.modules.dlinear import BackboneDLinear -from ...nn.modules.saits import SaitsLoss +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding class _DLinear(nn.Module): @@ -36,8 +35,12 @@ def __init__( self.backbone = BackboneDLinear(n_steps, n_features, individual, d_model) if not individual: - self.linear_seasonal_embedding = nn.Linear(n_features * 2, d_model) - self.linear_trend_embedding = nn.Linear(n_features * 2, d_model) + self.seasonal_saits_embedding = SaitsEmbedding( + n_features * 2, d_model, with_pos=False + ) + self.trend_saits_embedding = SaitsEmbedding( + n_features * 2, d_model, with_pos=False + ) self.linear_seasonal_output = nn.Linear(d_model, n_features) self.linear_trend_output = nn.Linear(d_model, n_features) @@ -53,14 +56,12 @@ def forward(self, inputs: dict, training: bool = True) -> dict: if not self.individual: # WDU: the original DLinear paper isn't proposed for imputation task. Hence the model doesn't take # the missing mask into account, which means, in the process, the model doesn't know which part of - # the input data is missing, and this may hurt the model's imputation performance. Therefore, I add the - # embedding layers to project the concatenation of features and masks into a hidden space, as well as + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as # the output layers to project the seasonal and trend from the hidden space to the original space. # But this is only for the non-individual mode. - seasonal_init = torch.cat([seasonal_init, missing_mask], dim=2) - trend_init = torch.cat([trend_init, missing_mask], dim=2) - seasonal_init = self.linear_seasonal_embedding(seasonal_init) - trend_init = self.linear_trend_embedding(trend_init) + seasonal_init = self.seasonal_saits_embedding(seasonal_init, missing_mask) + trend_init = self.trend_saits_embedding(trend_init, missing_mask) seasonal_output, trend_output = self.backbone(seasonal_init, trend_init) diff --git a/pypots/imputation/etsformer/core.py b/pypots/imputation/etsformer/core.py index a910033b..c18034a2 100644 --- a/pypots/imputation/etsformer/core.py +++ b/pypots/imputation/etsformer/core.py @@ -5,7 +5,6 @@ # Created by Wenjie Du # License: BSD-3-Clause -import torch import torch.nn as nn from ...nn.modules.etsformer import ( @@ -14,8 +13,7 @@ ETSformerDecoderLayer, ETSformerDecoder, ) -from ...nn.modules.saits import SaitsLoss -from ...nn.modules.transformer.embedding import DataEmbedding +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding class _ETSformer(nn.Module): @@ -36,9 +34,11 @@ def __init__( ): super().__init__() - self.enc_embedding = DataEmbedding( + self.saits_embedding = SaitsEmbedding( n_features * 2, d_model, + with_pos=True, + n_max_steps=n_steps, dropout=dropout, ) @@ -81,13 +81,10 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # WDU: the original ETSformer paper isn't proposed for imputation task. Hence the model doesn't take # the missing mask into account, which means, in the process, the model doesn't know which part of - # the input data is missing, and this may hurt the model's imputation performance. Therefore, I add the - # embedding layers to project the concatenation of features and masks into a hidden space, as well as + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as # the output layers to project back from the hidden space to the original space. - - # the same as SAITS, concatenate the time series data and the missing mask for embedding - input_X = torch.cat([X, missing_mask], dim=2) - res = self.enc_embedding(input_X) + res = self.saits_embedding(X, missing_mask) # ETSformer encoder processing level, growths, seasons = self.encoder(res, X, attn_mask=None) diff --git a/pypots/imputation/fedformer/core.py b/pypots/imputation/fedformer/core.py index 93ba7590..06799c7a 100644 --- a/pypots/imputation/fedformer/core.py +++ b/pypots/imputation/fedformer/core.py @@ -5,12 +5,10 @@ # Created by Wenjie Du # License: BSD-3-Clause -import torch import torch.nn as nn from ...nn.modules.fedformer import FEDformerEncoder -from ...nn.modules.saits import SaitsLoss -from ...nn.modules.transformer.embedding import DataEmbedding +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding class _FEDformer(nn.Module): @@ -33,9 +31,11 @@ def __init__( ): super().__init__() - self.enc_embedding = DataEmbedding( + self.saits_embedding = SaitsEmbedding( n_features * 2, d_model, + with_pos=True, + n_max_steps=n_steps, dropout=dropout, ) @@ -62,13 +62,10 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # WDU: the original FEDformer paper isn't proposed for imputation task. Hence the model doesn't take # the missing mask into account, which means, in the process, the model doesn't know which part of - # the input data is missing, and this may hurt the model's imputation performance. Therefore, I add the - # embedding layers to project the concatenation of features and masks into a hidden space, as well as + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as # the output layers to project back from the hidden space to the original space. - - # the same as SAITS, concatenate the time series data and the missing mask for embedding - input_X = torch.cat([X, missing_mask], dim=2) - enc_out = self.enc_embedding(input_X) + enc_out = self.saits_embedding(X, missing_mask) # FEDformer encoder processing enc_out, attns = self.encoder(enc_out) diff --git a/pypots/imputation/film/core.py b/pypots/imputation/film/core.py index d081f853..b292e8f7 100644 --- a/pypots/imputation/film/core.py +++ b/pypots/imputation/film/core.py @@ -5,7 +5,6 @@ # Created by Wenjie Du # License: BSD-3-Clause -import torch import torch.nn as nn from ...nn.modules.film import BackboneFiLM @@ -29,10 +28,9 @@ def __init__( ): super().__init__() - self.enc_embedding = DataEmbedding( + self.saits_embedding = DataEmbedding( n_features * 2, d_model, - dropout=0, with_pos=False, ) self.backbone = BackboneFiLM( @@ -55,13 +53,10 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # WDU: the original FiLM paper isn't proposed for imputation task. Hence the model doesn't take # the missing mask into account, which means, in the process, the model doesn't know which part of - # the input data is missing, and this may hurt the model's imputation performance. Therefore, I add the - # embedding layers to project the concatenation of features and masks into a hidden space, as well as + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as # the output layers to project back from the hidden space to the original space. - - # the same as SAITS, concatenate the time series data and the missing mask for embedding - input_X = torch.cat([X, missing_mask], dim=2) - X_embedding = self.enc_embedding(input_X) + X_embedding = self.saits_embedding(X, missing_mask) # FiLM processing backbone_output = self.backbone(X_embedding) diff --git a/pypots/imputation/frets/core.py b/pypots/imputation/frets/core.py index 53a4ecfc..d5940a91 100644 --- a/pypots/imputation/frets/core.py +++ b/pypots/imputation/frets/core.py @@ -5,12 +5,10 @@ # Created by Wenjie Du # License: BSD-3-Clause -import torch import torch.nn as nn from ...nn.modules.frets import BackboneFreTS -from ...nn.modules.saits import SaitsLoss -from ...nn.modules.transformer.embedding import DataEmbedding +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding class _FreTS(nn.Module): @@ -28,10 +26,9 @@ def __init__( self.n_steps = n_steps - self.enc_embedding = DataEmbedding( + self.saits_embedding = SaitsEmbedding( n_features * 2, embed_size, - dropout=0, with_pos=False, ) self.backbone = BackboneFreTS( @@ -52,13 +49,10 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # WDU: the original FreTS paper isn't proposed for imputation task. Hence the model doesn't take # the missing mask into account, which means, in the process, the model doesn't know which part of - # the input data is missing, and this may hurt the model's imputation performance. Therefore, I add the - # embedding layers to project the concatenation of features and masks into a hidden space, as well as + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as # the output layers to project back from the hidden space to the original space. - - # the same as SAITS, concatenate the time series data and the missing mask for embedding - input_X = torch.cat([X, missing_mask], dim=2) - enc_out = self.enc_embedding(input_X) + enc_out = self.saits_embedding(X, missing_mask) # FreTS processing backbone_output = self.backbone(enc_out) diff --git a/pypots/imputation/informer/core.py b/pypots/imputation/informer/core.py index 59de782d..978c4f16 100644 --- a/pypots/imputation/informer/core.py +++ b/pypots/imputation/informer/core.py @@ -5,7 +5,6 @@ # Created by Wenjie Du # License: BSD-3-Clause -import torch import torch.nn as nn from ...nn.modules.informer import ( @@ -14,9 +13,8 @@ InformerEncoderLayer, InformerEncoder, ) -from ...nn.modules.saits import SaitsLoss +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding from ...nn.modules.transformer import MultiHeadAttention -from ...nn.modules.transformer.embedding import DataEmbedding class _Informer(nn.Module): @@ -37,9 +35,11 @@ def __init__( ): super().__init__() - self.enc_embedding = DataEmbedding( + self.saits_embedding = SaitsEmbedding( n_features * 2, d_model, + with_pos=True, + n_max_steps=n_steps, dropout=dropout, ) self.encoder = InformerEncoder( @@ -73,13 +73,10 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # WDU: the original Informer paper isn't proposed for imputation task. Hence the model doesn't take # the missing mask into account, which means, in the process, the model doesn't know which part of - # the input data is missing, and this may hurt the model's imputation performance. Therefore, I add the - # embedding layers to project the concatenation of features and masks into a hidden space, as well as + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as # the output layers to project back from the hidden space to the original space. - - # the same as SAITS, concatenate the time series data and the missing mask for embedding - input_X = torch.cat([X, missing_mask], dim=2) - enc_out = self.enc_embedding(input_X) + enc_out = self.saits_embedding(X, missing_mask) # Informer encoder processing enc_out, attns = self.encoder(enc_out) diff --git a/pypots/imputation/itransformer/core.py b/pypots/imputation/itransformer/core.py index ec67df0a..f0e69044 100644 --- a/pypots/imputation/itransformer/core.py +++ b/pypots/imputation/itransformer/core.py @@ -8,7 +8,7 @@ import torch import torch.nn as nn -from ...nn.modules.saits import SaitsLoss +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding from ...nn.modules.transformer import TransformerEncoder @@ -34,8 +34,9 @@ def __init__( self.ORT_weight = ORT_weight self.MIT_weight = MIT_weight - self.embedding = nn.Linear(n_steps, d_model) - self.dropout = nn.Dropout(dropout) + self.saits_embedding = SaitsEmbedding( + n_steps, d_model, with_pos=False, dropout=dropout + ) self.encoder = TransformerEncoder( n_layers, d_model, @@ -54,12 +55,15 @@ def __init__( def forward(self, inputs: dict, training: bool = True) -> dict: X, missing_mask = inputs["X"], inputs["missing_mask"] - # apply the SAITS embedding strategy, concatenate X and missing mask for input + # WDU: the original Informer paper isn't proposed for imputation task. Hence the model doesn't take + # the missing mask into account, which means, in the process, the model doesn't know which part of + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as + # the output layers to project back from the hidden space to the original space. input_X = torch.cat([X.permute(0, 2, 1), missing_mask.permute(0, 2, 1)], dim=1) + input_X = self.saits_embedding(input_X) # Transformer encoder processing - input_X = self.embedding(input_X) - input_X = self.dropout(input_X) enc_output, _ = self.encoder(input_X) # project the representation from the d_model-dimensional space to the original data space for output reconstruction = self.output_projection(enc_output) diff --git a/pypots/imputation/patchtst/core.py b/pypots/imputation/patchtst/core.py index dde61e4e..6dc70615 100644 --- a/pypots/imputation/patchtst/core.py +++ b/pypots/imputation/patchtst/core.py @@ -5,11 +5,10 @@ # Created by Wenjie Du # License: BSD-3-Clause -import torch import torch.nn as nn from ...nn.modules.patchtst import PatchEmbedding, PatchtstEncoder, PredictionHead -from ...nn.modules.saits import SaitsLoss +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding class _PatchTST(nn.Module): @@ -35,7 +34,7 @@ def __init__( n_patches = int((n_steps - patch_len) / stride + 2) # number of patches padding = stride - self.embedding = nn.Linear(n_features * 2, d_model) + self.saits_embedding = SaitsEmbedding(n_features * 2, d_model, with_pos=False) self.patch_embedding = PatchEmbedding( d_model, patch_len, stride, padding, dropout ) @@ -51,12 +50,12 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # WDU: the original PatchTST paper isn't proposed for imputation task. Hence the model doesn't take # the missing mask into account, which means, in the process, the model doesn't know which part of - # the input data is missing, and this may hurt the model's imputation performance. Therefore, I add the - # embedding layers to project the concatenation of features and masks into a hidden space, as well as + # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the + # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as # the output layers to project back from the hidden space to the original space. + input_X = self.saits_embedding(X, missing_mask) - # do patching and embedding - input_X = self.embedding(torch.cat([X, missing_mask], dim=2)) + # do patch embedding enc_out = self.patch_embedding( input_X.permute(0, 2, 1) ) # [bz * d_model, n_patches, d_model] diff --git a/pypots/imputation/transformer/core.py b/pypots/imputation/transformer/core.py index 74693f4f..01a9d487 100644 --- a/pypots/imputation/transformer/core.py +++ b/pypots/imputation/transformer/core.py @@ -13,11 +13,10 @@ # Created by Wenjie Du # License: BSD-3-Clause -import torch import torch.nn as nn -from ...nn.modules.saits import SaitsLoss -from ...nn.modules.transformer import TransformerEncoder, PositionalEncoding +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding +from ...nn.modules.transformer import TransformerEncoder class _Transformer(nn.Module): @@ -41,9 +40,13 @@ def __init__( self.ORT_weight = ORT_weight self.MIT_weight = MIT_weight - self.embedding = nn.Linear(n_features * 2, d_model) - self.dropout = nn.Dropout(dropout) - self.position_enc = PositionalEncoding(d_model, n_positions=n_steps) + self.saits_embedding = SaitsEmbedding( + n_features * 2, + d_model, + with_pos=True, + n_max_steps=n_steps, + dropout=dropout, + ) self.encoder = TransformerEncoder( n_layers, d_model, @@ -63,11 +66,9 @@ def forward(self, inputs: dict, training: bool = True) -> dict: X, missing_mask = inputs["X"], inputs["missing_mask"] # apply the SAITS embedding strategy, concatenate X and missing mask for input - input_X = torch.cat([X, missing_mask], dim=2) + input_X = self.saits_embedding(X, missing_mask) # Transformer encoder processing - input_X = self.embedding(input_X) - input_X = self.dropout(self.position_enc(input_X)) enc_output, _ = self.encoder(input_X) # project the representation from the d_model-dimensional space to the original data space for output reconstruction = self.output_projection(enc_output) diff --git a/pypots/nn/modules/saits/embedding.py b/pypots/nn/modules/saits/embedding.py index 323c954a..fb4c2b04 100644 --- a/pypots/nn/modules/saits/embedding.py +++ b/pypots/nn/modules/saits/embedding.py @@ -52,9 +52,9 @@ def __init__( ) self.dropout = nn.Dropout(p=dropout) if dropout > 0 else None - def forward(self, X, missing_mask=None, concat_dim=2, embedding_dim=2): + def forward(self, X, missing_mask=None): if missing_mask is not None: - X = torch.cat([X, missing_mask], dim=concat_dim) + X = torch.cat([X, missing_mask], dim=2) X_embedding = self.embedding_layer(X) From dfd95f757c063f84cfdd529629fc52aa3e007f41 Mon Sep 17 00:00:00 2001 From: Wenjie Du Date: Tue, 30 Apr 2024 14:00:32 +0800 Subject: [PATCH 4/5] fix: use SaitsEmbedding in FiLM; --- pypots/imputation/film/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pypots/imputation/film/core.py b/pypots/imputation/film/core.py index b292e8f7..bf4b7845 100644 --- a/pypots/imputation/film/core.py +++ b/pypots/imputation/film/core.py @@ -8,8 +8,7 @@ import torch.nn as nn from ...nn.modules.film import BackboneFiLM -from ...nn.modules.saits import SaitsLoss -from ...nn.modules.transformer.embedding import DataEmbedding +from ...nn.modules.saits import SaitsLoss, SaitsEmbedding class _FiLM(nn.Module): @@ -28,7 +27,7 @@ def __init__( ): super().__init__() - self.saits_embedding = DataEmbedding( + self.saits_embedding = SaitsEmbedding( n_features * 2, d_model, with_pos=False, @@ -55,7 +54,8 @@ def forward(self, inputs: dict, training: bool = True) -> dict: # the missing mask into account, which means, in the process, the model doesn't know which part of # the input data is missing, and this may hurt the model's imputation performance. Therefore, I apply the # SAITS embedding method to project the concatenation of features and masks into a hidden space, as well as - # the output layers to project back from the hidden space to the original space. + # the output layers to pro + # ject back from the hidden space to the original space. X_embedding = self.saits_embedding(X, missing_mask) # FiLM processing From f332a0ce6a07c07fdc412bc20b574d3d35d3c074 Mon Sep 17 00:00:00 2001 From: Wenjie Du Date: Tue, 30 Apr 2024 15:39:21 +0800 Subject: [PATCH 5/5] docs: update the docs; --- README.md | 8 +++++--- pypots/classification/brits/__init__.py | 5 +++++ pypots/classification/brits/core.py | 12 ++---------- pypots/classification/brits/data.py | 2 +- pypots/classification/brits/model.py | 9 --------- pypots/classification/grud/__init__.py | 4 ++++ pypots/classification/grud/core.py | 7 ++----- pypots/classification/grud/data.py | 2 +- pypots/classification/grud/model.py | 4 ---- pypots/classification/raindrop/__init__.py | 4 ++++ pypots/classification/raindrop/core.py | 7 ++----- pypots/classification/raindrop/data.py | 2 +- pypots/classification/raindrop/model.py | 4 ---- pypots/classification/template/core.py | 6 ++---- pypots/classification/template/data.py | 2 +- pypots/classification/template/model.py | 2 -- pypots/clustering/crli/__init__.py | 4 ++++ pypots/clustering/crli/core.py | 8 ++------ pypots/clustering/crli/data.py | 2 +- pypots/clustering/crli/model.py | 4 ---- pypots/clustering/template/core.py | 6 ++---- pypots/clustering/template/data.py | 2 +- pypots/clustering/template/model.py | 2 -- pypots/clustering/vader/__init__.py | 4 ++++ pypots/clustering/vader/core.py | 8 ++------ pypots/clustering/vader/data.py | 2 +- pypots/clustering/vader/model.py | 5 ----- pypots/forecasting/bttf/__init__.py | 4 ++++ pypots/forecasting/bttf/core.py | 12 +----------- pypots/forecasting/bttf/model.py | 8 -------- pypots/forecasting/bttf/submodules.py | 9 --------- pypots/forecasting/csdi/__init__.py | 4 ++++ pypots/forecasting/csdi/core.py | 5 +++++ pypots/forecasting/csdi/data.py | 2 +- pypots/forecasting/csdi/model.py | 8 -------- pypots/forecasting/template/core.py | 6 ++---- pypots/forecasting/template/data.py | 2 +- pypots/forecasting/template/model.py | 2 -- pypots/imputation/autoformer/__init__.py | 4 ++++ pypots/imputation/autoformer/core.py | 3 ++- pypots/imputation/autoformer/model.py | 8 -------- pypots/imputation/brits/__init__.py | 5 +++++ pypots/imputation/brits/core.py | 12 ++---------- pypots/imputation/brits/data.py | 2 +- pypots/imputation/brits/model.py | 9 --------- pypots/imputation/crossformer/__init__.py | 4 ++++ pypots/imputation/crossformer/core.py | 3 ++- pypots/imputation/crossformer/data.py | 2 +- pypots/imputation/crossformer/model.py | 9 --------- pypots/imputation/csdi/__init__.py | 4 ++++ pypots/imputation/csdi/core.py | 5 +++++ pypots/imputation/csdi/data.py | 2 +- pypots/imputation/csdi/model.py | 8 -------- pypots/imputation/dlinear/__init__.py | 3 +++ pypots/imputation/dlinear/core.py | 3 ++- pypots/imputation/dlinear/data.py | 2 +- pypots/imputation/dlinear/model.py | 9 --------- pypots/imputation/etsformer/__init__.py | 4 ++++ pypots/imputation/etsformer/core.py | 3 ++- pypots/imputation/etsformer/data.py | 2 +- pypots/imputation/etsformer/model.py | 8 -------- pypots/imputation/fedformer/__init__.py | 4 ++++ pypots/imputation/fedformer/core.py | 3 ++- pypots/imputation/fedformer/data.py | 2 +- pypots/imputation/fedformer/model.py | 8 -------- pypots/imputation/film/__init__.py | 2 +- pypots/imputation/film/core.py | 3 ++- pypots/imputation/film/data.py | 2 +- pypots/imputation/frets/__init__.py | 2 +- pypots/imputation/frets/core.py | 3 ++- pypots/imputation/frets/data.py | 2 +- pypots/imputation/gpvae/__init__.py | 4 ++++ pypots/imputation/gpvae/core.py | 6 ++---- pypots/imputation/gpvae/data.py | 2 +- pypots/imputation/gpvae/model.py | 4 ---- pypots/imputation/informer/__init__.py | 4 ++++ pypots/imputation/informer/core.py | 3 ++- pypots/imputation/informer/data.py | 2 +- pypots/imputation/informer/model.py | 9 --------- pypots/imputation/itransformer/__init__.py | 2 +- pypots/imputation/itransformer/core.py | 3 ++- pypots/imputation/itransformer/data.py | 2 +- pypots/imputation/itransformer/model.py | 10 ---------- pypots/imputation/locf/core.py | 2 +- pypots/imputation/mrnn/__init__.py | 5 +++++ pypots/imputation/mrnn/core.py | 5 ++--- pypots/imputation/mrnn/data.py | 2 +- pypots/imputation/mrnn/model.py | 7 ------- pypots/imputation/patchtst/__init__.py | 4 ++++ pypots/imputation/patchtst/core.py | 3 ++- pypots/imputation/patchtst/data.py | 2 +- pypots/imputation/patchtst/model.py | 5 ----- pypots/imputation/saits/__init__.py | 4 ++++ pypots/imputation/saits/core.py | 10 ++-------- pypots/imputation/saits/data.py | 2 +- pypots/imputation/saits/model.py | 8 -------- pypots/imputation/template/core.py | 5 ++--- pypots/imputation/template/data.py | 2 +- pypots/imputation/template/model.py | 2 -- pypots/imputation/timesnet/__init__.py | 4 ++++ pypots/imputation/timesnet/data.py | 2 +- pypots/imputation/timesnet/model.py | 8 -------- pypots/imputation/transformer/__init__.py | 3 +++ pypots/imputation/transformer/core.py | 10 ++-------- pypots/imputation/transformer/data.py | 2 +- pypots/imputation/transformer/model.py | 8 -------- pypots/imputation/usgan/core.py | 7 ++----- pypots/imputation/usgan/data.py | 2 +- pypots/imputation/usgan/model.py | 4 ---- pypots/nn/modules/autoformer/__init__.py | 4 ++++ pypots/nn/modules/brits/__init__.py | 5 +++++ pypots/nn/modules/brits/backbone.py | 9 --------- pypots/nn/modules/brits/layers.py | 9 --------- pypots/nn/modules/crli/__init__.py | 3 +++ pypots/nn/modules/crli/layers.py | 4 ---- pypots/nn/modules/crossformer/__init__.py | 4 ++++ pypots/nn/modules/csdi/__init__.py | 4 ++++ pypots/nn/modules/csdi/layers.py | 2 +- pypots/nn/modules/dlinear/__init__.py | 3 +++ pypots/nn/modules/etsformer/__init__.py | 4 ++++ pypots/nn/modules/fedformer/__init__.py | 4 ++++ pypots/nn/modules/film/__init__.py | 3 +++ pypots/nn/modules/frets/__init__.py | 4 ++++ pypots/nn/modules/gpvae/__init__.py | 4 ++++ pypots/nn/modules/grud/__init__.py | 12 ++++++++++-- pypots/nn/modules/informer/__init__.py | 4 ++++ pypots/nn/modules/mrnn/__init__.py | 5 +++++ pypots/nn/modules/mrnn/backbone.py | 4 +--- pypots/nn/modules/mrnn/layers.py | 2 +- pypots/nn/modules/patchtst/__init__.py | 4 ++++ pypots/nn/modules/raindrop/__init__.py | 4 ++++ pypots/nn/modules/raindrop/backbone.py | 4 ---- pypots/nn/modules/raindrop/layers.py | 4 ---- pypots/nn/modules/saits/__init__.py | 4 ++++ pypots/nn/modules/saits/backbone.py | 6 ------ pypots/nn/modules/timesnet/__init__.py | 4 ++++ pypots/nn/modules/transformer/__init__.py | 5 +++++ pypots/nn/modules/transformer/attention.py | 2 +- pypots/nn/modules/transformer/embedding.py | 2 +- pypots/nn/modules/usgan/backbone.py | 4 ---- pypots/nn/modules/vader/__init__.py | 4 ++++ pypots/nn/modules/vader/backbone.py | 5 ----- pypots/nn/modules/vader/layers.py | 5 ----- 143 files changed, 273 insertions(+), 380 deletions(-) diff --git a/README.md b/README.md index 450bb32d..e8547f11 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The rest of this readme file is organized as follows: ## ❖ Available Algorithms PyPOTS supports imputation, classification, clustering, forecasting, and anomaly detection tasks on multivariate partially-observed -time series with missing values. The table below shows the availability of each algorithm in PyPOTS for different tasks. +time series with missing values. The table below shows the availability of each algorithm (sorted by Year) in PyPOTS for different tasks. The symbol ✅ indicates the algorithm is available for the corresponding task (note that models will be continuously updated in the future to handle tasks that are not currently supported. Stay tuned❗️). The task types are abbreviated as follows: **`IMPU`**: Imputation; **`FORE`**: Forecasting; @@ -93,8 +93,10 @@ The paper references are all listed at the bottom of this readme file. This functionality is implemented with the [Microsoft NNI](https://github.com/microsoft/nni) framework. You may want to refer to our time-series imputation survey repo [Awesome_Imputation](https://github.com/WenjieDu/Awesome_Imputation) to see how to config and tune the hyperparameters. -🔥 Note that Transformer, Crossformer, PatchTST, DLinear, ETSformer, FEDformer, Informer, Autoformer are not proposed as imputation methods in their original papers, -and they cannot accept POTS as input. **To make them applicable on POTS data, we apply the embedding strategy and training approach (ORT+MIT) + +🔥 Note that Transformer, iTransformer, FreTS, Crossformer, PatchTST, DLinear, ETSformer, FiLM, FEDformer, Informer, Autoformer +are not proposed as imputation methods in their original papers, and they cannot accept POTS as input. +**To make them applicable on POTS data, we apply the embedding strategy and training approach (ORT+MIT) the same as we did in [SAITS paper](https://arxiv.org/pdf/2202.08516).** | **Type** | **Algo** | **IMPU** | **FORE** | **CLAS** | **CLUS** | **ANOD** | **Year** | diff --git a/pypots/classification/brits/__init__.py b/pypots/classification/brits/__init__.py index e077f7cc..e8a70a17 100644 --- a/pypots/classification/brits/__init__.py +++ b/pypots/classification/brits/__init__.py @@ -7,6 +7,11 @@ In Advances in Neural Information Processing Systems, volume 31. Curran Associates, Inc., 2018. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/caow13/BRITS. +The bugs in the original implementation are fixed here. + """ # Created by Wenjie Du diff --git a/pypots/classification/brits/core.py b/pypots/classification/brits/core.py index ce9f0e43..ebd1bae3 100644 --- a/pypots/classification/brits/core.py +++ b/pypots/classification/brits/core.py @@ -1,14 +1,6 @@ """ -The implementation of BRITS for the partially-observed time-series classification task. - -Refer to the paper "Cao, W., Wang, D., Li, J., Zhou, H., Li, L., & Li, Y. (2018). -BRITS: Bidirectional Recurrent Imputation for Time Series. NeurIPS 2018." - -Notes ------ -Partial implementation uses code from https://github.com/caow13/BRITS. The bugs in the original implementation -are fixed here. - +The core wrapper assembles the submodules of BRITS classification model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/classification/brits/data.py b/pypots/classification/brits/data.py index 663ba81d..ec296513 100644 --- a/pypots/classification/brits/data.py +++ b/pypots/classification/brits/data.py @@ -1,5 +1,5 @@ """ -Dataset class for model BRITS. +Dataset class for the classification model BRITS. """ # Created by Wenjie Du diff --git a/pypots/classification/brits/model.py b/pypots/classification/brits/model.py index fa0ad349..537970e8 100644 --- a/pypots/classification/brits/model.py +++ b/pypots/classification/brits/model.py @@ -1,15 +1,6 @@ """ The implementation of BRITS for the partially-observed time-series classification task. -Refer to the paper "Wei Cao, Dong Wang, Jian Li, Hao Zhou, Lei Li, and Yitan Li. -BRITS: Bidirectional recurrent imputation for time series. -In Advances in Neural Information Processing Systems, volume 31. Curran Associates, Inc., 2018." - -Notes ------ -Partial implementation uses code from https://github.com/caow13/BRITS. -The bugs in the original implementation are fixed here. - """ # Created by Wenjie Du diff --git a/pypots/classification/grud/__init__.py b/pypots/classification/grud/__init__.py index a1821131..d131465a 100644 --- a/pypots/classification/grud/__init__.py +++ b/pypots/classification/grud/__init__.py @@ -7,6 +7,10 @@ Scientific Reports, 8(1):6085, April 2018. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/PeterChe1990/GRU-D + """ # Created by Wenjie Du diff --git a/pypots/classification/grud/core.py b/pypots/classification/grud/core.py index faae15b3..16cd2723 100644 --- a/pypots/classification/grud/core.py +++ b/pypots/classification/grud/core.py @@ -1,9 +1,6 @@ """ -The implementation of GRU-D for the partially-observed time-series imputation task. - -Refer to the paper "Che, Z., Purushotham, S., Cho, K., Sontag, D.A., & Liu, Y. (2018). -Recurrent Neural Networks for Multivariate Time Series with Missing Values. Scientific Reports." - +The core wrapper assembles the submodules of GRU-D classification model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/classification/grud/data.py b/pypots/classification/grud/data.py index 99401310..cf743993 100644 --- a/pypots/classification/grud/data.py +++ b/pypots/classification/grud/data.py @@ -1,5 +1,5 @@ """ -Dataset class for model GRU-D. +Dataset class for the classification model GRU-D. """ # Created by Wenjie Du diff --git a/pypots/classification/grud/model.py b/pypots/classification/grud/model.py index a18e78c8..e7b2ef91 100644 --- a/pypots/classification/grud/model.py +++ b/pypots/classification/grud/model.py @@ -1,10 +1,6 @@ """ The implementation of GRU-D for the partially-observed time-series classification task. -Refer to the paper "Zhengping Che, Sanjay Purushotham, Kyunghyun Cho, David Sontag, and Yan Liu. -Recurrent Neural Networks for Multivariate Time Series with Missing Values. -Scientific Reports, 8(1):6085, April 2018." - """ # Created by Wenjie Du diff --git a/pypots/classification/raindrop/__init__.py b/pypots/classification/raindrop/__init__.py index 117327e2..c7adeb4f 100644 --- a/pypots/classification/raindrop/__init__.py +++ b/pypots/classification/raindrop/__init__.py @@ -7,6 +7,10 @@ In ICLR, 2022. `_ +Notes +----- +This implementation is inspired by the official one the official implementation https://github.com/mims-harvard/Raindrop + """ # Created by Wenjie Du diff --git a/pypots/classification/raindrop/core.py b/pypots/classification/raindrop/core.py index d8709ef9..dbe2b653 100644 --- a/pypots/classification/raindrop/core.py +++ b/pypots/classification/raindrop/core.py @@ -1,9 +1,6 @@ """ -The implementation of Raindrop for the partially-observed time-series classification task. - -Refer to the paper "Zhang, X., Zeman, M., Tsiligkaridis, T., & Zitnik, M. (2022). -Graph-Guided Network for Irregularly Sampled Multivariate Time Series. ICLR 2022." - +The core wrapper assembles the submodules of Raindrop classification model +and takes over the forward progress of the algorithm. """ diff --git a/pypots/classification/raindrop/data.py b/pypots/classification/raindrop/data.py index 9449976f..429546ef 100644 --- a/pypots/classification/raindrop/data.py +++ b/pypots/classification/raindrop/data.py @@ -1,5 +1,5 @@ """ -Dataset class for model Raindrop. +Dataset class for the classification model Raindrop. """ # Created by Wenjie Du diff --git a/pypots/classification/raindrop/model.py b/pypots/classification/raindrop/model.py index 8def46d2..9adaeaa0 100644 --- a/pypots/classification/raindrop/model.py +++ b/pypots/classification/raindrop/model.py @@ -1,10 +1,6 @@ """ The implementation of Raindrop for the partially-observed time-series classification task. -Refer to the paper "Xiang Zhang, Marko Zeman, Theodoros Tsiligkaridis, and Marinka Zitnik. -Graph-guided network for irregularly sampled multivariate time series. -In ICLR, 2022." - """ diff --git a/pypots/classification/template/core.py b/pypots/classification/template/core.py index 157cc4d9..814a6cc7 100644 --- a/pypots/classification/template/core.py +++ b/pypots/classification/template/core.py @@ -1,8 +1,6 @@ """ -The implementation of YourNewModel for the partially-observed time-series classification task. - -Refer to the paper "Your paper citation". - +The core wrapper assembles the submodules of YourNewModel classification model +and takes over the forward progress of the algorithm. """ # Created by Your Name TODO: modify the author information. diff --git a/pypots/classification/template/data.py b/pypots/classification/template/data.py index 3c4ca97e..cc6a2745 100644 --- a/pypots/classification/template/data.py +++ b/pypots/classification/template/data.py @@ -1,5 +1,5 @@ """ -Dataset class for YourNewModel. +Dataset class for the classification model YourNewModel. TODO: modify the above description for your model's dataset class. diff --git a/pypots/classification/template/model.py b/pypots/classification/template/model.py index 40f6b252..c1cf194f 100644 --- a/pypots/classification/template/model.py +++ b/pypots/classification/template/model.py @@ -1,8 +1,6 @@ """ The implementation of YourNewModel for the partially-observed time-series classification task. -Refer to the paper "Your paper citation". - TODO: modify the above description with your model's information. """ diff --git a/pypots/clustering/crli/__init__.py b/pypots/clustering/crli/__init__.py index d86b0d37..32605412 100644 --- a/pypots/clustering/crli/__init__.py +++ b/pypots/clustering/crli/__init__.py @@ -8,6 +8,10 @@ `_ +Notes +----- +This implementation is inspired by the official one https://github.com/qianlima-lab/CRLI + """ # Created by Wenjie Du diff --git a/pypots/clustering/crli/core.py b/pypots/clustering/crli/core.py index 8e45357e..755d9ff7 100644 --- a/pypots/clustering/crli/core.py +++ b/pypots/clustering/crli/core.py @@ -1,10 +1,6 @@ """ -The implementation of CRLI (Clustering Representation Learning on Incomplete time-series data) for -the partially-observed time-series clustering task. - -Refer to the paper "Ma, Q., Chen, C., Li, S., & Cottrell, G. W. (2021). -Learning Representations for Incomplete Time Series Clustering. AAAI 2021." - +The core wrapper assembles the submodules of CRLI clustering model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/clustering/crli/data.py b/pypots/clustering/crli/data.py index cf8976a2..d64a8ae9 100644 --- a/pypots/clustering/crli/data.py +++ b/pypots/clustering/crli/data.py @@ -1,5 +1,5 @@ """ -Dataset class for model GRU-D. +Dataset class for the clustering model CRLI. """ # Created by Wenjie Du diff --git a/pypots/clustering/crli/model.py b/pypots/clustering/crli/model.py index 90651ca0..f6076f0c 100644 --- a/pypots/clustering/crli/model.py +++ b/pypots/clustering/crli/model.py @@ -2,10 +2,6 @@ The implementation of CRLI (Clustering Representation Learning on Incomplete time-series data) for the partially-observed time-series clustering task. -Refer to the paper "Qianli Ma, Chuxin Chen, Sen Li, and Garrison W. Cottrell. -Learning Representations for Incomplete Time Series Clustering. -In AAAI, 35(10):8837–8846, May 2021." - """ # Created by Wenjie Du diff --git a/pypots/clustering/template/core.py b/pypots/clustering/template/core.py index 524a7a3c..caa2e5a8 100644 --- a/pypots/clustering/template/core.py +++ b/pypots/clustering/template/core.py @@ -1,8 +1,6 @@ """ -The implementation of YourNewModel for the partially-observed time-series clustering task. - -Refer to the paper "Your paper citation". - +The core wrapper assembles the submodules of YourNewModel clustering model +and takes over the forward progress of the algorithm. """ # Created by Your Name TODO: modify the author information. diff --git a/pypots/clustering/template/data.py b/pypots/clustering/template/data.py index 3c4ca97e..b9fc051a 100644 --- a/pypots/clustering/template/data.py +++ b/pypots/clustering/template/data.py @@ -1,5 +1,5 @@ """ -Dataset class for YourNewModel. +Dataset class for the clustering model YourNewModel. TODO: modify the above description for your model's dataset class. diff --git a/pypots/clustering/template/model.py b/pypots/clustering/template/model.py index 65bc8aa9..0ed75220 100644 --- a/pypots/clustering/template/model.py +++ b/pypots/clustering/template/model.py @@ -1,8 +1,6 @@ """ The implementation of YourNewModel for the partially-observed time-series clustering task. -Refer to the paper "Your paper citation". - TODO: modify the above description with your model's information. """ diff --git a/pypots/clustering/vader/__init__.py b/pypots/clustering/vader/__init__.py index a97aeded..f671a264 100644 --- a/pypots/clustering/vader/__init__.py +++ b/pypots/clustering/vader/__init__.py @@ -8,6 +8,10 @@ GigaScience, 8(11):giz134, November 2019. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/johanndejong/VaDER + """ # Created by Wenjie Du diff --git a/pypots/clustering/vader/core.py b/pypots/clustering/vader/core.py index 3d6df970..52843b72 100644 --- a/pypots/clustering/vader/core.py +++ b/pypots/clustering/vader/core.py @@ -1,10 +1,6 @@ """ -The implementation of VaDER for the partially-observed time-series clustering task. - -Refer to the paper "Jong, J.D., Emon, M.A., Wu, P., Karki, R., Sood, M., Godard, P., Ahmad, A., Vrooman, H.A., -Hofmann-Apitius, M., & Fröhlich, H. (2019). -Deep learning for clustering of multivariate clinical patient trajectories with missing values. GigaScience." - +The core wrapper assembles the submodules of VaDER clustering model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/clustering/vader/data.py b/pypots/clustering/vader/data.py index ea718397..21b27024 100644 --- a/pypots/clustering/vader/data.py +++ b/pypots/clustering/vader/data.py @@ -1,5 +1,5 @@ """ -Dataset class for model GRU-D. +Dataset class for the clustering model VaDER. """ # Created by Wenjie Du diff --git a/pypots/clustering/vader/model.py b/pypots/clustering/vader/model.py index a9b151dc..3a895ca8 100644 --- a/pypots/clustering/vader/model.py +++ b/pypots/clustering/vader/model.py @@ -1,11 +1,6 @@ """ The implementation of VaDER for the partially-observed time-series clustering task. -Refer to the paper "Johann de Jong, Mohammad Asif Emon, Ping Wu, Reagon Karki, Meemansa Sood, Patrice Godard, -Ashar Ahmad, Henri Vrooman, Martin Hofmann-Apitius, and Holger Fröhlich. -Deep learning for clustering of multivariate clinical patient trajectories with missing values. -GigaScience, 8(11):giz134, November 2019." - """ # Created by Wenjie Du diff --git a/pypots/forecasting/bttf/__init__.py b/pypots/forecasting/bttf/__init__.py index 31522f67..3ed7a57d 100644 --- a/pypots/forecasting/bttf/__init__.py +++ b/pypots/forecasting/bttf/__init__.py @@ -7,6 +7,10 @@ IEEE Transactions on Pattern Analysis and Machine Intelligence, pages 1–1, 2021. `_ +Notes +----- +This numpy implementation is the same with the official one from https://github.com/xinychen/transdim. + """ # Created by Wenjie Du diff --git a/pypots/forecasting/bttf/core.py b/pypots/forecasting/bttf/core.py index ffd8ef67..f0fcccc2 100644 --- a/pypots/forecasting/bttf/core.py +++ b/pypots/forecasting/bttf/core.py @@ -1,15 +1,5 @@ """ -The implementation of BTTF (Bayesian Temporal Tensor Factorization) for the partially-observed time-series -forecasting task. - -Refer to the paper "Chen, X., & Sun, L. (2021). -Bayesian Temporal Factorization for Multidimensional Time Series Prediction. -IEEE transactions on pattern analysis and machine intelligence." - -Notes ------ -This numpy implementation is the same with the official one from https://github.com/xinychen/transdim. - +The core wrapper assembles the submodules of BTTF forecasting model. """ # Created by Wenjie Du diff --git a/pypots/forecasting/bttf/model.py b/pypots/forecasting/bttf/model.py index ab530058..40d5e714 100644 --- a/pypots/forecasting/bttf/model.py +++ b/pypots/forecasting/bttf/model.py @@ -2,14 +2,6 @@ The implementation of BTTF (Bayesian Temporal Tensor Factorization) for the partially-observed time-series forecasting task. -Refer to the paper "Xinyu Chen and Lijun Sun. -Bayesian Temporal Factorization for Multidimensional Time Series Prediction. -IEEE Transactions on Pattern Analysis and Machine Intelligence, pages 1–1, 2021." - -Notes ------ -This numpy implementation is the same with the official one from https://github.com/xinychen/transdim. - """ # Created by Wenjie Du diff --git a/pypots/forecasting/bttf/submodules.py b/pypots/forecasting/bttf/submodules.py index 42a670fd..3a73408e 100644 --- a/pypots/forecasting/bttf/submodules.py +++ b/pypots/forecasting/bttf/submodules.py @@ -1,13 +1,4 @@ """ -The implementation of the modules for BTTF (Bayesian Temporal Tensor Factorization). - -Refer to the paper "Chen, X., & Sun, L. (2021). -Bayesian Temporal Factorization for Multidimensional Time Series Prediction. -IEEE transactions on pattern analysis and machine intelligence." - -Notes ------ -This numpy implementation is the same with the official one from https://github.com/xinychen/transdim. """ diff --git a/pypots/forecasting/csdi/__init__.py b/pypots/forecasting/csdi/__init__.py index fbaae9a5..9d2cd0a5 100644 --- a/pypots/forecasting/csdi/__init__.py +++ b/pypots/forecasting/csdi/__init__.py @@ -7,6 +7,10 @@ In NeurIPS, 2021. `_ +Notes +----- +This implementation is inspired by the official one the official implementation https://github.com/ermongroup/CSDI + """ # Created by Wenjie Du diff --git a/pypots/forecasting/csdi/core.py b/pypots/forecasting/csdi/core.py index c66c99fd..e488cb20 100644 --- a/pypots/forecasting/csdi/core.py +++ b/pypots/forecasting/csdi/core.py @@ -1,3 +1,8 @@ +""" +The core wrapper assembles the submodules of CSDI forecasting model +and takes over the forward progress of the algorithm. +""" + # Created by Wenjie Du # License: BSD-3-Clause diff --git a/pypots/forecasting/csdi/data.py b/pypots/forecasting/csdi/data.py index d39bfb92..5f91f842 100644 --- a/pypots/forecasting/csdi/data.py +++ b/pypots/forecasting/csdi/data.py @@ -1,5 +1,5 @@ """ - +Dataset class for the forecasting model CSDI. """ # Created by Wenjie Du diff --git a/pypots/forecasting/csdi/model.py b/pypots/forecasting/csdi/model.py index 68f6a412..21db4719 100644 --- a/pypots/forecasting/csdi/model.py +++ b/pypots/forecasting/csdi/model.py @@ -1,14 +1,6 @@ """ The implementation of CSDI for the partially-observed time-series forecasting task. -Refer to the paper "Yusuke Tashiro, Jiaming Song, Yang Song, and Stefano Ermon. -CSDI: Conditional Score-based Diffusion Models for Probabilistic Time Series Imputation. -In NeurIPS, 2021." - -Notes ------ -Partial implementation uses code from the official implementation https://github.com/ermongroup/CSDI. - """ # Created by Wenjie Du diff --git a/pypots/forecasting/template/core.py b/pypots/forecasting/template/core.py index 55cf3ada..a0cb7deb 100644 --- a/pypots/forecasting/template/core.py +++ b/pypots/forecasting/template/core.py @@ -1,8 +1,6 @@ """ -The implementation of YourNewModel for the partially-observed time-series forecasting task. - -Refer to the paper "Your paper citation". - +The core wrapper assembles the submodules of YourNewModel forecasting model +and takes over the forward progress of the algorithm. """ # Created by Your Name TODO: modify the author information. diff --git a/pypots/forecasting/template/data.py b/pypots/forecasting/template/data.py index 3c4ca97e..aacd0cae 100644 --- a/pypots/forecasting/template/data.py +++ b/pypots/forecasting/template/data.py @@ -1,5 +1,5 @@ """ -Dataset class for YourNewModel. +Dataset class for the forecasting model YourNewModel. TODO: modify the above description for your model's dataset class. diff --git a/pypots/forecasting/template/model.py b/pypots/forecasting/template/model.py index 890c3fde..fd817694 100644 --- a/pypots/forecasting/template/model.py +++ b/pypots/forecasting/template/model.py @@ -1,8 +1,6 @@ """ The implementation of YourNewModel for the partially-observed time-series forecasting task. -Refer to the paper "Your paper citation". - TODO: modify the above description with your model's information. """ diff --git a/pypots/imputation/autoformer/__init__.py b/pypots/imputation/autoformer/__init__.py index ccd5ce26..f5cddb0c 100644 --- a/pypots/imputation/autoformer/__init__.py +++ b/pypots/imputation/autoformer/__init__.py @@ -7,6 +7,10 @@ In Advances in Neural Information Processing Systems, volume 34, pages 22419–22430. Curran Associates, Inc., 2021. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/thuml/Autoformer + """ # Created by Wenjie Du diff --git a/pypots/imputation/autoformer/core.py b/pypots/imputation/autoformer/core.py index 7d1fa1f0..9880c52c 100644 --- a/pypots/imputation/autoformer/core.py +++ b/pypots/imputation/autoformer/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of Autoformer imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/autoformer/model.py b/pypots/imputation/autoformer/model.py index edafd0a5..323d6b01 100644 --- a/pypots/imputation/autoformer/model.py +++ b/pypots/imputation/autoformer/model.py @@ -1,14 +1,6 @@ """ The implementation of Autoformer for the partially-observed time-series imputation task. -Refer to the paper "Haixu Wu, Jiehui Xu, Jianmin Wang, and Mingsheng Long. -Autoformer: Decomposition transformers with autocorrelation for long-term series forecasting. -In Advances in Neural Information Processing Systems, volume 34, pages 22419–22430. Curran Associates, Inc., 2021." - -Notes ------ -Partial implementation uses code from https://github.com/thuml/Autoformer - """ # Created by Wenjie Du diff --git a/pypots/imputation/brits/__init__.py b/pypots/imputation/brits/__init__.py index 0d89c285..43d10f3a 100644 --- a/pypots/imputation/brits/__init__.py +++ b/pypots/imputation/brits/__init__.py @@ -7,6 +7,11 @@ In Advances in Neural Information Processing Systems, volume 31. Curran Associates, Inc., 2018. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/caow13/BRITS +The bugs in the original implementation are fixed here. + """ # Created by Wenjie Du diff --git a/pypots/imputation/brits/core.py b/pypots/imputation/brits/core.py index aa715d2d..9d1734c4 100644 --- a/pypots/imputation/brits/core.py +++ b/pypots/imputation/brits/core.py @@ -1,14 +1,6 @@ """ -The implementation of BRITS for the partially-observed time-series imputation task. - -Refer to the paper "Cao, W., Wang, D., Li, J., Zhou, H., Li, L., & Li, Y. (2018). -BRITS: Bidirectional Recurrent Imputation for Time Series. NeurIPS 2018." - -Notes ------ -Partial implementation uses code from https://github.com/caow13/BRITS. The bugs in the original implementation -are fixed here. - +The core wrapper assembles the submodules of BRITS imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/brits/data.py b/pypots/imputation/brits/data.py index 589a5d5f..72915017 100644 --- a/pypots/imputation/brits/data.py +++ b/pypots/imputation/brits/data.py @@ -1,5 +1,5 @@ """ -Dataset class for model BRITS. +Dataset class for the imputation model BRITS. """ # Created by Wenjie Du diff --git a/pypots/imputation/brits/model.py b/pypots/imputation/brits/model.py index 68d71355..7c382402 100644 --- a/pypots/imputation/brits/model.py +++ b/pypots/imputation/brits/model.py @@ -1,15 +1,6 @@ """ The implementation of BRITS for the partially-observed time-series imputation task. -Refer to the paper "Wei Cao, Dong Wang, Jian Li, Hao Zhou, Lei Li, and Yitan Li. -BRITS: Bidirectional recurrent imputation for time series. -In Advances in Neural Information Processing Systems, volume 31. Curran Associates, Inc., 2018." - -Notes ------ -Partial implementation uses code from https://github.com/caow13/BRITS. -The bugs in the original implementation are fixed here. - """ # Created by Wenjie Du diff --git a/pypots/imputation/crossformer/__init__.py b/pypots/imputation/crossformer/__init__.py index ea67b5ab..238cc7a1 100644 --- a/pypots/imputation/crossformer/__init__.py +++ b/pypots/imputation/crossformer/__init__.py @@ -7,6 +7,10 @@ In The 11th ICLR, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/Thinklab-SJTU/Crossformer + """ # Created by Wenjie Du diff --git a/pypots/imputation/crossformer/core.py b/pypots/imputation/crossformer/core.py index a2d28cb4..a410b287 100644 --- a/pypots/imputation/crossformer/core.py +++ b/pypots/imputation/crossformer/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of Crossformer imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/crossformer/data.py b/pypots/imputation/crossformer/data.py index 6bbc771d..22cb0295 100644 --- a/pypots/imputation/crossformer/data.py +++ b/pypots/imputation/crossformer/data.py @@ -1,5 +1,5 @@ """ -Dataset class for Crossformer. +Dataset class for the imputation model Crossformer. """ # Created by Wenjie Du diff --git a/pypots/imputation/crossformer/model.py b/pypots/imputation/crossformer/model.py index a2076d95..7a158bee 100644 --- a/pypots/imputation/crossformer/model.py +++ b/pypots/imputation/crossformer/model.py @@ -1,15 +1,6 @@ """ The implementation of Crossformer for the partially-observed time-series imputation task. -Refer to the paper "Yunhao Zhang and Junchi Yan. -Crossformer: Transformer utilizing cross-dimension dependency for multivariate time series forecasting. -In The 11th ICLR, 2023." - -Notes ------ -Partial implementation uses code from -https://github.com/Thinklab-SJTU/Crossformer and https://github.com/thuml/Time-Series-Library - """ # Created by Wenjie Du diff --git a/pypots/imputation/csdi/__init__.py b/pypots/imputation/csdi/__init__.py index aa12eaaf..bb3f7160 100644 --- a/pypots/imputation/csdi/__init__.py +++ b/pypots/imputation/csdi/__init__.py @@ -7,6 +7,10 @@ In NeurIPS, 2021. `_ +Notes +----- +This implementation is inspired by the official one the official implementation https://github.com/ermongroup/CSDI. + """ # Created by Wenjie Du diff --git a/pypots/imputation/csdi/core.py b/pypots/imputation/csdi/core.py index fd3cdd84..a80acce3 100644 --- a/pypots/imputation/csdi/core.py +++ b/pypots/imputation/csdi/core.py @@ -1,3 +1,8 @@ +""" +The core wrapper assembles the submodules of CSDI imputation model +and takes over the forward progress of the algorithm. +""" + # Created by Wenjie Du # License: BSD-3-Clause diff --git a/pypots/imputation/csdi/data.py b/pypots/imputation/csdi/data.py index 03d07923..491a8d0a 100644 --- a/pypots/imputation/csdi/data.py +++ b/pypots/imputation/csdi/data.py @@ -1,5 +1,5 @@ """ - +Dataset class for the imputation model CSDI. """ # Created by Wenjie Du diff --git a/pypots/imputation/csdi/model.py b/pypots/imputation/csdi/model.py index bcfdc29d..fca7d554 100644 --- a/pypots/imputation/csdi/model.py +++ b/pypots/imputation/csdi/model.py @@ -1,14 +1,6 @@ """ The implementation of CSDI for the partially-observed time-series imputation task. -Refer to the paper "Yusuke Tashiro, Jiaming Song, Yang Song, and Stefano Ermon. -CSDI: Conditional Score-based Diffusion Models for Probabilistic Time Series Imputation. -In NeurIPS, 2021." - -Notes ------ -Partial implementation uses code from the official implementation https://github.com/ermongroup/CSDI. - """ # Created by Wenjie Du diff --git a/pypots/imputation/dlinear/__init__.py b/pypots/imputation/dlinear/__init__.py index 7cae6e91..5eaa3f34 100644 --- a/pypots/imputation/dlinear/__init__.py +++ b/pypots/imputation/dlinear/__init__.py @@ -7,6 +7,9 @@ In AAAI, volume 37, pages 11121–11128, Jun. 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/cure-lab/LTSF-Linear """ diff --git a/pypots/imputation/dlinear/core.py b/pypots/imputation/dlinear/core.py index 8adfbe88..78d3bcbd 100644 --- a/pypots/imputation/dlinear/core.py +++ b/pypots/imputation/dlinear/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of DLinear imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/dlinear/data.py b/pypots/imputation/dlinear/data.py index b47cb439..721e93b5 100644 --- a/pypots/imputation/dlinear/data.py +++ b/pypots/imputation/dlinear/data.py @@ -1,5 +1,5 @@ """ -Dataset class for DLinear. +Dataset class for the imputation model DLinear. """ # Created by Wenjie Du diff --git a/pypots/imputation/dlinear/model.py b/pypots/imputation/dlinear/model.py index af7ba286..5eb969bb 100644 --- a/pypots/imputation/dlinear/model.py +++ b/pypots/imputation/dlinear/model.py @@ -1,15 +1,6 @@ """ The implementation of DLinear for the partially-observed time-series imputation task. -Refer to the paper "Ailing Zeng, Muxi Chen, Lei Zhang, and Qiang Xu. -Are transformers effective for time series forecasting? -In AAAI, volume 37, pages 11121–11128, Jun. 2023.". - -Notes ------ -Partial implementation uses code from -https://github.com/cure-lab/LTSF-Linear and https://github.com/thuml/Time-Series-Library - """ # Created by Wenjie Du diff --git a/pypots/imputation/etsformer/__init__.py b/pypots/imputation/etsformer/__init__.py index ea42f5dc..53cd8c5c 100644 --- a/pypots/imputation/etsformer/__init__.py +++ b/pypots/imputation/etsformer/__init__.py @@ -7,6 +7,10 @@ In ICLR, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/salesforce/ETSformer + """ # Created by Wenjie Du diff --git a/pypots/imputation/etsformer/core.py b/pypots/imputation/etsformer/core.py index c18034a2..ce1e8cde 100644 --- a/pypots/imputation/etsformer/core.py +++ b/pypots/imputation/etsformer/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of ETSformer imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/etsformer/data.py b/pypots/imputation/etsformer/data.py index 19503a4d..c902c535 100644 --- a/pypots/imputation/etsformer/data.py +++ b/pypots/imputation/etsformer/data.py @@ -1,5 +1,5 @@ """ -Dataset class for ETSformer. +Dataset class for the imputation model ETSformer. """ # Created by Wenjie Du diff --git a/pypots/imputation/etsformer/model.py b/pypots/imputation/etsformer/model.py index 94a253e1..cb0eb5f5 100644 --- a/pypots/imputation/etsformer/model.py +++ b/pypots/imputation/etsformer/model.py @@ -1,14 +1,6 @@ """ The implementation of ETSformer for the partially-observed time-series imputation task. -Refer to the paper "Gerald Woo, Chenghao Liu, Doyen Sahoo, Akshat Kumar, and Steven Hoi. -ETSformer: Exponential smoothing transformers for time-series forecasting. -In ICLR, 2023." - -Notes ------ -Partial implementation uses code from https://github.com/salesforce/ETSformer - """ # Created by Wenjie Du diff --git a/pypots/imputation/fedformer/__init__.py b/pypots/imputation/fedformer/__init__.py index d37fb0e8..19b955ed 100644 --- a/pypots/imputation/fedformer/__init__.py +++ b/pypots/imputation/fedformer/__init__.py @@ -7,6 +7,10 @@ In ICML, volume 162 of Proceedings of Machine Learning Research, pages 27268–27286. PMLR, 17–23 Jul 2022. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/MAZiqing/FEDformer + """ # Created by Wenjie Du diff --git a/pypots/imputation/fedformer/core.py b/pypots/imputation/fedformer/core.py index 06799c7a..3cb9abf5 100644 --- a/pypots/imputation/fedformer/core.py +++ b/pypots/imputation/fedformer/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of FEDformer imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/fedformer/data.py b/pypots/imputation/fedformer/data.py index f8d79217..ae3ff280 100644 --- a/pypots/imputation/fedformer/data.py +++ b/pypots/imputation/fedformer/data.py @@ -1,5 +1,5 @@ """ -Dataset class for FEDformer. +Dataset class for the imputation model FEDformer. """ # Created by Wenjie Du diff --git a/pypots/imputation/fedformer/model.py b/pypots/imputation/fedformer/model.py index dfda3740..11c8a325 100644 --- a/pypots/imputation/fedformer/model.py +++ b/pypots/imputation/fedformer/model.py @@ -1,14 +1,6 @@ """ The implementation of FEDformer for the partially-observed time-series imputation task. -Refer to the paper "Tian Zhou, Ziqing Ma, Qingsong Wen, Xue Wang, Liang Sun, and Rong Jin. -FEDformer: Frequency enhanced decomposed transformer for long-term series forecasting. -In ICML, volume 162 of Proceedings of Machine Learning Research, pages 27268–27286. PMLR, 17–23 Jul 2022.". - -Notes ------ -Partial implementation uses code from https://github.com/MAZiqing/FEDformer - """ # Created by Wenjie Du diff --git a/pypots/imputation/film/__init__.py b/pypots/imputation/film/__init__.py index d654155f..28e913f7 100644 --- a/pypots/imputation/film/__init__.py +++ b/pypots/imputation/film/__init__.py @@ -9,7 +9,7 @@ Notes ----- -Partial implementation uses code from https://github.com/tianzhou2011/FiLM +This implementation is inspired by the official one https://github.com/tianzhou2011/FiLM """ diff --git a/pypots/imputation/film/core.py b/pypots/imputation/film/core.py index bf4b7845..2e48f8c2 100644 --- a/pypots/imputation/film/core.py +++ b/pypots/imputation/film/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of FiLM imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/film/data.py b/pypots/imputation/film/data.py index a0570365..4fc7d7a7 100644 --- a/pypots/imputation/film/data.py +++ b/pypots/imputation/film/data.py @@ -1,5 +1,5 @@ """ -Dataset class for FiLM. +Dataset class for the imputation model FiLM. """ # Created by Wenjie Du diff --git a/pypots/imputation/frets/__init__.py b/pypots/imputation/frets/__init__.py index 2c690829..94273c50 100644 --- a/pypots/imputation/frets/__init__.py +++ b/pypots/imputation/frets/__init__.py @@ -9,7 +9,7 @@ Notes ----- -Partial implementation uses code from https://github.com/aikunyi/FreTS +This implementation is inspired by the official one https://github.com/aikunyi/FreTS """ diff --git a/pypots/imputation/frets/core.py b/pypots/imputation/frets/core.py index d5940a91..488880d9 100644 --- a/pypots/imputation/frets/core.py +++ b/pypots/imputation/frets/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of FreTS imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/frets/data.py b/pypots/imputation/frets/data.py index 203131f0..a8972b58 100644 --- a/pypots/imputation/frets/data.py +++ b/pypots/imputation/frets/data.py @@ -1,5 +1,5 @@ """ -Dataset class for FreTS. +Dataset class for the imputation model FreTS. """ # Created by Wenjie Du diff --git a/pypots/imputation/gpvae/__init__.py b/pypots/imputation/gpvae/__init__.py index 592ab185..3ab75cfa 100644 --- a/pypots/imputation/gpvae/__init__.py +++ b/pypots/imputation/gpvae/__init__.py @@ -7,6 +7,10 @@ In International conference on artificial intelligence and statistics, pages 1651–1661. PMLR, 2020. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/ratschlab/GP-VAE + """ # Created by Jun Wang diff --git a/pypots/imputation/gpvae/core.py b/pypots/imputation/gpvae/core.py index 72829287..79b8f724 100644 --- a/pypots/imputation/gpvae/core.py +++ b/pypots/imputation/gpvae/core.py @@ -1,8 +1,6 @@ """ -The implementation of GP-VAE for the partially-observed time-series imputation task. - -Refer to the paper Fortuin V, Baranchuk D, Rätsch G, et al. -GP-VAE: Deep probabilistic time series imputation. AISTATS. PMLR, 2020: 1651-1661. +The core wrapper assembles the submodules of GP-VAE imputation model +and takes over the forward progress of the algorithm. """ diff --git a/pypots/imputation/gpvae/data.py b/pypots/imputation/gpvae/data.py index 27b3b456..66665c9e 100644 --- a/pypots/imputation/gpvae/data.py +++ b/pypots/imputation/gpvae/data.py @@ -1,5 +1,5 @@ """ -Dataset class for model GP-VAE. +Dataset class for the imputation model GP-VAE. """ # Created by Jun Wang and Wenjie Du diff --git a/pypots/imputation/gpvae/model.py b/pypots/imputation/gpvae/model.py index 1ff234c9..5e8058bb 100644 --- a/pypots/imputation/gpvae/model.py +++ b/pypots/imputation/gpvae/model.py @@ -1,10 +1,6 @@ """ The implementation of GP-VAE for the partially-observed time-series imputation task. -Refer to the paper "Vincent Fortuin, Dmitry Baranchuk, Gunnar Rätsch, and Stephan Mandt. -GP-VAE: Deep probabilistic time series imputation. -In International conference on artificial intelligence and statistics, pages 1651–1661. PMLR, 2020." - """ # Created by Jun Wang and Wenjie Du diff --git a/pypots/imputation/informer/__init__.py b/pypots/imputation/informer/__init__.py index 298d2345..77189592 100644 --- a/pypots/imputation/informer/__init__.py +++ b/pypots/imputation/informer/__init__.py @@ -7,6 +7,10 @@ In Proceedings of the AAAI conference on artificial intelligence, volume 35, pages 11106–11115, 2021. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/zhouhaoyi/Informer2020 + """ # Created by Wenjie Du diff --git a/pypots/imputation/informer/core.py b/pypots/imputation/informer/core.py index 978c4f16..0084d2d6 100644 --- a/pypots/imputation/informer/core.py +++ b/pypots/imputation/informer/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of Informer imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/informer/data.py b/pypots/imputation/informer/data.py index 7624c41d..24fc2bdd 100644 --- a/pypots/imputation/informer/data.py +++ b/pypots/imputation/informer/data.py @@ -1,5 +1,5 @@ """ -Dataset class for Informer. +Dataset class for the imputation model Informer. """ # Created by Wenjie Du diff --git a/pypots/imputation/informer/model.py b/pypots/imputation/informer/model.py index 9429485e..d60770c7 100644 --- a/pypots/imputation/informer/model.py +++ b/pypots/imputation/informer/model.py @@ -1,15 +1,6 @@ """ The implementation of Informer for the partially-observed time-series imputation task. -Refer to the paper "Haoyi Zhou, Shanghang Zhang, Jieqi Peng, Shuai Zhang, Jianxin Li, Hui Xiong, and Wancai Zhang. -Informer: Beyond efficient transformer for long sequence time-series forecasting. -In Proceedings of the AAAI conference on artificial intelligence, volume 35, pages 11106–11115, 2021.". - -Notes ------ -Partial implementation uses code from -https://github.com/zhouhaoyi/Informer2020 and https://github.com/thuml/Time-Series-Library - """ # Created by Wenjie Du diff --git a/pypots/imputation/itransformer/__init__.py b/pypots/imputation/itransformer/__init__.py index 98ddb2b2..5ac77983 100644 --- a/pypots/imputation/itransformer/__init__.py +++ b/pypots/imputation/itransformer/__init__.py @@ -9,7 +9,7 @@ Notes ----- -Partial implementation uses code from https://github.com/thuml/iTransformer +This implementation is inspired by the official one https://github.com/thuml/iTransformer """ diff --git a/pypots/imputation/itransformer/core.py b/pypots/imputation/itransformer/core.py index f0e69044..bd620fdf 100644 --- a/pypots/imputation/itransformer/core.py +++ b/pypots/imputation/itransformer/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of iTransformer imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/itransformer/data.py b/pypots/imputation/itransformer/data.py index 069df81f..e7ee4579 100644 --- a/pypots/imputation/itransformer/data.py +++ b/pypots/imputation/itransformer/data.py @@ -1,5 +1,5 @@ """ -Dataset class for self-attention models trained with MIT (masked imputation task) task. +Dataset class for the imputation model iTransformer. """ # Created by Wenjie Du diff --git a/pypots/imputation/itransformer/model.py b/pypots/imputation/itransformer/model.py index 9dda2549..757d0e27 100644 --- a/pypots/imputation/itransformer/model.py +++ b/pypots/imputation/itransformer/model.py @@ -1,16 +1,6 @@ """ The package of the partially-observed time-series imputation model iTransformer. -Refer to the papers -`Liu, Yong, Tengge Hu, Haoran Zhang, Haixu Wu, Shiyu Wang, Lintao Ma, and Mingsheng Long. -"iTransformer: Inverted transformers are effective for time series forecasting." -ICLR 2024. -`_ - -Notes ------ -Partial implementation uses code from https://github.com/thuml/iTransformer - """ # Created by Wenjie Du diff --git a/pypots/imputation/locf/core.py b/pypots/imputation/locf/core.py index 8c12ffc9..f9b0d087 100644 --- a/pypots/imputation/locf/core.py +++ b/pypots/imputation/locf/core.py @@ -1,5 +1,5 @@ """ - +The core wrapper assembles the submodules of LOCF imputation model. """ # Created by Wenjie Du diff --git a/pypots/imputation/mrnn/__init__.py b/pypots/imputation/mrnn/__init__.py index bf4f805c..6de006d7 100644 --- a/pypots/imputation/mrnn/__init__.py +++ b/pypots/imputation/mrnn/__init__.py @@ -7,6 +7,11 @@ IEEE Transactions on Biomedical Engineering, 66(5):14771490, 2019. `_ +Notes +----- +This implementation is inspired by the official one +https://github.com/jsyoon0823/MRNN and https://github.com/WenjieDu/SAITS + """ # Created by Wenjie Du diff --git a/pypots/imputation/mrnn/core.py b/pypots/imputation/mrnn/core.py index e2ed4fdf..0cdf7084 100644 --- a/pypots/imputation/mrnn/core.py +++ b/pypots/imputation/mrnn/core.py @@ -1,7 +1,6 @@ """ -PyTorch MRNN model for the time-series imputation task. -This implementation is inspired by the official one https://github.com/jsyoon0823/MRNN. -Some part of the code is from https://github.com/WenjieDu/SAITS. +The core wrapper assembles the submodules of M-RNN imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/mrnn/data.py b/pypots/imputation/mrnn/data.py index cb228f53..2d9a39d7 100644 --- a/pypots/imputation/mrnn/data.py +++ b/pypots/imputation/mrnn/data.py @@ -1,5 +1,5 @@ """ -Dataset class for model MRNN. +Dataset class for the imputation model M-RNN. """ # Created by Wenjie Du diff --git a/pypots/imputation/mrnn/model.py b/pypots/imputation/mrnn/model.py index 378cd5c3..46f35a03 100644 --- a/pypots/imputation/mrnn/model.py +++ b/pypots/imputation/mrnn/model.py @@ -1,13 +1,6 @@ """ PyTorch M-RNN model for the time-series imputation task. -Refer to the paper "Jinsung Yoon, William R. Zame, and Mihaela van der Schaar. -Estimating missing data in temporal data streams using multi-directional recurrent neural networks. -EEE Transactions on Biomedical Engineering, 66(5):14771490, 2019." - -This implementation is inspired by the official one https://github.com/jsyoon0823/MRNN. -Some part of the code is from https://github.com/WenjieDu/SAITS. - """ # Created by Wenjie Du diff --git a/pypots/imputation/patchtst/__init__.py b/pypots/imputation/patchtst/__init__.py index 13c34f93..2752bde8 100644 --- a/pypots/imputation/patchtst/__init__.py +++ b/pypots/imputation/patchtst/__init__.py @@ -7,6 +7,10 @@ In ICLR, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/yuqinie98/PatchTST + """ # Created by Wenjie Du diff --git a/pypots/imputation/patchtst/core.py b/pypots/imputation/patchtst/core.py index 6dc70615..8d394384 100644 --- a/pypots/imputation/patchtst/core.py +++ b/pypots/imputation/patchtst/core.py @@ -1,5 +1,6 @@ """ - +The core wrapper assembles the submodules of PatchTST imputation model +and takes over the forward progress of the algorithm. """ # Created by Wenjie Du diff --git a/pypots/imputation/patchtst/data.py b/pypots/imputation/patchtst/data.py index 4ccb1e72..c0df9554 100644 --- a/pypots/imputation/patchtst/data.py +++ b/pypots/imputation/patchtst/data.py @@ -1,5 +1,5 @@ """ -Dataset class for PatchTST. +Dataset class for the imputation model PatchTST. """ # Created by Wenjie Du diff --git a/pypots/imputation/patchtst/model.py b/pypots/imputation/patchtst/model.py index b4c72c4d..d1b5274e 100644 --- a/pypots/imputation/patchtst/model.py +++ b/pypots/imputation/patchtst/model.py @@ -5,11 +5,6 @@ A time series is worth 64 words: Long-term forecasting with transformers. In The ICLR, 2023." -Notes ------ -Partial implementation uses code from -https://github.com/yuqinie98/PatchTST and https://github.com/thuml/Time-Series-Library - """ # Created by Wenjie Du diff --git a/pypots/imputation/saits/__init__.py b/pypots/imputation/saits/__init__.py index 3d9b4312..c4aba1cc 100644 --- a/pypots/imputation/saits/__init__.py +++ b/pypots/imputation/saits/__init__.py @@ -7,6 +7,10 @@ Expert Systems with Applications, 219:119619, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/WenjieDu/SAITS + """ # Created by Wenjie Du diff --git a/pypots/imputation/saits/core.py b/pypots/imputation/saits/core.py index 89c5e731..7e793583 100644 --- a/pypots/imputation/saits/core.py +++ b/pypots/imputation/saits/core.py @@ -1,12 +1,6 @@ """ -The implementation of SAITS for the partially-observed time-series imputation task. - -Refer to the paper "Du, W., Cote, D., & Liu, Y. (2023). SAITS: Self-Attention-based Imputation for Time Series. -Expert systems with applications." - -Notes ------ -Partial implementation uses code from https://github.com/WenjieDu/SAITS. +The core wrapper assembles the submodules of SAITS imputation model +and takes over the forward progress of the algorithm. """ diff --git a/pypots/imputation/saits/data.py b/pypots/imputation/saits/data.py index 0c25b0f8..ebc1b163 100644 --- a/pypots/imputation/saits/data.py +++ b/pypots/imputation/saits/data.py @@ -1,5 +1,5 @@ """ -Dataset class for self-attention models trained with MIT (masked imputation task) task. +Dataset class for the imputation model SAITS. """ # Created by Wenjie Du diff --git a/pypots/imputation/saits/model.py b/pypots/imputation/saits/model.py index a45927f6..eb8c22f2 100644 --- a/pypots/imputation/saits/model.py +++ b/pypots/imputation/saits/model.py @@ -1,14 +1,6 @@ """ The implementation of SAITS for the partially-observed time-series imputation task. -Refer to the paper "Wenjie Du, David Cote, and Yan Liu. -SAITS: Self-Attention-based Imputation for Time Series. -Expert Systems with Applications, 219:119619, 2023." - -Notes ------ -Partial implementation uses code from https://github.com/WenjieDu/SAITS. - """ # Created by Wenjie Du diff --git a/pypots/imputation/template/core.py b/pypots/imputation/template/core.py index 4bfd259d..81c59a61 100644 --- a/pypots/imputation/template/core.py +++ b/pypots/imputation/template/core.py @@ -1,7 +1,6 @@ """ -The implementation of YourNewModel for the partially-observed time-series imputation task. - -Refer to the paper "Your paper citation". +The core wrapper assembles the submodules of YourNewModel imputation model +and takes over the forward progress of the algorithm. """ diff --git a/pypots/imputation/template/data.py b/pypots/imputation/template/data.py index 3c4ca97e..ec0b6957 100644 --- a/pypots/imputation/template/data.py +++ b/pypots/imputation/template/data.py @@ -1,5 +1,5 @@ """ -Dataset class for YourNewModel. +Dataset class for the imputation model YourNewModel. TODO: modify the above description for your model's dataset class. diff --git a/pypots/imputation/template/model.py b/pypots/imputation/template/model.py index 170e140b..9f135893 100644 --- a/pypots/imputation/template/model.py +++ b/pypots/imputation/template/model.py @@ -1,8 +1,6 @@ """ The implementation of YourNewModel for the partially-observed time-series imputation task. -Refer to the paper "Your paper citation". - TODO: modify the above description with your model's information. """ diff --git a/pypots/imputation/timesnet/__init__.py b/pypots/imputation/timesnet/__init__.py index 6016a73b..dc2f7a55 100644 --- a/pypots/imputation/timesnet/__init__.py +++ b/pypots/imputation/timesnet/__init__.py @@ -7,6 +7,10 @@ In ICLR, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/thuml/Time-Series-Library + """ # Created by Wenjie Du diff --git a/pypots/imputation/timesnet/data.py b/pypots/imputation/timesnet/data.py index f65632c9..5e4e32d2 100644 --- a/pypots/imputation/timesnet/data.py +++ b/pypots/imputation/timesnet/data.py @@ -1,5 +1,5 @@ """ -Dataset class for TimesNet. +Dataset class for the imputation model TimesNet. """ # Created by Wenjie Du diff --git a/pypots/imputation/timesnet/model.py b/pypots/imputation/timesnet/model.py index 419470b2..846a6c5a 100644 --- a/pypots/imputation/timesnet/model.py +++ b/pypots/imputation/timesnet/model.py @@ -1,14 +1,6 @@ """ The implementation of TimesNet for the partially-observed time-series imputation task. -Refer to the paper "Haixu Wu, Tengge Hu, Yong Liu, Hang Zhou, Jianmin Wang, and Mingsheng Long. -TimesNet: Temporal 2D-Variation Modeling for General Time Series Analysis. -In ICLR, 2023." - -Notes ------ -Partial implementation uses code from https://github.com/thuml/Time-Series-Library. - """ # Created by Wenjie Du diff --git a/pypots/imputation/transformer/__init__.py b/pypots/imputation/transformer/__init__.py index fa086173..a5f98a0a 100644 --- a/pypots/imputation/transformer/__init__.py +++ b/pypots/imputation/transformer/__init__.py @@ -13,6 +13,9 @@ Expert Systems with Applications, 219:119619, 2023. `_ +Notes +----- +This implementation is inspired by https://github.com/WenjieDu/SAITS """ # Created by Wenjie Du diff --git a/pypots/imputation/transformer/core.py b/pypots/imputation/transformer/core.py index 01a9d487..dc0d7abe 100644 --- a/pypots/imputation/transformer/core.py +++ b/pypots/imputation/transformer/core.py @@ -1,12 +1,6 @@ """ -The implementation of Transformer for the partially-observed time-series imputation task. - -Refer to the paper "Du, W., Cote, D., & Liu, Y. (2023). SAITS: Self-Attention-based Imputation for Time Series. -Expert systems with applications." - -Notes ------ -Partial implementation uses code from https://github.com/WenjieDu/SAITS. +The core wrapper assembles the submodules of Transformer imputation model +and takes over the forward progress of the algorithm. """ diff --git a/pypots/imputation/transformer/data.py b/pypots/imputation/transformer/data.py index d8751050..a5a6ddbc 100644 --- a/pypots/imputation/transformer/data.py +++ b/pypots/imputation/transformer/data.py @@ -1,5 +1,5 @@ """ -Dataset class for self-attention models trained with MIT (masked imputation task) task. +Dataset class for the imputation model Transformer. """ # Created by Wenjie Du diff --git a/pypots/imputation/transformer/model.py b/pypots/imputation/transformer/model.py index 46ee13ab..cf3c2178 100644 --- a/pypots/imputation/transformer/model.py +++ b/pypots/imputation/transformer/model.py @@ -1,14 +1,6 @@ """ The implementation of Transformer for the partially-observed time-series imputation task. -Refer to the paper "Wenjie Du, David Cote, and Yan Liu. -SAITS: Self-Attention-based Imputation for Time Series. -Expert Systems with Applications, 219:119619, 2023." - -Notes ------ -Partial implementation uses code from https://github.com/WenjieDu/SAITS. - """ # Created by Wenjie Du diff --git a/pypots/imputation/usgan/core.py b/pypots/imputation/usgan/core.py index 25fbb345..a0b04ade 100644 --- a/pypots/imputation/usgan/core.py +++ b/pypots/imputation/usgan/core.py @@ -1,9 +1,6 @@ """ -The implementation of USGAN for the partially-observed time-series imputation task. - -Refer to the paper "Miao, X., Wu, Y., Wang, J., Gao, Y., Mao, X., & Yin, J. (2021). -Generative Semi-supervised Learning for Multivariate Time Series Imputation. AAAI 2021." - +The core wrapper assembles the submodules of USGAN imputation model +and takes over the forward progress of the algorithm. """ # Created by Jun Wang and Wenjie Du diff --git a/pypots/imputation/usgan/data.py b/pypots/imputation/usgan/data.py index 40e6ee77..9c50bffb 100644 --- a/pypots/imputation/usgan/data.py +++ b/pypots/imputation/usgan/data.py @@ -1,5 +1,5 @@ """ -Dataset class for model USGAN. +Dataset class for the imputation model USGAN. """ # Created by Jun Wang and Wenjie Du diff --git a/pypots/imputation/usgan/model.py b/pypots/imputation/usgan/model.py index 1f684e92..7a4aeded 100644 --- a/pypots/imputation/usgan/model.py +++ b/pypots/imputation/usgan/model.py @@ -1,10 +1,6 @@ """ The implementation of USGAN for the partially-observed time-series imputation task. -Refer to the paper "Xiaoye Miao, Yangyang Wu, Jun Wang, Yunjun Gao, Xudong Mao, and Jianwei Yin. -Generative Semi-supervised Learning for Multivariate Time Series Imputation. -In AAAI, 35(10):8983–8991, May 2021." - """ # Created by Jun Wang and Wenjie Du diff --git a/pypots/nn/modules/autoformer/__init__.py b/pypots/nn/modules/autoformer/__init__.py index 570adecf..8ab5d103 100644 --- a/pypots/nn/modules/autoformer/__init__.py +++ b/pypots/nn/modules/autoformer/__init__.py @@ -7,6 +7,10 @@ In Advances in Neural Information Processing Systems, volume 34, pages 22419–22430. Curran Associates, Inc., 2021. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/thuml/Autoformer + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/brits/__init__.py b/pypots/nn/modules/brits/__init__.py index a85ff1e2..9c2b86aa 100644 --- a/pypots/nn/modules/brits/__init__.py +++ b/pypots/nn/modules/brits/__init__.py @@ -7,6 +7,11 @@ In Advances in Neural Information Processing Systems, volume 31. Curran Associates, Inc., 2018. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/caow13/BRITS +The bugs in the original implementation are fixed here. + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/brits/backbone.py b/pypots/nn/modules/brits/backbone.py index a22cffc1..3a8a87d1 100644 --- a/pypots/nn/modules/brits/backbone.py +++ b/pypots/nn/modules/brits/backbone.py @@ -1,13 +1,4 @@ """ -The implementation of BRITS for the partially-observed time-series imputation task. - -Refer to the paper "Cao, W., Wang, D., Li, J., Zhou, H., Li, L., & Li, Y. (2018). -BRITS: Bidirectional Recurrent Imputation for Time Series. NeurIPS 2018." - -Notes ------ -Partial implementation uses code from https://github.com/caow13/BRITS. The bugs in the original implementation -are fixed here. """ diff --git a/pypots/nn/modules/brits/layers.py b/pypots/nn/modules/brits/layers.py index a0ef4050..03ada5b8 100644 --- a/pypots/nn/modules/brits/layers.py +++ b/pypots/nn/modules/brits/layers.py @@ -1,13 +1,4 @@ """ -The implementation of the modules for BRITS. - -Refer to the paper "Cao, W., Wang, D., Li, J., Zhou, H., Li, L., & Li, Y. (2018). -BRITS: Bidirectional Recurrent Imputation for Time Series. NeurIPS 2018." - -Notes ------ -Partial implementation uses code from https://github.com/caow13/BRITS. The bugs in the original implementation -are fixed here. """ diff --git a/pypots/nn/modules/crli/__init__.py b/pypots/nn/modules/crli/__init__.py index 08ace790..e9aaf8cf 100644 --- a/pypots/nn/modules/crli/__init__.py +++ b/pypots/nn/modules/crli/__init__.py @@ -7,6 +7,9 @@ In AAAI, 35(10):8837–8846, May 2021. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/qianlima-lab/CRLI """ diff --git a/pypots/nn/modules/crli/layers.py b/pypots/nn/modules/crli/layers.py index f5108fe2..d9558c32 100644 --- a/pypots/nn/modules/crli/layers.py +++ b/pypots/nn/modules/crli/layers.py @@ -1,8 +1,4 @@ """ -The implementation of the modules for CRLI (Clustering Representation Learning on Incomplete time-series data). - -Refer to the paper "Ma, Q., Chen, C., Li, S., & Cottrell, G. W. (2021). -Learning Representations for Incomplete Time Series Clustering. AAAI 2021." """ diff --git a/pypots/nn/modules/crossformer/__init__.py b/pypots/nn/modules/crossformer/__init__.py index ddcb6933..6473d52d 100644 --- a/pypots/nn/modules/crossformer/__init__.py +++ b/pypots/nn/modules/crossformer/__init__.py @@ -7,6 +7,10 @@ In The 11th ICLR, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/Thinklab-SJTU/Crossformer + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/csdi/__init__.py b/pypots/nn/modules/csdi/__init__.py index 2efaaa5c..201315fa 100644 --- a/pypots/nn/modules/csdi/__init__.py +++ b/pypots/nn/modules/csdi/__init__.py @@ -7,6 +7,10 @@ In NeurIPS, 2021. `_ +Notes +----- +This implementation is inspired by the official one the official implementation https://github.com/ermongroup/CSDI. + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/csdi/layers.py b/pypots/nn/modules/csdi/layers.py index c17c044f..dfaacf20 100644 --- a/pypots/nn/modules/csdi/layers.py +++ b/pypots/nn/modules/csdi/layers.py @@ -1,5 +1,5 @@ """ -Modules for CSDI model. + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/dlinear/__init__.py b/pypots/nn/modules/dlinear/__init__.py index 71508644..7f4be66e 100644 --- a/pypots/nn/modules/dlinear/__init__.py +++ b/pypots/nn/modules/dlinear/__init__.py @@ -7,6 +7,9 @@ In AAAI, volume 37, pages 11121–11128, Jun. 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/cure-lab/LTSF-Linear """ diff --git a/pypots/nn/modules/etsformer/__init__.py b/pypots/nn/modules/etsformer/__init__.py index bfdea42e..d04997ab 100644 --- a/pypots/nn/modules/etsformer/__init__.py +++ b/pypots/nn/modules/etsformer/__init__.py @@ -7,6 +7,10 @@ In ICLR, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/salesforce/ETSformer + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/fedformer/__init__.py b/pypots/nn/modules/fedformer/__init__.py index 76a71316..ac64bc66 100644 --- a/pypots/nn/modules/fedformer/__init__.py +++ b/pypots/nn/modules/fedformer/__init__.py @@ -7,6 +7,10 @@ In ICML, volume 162 of Proceedings of Machine Learning Research, pages 27268–27286. PMLR, 17–23 Jul 2022. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/MAZiqing/FEDformer + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/film/__init__.py b/pypots/nn/modules/film/__init__.py index 9399a9af..4f97f20b 100644 --- a/pypots/nn/modules/film/__init__.py +++ b/pypots/nn/modules/film/__init__.py @@ -7,6 +7,9 @@ In Advances in Neural Information Processing Systems 35 (2022): 12677-12690. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/tianzhou2011/FiLM """ diff --git a/pypots/nn/modules/frets/__init__.py b/pypots/nn/modules/frets/__init__.py index 7ebc1dce..04e9f121 100644 --- a/pypots/nn/modules/frets/__init__.py +++ b/pypots/nn/modules/frets/__init__.py @@ -7,6 +7,10 @@ Advances in Neural Information Processing Systems 36 (2024). `_ +Notes +----- +This implementation is inspired by the official one https://github.com/aikunyi/FreTS + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/gpvae/__init__.py b/pypots/nn/modules/gpvae/__init__.py index 241ae4cb..f3eb1fc3 100644 --- a/pypots/nn/modules/gpvae/__init__.py +++ b/pypots/nn/modules/gpvae/__init__.py @@ -7,6 +7,10 @@ In International conference on artificial intelligence and statistics, pages 1651–1661. PMLR, 2020. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/ratschlab/GP-VAE + """ # Created by Jun Wang and Wenjie Du diff --git a/pypots/nn/modules/grud/__init__.py b/pypots/nn/modules/grud/__init__.py index ed797c56..23f84c33 100644 --- a/pypots/nn/modules/grud/__init__.py +++ b/pypots/nn/modules/grud/__init__.py @@ -1,8 +1,16 @@ """ The package including the modules of GRU-D. -Refer to the paper "Che, Z., Purushotham, S., Cho, K., Sontag, D.A., & Liu, Y. (2018). -Recurrent Neural Networks for Multivariate Time Series with Missing Values. Scientific Reports." +Refer to the paper +`Zhengping Che, Sanjay Purushotham, Kyunghyun Cho, David Sontag, and Yan Liu. +Recurrent Neural Networks for Multivariate Time Series with Missing Values. +Scientific Reports, 8(1):6085, April 2018. +`_ + +Notes +----- +This implementation is inspired by the official one https://github.com/PeterChe1990/GRU-D + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/informer/__init__.py b/pypots/nn/modules/informer/__init__.py index 5915d8c5..9483c9c5 100644 --- a/pypots/nn/modules/informer/__init__.py +++ b/pypots/nn/modules/informer/__init__.py @@ -7,6 +7,10 @@ In Proceedings of the AAAI conference on artificial intelligence, volume 35, pages 11106–11115, 2021. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/zhouhaoyi/Informer2020 + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/mrnn/__init__.py b/pypots/nn/modules/mrnn/__init__.py index d5506c9c..094b9325 100644 --- a/pypots/nn/modules/mrnn/__init__.py +++ b/pypots/nn/modules/mrnn/__init__.py @@ -7,6 +7,11 @@ IEEE Transactions on Biomedical Engineering, 66(5):14771490, 2019. `_ +Notes +----- +This implementation is inspired by the official one +https://github.com/jsyoon0823/MRNN and https://github.com/WenjieDu/SAITS + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/mrnn/backbone.py b/pypots/nn/modules/mrnn/backbone.py index f88b9d73..2478da28 100644 --- a/pypots/nn/modules/mrnn/backbone.py +++ b/pypots/nn/modules/mrnn/backbone.py @@ -1,7 +1,5 @@ """ -PyTorch MRNN model for the time-series imputation task. -This implementation is inspired by the official one https://github.com/jsyoon0823/MRNN. -Some part of the code is from https://github.com/WenjieDu/SAITS. + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/mrnn/layers.py b/pypots/nn/modules/mrnn/layers.py index 40e573dd..9d48d977 100644 --- a/pypots/nn/modules/mrnn/layers.py +++ b/pypots/nn/modules/mrnn/layers.py @@ -1,5 +1,5 @@ """ -The layers of the M-RNN model. + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/patchtst/__init__.py b/pypots/nn/modules/patchtst/__init__.py index 7b374d84..edb4a956 100644 --- a/pypots/nn/modules/patchtst/__init__.py +++ b/pypots/nn/modules/patchtst/__init__.py @@ -7,6 +7,10 @@ In ICLR, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/yuqinie98/PatchTST + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/raindrop/__init__.py b/pypots/nn/modules/raindrop/__init__.py index c8aec3ee..7d4cfb3d 100644 --- a/pypots/nn/modules/raindrop/__init__.py +++ b/pypots/nn/modules/raindrop/__init__.py @@ -7,6 +7,10 @@ In ICLR, 2022. `_ +Notes +----- +This implementation is inspired by the official one the official implementation https://github.com/mims-harvard/Raindrop + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/raindrop/backbone.py b/pypots/nn/modules/raindrop/backbone.py index 3d16ce84..50af4fe0 100644 --- a/pypots/nn/modules/raindrop/backbone.py +++ b/pypots/nn/modules/raindrop/backbone.py @@ -1,8 +1,4 @@ """ -The implementation of Raindrop for the partially-observed time-series classification task. - -Refer to the paper "Zhang, X., Zeman, M., Tsiligkaridis, T., & Zitnik, M. (2022). -Graph-Guided Network for Irregularly Sampled Multivariate Time Series. ICLR 2022." """ diff --git a/pypots/nn/modules/raindrop/layers.py b/pypots/nn/modules/raindrop/layers.py index e21f7a0f..d70fba66 100644 --- a/pypots/nn/modules/raindrop/layers.py +++ b/pypots/nn/modules/raindrop/layers.py @@ -1,8 +1,4 @@ """ -The implementation of the modules for Raindrop. - -Refer to the paper "Zhang, X., Zeman, M., Tsiligkaridis, T., & Zitnik, M. (2022). -Graph-Guided Network for Irregularly Sampled Multivariate Time Series. ICLR 2022." """ diff --git a/pypots/nn/modules/saits/__init__.py b/pypots/nn/modules/saits/__init__.py index 84fd811e..bf5f91ac 100644 --- a/pypots/nn/modules/saits/__init__.py +++ b/pypots/nn/modules/saits/__init__.py @@ -7,6 +7,10 @@ Expert Systems with Applications, 219:119619, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/WenjieDu/SAITS + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/saits/backbone.py b/pypots/nn/modules/saits/backbone.py index b0aee630..fdf45f3c 100644 --- a/pypots/nn/modules/saits/backbone.py +++ b/pypots/nn/modules/saits/backbone.py @@ -1,10 +1,4 @@ """ -Refer to the paper "Du, W., Cote, D., & Liu, Y. (2023). SAITS: Self-Attention-based Imputation for Time Series. -Expert systems with applications." - -Notes ------ -Partial implementation uses code from https://github.com/WenjieDu/SAITS. """ diff --git a/pypots/nn/modules/timesnet/__init__.py b/pypots/nn/modules/timesnet/__init__.py index d6aefda6..ee79fb51 100644 --- a/pypots/nn/modules/timesnet/__init__.py +++ b/pypots/nn/modules/timesnet/__init__.py @@ -7,6 +7,10 @@ In ICLR, 2023. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/thuml/Time-Series-Library + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/transformer/__init__.py b/pypots/nn/modules/transformer/__init__.py index 40078a42..97fe07c2 100644 --- a/pypots/nn/modules/transformer/__init__.py +++ b/pypots/nn/modules/transformer/__init__.py @@ -13,6 +13,11 @@ Expert Systems with Applications, 219:119619, 2023. `_ + +Notes +----- +This implementation is inspired by https://github.com/WenjieDu/SAITS + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/transformer/attention.py b/pypots/nn/modules/transformer/attention.py index e83a2bb0..1d3b0a52 100644 --- a/pypots/nn/modules/transformer/attention.py +++ b/pypots/nn/modules/transformer/attention.py @@ -3,7 +3,7 @@ Notes ----- -Partial implementation uses code from https://github.com/WenjieDu/SAITS, +This implementation is inspired by the official one https://github.com/WenjieDu/SAITS, and https://github.com/jadore801120/attention-is-all-you-need-pytorch. """ diff --git a/pypots/nn/modules/transformer/embedding.py b/pypots/nn/modules/transformer/embedding.py index 68b8b93f..6212e0ae 100644 --- a/pypots/nn/modules/transformer/embedding.py +++ b/pypots/nn/modules/transformer/embedding.py @@ -2,7 +2,7 @@ Embedding methods for Transformer models are put here. -Partial implementation uses code from https://github.com/zhouhaoyi/Informer2020/blob/main/models/embed.py +This implementation is inspired by the official one https://github.com/zhouhaoyi/Informer2020/blob/main/models/embed.py """ # Created by Wenjie Du diff --git a/pypots/nn/modules/usgan/backbone.py b/pypots/nn/modules/usgan/backbone.py index 38ff15ad..42d7f430 100644 --- a/pypots/nn/modules/usgan/backbone.py +++ b/pypots/nn/modules/usgan/backbone.py @@ -1,8 +1,4 @@ """ -The implementation of USGAN for the partially-observed time-series imputation task. - -Refer to the paper "Miao, X., Wu, Y., Wang, J., Gao, Y., Mao, X., & Yin, J. (2021). -Generative Semi-supervised Learning for Multivariate Time Series Imputation. AAAI 2021." """ diff --git a/pypots/nn/modules/vader/__init__.py b/pypots/nn/modules/vader/__init__.py index 260ff58f..d850f5c8 100644 --- a/pypots/nn/modules/vader/__init__.py +++ b/pypots/nn/modules/vader/__init__.py @@ -8,6 +8,10 @@ GigaScience, 8(11):giz134, November 2019. `_ +Notes +----- +This implementation is inspired by the official one https://github.com/johanndejong/VaDER + """ # Created by Wenjie Du diff --git a/pypots/nn/modules/vader/backbone.py b/pypots/nn/modules/vader/backbone.py index b711ee3b..7c2d6639 100644 --- a/pypots/nn/modules/vader/backbone.py +++ b/pypots/nn/modules/vader/backbone.py @@ -1,9 +1,4 @@ """ -The implementation of VaDER for the partially-observed time-series clustering task. - -Refer to the paper "Jong, J.D., Emon, M.A., Wu, P., Karki, R., Sood, M., Godard, P., Ahmad, A., Vrooman, H.A., -Hofmann-Apitius, M., & Fröhlich, H. (2019). -Deep learning for clustering of multivariate clinical patient trajectories with missing values. GigaScience." """ diff --git a/pypots/nn/modules/vader/layers.py b/pypots/nn/modules/vader/layers.py index 5e4f7055..a3e53b87 100644 --- a/pypots/nn/modules/vader/layers.py +++ b/pypots/nn/modules/vader/layers.py @@ -1,9 +1,4 @@ """ -The implementation of the modules for VaDER. - -Refer to the paper "Jong, J.D., Emon, M.A., Wu, P., Karki, R., Sood, M., Godard, P., Ahmad, A., Vrooman, H.A., -Hofmann-Apitius, M., & Fröhlich, H. (2019). -Deep learning for clustering of multivariate clinical patient trajectories with missing values. GigaScience." """