From fb2d4d3d6304f9f0037f174c54707b18ae0c163e Mon Sep 17 00:00:00 2001 From: Tianxiang Zhan <1665186372@qq.com> Date: Fri, 6 Sep 2024 17:51:16 +0800 Subject: [PATCH 1/4] Add TEFN model. --- README.md | 260 +++++++++++++++++------- README_zh.md | 310 +++++++++++++++++++---------- docs/references.bib | 7 + pypots/imputation/__init__.py | 3 +- pypots/imputation/tefn/__init__.py | 24 +++ pypots/imputation/tefn/core.py | 59 ++++++ pypots/imputation/tefn/data.py | 24 +++ pypots/imputation/tefn/model.py | 284 ++++++++++++++++++++++++++ pypots/nn/modules/tefn/__init__.py | 24 +++ pypots/nn/modules/tefn/backbone.py | 32 +++ pypots/nn/modules/tefn/layers.py | 23 +++ 11 files changed, 869 insertions(+), 181 deletions(-) create mode 100644 pypots/imputation/tefn/__init__.py create mode 100644 pypots/imputation/tefn/core.py create mode 100644 pypots/imputation/tefn/data.py create mode 100644 pypots/imputation/tefn/model.py create mode 100644 pypots/nn/modules/tefn/__init__.py create mode 100644 pypots/nn/modules/tefn/backbone.py create mode 100644 pypots/nn/modules/tefn/layers.py diff --git a/README.md b/README.md index 4bf1684a..58ff8bfd 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,11 @@ This makes partially-observed time series (POTS) a pervasive problem in open-wor data analysis. Although this problem is important, the area of machine learning on POTS still lacks a dedicated toolkit. PyPOTS is created to fill in this blank. -⦿ `Mission`: PyPOTS (pronounced "Pie Pots") is born to become a handy toolbox that is going to make machine learning on POTS easy rather than +⦿ `Mission`: PyPOTS (pronounced "Pie Pots") is born to become a handy toolbox that is going to make machine learning on +POTS easy rather than tedious, to help engineers and researchers focus more on the core problems in their hands rather than on how to deal -with the missing parts in their data. PyPOTS will keep integrating classical and the latest state-of-the-art machine learning +with the missing parts in their data. PyPOTS will keep integrating classical and the latest state-of-the-art machine +learning algorithms for partially-observed multivariate time series. For sure, besides various algorithms, PyPOTS is going to have unified APIs together with detailed documentation and interactive examples across algorithms as tutorials. @@ -88,20 +90,26 @@ The rest of this readme file is organized as follows: [**❖ Contribution**](#-contribution), [**❖ Community**](#-community). - ## ❖ Available Algorithms -PyPOTS supports imputation, classification, clustering, forecasting, and anomaly detection tasks on multivariate partially-observed -time series with missing values. The table below shows the availability of each algorithm (sorted by Year) in PyPOTS for different tasks. -The symbol `✅` indicates the algorithm is available for the corresponding task (note that models will be continuously updated + +PyPOTS supports imputation, classification, clustering, forecasting, and anomaly detection tasks on multivariate +partially-observed +time series with missing values. The table below shows the availability of each algorithm (sorted by Year) in PyPOTS for +different tasks. +The symbol `✅` indicates the algorithm is available for the corresponding task (note that models will be continuously +updated in the future to handle tasks that are not currently supported. Stay tuned❗️). 🌟 Since **v0.2**, all neural-network models in PyPOTS has got hyperparameter-optimization support. -This functionality is implemented with the [Microsoft NNI](https://github.com/microsoft/nni) framework. You may want to refer to our time-series +This functionality is implemented with the [Microsoft NNI](https://github.com/microsoft/nni) framework. You may want to +refer to our time-series imputation survey repo [Awesome_Imputation](https://github.com/WenjieDu/Awesome_Imputation) to see how to config and tune the hyperparameters. -🔥 Note that all models whose name with `🧑‍🔧` in the table (e.g. Transformer, iTransformer, Informer etc.) are not originally -proposed as algorithms for POTS data in their papers, and they cannot directly accept time series with missing values as input, +🔥 Note that all models whose name with `🧑‍🔧` in the table (e.g. Transformer, iTransformer, Informer etc.) are not +originally +proposed as algorithms for POTS data in their papers, and they cannot directly accept time series with missing values as +input, let alone imputation. **To make them applicable to POTS data, we specifically apply the embedding strategy and training approach (ORT+MIT) the same as we did in [the SAITS paper](https://arxiv.org/pdf/2202.08516)[^1].** @@ -116,6 +124,7 @@ The paper references and links are all listed at the bottom of this file. | **Type** | **Algo** | **IMPU** | **FORE** | **CLAS** | **CLUS** | **ANOD** | **Year - Venue** | |:--------------|:---------------------------------------------------------------------------------------------------------------------------------|:--------:|:--------:|:--------:|:--------:|:--------:|:-------------------| | LLM | Time-Series.AI [^36] | ✅ | ✅ | ✅ | ✅ | ✅ | `Later in 2024` | +| Neural Net | TEFN[^39] | ✅ | | | | | `2024 - arXiv` | | Neural Net | TimeMixer[^37] | ✅ | | | | | `2024 - ICLR` | | Neural Net | iTransformer🧑‍🔧[^24] | ✅ | | | | | `2024 - ICLR` | | Neural Net | ModernTCN[^38] | ✅ | | | | | `2024 - ICLR` | @@ -157,11 +166,14 @@ The paper references and links are all listed at the bottom of this file. | Naive | Mean | ✅ | | | | | | | Naive | Median | ✅ | | | | | | -💯 Contribute your model right now to increase your research impact! PyPOTS downloads are increasing rapidly (**[300K+ in total and 1K+ daily on PyPI so far](https://www.pepy.tech/projects/pypots)**), +💯 Contribute your model right now to increase your research impact! PyPOTS downloads are increasing rapidly (* +*[300K+ in total and 1K+ daily on PyPI so far](https://www.pepy.tech/projects/pypots)**), and your work will be widely used and cited by the community. -Refer to the [contribution guide](https://github.com/WenjieDu/PyPOTS#-contribution) to see how to include your model in PyPOTS. +Refer to the [contribution guide](https://github.com/WenjieDu/PyPOTS#-contribution) to see how to include your model in +PyPOTS. ## ❖ PyPOTS Ecosystem + At PyPOTS, things are related to coffee, which we're familiar with. Yes, this is a coffee universe! As you can see, there is a coffee pot in the PyPOTS logo. And what else? Please read on ;-) @@ -170,17 +182,21 @@ And what else? Please read on ;-) TSDB logo -👈 Time series datasets are taken as coffee beans at PyPOTS, and POTS datasets are incomplete coffee beans with missing parts that have their own meanings. +👈 Time series datasets are taken as coffee beans at PyPOTS, and POTS datasets are incomplete coffee beans with missing +parts that have their own meanings. To make various public time-series datasets readily available to users, Time Series Data Beans (TSDB) is created to make loading time-series datasets super easy! -Visit [TSDB](https://github.com/WenjieDu/TSDB) right now to know more about this handy tool 🛠, and it now supports a total of 172 open-source datasets! +Visit [TSDB](https://github.com/WenjieDu/TSDB) right now to know more about this handy tool 🛠, and it now supports a +total of 172 open-source datasets! PyGrinder logo -👉 To simulate the real-world data beans with missingness, the ecosystem library [PyGrinder](https://github.com/WenjieDu/PyGrinder), -a toolkit helping grind your coffee beans into incomplete ones, is created. Missing patterns fall into three categories according to Robin's theory[^13]: +👉 To simulate the real-world data beans with missingness, the ecosystem +library [PyGrinder](https://github.com/WenjieDu/PyGrinder), +a toolkit helping grind your coffee beans into incomplete ones, is created. Missing patterns fall into three categories +according to Robin's theory[^13]: MCAR (missing completely at random), MAR (missing at random), and MNAR (missing not at random). PyGrinder supports all of them and additional functionalities related to missingness. With PyGrinder, you can introduce synthetic missing values into your datasets with a single line of code. @@ -189,15 +205,18 @@ With PyGrinder, you can introduce synthetic missing values into your datasets wi BenchPOTS logo -👈 To fairly evaluate the performance of PyPOTS algorithms, the benchmarking suite [BenchPOTS](https://github.com/WenjieDu/BenchPOTS) is created, -which provides standard and unified data-preprocessing pipelines to prepare datasets for measuring the performance of different +👈 To fairly evaluate the performance of PyPOTS algorithms, the benchmarking +suite [BenchPOTS](https://github.com/WenjieDu/BenchPOTS) is created, +which provides standard and unified data-preprocessing pipelines to prepare datasets for measuring the performance of +different POTS algorithms on various tasks. BrewPOTS logo -👉 Now the beans, grinder, and pot are ready, please have a seat on the bench and let's think about how to brew us a cup of coffee. +👉 Now the beans, grinder, and pot are ready, please have a seat on the bench and let's think about how to brew us a cup +of coffee. Tutorials are necessary! Considering the future workload, PyPOTS tutorials are released in a single repo, and you can find them in [BrewPOTS](https://github.com/WenjieDu/BrewPOTS). Take a look at it now, and learn how to brew your POTS datasets! @@ -210,11 +229,13 @@ Take a look at it now, and learn how to brew your POTS datasets! ☕️ Welcome to the universe of PyPOTS. Enjoy it and have fun!

- ## ❖ Installation -You can refer to [the installation instruction](https://docs.pypots.com/en/latest/install.html) in PyPOTS documentation for a guideline with more details. -PyPOTS is available on both [PyPI](https://pypi.python.org/pypi/pypots) and [Anaconda](https://anaconda.org/conda-forge/pypots). +You can refer to [the installation instruction](https://docs.pypots.com/en/latest/install.html) in PyPOTS documentation +for a guideline with more details. + +PyPOTS is available on both [PyPI](https://pypi.python.org/pypi/pypots) +and [Anaconda](https://anaconda.org/conda-forge/pypots). You can install PyPOTS like below as well as TSDB and PyGrinder: ``` bash @@ -229,9 +250,10 @@ conda install -c conda-forge pypots # the first time installation conda update -c conda-forge pypots # update pypots to the latest version ``` - ## ❖ Usage -Besides [BrewPOTS](https://github.com/WenjieDu/BrewPOTS), you can also find a simple and quick-start tutorial notebook on Google Colab + +Besides [BrewPOTS](https://github.com/WenjieDu/BrewPOTS), you can also find a simple and quick-start tutorial notebook +on Google Colab Colab tutorials . If you have further questions, please refer to PyPOTS documentation [docs.pypots.com](https://docs.pypots.com). @@ -271,23 +293,28 @@ mae = calc_mae(imputation, np.nan_to_num(X_ori), indicating_mask) # calculate m saits.save("save_it_here/saits_physionet2012.pypots") # save the model for future use saits.load("save_it_here/saits_physionet2012.pypots") # reload the serialized model file for following imputation or training ``` - + ## ❖ Citing PyPOTS + > [!TIP] > **[Updates in Jun 2024]** 😎 The 1st comprehensive time-seres imputation benchmark paper [TSI-Bench: Benchmarking Time Series Imputation](https://arxiv.org/abs/2406.12747) now is public available. -The code is open source in the repo [Awesome_Imputation](https://github.com/WenjieDu/Awesome_Imputation). -With nearly 35,000 experiments, we provide a comprehensive benchmarking study on 28 imputation methods, 3 missing patterns (points, sequences, blocks), -various missing rates, and 8 real-world datasets. +> The code is open source in the repo [Awesome_Imputation](https://github.com/WenjieDu/Awesome_Imputation). +> With nearly 35,000 experiments, we provide a comprehensive benchmarking study on 28 imputation methods, 3 missing +> patterns (points, sequences, blocks), +> various missing rates, and 8 real-world datasets. > -> **[Updates in Feb 2024]** 🎉 Our survey paper [Deep Learning for Multivariate Time Series Imputation: A Survey](https://arxiv.org/abs/2402.04059) has been released on arXiv. -We comprehensively review the literature of the state-of-the-art deep-learning imputation methods for time series, -provide a taxonomy for them, and discuss the challenges and future directions in this field. +> **[Updates in Feb 2024]** 🎉 Our survey +> paper [Deep Learning for Multivariate Time Series Imputation: A Survey](https://arxiv.org/abs/2402.04059) has been +> released on arXiv. +> We comprehensively review the literature of the state-of-the-art deep-learning imputation methods for time series, +> provide a taxonomy for them, and discuss the challenges and future directions in this field. The paper introducing PyPOTS is available [on arXiv](https://arxiv.org/abs/2305.18811), -A short version of it is accepted by the 9th SIGKDD international workshop on Mining and Learning from Time Series ([MiLeTS'23](https://kdd-milets.github.io/milets2023/))). +A short version of it is accepted by the 9th SIGKDD international workshop on Mining and Learning from Time +Series ([MiLeTS'23](https://kdd-milets.github.io/milets2023/))). **Additionally**, PyPOTS has been included as a [PyTorch Ecosystem](https://pytorch.org/ecosystem/) project. We are pursuing to publish it in prestigious academic venues, e.g. JMLR (track for [Machine Learning Open Source Software](https://www.jmlr.org/mloss/)). If you use PyPOTS in your work, @@ -304,13 +331,14 @@ journal={arXiv preprint arXiv:2305.18811}, year={2023}, } ``` + or > Wenjie Du. > PyPOTS: a Python toolbox for data mining on Partially-Observed Time Series. > arXiv, abs/2305.18811, 2023. - ## ❖ Contribution + You're very welcome to contribute to this exciting project! By committing your code, you'll @@ -319,7 +347,8 @@ By committing your code, you'll and help your work obtain more exposure and impact. Take a look at our [inclusion criteria](https://docs.pypots.com/en/latest/faq.html#inclusion-criteria). You can utilize the `template` folder in each task package (e.g. - [pypots/imputation/template](https://github.com/WenjieDu/PyPOTS/tree/main/pypots/imputation/template)) to quickly start; + [pypots/imputation/template](https://github.com/WenjieDu/PyPOTS/tree/main/pypots/imputation/template)) to quickly + start; 2. become one of [PyPOTS contributors](https://github.com/WenjieDu/PyPOTS/graphs/contributors) and be listed as a volunteer developer [on the PyPOTS website](https://pypots.com/about/#volunteer-developers); 3. get mentioned in PyPOTS [release notes](https://github.com/WenjieDu/PyPOTS/releases); @@ -345,11 +374,12 @@ Your star is your recognition to PyPOTS, and it matters! 👀 Check out a full list of our users' affiliations [on PyPOTS website here](https://pypots.com/users/)! - ## ❖ Community + We care about the feedback from our users, so we're building PyPOTS community on -- [Slack](https://join.slack.com/t/pypots-org/shared_invite/zt-1gq6ufwsi-p0OZdW~e9UW_IA4_f1OfxA). General discussion, Q&A, and our development team are here; +- [Slack](https://join.slack.com/t/pypots-org/shared_invite/zt-1gq6ufwsi-p0OZdW~e9UW_IA4_f1OfxA). General discussion, + Q&A, and our development team are here; - [LinkedIn](https://www.linkedin.com/company/pypots). Official announcements and news are here; - [WeChat (微信公众号)](https://mp.weixin.qq.com/s/X3ukIgL1QpNH8ZEXq1YifA). We also run a group chat on WeChat, and you can get the QR code from the official account after following it; @@ -359,43 +389,123 @@ PyPOTS community is open, transparent, and surely friendly. Let's work together [//]: # (Use APA reference style below) -[^1]: Du, W., Cote, D., & Liu, Y. (2023). [SAITS: Self-Attention-based Imputation for Time Series](https://doi.org/10.1016/j.eswa.2023.119619). *Expert systems with applications*. -[^2]: Vaswani, A., Shazeer, N.M., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A.N., Kaiser, L., & Polosukhin, I. (2017). [Attention is All you Need](https://papers.nips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html). *NeurIPS 2017*. -[^3]: Cao, W., Wang, D., Li, J., Zhou, H., Li, L., & Li, Y. (2018). [BRITS: Bidirectional Recurrent Imputation for Time Series](https://papers.nips.cc/paper/2018/hash/734e6bfcd358e25ac1db0a4241b95651-Abstract.html). *NeurIPS 2018*. -[^4]: Che, Z., Purushotham, S., Cho, K., Sontag, D.A., & Liu, Y. (2018). [Recurrent Neural Networks for Multivariate Time Series with Missing Values](https://www.nature.com/articles/s41598-018-24271-9). *Scientific Reports*. -[^5]: Zhang, X., Zeman, M., Tsiligkaridis, T., & Zitnik, M. (2022). [Graph-Guided Network for Irregularly Sampled Multivariate Time Series](https://arxiv.org/abs/2110.05357). *ICLR 2022*. -[^6]: Ma, Q., Chen, C., Li, S., & Cottrell, G. W. (2021). [Learning Representations for Incomplete Time Series Clustering](https://ojs.aaai.org/index.php/AAAI/article/view/17070). *AAAI 2021*. -[^7]: Jong, J.D., Emon, M.A., Wu, P., Karki, R., Sood, M., Godard, P., Ahmad, A., Vrooman, H.A., Hofmann-Apitius, M., & Fröhlich, H. (2019). [Deep learning for clustering of multivariate clinical patient trajectories with missing values](https://academic.oup.com/gigascience/article/8/11/giz134/5626377). *GigaScience*. -[^8]: Chen, X., & Sun, L. (2021). [Bayesian Temporal Factorization for Multidimensional Time Series Prediction](https://arxiv.org/abs/1910.06366). *IEEE transactions on pattern analysis and machine intelligence*. -[^9]: Yoon, J., Zame, W. R., & van der Schaar, M. (2019). [Estimating Missing Data in Temporal Data Streams Using Multi-Directional Recurrent Neural Networks](https://ieeexplore.ieee.org/document/8485748). *IEEE Transactions on Biomedical Engineering*. -[^10]: Miao, X., Wu, Y., Wang, J., Gao, Y., Mao, X., & Yin, J. (2021). [Generative Semi-supervised Learning for Multivariate Time Series Imputation](https://ojs.aaai.org/index.php/AAAI/article/view/17086). *AAAI 2021*. -[^11]: Fortuin, V., Baranchuk, D., Raetsch, G. & Mandt, S. (2020). [GP-VAE: Deep Probabilistic Time Series Imputation](https://proceedings.mlr.press/v108/fortuin20a.html). *AISTATS 2020*. -[^12]: Tashiro, Y., Song, J., Song, Y., & Ermon, S. (2021). [CSDI: Conditional Score-based Diffusion Models for Probabilistic Time Series Imputation](https://proceedings.neurips.cc/paper/2021/hash/cfe8504bda37b575c70ee1a8276f3486-Abstract.html). *NeurIPS 2021*. -[^13]: Rubin, D. B. (1976). [Inference and missing data](https://academic.oup.com/biomet/article-abstract/63/3/581/270932). *Biometrika*. -[^14]: Wu, H., Hu, T., Liu, Y., Zhou, H., Wang, J., & Long, M. (2023). [TimesNet: Temporal 2d-variation modeling for general time series analysis](https://openreview.net/forum?id=ju_Uqw384Oq). *ICLR 2023* -[^15]: Wu, H., Xu, J., Wang, J., & Long, M. (2021). [Autoformer: Decomposition transformers with auto-correlation for long-term series forecasting](https://proceedings.neurips.cc/paper/2021/hash/bcc0d400288793e8bdcd7c19a8ac0c2b-Abstract.html). *NeurIPS 2021*. -[^16]: Zhang, Y., & Yan, J. (2023). [Crossformer: Transformer utilizing cross-dimension dependency for multivariate time series forecasting](https://openreview.net/forum?id=vSVLM2j9eie). *ICLR 2023*. -[^17]: Zeng, A., Chen, M., Zhang, L., & Xu, Q. (2023). [Are transformers effective for time series forecasting?](https://ojs.aaai.org/index.php/AAAI/article/view/26317). *AAAI 2023* -[^18]: Nie, Y., Nguyen, N. H., Sinthong, P., & Kalagnanam, J. (2023). [A time series is worth 64 words: Long-term forecasting with transformers](https://openreview.net/forum?id=Jbdc0vTOcol). *ICLR 2023* -[^19]: Woo, G., Liu, C., Sahoo, D., Kumar, A., & Hoi, S. (2023). [ETSformer: Exponential Smoothing Transformers for Time-series Forecasting](https://openreview.net/forum?id=5m_3whfo483). *ICLR 2023* -[^20]: Zhou, T., Ma, Z., Wen, Q., Wang, X., Sun, L., & Jin, R. (2022). [FEDformer: Frequency enhanced decomposed transformer for long-term series forecasting](https://proceedings.mlr.press/v162/zhou22g.html). *ICML 2022*. -[^21]: Zhou, H., Zhang, S., Peng, J., Zhang, S., Li, J., Xiong, H., & Zhang, W. (2021). [Informer: Beyond efficient transformer for long sequence time-series forecasting](https://ojs.aaai.org/index.php/AAAI/article/view/17325). *AAAI 2021*. -[^22]: Zhou, T., Ma, Z., Wen, Q., Sun, L., Yao, T., Yin, W., & Jin, R. (2022). [FiLM: Frequency improved Legendre Memory Model for Long-term Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2022/hash/524ef58c2bd075775861234266e5e020-Abstract-Conference.html). *NeurIPS 2022*. -[^23]: Yi, K., Zhang, Q., Fan, W., Wang, S., Wang, P., He, H., An, N., Lian, D., Cao, L., & Niu, Z. (2023). [Frequency-domain MLPs are More Effective Learners in Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2023/hash/f1d16af76939f476b5f040fd1398c0a3-Abstract-Conference.html). *NeurIPS 2023*. -[^24]: Liu, Y., Hu, T., Zhang, H., Wu, H., Wang, S., Ma, L., & Long, M. (2024). [iTransformer: Inverted Transformers Are Effective for Time Series Forecasting](https://openreview.net/forum?id=JePfAI8fah). *ICLR 2024*. -[^25]: Liu, Y., Wu, H., Wang, J., & Long, M. (2022). [Non-stationary Transformers: Exploring the Stationarity in Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2022/hash/4054556fcaa934b0bf76da52cf4f92cb-Abstract-Conference.html). *NeurIPS 2022*. -[^26]: Liu, S., Yu, H., Liao, C., Li, J., Lin, W., Liu, A. X., & Dustdar, S. (2022). [Pyraformer: Low-Complexity Pyramidal Attention for Long-Range Time Series Modeling and Forecasting](https://openreview.net/forum?id=0EXmFzUn5I). *ICLR 2022*. -[^27]: Wang, H., Peng, J., Huang, F., Wang, J., Chen, J., & Xiao, Y. (2023). [MICN: Multi-scale Local and Global Context Modeling for Long-term Series Forecasting](https://openreview.net/forum?id=zt53IDUR1U). *ICLR 2023*. -[^28]: Das, A., Kong, W., Leach, A., Mathur, S., Sen, R., & Yu, R. (2023). [Long-term Forecasting with TiDE: Time-series Dense Encoder](https://openreview.net/forum?id=pCbC3aQB5W). *TMLR 2023*. -[^29]: Liu, Y., Li, C., Wang, J., & Long, M. (2023). [Koopa: Learning Non-stationary Time Series Dynamics with Koopman Predictors](https://proceedings.neurips.cc/paper_files/paper/2023/hash/28b3dc0970fa4624a63278a4268de997-Abstract-Conference.html). *NeurIPS 2023*. -[^30]: Liu, M., Zeng, A., Chen, M., Xu, Z., Lai, Q., Ma, L., & Xu, Q. (2022). [SCINet: Time Series Modeling and Forecasting with Sample Convolution and Interaction](https://proceedings.neurips.cc/paper_files/paper/2022/hash/266983d0949aed78a16fa4782237dea7-Abstract-Conference.html). *NeurIPS 2022*. -[^31]: Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2022). [Reversible Instance Normalization for Accurate Time-Series Forecasting against Distribution Shift](https://openreview.net/forum?id=cGDAkQo1C0p). *ICLR 2022*. -[^32]: Kitaev, N., Kaiser, Ł., & Levskaya, A. (2020). [Reformer: The Efficient Transformer](https://openreview.net/forum?id=0EXmFzUn5I). *ICLR 2020*. -[^33]: Cao, D., Wang, Y., Duan, J., Zhang, C., Zhu, X., Huang, C., Tong, Y., Xu, B., Bai, J., Tong, J., & Zhang, Q. (2020). [Spectral Temporal Graph Neural Network for Multivariate Time-series Forecasting](https://proceedings.neurips.cc/paper/2020/hash/cdf6581cb7aca4b7e19ef136c6e601a5-Abstract.html). *NeurIPS 2020*. -[^34]: Nie, T., Qin, G., Mei, Y., & Sun, J. (2024). [ImputeFormer: Low Rankness-Induced Transformers for Generalizable Spatiotemporal Imputation](https://arxiv.org/abs/2312.01728). *KDD 2024*. -[^35]: Bai, S., Kolter, J. Z., & Koltun, V. (2018). [An empirical evaluation of generic convolutional and recurrent networks for sequence modeling](https://arxiv.org/abs/1803.01271). *arXiv 2018*. -[^36]: Project Gungnir, the world 1st LLM for time-series multitask modeling, will meet you soon. 🚀 Missing values and variable lengths in your datasets? - Hard to perform multitask learning with your time series? Not problems no longer. We'll open application for public beta test recently ;-) Follow us, and stay tuned! - Time-Series.AI -[^37]: Wang, S., Wu, H., Shi, X., Hu, T., Luo, H., Ma, L., ... & ZHOU, J. (2024). [TimeMixer: Decomposable Multiscale Mixing for Time Series Forecasting](https://openreview.net/forum?id=7oLshfEIC2). *ICLR 2024* -[^38]: Luo, D., & Wang X. (2024). [ModernTCN: A Modern Pure Convolution Structure for General Time Series Analysis](https://openreview.net/forum?id=vpJMJerXHU). *ICLR 2024* +[^1]: Du, W., Cote, D., & Liu, Y. ( +2023). [SAITS: Self-Attention-based Imputation for Time Series](https://doi.org/10.1016/j.eswa.2023.119619). *Expert +systems with applications*. +[^2]: Vaswani, A., Shazeer, N.M., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A.N., Kaiser, L., & Polosukhin, I. ( +2017). [Attention is All you Need](https://papers.nips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html). +*NeurIPS 2017*. +[^3]: Cao, W., Wang, D., Li, J., Zhou, H., Li, L., & Li, Y. ( +2018). [BRITS: Bidirectional Recurrent Imputation for Time Series](https://papers.nips.cc/paper/2018/hash/734e6bfcd358e25ac1db0a4241b95651-Abstract.html). +*NeurIPS 2018*. +[^4]: Che, Z., Purushotham, S., Cho, K., Sontag, D.A., & Liu, Y. ( +2018). [Recurrent Neural Networks for Multivariate Time Series with Missing Values](https://www.nature.com/articles/s41598-018-24271-9). +*Scientific Reports*. +[^5]: Zhang, X., Zeman, M., Tsiligkaridis, T., & Zitnik, M. ( +2022). [Graph-Guided Network for Irregularly Sampled Multivariate Time Series](https://arxiv.org/abs/2110.05357). *ICLR +2022*. +[^6]: Ma, Q., Chen, C., Li, S., & Cottrell, G. W. ( +2021). [Learning Representations for Incomplete Time Series Clustering](https://ojs.aaai.org/index.php/AAAI/article/view/17070). +*AAAI 2021*. +[^7]: Jong, J.D., Emon, M.A., Wu, P., Karki, R., Sood, M., Godard, P., Ahmad, A., Vrooman, H.A., Hofmann-Apitius, M., & +Fröhlich, H. ( +2019). [Deep learning for clustering of multivariate clinical patient trajectories with missing values](https://academic.oup.com/gigascience/article/8/11/giz134/5626377). +*GigaScience*. +[^8]: Chen, X., & Sun, L. ( +2021). [Bayesian Temporal Factorization for Multidimensional Time Series Prediction](https://arxiv.org/abs/1910.06366). +*IEEE transactions on pattern analysis and machine intelligence*. +[^9]: Yoon, J., Zame, W. R., & van der Schaar, M. ( +2019). [Estimating Missing Data in Temporal Data Streams Using Multi-Directional Recurrent Neural Networks](https://ieeexplore.ieee.org/document/8485748). +*IEEE Transactions on Biomedical Engineering*. +[^10]: Miao, X., Wu, Y., Wang, J., Gao, Y., Mao, X., & Yin, J. ( +2021). [Generative Semi-supervised Learning for Multivariate Time Series Imputation](https://ojs.aaai.org/index.php/AAAI/article/view/17086). +*AAAI 2021*. +[^11]: Fortuin, V., Baranchuk, D., Raetsch, G. & Mandt, S. ( +2020). [GP-VAE: Deep Probabilistic Time Series Imputation](https://proceedings.mlr.press/v108/fortuin20a.html). *AISTATS +2020*. +[^12]: Tashiro, Y., Song, J., Song, Y., & Ermon, S. ( +2021). [CSDI: Conditional Score-based Diffusion Models for Probabilistic Time Series Imputation](https://proceedings.neurips.cc/paper/2021/hash/cfe8504bda37b575c70ee1a8276f3486-Abstract.html). +*NeurIPS 2021*. +[^13]: Rubin, D. B. ( +1976). [Inference and missing data](https://academic.oup.com/biomet/article-abstract/63/3/581/270932). *Biometrika*. +[^14]: Wu, H., Hu, T., Liu, Y., Zhou, H., Wang, J., & Long, M. ( +2023). [TimesNet: Temporal 2d-variation modeling for general time series analysis](https://openreview.net/forum?id=ju_Uqw384Oq). +*ICLR 2023* +[^15]: Wu, H., Xu, J., Wang, J., & Long, M. ( +2021). [Autoformer: Decomposition transformers with auto-correlation for long-term series forecasting](https://proceedings.neurips.cc/paper/2021/hash/bcc0d400288793e8bdcd7c19a8ac0c2b-Abstract.html). +*NeurIPS 2021*. +[^16]: Zhang, Y., & Yan, J. ( +2023). [Crossformer: Transformer utilizing cross-dimension dependency for multivariate time series forecasting](https://openreview.net/forum?id=vSVLM2j9eie). +*ICLR 2023*. +[^17]: Zeng, A., Chen, M., Zhang, L., & Xu, Q. ( +2023). [Are transformers effective for time series forecasting?](https://ojs.aaai.org/index.php/AAAI/article/view/26317). +*AAAI 2023* +[^18]: Nie, Y., Nguyen, N. H., Sinthong, P., & Kalagnanam, J. ( +2023). [A time series is worth 64 words: Long-term forecasting with transformers](https://openreview.net/forum?id=Jbdc0vTOcol). +*ICLR 2023* +[^19]: Woo, G., Liu, C., Sahoo, D., Kumar, A., & Hoi, S. ( +2023). [ETSformer: Exponential Smoothing Transformers for Time-series Forecasting](https://openreview.net/forum?id=5m_3whfo483). +*ICLR 2023* +[^20]: Zhou, T., Ma, Z., Wen, Q., Wang, X., Sun, L., & Jin, R. ( +2022). [FEDformer: Frequency enhanced decomposed transformer for long-term series forecasting](https://proceedings.mlr.press/v162/zhou22g.html). +*ICML 2022*. +[^21]: Zhou, H., Zhang, S., Peng, J., Zhang, S., Li, J., Xiong, H., & Zhang, W. ( +2021). [Informer: Beyond efficient transformer for long sequence time-series forecasting](https://ojs.aaai.org/index.php/AAAI/article/view/17325). +*AAAI 2021*. +[^22]: Zhou, T., Ma, Z., Wen, Q., Sun, L., Yao, T., Yin, W., & Jin, R. ( +2022). [FiLM: Frequency improved Legendre Memory Model for Long-term Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2022/hash/524ef58c2bd075775861234266e5e020-Abstract-Conference.html). +*NeurIPS 2022*. +[^23]: Yi, K., Zhang, Q., Fan, W., Wang, S., Wang, P., He, H., An, N., Lian, D., Cao, L., & Niu, Z. ( +2023). [Frequency-domain MLPs are More Effective Learners in Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2023/hash/f1d16af76939f476b5f040fd1398c0a3-Abstract-Conference.html). +*NeurIPS 2023*. +[^24]: Liu, Y., Hu, T., Zhang, H., Wu, H., Wang, S., Ma, L., & Long, M. ( +2024). [iTransformer: Inverted Transformers Are Effective for Time Series Forecasting](https://openreview.net/forum?id=JePfAI8fah). +*ICLR 2024*. +[^25]: Liu, Y., Wu, H., Wang, J., & Long, M. ( +2022). [Non-stationary Transformers: Exploring the Stationarity in Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2022/hash/4054556fcaa934b0bf76da52cf4f92cb-Abstract-Conference.html). +*NeurIPS 2022*. +[^26]: Liu, S., Yu, H., Liao, C., Li, J., Lin, W., Liu, A. X., & Dustdar, S. ( +2022). [Pyraformer: Low-Complexity Pyramidal Attention for Long-Range Time Series Modeling and Forecasting](https://openreview.net/forum?id=0EXmFzUn5I). +*ICLR 2022*. +[^27]: Wang, H., Peng, J., Huang, F., Wang, J., Chen, J., & Xiao, Y. ( +2023). [MICN: Multi-scale Local and Global Context Modeling for Long-term Series Forecasting](https://openreview.net/forum?id=zt53IDUR1U). +*ICLR 2023*. +[^28]: Das, A., Kong, W., Leach, A., Mathur, S., Sen, R., & Yu, R. ( +2023). [Long-term Forecasting with TiDE: Time-series Dense Encoder](https://openreview.net/forum?id=pCbC3aQB5W). *TMLR +2023*. +[^29]: Liu, Y., Li, C., Wang, J., & Long, M. ( +2023). [Koopa: Learning Non-stationary Time Series Dynamics with Koopman Predictors](https://proceedings.neurips.cc/paper_files/paper/2023/hash/28b3dc0970fa4624a63278a4268de997-Abstract-Conference.html). +*NeurIPS 2023*. +[^30]: Liu, M., Zeng, A., Chen, M., Xu, Z., Lai, Q., Ma, L., & Xu, Q. ( +2022). [SCINet: Time Series Modeling and Forecasting with Sample Convolution and Interaction](https://proceedings.neurips.cc/paper_files/paper/2022/hash/266983d0949aed78a16fa4782237dea7-Abstract-Conference.html). +*NeurIPS 2022*. +[^31]: Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. ( +2022). [Reversible Instance Normalization for Accurate Time-Series Forecasting against Distribution Shift](https://openreview.net/forum?id=cGDAkQo1C0p). +*ICLR 2022*. +[^32]: Kitaev, N., Kaiser, Ł., & Levskaya, A. ( +2020). [Reformer: The Efficient Transformer](https://openreview.net/forum?id=0EXmFzUn5I). *ICLR 2020*. +[^33]: Cao, D., Wang, Y., Duan, J., Zhang, C., Zhu, X., Huang, C., Tong, Y., Xu, B., Bai, J., Tong, J., & Zhang, Q. ( +2020). [Spectral Temporal Graph Neural Network for Multivariate Time-series Forecasting](https://proceedings.neurips.cc/paper/2020/hash/cdf6581cb7aca4b7e19ef136c6e601a5-Abstract.html). +*NeurIPS 2020*. +[^34]: Nie, T., Qin, G., Mei, Y., & Sun, J. ( +2024). [ImputeFormer: Low Rankness-Induced Transformers for Generalizable Spatiotemporal Imputation](https://arxiv.org/abs/2312.01728). +*KDD 2024*. +[^35]: Bai, S., Kolter, J. Z., & Koltun, V. ( +2018). [An empirical evaluation of generic convolutional and recurrent networks for sequence modeling](https://arxiv.org/abs/1803.01271). +*arXiv 2018*. +[^36]: Project Gungnir, the world 1st LLM for time-series multitask modeling, will meet you soon. 🚀 Missing values and +variable lengths in your datasets? +Hard to perform multitask learning with your time series? Not problems no longer. We'll open application for public beta +test recently ;-) Follow us, and stay tuned! + +Time-Series.AI +[^37]: Wang, S., Wu, H., Shi, X., Hu, T., Luo, H., Ma, L., ... & ZHOU, J. ( +2024). [TimeMixer: Decomposable Multiscale Mixing for Time Series Forecasting](https://openreview.net/forum?id=7oLshfEIC2). +*ICLR 2024* +[^38]: Luo, D., & Wang X. ( +2024). [ModernTCN: A Modern Pure Convolution Structure for General Time Series Analysis](https://openreview.net/forum?id=vpJMJerXHU). +*ICLR 2024* +[^39]: Zhan, T., He, Y., Li, Z., & Deng, Y. ( +2024). [Time Evidence Fusion Network: Multi-source View in Long-Term Time Series Forecasting](https://arxiv.org/abs/2405.06419). +*arXiv 2024* + diff --git a/README_zh.md b/README_zh.md index 4d55e68d..d43b5e83 100644 --- a/README_zh.md +++ b/README_zh.md @@ -86,67 +86,74 @@ [**❖ 贡献声明**](#-贡献声明), [**❖ 社区组织**](#-社区组织)。 - ## ❖ 支持的算法 + PyPOTS当前支持多变量POTS数据的插补，预测，分类，聚类以及异常检测五类任务。下表描述了当前PyPOTS中所集成的算法以及对应不同任务的可用性。 符号`✅`表示该算法当前可用于相应的任务（注意，目前模型尚不支持的任务在未来版本中可能会逐步添加，敬请关注！)。 算法的参考文献以及论文链接在该文档底部可以找到。 -🌟 自**v0.2**版本开始, PyPOTS中所有神经网络模型都支持超参数调优。该功能基于[微软的NNI](https://github.com/microsoft/nni)框架实现。 -你可以通过参考我们的时间序列插补综述项目的代码[Awesome_Imputation](https://github.com/WenjieDu/Awesome_Imputation)来了解如何使用PyPOTS调优模型的超参。 +🌟 自**v0.2**版本开始, PyPOTS中所有神经网络模型都支持超参数调优。该功能基于[微软的NNI](https://github.com/microsoft/nni) +框架实现。 +你可以通过参考我们的时间序列插补综述项目的代码[Awesome_Imputation](https://github.com/WenjieDu/Awesome_Imputation) +来了解如何使用PyPOTS调优模型的超参。 🔥 请注意: 表格中名称带有`🧑‍🔧`的模型（例如Transformer, iTransformer, Informer等）在它们的原始论文中并非作为可以处理POTS数据的算法提出， 所以这些模型的输入中不能带有缺失值，无法接受POTS数据作为输入，更加不是插补算法。 -**为了使上述模型能够适用于POTS数据，我们采用了与[SAITS论文](https://arxiv.org/pdf/2202.08516)[^1]中相同的embedding策略和训练方法（ORT+MIT）对它们进行改进**。 - -| **类型** | **算法** | **插补** | **预测** | **分类** | **聚类** | **异常检测** | **年份 - 刊物** | -|:--------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------|:------:|:------:|:------:|:------:|:--------:|:-------------------| -| LLM | Time-Series.AI [^36] | ✅ | ✅ | ✅ | ✅ | ✅ | `Later in 2024` | -| Neural Net | TimeMixer[^37] | ✅ | | | | | `2024 - ICLR` | -| Neural Net | iTransformer🧑‍🔧[^24] | ✅ | | | | | `2024 - ICLR` | -| Neural Net | ModernTCN[^38] | ✅ | | | | | `2024 - ICLR` | -| Neural Net | ImputeFormer🧑‍🔧[^34] | ✅ | | | | | `2024 - KDD` | -| Neural Net | SAITS[^1] | ✅ | | | | | `2023 - ESWA` | -| Neural Net | FreTS🧑‍🔧[^23] | ✅ | | | | | `2023 - NeurIPS` | -| Neural Net | Koopa🧑‍🔧[^29] | ✅ | | | | | `2023 - NeurIPS` | -| Neural Net | Crossformer🧑‍🔧[^16] | ✅ | | | | | `2023 - ICLR` | -| Neural Net | TimesNet[^14] | ✅ | | | | | `2023 - ICLR` | -| Neural Net | PatchTST🧑‍🔧[^18] | ✅ | | | | | `2023 - ICLR` | -| Neural Net | ETSformer🧑‍🔧[^19] | ✅ | | | | | `2023 - ICLR` | -| Neural Net | MICN🧑‍🔧[^27] | ✅ | | | | | `2023 - ICLR` | -| Neural Net | DLinear🧑‍🔧[^17] | ✅ | | | | | `2023 - AAAI` | -| Neural Net | TiDE🧑‍🔧[^28] | ✅ | | | | | `2023 - TMLR` | -| Neural Net | SCINet🧑‍🔧[^30] | ✅ | | | | | `2022 - NeurIPS` | -| Neural Net | Nonstationary Tr.🧑‍🔧[^25] | ✅ | | | | | `2022 - NeurIPS` | -| Neural Net | FiLM🧑‍🔧[^22] | ✅ | | | | | `2022 - NeurIPS` | -| Neural Net | RevIN_SCINet🧑‍🔧[^31] | ✅ | | | | | `2022 - ICLR` | -| Neural Net | Pyraformer🧑‍🔧[^26] | ✅ | | | | | `2022 - ICLR` | -| Neural Net | Raindrop[^5] | | | ✅ | | | `2022 - ICLR` | -| Neural Net | FEDformer🧑‍🔧[^20] | ✅ | | | | | `2022 - ICML` | -| Neural Net | Autoformer🧑‍🔧[^15] | ✅ | | | | | `2021 - NeurIPS` | -| Neural Net | CSDI[^12] | ✅ | ✅ | | | | `2021 - NeurIPS` | -| Neural Net | Informer🧑‍🔧[^21] | ✅ | | | | | `2021 - AAAI` | -| Neural Net | US-GAN[^10] | ✅ | | | | | `2021 - AAAI` | -| Neural Net | CRLI[^6] | | | | ✅ | | `2021 - AAAI` | -| Probabilistic | BTTF[^8] | | ✅ | | | | `2021 - TPAMI` | -| Neural Net | StemGNN🧑‍🔧[^33] | ✅ | | | | | `2020 - NeurIPS` | -| Neural Net | Reformer🧑‍🔧[^32] | ✅ | | | | | `2020 - ICLR` | -| Neural Net | GP-VAE[^11] | ✅ | | | | | `2020 - AISTATS` | -| Neural Net | VaDER[^7] | | | | ✅ | | `2019 - GigaSci.` | -| Neural Net | M-RNN[^9] | ✅ | | | | | `2019 - TBME` | -| Neural Net | BRITS[^3] | ✅ | | ✅ | | | `2018 - NeurIPS` | -| Neural Net | GRU-D[^4] | ✅ | | ✅ | | | `2018 - Sci. Rep.` | -| Neural Net | TCN🧑‍🔧[^35] | ✅ | | | | | `2018 - arXiv` | -| Neural Net | Transformer🧑‍🔧[^2] | ✅ | | | | | `2017 - NeurIPS` | -| Naive | Lerp | ✅ | | | | | | -| Naive | LOCF/NOCB | ✅ | | | | | | -| Naive | Mean | ✅ | | | | | | -| Naive | Median | ✅ | | | | | | - -💯 现在贡献你的模型来增加你的研究影响力！PyPOTS的下载量正在迅速增长(**[目前PyPI上总共超过30万次且每日超1000的下载](https://www.pepy.tech/projects/pypots)**), -你的工作将被社区广泛使用和引用。请参阅[贡献指南](https://github.com/WenjieDu/PyPOTS/blob/main/README_zh.md#-%E8%B4%A1%E7%8C%AE%E5%A3%B0%E6%98%8E)，了解如何将模型包含在PyPOTS中。 +**为了使上述模型能够适用于POTS数据，我们采用了与[SAITS论文](https://arxiv.org/pdf/2202.08516)[^1] +中相同的embedding策略和训练方法（ORT+MIT）对它们进行改进**。 + +| **类型** | **算法** | **插补** | **预测** | **分类** | **聚类** | **异常检测** | **年份 - 刊物** | +|:--------------|:---------------------------------------------------------------------------------------------------------------------------------|:------:|:------:|:------:|:------:|:--------:|:-------------------| +| LLM | Time-Series.AI [^36] | ✅ | ✅ | ✅ | ✅ | ✅ | `Later in 2024` | +| Neural Net | TEFN[^39] | ✅ | | | | | `2024 - arXiv` | +| Neural Net | TimeMixer[^37] | ✅ | | | | | `2024 - ICLR` | +| Neural Net | iTransformer🧑‍🔧[^24] | ✅ | | | | | `2024 - ICLR` | +| Neural Net | ModernTCN[^38] | ✅ | | | | | `2024 - ICLR` | +| Neural Net | ImputeFormer🧑‍🔧[^34] | ✅ | | | | | `2024 - KDD` | +| Neural Net | SAITS[^1] | ✅ | | | | | `2023 - ESWA` | +| Neural Net | FreTS🧑‍🔧[^23] | ✅ | | | | | `2023 - NeurIPS` | +| Neural Net | Koopa🧑‍🔧[^29] | ✅ | | | | | `2023 - NeurIPS` | +| Neural Net | Crossformer🧑‍🔧[^16] | ✅ | | | | | `2023 - ICLR` | +| Neural Net | TimesNet[^14] | ✅ | | | | | `2023 - ICLR` | +| Neural Net | PatchTST🧑‍🔧[^18] | ✅ | | | | | `2023 - ICLR` | +| Neural Net | ETSformer🧑‍🔧[^19] | ✅ | | | | | `2023 - ICLR` | +| Neural Net | MICN🧑‍🔧[^27] | ✅ | | | | | `2023 - ICLR` | +| Neural Net | DLinear🧑‍🔧[^17] | ✅ | | | | | `2023 - AAAI` | +| Neural Net | TiDE🧑‍🔧[^28] | ✅ | | | | | `2023 - TMLR` | +| Neural Net | SCINet🧑‍🔧[^30] | ✅ | | | | | `2022 - NeurIPS` | +| Neural Net | Nonstationary Tr.🧑‍🔧[^25] | ✅ | | | | | `2022 - NeurIPS` | +| Neural Net | FiLM🧑‍🔧[^22] | ✅ | | | | | `2022 - NeurIPS` | +| Neural Net | RevIN_SCINet🧑‍🔧[^31] | ✅ | | | | | `2022 - ICLR` | +| Neural Net | Pyraformer🧑‍🔧[^26] | ✅ | | | | | `2022 - ICLR` | +| Neural Net | Raindrop[^5] | | | ✅ | | | `2022 - ICLR` | +| Neural Net | FEDformer🧑‍🔧[^20] | ✅ | | | | | `2022 - ICML` | +| Neural Net | Autoformer🧑‍🔧[^15] | ✅ | | | | | `2021 - NeurIPS` | +| Neural Net | CSDI[^12] | ✅ | ✅ | | | | `2021 - NeurIPS` | +| Neural Net | Informer🧑‍🔧[^21] | ✅ | | | | | `2021 - AAAI` | +| Neural Net | US-GAN[^10] | ✅ | | | | | `2021 - AAAI` | +| Neural Net | CRLI[^6] | | | | ✅ | | `2021 - AAAI` | +| Probabilistic | BTTF[^8] | | ✅ | | | | `2021 - TPAMI` | +| Neural Net | StemGNN🧑‍🔧[^33] | ✅ | | | | | `2020 - NeurIPS` | +| Neural Net | Reformer🧑‍🔧[^32] | ✅ | | | | | `2020 - ICLR` | +| Neural Net | GP-VAE[^11] | ✅ | | | | | `2020 - AISTATS` | +| Neural Net | VaDER[^7] | | | | ✅ | | `2019 - GigaSci.` | +| Neural Net | M-RNN[^9] | ✅ | | | | | `2019 - TBME` | +| Neural Net | BRITS[^3] | ✅ | | ✅ | | | `2018 - NeurIPS` | +| Neural Net | GRU-D[^4] | ✅ | | ✅ | | | `2018 - Sci. Rep.` | +| Neural Net | TCN🧑‍🔧[^35] | ✅ | | | | | `2018 - arXiv` | +| Neural Net | Transformer🧑‍🔧[^2] | ✅ | | | | | `2017 - NeurIPS` | +| Naive | Lerp | ✅ | | | | | | +| Naive | LOCF/NOCB | ✅ | | | | | | +| Naive | Mean | ✅ | | | | | | +| Naive | Median | ✅ | | | | | | + +💯 现在贡献你的模型来增加你的研究影响力！PyPOTS的下载量正在迅速增长(* +*[目前PyPI上总共超过30万次且每日超1000的下载](https://www.pepy.tech/projects/pypots)**), +你的工作将被社区广泛使用和引用。请参阅[贡献指南](https://github.com/WenjieDu/PyPOTS/blob/main/README_zh.md#-%E8%B4%A1%E7%8C%AE%E5%A3%B0%E6%98%8E) +，了解如何将模型包含在PyPOTS中。 ## ❖ PyPOTS生态系统 + 在PyPOTS生态系统中，一切都与我们熟悉的咖啡息息相关，甚至可以将其视为一杯咖啡的诞生过程！ 如你所见，PyPOTS的标志中有一个咖啡壶。除此之外还需要什么呢？请接着看下去、 @@ -155,23 +162,28 @@ PyPOTS当前支持多变量POTS数据的插补，预测，分类，聚类以及 👈 在PyPOTS中，数据可以被看作是咖啡豆，而写的携带缺失值的POTS数据则是不完整的咖啡豆。 -为了让用户能够轻松使用各种开源的时间序列数据集，我们创建了开源时间序列数据集的仓库 Time Series Data Beans (TSDB)（可以将其视为咖啡豆仓库）， +为了让用户能够轻松使用各种开源的时间序列数据集，我们创建了开源时间序列数据集的仓库 Time Series Data Beans (TSDB) +（可以将其视为咖啡豆仓库）， TSDB让加载开源时序数据集变得超级简单！访问 [TSDB](https://github.com/WenjieDu/TSDB)，了解更多关于TSDB的信息，目前总共支持172个开源数据集！ PyGrinder logo -👉 为了在真实数据中模拟缺失进而获得不完整的咖啡豆，我们创建了生态系统中的另一个仓库[PyGrinder](https://github.com/WenjieDu/PyGrinder)（可以将其视为磨豆机）， +👉 +为了在真实数据中模拟缺失进而获得不完整的咖啡豆，我们创建了生态系统中的另一个仓库[PyGrinder](https://github.com/WenjieDu/PyGrinder) +（可以将其视为磨豆机）， 帮助你在数据集中模拟缺失数据，用于评估机器学习算法。根据Robin的理论[^13]，缺失模式分为三类： -完全随机缺失（missing completely at random，简称为MCAR）、随机缺失（missing at random，简称为MAR）和非随机缺失（missing not at random，简称为MNAR ）。 +完全随机缺失（missing completely at random，简称为MCAR）、随机缺失（missing at random，简称为MAR）和非随机缺失（missing not at +random，简称为MNAR ）。 PyGrinder支持以上所有模式并提供与缺失相关的其他功能函数。通过PyGrinder，你可以仅仅通过一行代码就将模拟缺失引入你的数据集中。 BenchPOTS logo -👈 为了评估机器学习算法在POTS数据上的性能，我们创建了生态系统中的另一个仓库[BenchPOTS](https://github.com/WenjieDu/BenchPOTS), +👈 +为了评估机器学习算法在POTS数据上的性能，我们创建了生态系统中的另一个仓库[BenchPOTS](https://github.com/WenjieDu/BenchPOTS), 其提供了标准且统一的数据预处理管道来帮助你在多种任务上衡量不同POTS算法的性能。 @@ -190,8 +202,8 @@ PyGrinder支持以上所有模式并提供与缺失相关的其他功能函数 ☕️ 欢迎来到 PyPOTS 生态系统 !

- ## ❖ 安装教程 + 你可以参考PyPOTS文档中的 [安装说明](https://docs.pypots.com/en/latest/install.html) 以获取更详细的指南。 PyPOTS可以在 [PyPI](https://pypi.python.org/pypi/pypots) 和 [Anaconda](https://anaconda.org/conda-forge/pypots) 上安装。 你可以按照以下方式安装PyPOTS（同样适用于TSDB以及PyGrinder）： @@ -208,8 +220,8 @@ conda install -c conda-forge pypots # 首次安装 conda update -c conda-forge pypots # 更新为最新版本 ``` - ## ❖ 使用案例 + 除了[BrewPOTS](https://github.com/WenjieDu/BrewPOTS)之外, 你还可以在Google Colab Colab tutorials @@ -250,24 +262,31 @@ mae = calc_mae(imputation, np.nan_to_num(X_ori), indicating_mask) # 计算人 saits.save("save_it_here/saits_physionet2012.pypots") # 保存模型 saits.load("save_it_here/saits_physionet2012.pypots") # 你随时可以重新加载保存的模型文件以进行后续的插补或训练 ``` - + ## ❖ 引用PyPOTS + > [!TIP] -> **[2024年6月更新]** 😎 第一个全面的时间序列插补基准论文[TSI-Bench: Benchmarking Time Series Imputation](https://arxiv.org/abs/2406.12747)现在来了。 -> 所有代码开源在[Awesome_Imputation](https://github.com/WenjieDu/Awesome_Imputation)仓库中。通过近35,000个实验，我们对28种imputation方法，3种缺失模式(点，序列，块)，各种缺失率，和8个真实数据集进行了全面的基准研究。 +> **[2024年6月更新]** 😎 +> 第一个全面的时间序列插补基准论文[TSI-Bench: Benchmarking Time Series Imputation](https://arxiv.org/abs/2406.12747) +> 现在来了。 +> 所有代码开源在[Awesome_Imputation](https://github.com/WenjieDu/Awesome_Imputation) +> 仓库中。通过近35,000个实验，我们对28种imputation方法，3种缺失模式(点，序列，块)，各种缺失率，和8个真实数据集进行了全面的基准研究。 > -> **[2024年2月更新]** 🎉 我们的综述论文[Deep Learning for Multivariate Time Series Imputation: A Survey](https://arxiv.org/abs/2402.04059) +> **[2024年2月更新]** 🎉 +> 我们的综述论文[Deep Learning for Multivariate Time Series Imputation: A Survey](https://arxiv.org/abs/2402.04059) > 已在 arXiv 上发布。我们全面调研总结了最新基于深度学习的时间序列插补方法文献并对现有的方法进行分类，此外，还讨论了该领域当前的挑战和未来发展方向。 -PyPOTS的论文可以[在arXiv上获取](https://arxiv.org/abs/2305.18811)，其5页的短版论文已被第9届SIGKDD international workshop on +PyPOTS的论文可以[在arXiv上获取](https://arxiv.org/abs/2305.18811)，其5页的短版论文已被第9届SIGKDD international workshop +on Mining and Learning from Time Series ([MiLeTS'23](https://kdd-milets.github.io/milets2023/))收录，与此同时， PyPOTS也已被纳入[PyTorch Ecosystem](https://pytorch.org/ecosystem/)。我们正在努力将其发表在更具影响力的学术刊物上， 如JMLR (track for [Machine Learning Open Source Software](https://www.jmlr.org/mloss/))。 如果你在工作中使用了PyPOTS，请按照以下格式引用我们的论文并为将项目设为星标🌟，以便让更多人关注到它，对此我们深表感谢🤗。 -据不完全统计，该[列表](https://scholar.google.com/scholar?as_ylo=2022&q=%E2%80%9CPyPOTS%E2%80%9D&hl=en>)为当前使用PyPOTS并在其论文中引用PyPOTS的科学研究项目 +据不完全统计，该[列表](https://scholar.google.com/scholar?as_ylo=2022&q=%E2%80%9CPyPOTS%E2%80%9D&hl=en>) +为当前使用PyPOTS并在其论文中引用PyPOTS的科学研究项目 ```bibtex @article{du2023pypots, @@ -277,13 +296,14 @@ journal={arXiv preprint arXiv:2305.18811}, year={2023}, } ``` + 或者 > Wenjie Du. (2023). > PyPOTS: a Python toolbox for data mining on Partially-Observed Time Series. > arXiv, abs/2305.18811. https://arxiv.org/abs/2305.18811 - ## ❖ 贡献声明 + 非常欢迎你为这个激动人心的项目做出贡献！ 通过提交你的代码，你将： @@ -316,11 +336,12 @@ year={2023}, 👀请在[PyPOTS网站](https://pypots.com/users/)上查看我们用户所属机构的完整列表！ - ## ❖ 社区组织 + 我们非常关心用户的反馈，因此我们正在建立PyPOTS社区: -- [Slack](https://join.slack.com/t/pypots-org/shared_invite/zt-1gq6ufwsi-p0OZdW~e9UW_IA4_f1OfxA)：你可以在这里进行日常讨论、问答以及与我们的开发团队交流； +- [Slack](https://join.slack.com/t/pypots-org/shared_invite/zt-1gq6ufwsi-p0OZdW~e9UW_IA4_f1OfxA) + ：你可以在这里进行日常讨论、问答以及与我们的开发团队交流； - [领英](https://www.linkedin.com/company/pypots)：你可以在这里获取官方公告和新闻； - [微信公众号](https://mp.weixin.qq.com/s/X3ukIgL1QpNH8ZEXq1YifA)：你可以关注官方公众号并加入微信群聊参与讨论以及获取最新动态； @@ -329,42 +350,121 @@ PyPOTS社区是一个开放、透明、友好的社区，让我们共同努力 [//]: # (Use APA reference style below) -[^1]: Du, W., Cote, D., & Liu, Y. (2023). [SAITS: Self-Attention-based Imputation for Time Series](https://doi.org/10.1016/j.eswa.2023.119619). *Expert systems with applications*. -[^2]: Vaswani, A., Shazeer, N.M., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A.N., Kaiser, L., & Polosukhin, I. (2017). [Attention is All you Need](https://papers.nips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html). *NeurIPS 2017*. -[^3]: Cao, W., Wang, D., Li, J., Zhou, H., Li, L., & Li, Y. (2018). [BRITS: Bidirectional Recurrent Imputation for Time Series](https://papers.nips.cc/paper/2018/hash/734e6bfcd358e25ac1db0a4241b95651-Abstract.html). *NeurIPS 2018*. -[^4]: Che, Z., Purushotham, S., Cho, K., Sontag, D.A., & Liu, Y. (2018). [Recurrent Neural Networks for Multivariate Time Series with Missing Values](https://www.nature.com/articles/s41598-018-24271-9). *Scientific Reports*. -[^5]: Zhang, X., Zeman, M., Tsiligkaridis, T., & Zitnik, M. (2022). [Graph-Guided Network for Irregularly Sampled Multivariate Time Series](https://arxiv.org/abs/2110.05357). *ICLR 2022*. -[^6]: Ma, Q., Chen, C., Li, S., & Cottrell, G. W. (2021). [Learning Representations for Incomplete Time Series Clustering](https://ojs.aaai.org/index.php/AAAI/article/view/17070). *AAAI 2021*. -[^7]: Jong, J.D., Emon, M.A., Wu, P., Karki, R., Sood, M., Godard, P., Ahmad, A., Vrooman, H.A., Hofmann-Apitius, M., & Fröhlich, H. (2019). [Deep learning for clustering of multivariate clinical patient trajectories with missing values](https://academic.oup.com/gigascience/article/8/11/giz134/5626377). *GigaScience*. -[^8]: Chen, X., & Sun, L. (2021). [Bayesian Temporal Factorization for Multidimensional Time Series Prediction](https://arxiv.org/abs/1910.06366). *IEEE transactions on pattern analysis and machine intelligence*. -[^9]: Yoon, J., Zame, W. R., & van der Schaar, M. (2019). [Estimating Missing Data in Temporal Data Streams Using Multi-Directional Recurrent Neural Networks](https://ieeexplore.ieee.org/document/8485748). *IEEE Transactions on Biomedical Engineering*. -[^10]: Miao, X., Wu, Y., Wang, J., Gao, Y., Mao, X., & Yin, J. (2021). [Generative Semi-supervised Learning for Multivariate Time Series Imputation](https://ojs.aaai.org/index.php/AAAI/article/view/17086). *AAAI 2021*. -[^11]: Fortuin, V., Baranchuk, D., Raetsch, G. & Mandt, S. (2020). [GP-VAE: Deep Probabilistic Time Series Imputation](https://proceedings.mlr.press/v108/fortuin20a.html). *AISTATS 2020*. -[^12]: Tashiro, Y., Song, J., Song, Y., & Ermon, S. (2021). [CSDI: Conditional Score-based Diffusion Models for Probabilistic Time Series Imputation](https://proceedings.neurips.cc/paper/2021/hash/cfe8504bda37b575c70ee1a8276f3486-Abstract.html). *NeurIPS 2021*. -[^13]: Rubin, D. B. (1976). [Inference and missing data](https://academic.oup.com/biomet/article-abstract/63/3/581/270932). *Biometrika*. -[^14]: Wu, H., Hu, T., Liu, Y., Zhou, H., Wang, J., & Long, M. (2023). [TimesNet: Temporal 2d-variation modeling for general time series analysis](https://openreview.net/forum?id=ju_Uqw384Oq). *ICLR 2023* -[^15]: Wu, H., Xu, J., Wang, J., & Long, M. (2021). [Autoformer: Decomposition transformers with auto-correlation for long-term series forecasting](https://proceedings.neurips.cc/paper/2021/hash/bcc0d400288793e8bdcd7c19a8ac0c2b-Abstract.html). *NeurIPS 2021*. -[^16]: Zhang, Y., & Yan, J. (2023). [Crossformer: Transformer utilizing cross-dimension dependency for multivariate time series forecasting](https://openreview.net/forum?id=vSVLM2j9eie). *ICLR 2023*. -[^17]: Zeng, A., Chen, M., Zhang, L., & Xu, Q. (2023). [Are transformers effective for time series forecasting?](https://ojs.aaai.org/index.php/AAAI/article/view/26317). *AAAI 2023* -[^18]: Nie, Y., Nguyen, N. H., Sinthong, P., & Kalagnanam, J. (2023). [A time series is worth 64 words: Long-term forecasting with transformers](https://openreview.net/forum?id=Jbdc0vTOcol). *ICLR 2023* -[^19]: Woo, G., Liu, C., Sahoo, D., Kumar, A., & Hoi, S. (2023). [ETSformer: Exponential Smoothing Transformers for Time-series Forecasting](https://openreview.net/forum?id=5m_3whfo483). *ICLR 2023* -[^20]: Zhou, T., Ma, Z., Wen, Q., Wang, X., Sun, L., & Jin, R. (2022). [FEDformer: Frequency enhanced decomposed transformer for long-term series forecasting](https://proceedings.mlr.press/v162/zhou22g.html). *ICML 2022*. -[^21]: Zhou, H., Zhang, S., Peng, J., Zhang, S., Li, J., Xiong, H., & Zhang, W. (2021). [Informer: Beyond efficient transformer for long sequence time-series forecasting](https://ojs.aaai.org/index.php/AAAI/article/view/17325). *AAAI 2021*. -[^22]: Zhou, T., Ma, Z., Wen, Q., Sun, L., Yao, T., Yin, W., & Jin, R. (2022). [FiLM: Frequency improved Legendre Memory Model for Long-term Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2022/hash/524ef58c2bd075775861234266e5e020-Abstract-Conference.html). *NeurIPS 2022*. -[^23]: Yi, K., Zhang, Q., Fan, W., Wang, S., Wang, P., He, H., An, N., Lian, D., Cao, L., & Niu, Z. (2023). [Frequency-domain MLPs are More Effective Learners in Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2023/hash/f1d16af76939f476b5f040fd1398c0a3-Abstract-Conference.html). *NeurIPS 2023*. -[^24]: Liu, Y., Hu, T., Zhang, H., Wu, H., Wang, S., Ma, L., & Long, M. (2024). [iTransformer: Inverted Transformers Are Effective for Time Series Forecasting](https://openreview.net/forum?id=JePfAI8fah). *ICLR 2024*. -[^25]: Liu, Y., Wu, H., Wang, J., & Long, M. (2022). [Non-stationary Transformers: Exploring the Stationarity in Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2022/hash/4054556fcaa934b0bf76da52cf4f92cb-Abstract-Conference.html). *NeurIPS 2022*. -[^26]: Liu, S., Yu, H., Liao, C., Li, J., Lin, W., Liu, A. X., & Dustdar, S. (2022). [Pyraformer: Low-Complexity Pyramidal Attention for Long-Range Time Series Modeling and Forecasting](https://openreview.net/forum?id=0EXmFzUn5I). *ICLR 2022*. -[^27]: Wang, H., Peng, J., Huang, F., Wang, J., Chen, J., & Xiao, Y. (2023). [MICN: Multi-scale Local and Global Context Modeling for Long-term Series Forecasting](https://openreview.net/forum?id=zt53IDUR1U). *ICLR 2023*. -[^28]: Das, A., Kong, W., Leach, A., Mathur, S., Sen, R., & Yu, R. (2023). [Long-term Forecasting with TiDE: Time-series Dense Encoder](https://openreview.net/forum?id=pCbC3aQB5W). *TMLR 2023*. -[^29]: Liu, Y., Li, C., Wang, J., & Long, M. (2023). [Koopa: Learning Non-stationary Time Series Dynamics with Koopman Predictors](https://proceedings.neurips.cc/paper_files/paper/2023/hash/28b3dc0970fa4624a63278a4268de997-Abstract-Conference.html). *NeurIPS 2023*. -[^30]: Liu, M., Zeng, A., Chen, M., Xu, Z., Lai, Q., Ma, L., & Xu, Q. (2022). [SCINet: Time Series Modeling and Forecasting with Sample Convolution and Interaction](https://proceedings.neurips.cc/paper_files/paper/2022/hash/266983d0949aed78a16fa4782237dea7-Abstract-Conference.html). *NeurIPS 2022*. -[^31]: Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. (2022). [Reversible Instance Normalization for Accurate Time-Series Forecasting against Distribution Shift](https://openreview.net/forum?id=cGDAkQo1C0p). *ICLR 2022*. -[^32]: Kitaev, N., Kaiser, Ł., & Levskaya, A. (2020). [Reformer: The Efficient Transformer](https://openreview.net/forum?id=0EXmFzUn5I). *ICLR 2020*. -[^33]: Cao, D., Wang, Y., Duan, J., Zhang, C., Zhu, X., Huang, C., Tong, Y., Xu, B., Bai, J., Tong, J., & Zhang, Q. (2020). [Spectral Temporal Graph Neural Network for Multivariate Time-series Forecasting](https://proceedings.neurips.cc/paper/2020/hash/cdf6581cb7aca4b7e19ef136c6e601a5-Abstract.html). *NeurIPS 2020*. -[^34]: Nie, T., Qin, G., Mei, Y., & Sun, J. (2024). [ImputeFormer: Low Rankness-Induced Transformers for Generalizable Spatiotemporal Imputation](https://arxiv.org/abs/2312.01728). *KDD 2024*. -[^35]: Bai, S., Kolter, J. Z., & Koltun, V. (2018). [An empirical evaluation of generic convolutional and recurrent networks for sequence modeling](https://arxiv.org/abs/1803.01271). *arXiv 2018*. -[^36]: Gungnir项目，世界上第一个时间序列多任务大模型，将很快与大家见面。🚀 数据集存在缺少值且样本长短不一？多任务建模场景困难？都不再是问题，让我们的大模型来帮你解决。我们将在近期开放公测申请 ;-) 关注我们，敬请期待! - Time-Series.AI -[^37]: Wang, S., Wu, H., Shi, X., Hu, T., Luo, H., Ma, L., ... & ZHOU, J. (2024). [TimeMixer: Decomposable Multiscale Mixing for Time Series Forecasting](https://openreview.net/forum?id=7oLshfEIC2). *ICLR 2024* -[^38]: Luo, D., & Wang X. (2024). [ModernTCN: A Modern Pure Convolution Structure for General Time Series Analysis](https://openreview.net/forum?id=vpJMJerXHU). *ICLR 2024* +[^1]: Du, W., Cote, D., & Liu, Y. ( +2023). [SAITS: Self-Attention-based Imputation for Time Series](https://doi.org/10.1016/j.eswa.2023.119619). *Expert +systems with applications*. +[^2]: Vaswani, A., Shazeer, N.M., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A.N., Kaiser, L., & Polosukhin, I. ( +2017). [Attention is All you Need](https://papers.nips.cc/paper/2017/hash/3f5ee243547dee91fbd053c1c4a845aa-Abstract.html). +*NeurIPS 2017*. +[^3]: Cao, W., Wang, D., Li, J., Zhou, H., Li, L., & Li, Y. ( +2018). [BRITS: Bidirectional Recurrent Imputation for Time Series](https://papers.nips.cc/paper/2018/hash/734e6bfcd358e25ac1db0a4241b95651-Abstract.html). +*NeurIPS 2018*. +[^4]: Che, Z., Purushotham, S., Cho, K., Sontag, D.A., & Liu, Y. ( +2018). [Recurrent Neural Networks for Multivariate Time Series with Missing Values](https://www.nature.com/articles/s41598-018-24271-9). +*Scientific Reports*. +[^5]: Zhang, X., Zeman, M., Tsiligkaridis, T., & Zitnik, M. ( +2022). [Graph-Guided Network for Irregularly Sampled Multivariate Time Series](https://arxiv.org/abs/2110.05357). *ICLR +2022*. +[^6]: Ma, Q., Chen, C., Li, S., & Cottrell, G. W. ( +2021). [Learning Representations for Incomplete Time Series Clustering](https://ojs.aaai.org/index.php/AAAI/article/view/17070). +*AAAI 2021*. +[^7]: Jong, J.D., Emon, M.A., Wu, P., Karki, R., Sood, M., Godard, P., Ahmad, A., Vrooman, H.A., Hofmann-Apitius, M., & +Fröhlich, H. ( +2019). [Deep learning for clustering of multivariate clinical patient trajectories with missing values](https://academic.oup.com/gigascience/article/8/11/giz134/5626377). +*GigaScience*. +[^8]: Chen, X., & Sun, L. ( +2021). [Bayesian Temporal Factorization for Multidimensional Time Series Prediction](https://arxiv.org/abs/1910.06366). +*IEEE transactions on pattern analysis and machine intelligence*. +[^9]: Yoon, J., Zame, W. R., & van der Schaar, M. ( +2019). [Estimating Missing Data in Temporal Data Streams Using Multi-Directional Recurrent Neural Networks](https://ieeexplore.ieee.org/document/8485748). +*IEEE Transactions on Biomedical Engineering*. +[^10]: Miao, X., Wu, Y., Wang, J., Gao, Y., Mao, X., & Yin, J. ( +2021). [Generative Semi-supervised Learning for Multivariate Time Series Imputation](https://ojs.aaai.org/index.php/AAAI/article/view/17086). +*AAAI 2021*. +[^11]: Fortuin, V., Baranchuk, D., Raetsch, G. & Mandt, S. ( +2020). [GP-VAE: Deep Probabilistic Time Series Imputation](https://proceedings.mlr.press/v108/fortuin20a.html). *AISTATS +2020*. +[^12]: Tashiro, Y., Song, J., Song, Y., & Ermon, S. ( +2021). [CSDI: Conditional Score-based Diffusion Models for Probabilistic Time Series Imputation](https://proceedings.neurips.cc/paper/2021/hash/cfe8504bda37b575c70ee1a8276f3486-Abstract.html). +*NeurIPS 2021*. +[^13]: Rubin, D. B. ( +1976). [Inference and missing data](https://academic.oup.com/biomet/article-abstract/63/3/581/270932). *Biometrika*. +[^14]: Wu, H., Hu, T., Liu, Y., Zhou, H., Wang, J., & Long, M. ( +2023). [TimesNet: Temporal 2d-variation modeling for general time series analysis](https://openreview.net/forum?id=ju_Uqw384Oq). +*ICLR 2023* +[^15]: Wu, H., Xu, J., Wang, J., & Long, M. ( +2021). [Autoformer: Decomposition transformers with auto-correlation for long-term series forecasting](https://proceedings.neurips.cc/paper/2021/hash/bcc0d400288793e8bdcd7c19a8ac0c2b-Abstract.html). +*NeurIPS 2021*. +[^16]: Zhang, Y., & Yan, J. ( +2023). [Crossformer: Transformer utilizing cross-dimension dependency for multivariate time series forecasting](https://openreview.net/forum?id=vSVLM2j9eie). +*ICLR 2023*. +[^17]: Zeng, A., Chen, M., Zhang, L., & Xu, Q. ( +2023). [Are transformers effective for time series forecasting?](https://ojs.aaai.org/index.php/AAAI/article/view/26317). +*AAAI 2023* +[^18]: Nie, Y., Nguyen, N. H., Sinthong, P., & Kalagnanam, J. ( +2023). [A time series is worth 64 words: Long-term forecasting with transformers](https://openreview.net/forum?id=Jbdc0vTOcol). +*ICLR 2023* +[^19]: Woo, G., Liu, C., Sahoo, D., Kumar, A., & Hoi, S. ( +2023). [ETSformer: Exponential Smoothing Transformers for Time-series Forecasting](https://openreview.net/forum?id=5m_3whfo483). +*ICLR 2023* +[^20]: Zhou, T., Ma, Z., Wen, Q., Wang, X., Sun, L., & Jin, R. ( +2022). [FEDformer: Frequency enhanced decomposed transformer for long-term series forecasting](https://proceedings.mlr.press/v162/zhou22g.html). +*ICML 2022*. +[^21]: Zhou, H., Zhang, S., Peng, J., Zhang, S., Li, J., Xiong, H., & Zhang, W. ( +2021). [Informer: Beyond efficient transformer for long sequence time-series forecasting](https://ojs.aaai.org/index.php/AAAI/article/view/17325). +*AAAI 2021*. +[^22]: Zhou, T., Ma, Z., Wen, Q., Sun, L., Yao, T., Yin, W., & Jin, R. ( +2022). [FiLM: Frequency improved Legendre Memory Model for Long-term Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2022/hash/524ef58c2bd075775861234266e5e020-Abstract-Conference.html). +*NeurIPS 2022*. +[^23]: Yi, K., Zhang, Q., Fan, W., Wang, S., Wang, P., He, H., An, N., Lian, D., Cao, L., & Niu, Z. ( +2023). [Frequency-domain MLPs are More Effective Learners in Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2023/hash/f1d16af76939f476b5f040fd1398c0a3-Abstract-Conference.html). +*NeurIPS 2023*. +[^24]: Liu, Y., Hu, T., Zhang, H., Wu, H., Wang, S., Ma, L., & Long, M. ( +2024). [iTransformer: Inverted Transformers Are Effective for Time Series Forecasting](https://openreview.net/forum?id=JePfAI8fah). +*ICLR 2024*. +[^25]: Liu, Y., Wu, H., Wang, J., & Long, M. ( +2022). [Non-stationary Transformers: Exploring the Stationarity in Time Series Forecasting](https://proceedings.neurips.cc/paper_files/paper/2022/hash/4054556fcaa934b0bf76da52cf4f92cb-Abstract-Conference.html). +*NeurIPS 2022*. +[^26]: Liu, S., Yu, H., Liao, C., Li, J., Lin, W., Liu, A. X., & Dustdar, S. ( +2022). [Pyraformer: Low-Complexity Pyramidal Attention for Long-Range Time Series Modeling and Forecasting](https://openreview.net/forum?id=0EXmFzUn5I). +*ICLR 2022*. +[^27]: Wang, H., Peng, J., Huang, F., Wang, J., Chen, J., & Xiao, Y. ( +2023). [MICN: Multi-scale Local and Global Context Modeling for Long-term Series Forecasting](https://openreview.net/forum?id=zt53IDUR1U). +*ICLR 2023*. +[^28]: Das, A., Kong, W., Leach, A., Mathur, S., Sen, R., & Yu, R. ( +2023). [Long-term Forecasting with TiDE: Time-series Dense Encoder](https://openreview.net/forum?id=pCbC3aQB5W). *TMLR +2023*. +[^29]: Liu, Y., Li, C., Wang, J., & Long, M. ( +2023). [Koopa: Learning Non-stationary Time Series Dynamics with Koopman Predictors](https://proceedings.neurips.cc/paper_files/paper/2023/hash/28b3dc0970fa4624a63278a4268de997-Abstract-Conference.html). +*NeurIPS 2023*. +[^30]: Liu, M., Zeng, A., Chen, M., Xu, Z., Lai, Q., Ma, L., & Xu, Q. ( +2022). [SCINet: Time Series Modeling and Forecasting with Sample Convolution and Interaction](https://proceedings.neurips.cc/paper_files/paper/2022/hash/266983d0949aed78a16fa4782237dea7-Abstract-Conference.html). +*NeurIPS 2022*. +[^31]: Kim, T., Kim, J., Tae, Y., Park, C., Choi, J. H., & Choo, J. ( +2022). [Reversible Instance Normalization for Accurate Time-Series Forecasting against Distribution Shift](https://openreview.net/forum?id=cGDAkQo1C0p). +*ICLR 2022*. +[^32]: Kitaev, N., Kaiser, Ł., & Levskaya, A. ( +2020). [Reformer: The Efficient Transformer](https://openreview.net/forum?id=0EXmFzUn5I). *ICLR 2020*. +[^33]: Cao, D., Wang, Y., Duan, J., Zhang, C., Zhu, X., Huang, C., Tong, Y., Xu, B., Bai, J., Tong, J., & Zhang, Q. ( +2020). [Spectral Temporal Graph Neural Network for Multivariate Time-series Forecasting](https://proceedings.neurips.cc/paper/2020/hash/cdf6581cb7aca4b7e19ef136c6e601a5-Abstract.html). +*NeurIPS 2020*. +[^34]: Nie, T., Qin, G., Mei, Y., & Sun, J. ( +2024). [ImputeFormer: Low Rankness-Induced Transformers for Generalizable Spatiotemporal Imputation](https://arxiv.org/abs/2312.01728). +*KDD 2024*. +[^35]: Bai, S., Kolter, J. Z., & Koltun, V. ( +2018). [An empirical evaluation of generic convolutional and recurrent networks for sequence modeling](https://arxiv.org/abs/1803.01271). +*arXiv 2018*. +[^36]: Gungnir项目，世界上第一个时间序列多任务大模型，将很快与大家见面。🚀 +数据集存在缺少值且样本长短不一？多任务建模场景困难？都不再是问题，让我们的大模型来帮你解决。我们将在近期开放公测申请 ;-) +关注我们，敬请期待! + +Time-Series.AI +[^37]: Wang, S., Wu, H., Shi, X., Hu, T., Luo, H., Ma, L., ... & ZHOU, J. ( +2024). [TimeMixer: Decomposable Multiscale Mixing for Time Series Forecasting](https://openreview.net/forum?id=7oLshfEIC2). +*ICLR 2024* +[^38]: Luo, D., & Wang X. ( +2024). [ModernTCN: A Modern Pure Convolution Structure for General Time Series Analysis](https://openreview.net/forum?id=vpJMJerXHU). +*ICLR 2024* +[^39]: Zhan, T., He, Y., Li, Z., & Deng, Y. ( +2024). [Time Evidence Fusion Network: Multi-source View in Long-Term Time Series Forecasting](https://arxiv.org/abs/2405.06419). +*arXiv 2024* diff --git a/docs/references.bib b/docs/references.bib index abee97c8..08222b54 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -763,3 +763,10 @@ @article{bai2018tcn journal={arXiv preprint arXiv:1803.01271}, year={2018} } + +@article{zhan2024tefn, + title={Time Evidence Fusion Network: Multi-source View in Long-Term Time Series Forecasting}, + author={Zhan, Tianxiang and He, Yuanpeng and Li, Zhen and Deng, Yong}, + journal={arXiv preprint arXiv:2405.06419}, + year={2024} +} \ No newline at end of file diff --git a/pypots/imputation/__init__.py b/pypots/imputation/__init__.py index 8136b0f8..4ae883f8 100644 --- a/pypots/imputation/__init__.py +++ b/pypots/imputation/__init__.py @@ -5,7 +5,6 @@ # Created by Wenjie Du # License: BSD-3-Clause -# neural network imputation methods from .brits import BRITS from .csdi import CSDI from .gpvae import GPVAE @@ -44,6 +43,7 @@ from .mean import Mean from .median import Median from .lerp import Lerp +from .tefn import TEFN __all__ = [ # neural network imputation methods @@ -84,4 +84,5 @@ "Mean", "Median", "Lerp", + "TEFN" ] diff --git a/pypots/imputation/tefn/__init__.py b/pypots/imputation/tefn/__init__.py new file mode 100644 index 00000000..b42e681e --- /dev/null +++ b/pypots/imputation/tefn/__init__.py @@ -0,0 +1,24 @@ +""" +The package of the forecasting model TEFN. + +Refer to the paper +`Tianxiang Zhan, Yuanpeng He, Yong Deng, and Zhen Li. +Time Evidence Fusion Network: Multi-source View in Long-Term Time Series Forecasting. +In Arxiv, 2024. +`_ + +Notes +----- +This implementation is transfered from the official one https://github.com/ztxtech/Time-Evidence-Fusion-Network + +""" + +# Created by Tianxiang Zhan +# License: BSD-3-Clause + + +from .model import TEFN + +__all__ = [ + "TEFN", +] diff --git a/pypots/imputation/tefn/core.py b/pypots/imputation/tefn/core.py new file mode 100644 index 00000000..c6dc6d8b --- /dev/null +++ b/pypots/imputation/tefn/core.py @@ -0,0 +1,59 @@ +""" + +""" + +# Created by Tianxiang Zhan +# License: BSD-3-Clause + +import torch.nn as nn + +from ...nn.functional import nonstationary_norm, nonstationary_denorm +from ...nn.modules.tefn import BackboneTEFN +from ...utils.metrics import calc_mse + + +class _TEFN(nn.Module): + def __init__( + self, + n_steps, + n_features, + n_fod, + apply_nonstationary_norm, + ): + super().__init__() + + self.seq_len = n_steps + self.n_fod = n_fod + self.apply_nonstationary_norm = apply_nonstationary_norm + + self.model = BackboneTEFN( + n_steps, + n_features, + n_fod, + ) + + def forward(self, inputs: dict, training: bool = True) -> dict: + X, missing_mask = inputs["X"], inputs["missing_mask"] + + if self.apply_nonstationary_norm: + # Normalization from Non-stationary Transformer + X, means, stdev = nonstationary_norm(X, missing_mask) + + # TEFN processing + out = self.model(X) + + if self.apply_nonstationary_norm: + # De-Normalization from Non-stationary Transformer + out = nonstationary_denorm(out, means, stdev) + + imputed_data = missing_mask * X + (1 - missing_mask) * out + results = { + "imputed_data": imputed_data, + } + + if training: + # `loss` is always the item for backward propagating to update the model + loss = calc_mse(out, inputs["X_ori"], inputs["indicating_mask"]) + results["loss"] = loss + + return results diff --git a/pypots/imputation/tefn/data.py b/pypots/imputation/tefn/data.py new file mode 100644 index 00000000..3239a6e2 --- /dev/null +++ b/pypots/imputation/tefn/data.py @@ -0,0 +1,24 @@ +""" +Dataset class for the imputation model TEFN. +""" + +# Created by Tianxiang Zhan +# License: BSD-3-Clause + +from typing import Union + +from ..saits.data import DatasetForSAITS + + +class DatasetForTEFN(DatasetForSAITS): + """Actually TEFN uses the same data strategy as SAITS, needs MIT for training.""" + + def __init__( + self, + data: Union[dict, str], + return_X_ori: bool, + return_y: bool, + file_type: str = "hdf5", + rate: float = 0.2, + ): + super().__init__(data, return_X_ori, return_y, file_type, rate) diff --git a/pypots/imputation/tefn/model.py b/pypots/imputation/tefn/model.py new file mode 100644 index 00000000..e97b6f4e --- /dev/null +++ b/pypots/imputation/tefn/model.py @@ -0,0 +1,284 @@ +""" +The implementation of TEFN for the partially-observed time-series imputation task. + +""" + +# Created by Tianxiang Zhan +# License: BSD-3-Clause + +from typing import Union, Optional + +import numpy as np +import torch +from torch.utils.data import DataLoader + +from .core import _TEFN +from .data import DatasetForTEFN +from ..base import BaseNNImputer +from ...data.checking import key_in_data_set +from ...data.dataset import BaseDataset +from ...optim.adam import Adam +from ...optim.base import Optimizer + + +class TEFN(BaseNNImputer): + """The PyTorch implementation of the TEFN model. + TEFN is originally proposed by Zhan et al. in :cite:`zhan2024tefn`. + + Parameters + ---------- + n_steps : + The number of time steps in the time-series data sample. + + n_features : + The number of features in the time-series data sample. + + n_fod : + The number of frame of discernment in the TEFN model. + + apply_nonstationary_norm : + Whether to apply non-stationary normalization to the input data for TimesNet. + Please refer to :cite:`liu2022nonstationary` for details about non-stationary normalization, + which is not the idea of the original TimesNet paper. Hence, we make it optional and default not to use here. + + batch_size : + The batch size for training and evaluating the model. + + epochs : + The number of epochs for training the model. + + patience : + The patience for the early-stopping mechanism. Given a positive integer, the training process will be + stopped when the model does not perform better after that number of epochs. + Leaving it default as None will disable the early-stopping. + + optimizer : + The optimizer for model training. + If not given, will use a default Adam optimizer. + + num_workers : + The number of subprocesses to use for data loading. + `0` means data loading will be in the main process, i.e. there won't be subprocesses. + + device : + The device for the model to run on. It can be a string, a :class:`torch.device` object, or a list of them. + If not given, will try to use CUDA devices first (will use the default CUDA device if there are multiple), + then CPUs, considering CUDA and CPU are so far the main devices for people to train ML models. + If given a list of devices, e.g. ['cuda:0', 'cuda:1'], or [torch.device('cuda:0'), torch.device('cuda:1')] , the + model will be parallely trained on the multiple devices (so far only support parallel training on CUDA devices). + Other devices like Google TPU and Apple Silicon accelerator MPS may be added in the future. + + saving_path : + The path for automatically saving model checkpoints and tensorboard files (i.e. loss values recorded during + training into a tensorboard file). Will not save if not given. + + model_saving_strategy : + The strategy to save model checkpoints. It has to be one of [None, "best", "better", "all"]. + No model will be saved when it is set as None. + The "best" strategy will only automatically save the best model after the training finished. + The "better" strategy will automatically save the model during training whenever the model performs + better than in previous epochs. + The "all" strategy will save every model after each epoch training. + + verbose : + Whether to print out the training logs during the training process. + """ + + def __init__( + self, + n_steps: int, + n_features: int, + n_fod: int = 2, + apply_nonstationary_norm: bool = True, + batch_size: int = 32, + epochs: int = 100, + patience: int = None, + optimizer: Optional[Optimizer] = Adam(), + num_workers: int = 0, + device: Optional[Union[str, torch.device, list]] = None, + saving_path: str = None, + model_saving_strategy: Optional[str] = "best", + verbose: bool = True, + ): + super().__init__( + batch_size, + epochs, + patience, + num_workers, + device, + saving_path, + model_saving_strategy, + verbose, + ) + + self.n_steps = n_steps + self.n_features = n_features + # model hype-parameters + self.apply_nonstationary_norm = apply_nonstationary_norm + self.n_fod = n_fod + + # set up the model + self.model = _TEFN( + n_steps, + n_features, + n_fod, + self.apply_nonstationary_norm + ) + self._send_model_to_given_device() + self._print_model_size() + + # set up the optimizer + self.optimizer = optimizer + self.optimizer.init_optimizer(self.model.parameters()) + + def _assemble_input_for_training(self, data: list) -> dict: + ( + indices, + X, + missing_mask, + X_ori, + indicating_mask, + ) = self._send_data_to_given_device(data) + + inputs = { + "X": X, + "missing_mask": missing_mask, + "X_ori": X_ori, + "indicating_mask": indicating_mask, + } + + return inputs + + def _assemble_input_for_validating(self, data: list) -> dict: + return self._assemble_input_for_training(data) + + def _assemble_input_for_testing(self, data: list) -> dict: + indices, X, missing_mask = self._send_data_to_given_device(data) + + inputs = { + "X": X, + "missing_mask": missing_mask, + } + + return inputs + + def fit( + self, + train_set: Union[dict, str], + val_set: Optional[Union[dict, str]] = None, + file_type: str = "hdf5", + ) -> None: + # Step 1: wrap the input data with classes Dataset and DataLoader + training_set = DatasetForTEFN( + train_set, return_X_ori=False, return_y=False, file_type=file_type + ) + training_loader = DataLoader( + training_set, + batch_size=self.batch_size, + shuffle=True, + num_workers=self.num_workers, + ) + val_loader = None + if val_set is not None: + if not key_in_data_set("X_ori", val_set): + raise ValueError("val_set must contain 'X_ori' for model validation.") + val_set = DatasetForTEFN( + val_set, return_X_ori=True, return_y=False, file_type=file_type + ) + val_loader = DataLoader( + val_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + ) + + # Step 2: train the model and freeze it + self._train_model(training_loader, val_loader) + self.model.load_state_dict(self.best_model_dict) + self.model.eval() # set the model as eval status to freeze it. + + # Step 3: save the model if necessary + self._auto_save_model_if_necessary(confirm_saving=True) + + def predict( + self, + test_set: Union[dict, str], + file_type: str = "hdf5", + ) -> dict: + """Make predictions for the input data with the trained model. + + Parameters + ---------- + test_set : dict or str + The dataset for model validating, should be a dictionary including keys as 'X', + or a path string locating a data file supported by PyPOTS (e.g. h5 file). + If it is a dict, X should be array-like of shape [n_samples, sequence length (n_steps), n_features], + which is time-series data for validating, can contain missing values, and y should be array-like of shape + [n_samples], which is classification labels of X. + If it is a path string, the path should point to a data file, e.g. a h5 file, which contains + key-value pairs like a dict, and it has to include keys as 'X' and 'y'. + + file_type : + The type of the given file if test_set is a path string. + + Returns + ------- + file_type : + The dictionary containing the clustering results and latent variables if necessary. + + """ + # Step 1: wrap the input data with classes Dataset and DataLoader + self.model.eval() # set the model as eval status to freeze it. + test_set = BaseDataset( + test_set, + return_X_ori=False, + return_X_pred=False, + return_y=False, + file_type=file_type, + ) + test_loader = DataLoader( + test_set, + batch_size=self.batch_size, + shuffle=False, + num_workers=self.num_workers, + ) + imputation_collector = [] + + # Step 2: process the data with the model + with torch.no_grad(): + for idx, data in enumerate(test_loader): + inputs = self._assemble_input_for_testing(data) + results = self.model.forward(inputs, training=False) + imputation_collector.append(results["imputed_data"]) + + # Step 3: output collection and return + imputation = torch.cat(imputation_collector).cpu().detach().numpy() + result_dict = { + "imputation": imputation, + } + return result_dict + + def impute( + self, + test_set: Union[dict, str], + file_type: str = "hdf5", + ) -> np.ndarray: + """Impute missing values in the given data with the trained model. + + Parameters + ---------- + test_set : + The data samples for testing, should be array-like of shape [n_samples, sequence length (n_steps), + n_features], or a path string locating a data file, e.g. h5 file. + + file_type : + The type of the given file if X is a path string. + + Returns + ------- + array-like, shape [n_samples, sequence length (n_steps), n_features], + Imputed data. + """ + + result_dict = self.predict(test_set, file_type=file_type) + return result_dict["imputation"] diff --git a/pypots/nn/modules/tefn/__init__.py b/pypots/nn/modules/tefn/__init__.py new file mode 100644 index 00000000..f49a4c0f --- /dev/null +++ b/pypots/nn/modules/tefn/__init__.py @@ -0,0 +1,24 @@ +""" +The package of the forecasting model TEFN. + +Refer to the paper +`Tianxiang Zhan, Yuanpeng He, Yong Deng, and Zhen Li. +Time Evidence Fusion Network: Multi-source View in Long-Term Time Series Forecasting. +In Arxiv, 2024. +`_ + +Notes +----- +This implementation is transfered from the official one https://github.com/ztxtech/Time-Evidence-Fusion-Network + +""" + +# Created by Tianxiang Zhan +# License: BSD-3-Clause + + +from .backbone import BackboneTEFN + +__all__ = [ + "BackboneTEFN", +] diff --git a/pypots/nn/modules/tefn/backbone.py b/pypots/nn/modules/tefn/backbone.py new file mode 100644 index 00000000..78708a62 --- /dev/null +++ b/pypots/nn/modules/tefn/backbone.py @@ -0,0 +1,32 @@ +""" + +""" +import torch +import torch.nn as nn + +from .layers import EvidenceMachineKernel + + +# Created by Tianxiang Zhan > +# License: BSD-3-Clause + + +class BackboneTEFN(nn.Module): + + def __init__(self, + n_steps, + n_features, + n_fod): + super().__init__() + + self.n_steps = n_steps + self.n_features = n_features + self.n_fod = n_fod + + self.T_model = EvidenceMachineKernel(self.n_steps, self.n_fod) + self.C_model = EvidenceMachineKernel(self.n_features, self.n_fod) + + def forward(self, X) -> torch.Tensor: + X = self.T_model(X.permute(0, 2, 1)).permute(0, 2, 1, 3) + self.C_model(X) + X = torch.einsum('btcf->btc', X) + return X diff --git a/pypots/nn/modules/tefn/layers.py b/pypots/nn/modules/tefn/layers.py new file mode 100644 index 00000000..f0b8fa0f --- /dev/null +++ b/pypots/nn/modules/tefn/layers.py @@ -0,0 +1,23 @@ +""" + +""" + +import torch +import torch.nn as nn + + +# Created by Tianxiang Zhan > +# License: BSD-3-Clause + + +class EvidenceMachineKernel(nn.Module): + def __init__(self, C, F): + super(EvidenceMachineKernel, self).__init__() + self.C = C + self.F = 2 ** F + self.C_weight = nn.Parameter(torch.randn(self.C, self.F)) + self.C_bias = nn.Parameter(torch.randn(self.C, self.F)) + + def forward(self, x): + x = torch.einsum('btc,cf->btcf', x, self.C_weight) + self.C_bias + return x From bfc8a18e1df0b3f236f81ee550dd4c24f70fb1d5 Mon Sep 17 00:00:00 2001 From: Tianxiang Zhan <1665186372@qq.com> Date: Fri, 6 Sep 2024 18:19:50 +0800 Subject: [PATCH 2/4] Add the test script of TEFN model. --- tests/imputation/tefn.py | 123 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 tests/imputation/tefn.py diff --git a/tests/imputation/tefn.py b/tests/imputation/tefn.py new file mode 100644 index 00000000..e0edd0fe --- /dev/null +++ b/tests/imputation/tefn.py @@ -0,0 +1,123 @@ +""" +Test cases for TEFN imputation model. +""" + +# Created by Tianxiang Zhan +# License: BSD-3-Clause + + +import os.path +import unittest + +import numpy as np +import pytest + +from pypots.imputation import TEFN +from pypots.optim import Adam +from pypots.utils.logging import logger +from pypots.utils.metrics import calc_mse +from tests.global_test_config import ( + DATA, + EPOCHS, + DEVICE, + TRAIN_SET, + VAL_SET, + TEST_SET, + GENERAL_H5_TRAIN_SET_PATH, + GENERAL_H5_VAL_SET_PATH, + GENERAL_H5_TEST_SET_PATH, + RESULT_SAVING_DIR_FOR_IMPUTATION, + check_tb_and_model_checkpoints_existence, +) + + +class TestTEFN(unittest.TestCase): + logger.info("Running tests for an imputation model TEFN...") + + # set the log and model saving path + saving_path = os.path.join(RESULT_SAVING_DIR_FOR_IMPUTATION, "TEFN") + model_save_name = "saved_tefn_model.pypots" + + # initialize an Adam optimizer + optimizer = Adam(lr=1e-1, weight_decay=1e-5) + + # initialize a TEFN model + tefn = TEFN( + DATA["n_steps"], + DATA["n_features"], + n_fod=2, + epochs=EPOCHS, + saving_path=saving_path, + optimizer=optimizer, + device=DEVICE, + ) + + @pytest.mark.xdist_group(name="imputation-tefn") + def test_0_fit(self): + self.tefn.fit(TRAIN_SET, VAL_SET) + + @pytest.mark.xdist_group(name="imputation-tefn") + def test_1_impute(self): + imputation_results = self.tefn.predict(TEST_SET) + assert not np.isnan( + imputation_results["imputation"] + ).any(), "Output still has missing values after running impute()." + + test_MSE = calc_mse( + imputation_results["imputation"], + DATA["test_X_ori"], + DATA["test_X_indicating_mask"], + ) + logger.info(f"TEFN test_MSE: {test_MSE}") + + @pytest.mark.xdist_group(name="imputation-tefn") + def test_2_parameters(self): + assert hasattr(self.tefn, "model") and self.tefn.model is not None + + assert ( + hasattr(self.tefn, "optimizer") and self.tefn.optimizer is not None + ) + + assert hasattr(self.tefn, "best_loss") + self.assertNotEqual(self.tefn.best_loss, float("inf")) + + assert ( + hasattr(self.tefn, "best_model_dict") + and self.tefn.best_model_dict is not None + ) + + @pytest.mark.xdist_group(name="imputation-tefn") + def test_3_saving_path(self): + # whether the root saving dir exists, which should be created by save_log_into_tb_file + assert os.path.exists( + self.saving_path + ), f"file {self.saving_path} does not exist" + + # check if the tensorboard file and model checkpoints exist + check_tb_and_model_checkpoints_existence(self.tefn) + + # save the trained model into file, and check if the path exists + saved_model_path = os.path.join(self.saving_path, self.model_save_name) + self.tefn.save(saved_model_path) + + # test loading the saved model, not necessary, but need to test + self.tefn.load(saved_model_path) + + @pytest.mark.xdist_group(name="imputation-tefn") + def test_4_lazy_loading(self): + self.tefn.fit(GENERAL_H5_TRAIN_SET_PATH, GENERAL_H5_VAL_SET_PATH) + imputation_results = self.tefn.predict(GENERAL_H5_TEST_SET_PATH) + assert not np.isnan( + imputation_results["imputation"] + ).any(), "Output still has missing values after running impute()." + + test_MSE = calc_mse( + imputation_results["imputation"], + DATA["test_X_ori"], + DATA["test_X_indicating_mask"], + ) + logger.info(f"Lazy-loading TEFN test_MSE: {test_MSE}") + + +if __name__ == "__main__": + unittest.main() From a8ad2df2ea0f30f0ddfd865d24d2a832e5167724 Mon Sep 17 00:00:00 2001 From: Tianxiang Zhan <1665186372@qq.com> Date: Fri, 6 Sep 2024 18:26:37 +0800 Subject: [PATCH 3/4] Adjust TEFN's annotations --- pypots/imputation/tefn/model.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pypots/imputation/tefn/model.py b/pypots/imputation/tefn/model.py index e97b6f4e..c7912164 100644 --- a/pypots/imputation/tefn/model.py +++ b/pypots/imputation/tefn/model.py @@ -39,7 +39,8 @@ class TEFN(BaseNNImputer): apply_nonstationary_norm : Whether to apply non-stationary normalization to the input data for TimesNet. Please refer to :cite:`liu2022nonstationary` for details about non-stationary normalization, - which is not the idea of the original TimesNet paper. Hence, we make it optional and default not to use here. + which is not the idea of the original TimesNet paper. Hence, we make it optional + and default not to use here. batch_size : The batch size for training and evaluating the model. @@ -64,9 +65,9 @@ class TEFN(BaseNNImputer): The device for the model to run on. It can be a string, a :class:`torch.device` object, or a list of them. If not given, will try to use CUDA devices first (will use the default CUDA device if there are multiple), then CPUs, considering CUDA and CPU are so far the main devices for people to train ML models. - If given a list of devices, e.g. ['cuda:0', 'cuda:1'], or [torch.device('cuda:0'), torch.device('cuda:1')] , the - model will be parallely trained on the multiple devices (so far only support parallel training on CUDA devices). - Other devices like Google TPU and Apple Silicon accelerator MPS may be added in the future. + If given a list of devices, e.g. ['cuda:0', 'cuda:1'], or [torch.device('cuda:0'), torch.device('cuda:1')] , + the model will be parallely trained on the multiple devices (so far only support parallel training on CUDA + devices). Other devices like Google TPU and Apple Silicon accelerator MPS may be added in the future. saving_path : The path for automatically saving model checkpoints and tensorboard files (i.e. loss values recorded during From eed33c981d2f29eae1e8ec83a6d11ed820d7e98c Mon Sep 17 00:00:00 2001 From: Wenjie Du Date: Sun, 8 Sep 2024 22:03:17 +0800 Subject: [PATCH 4/4] refactor: adjust code and docs of TEFN; --- README.md | 6 +- README_zh.md | 5 +- docs/index.rst | 2 + pypots/imputation/__init__.py | 2 +- pypots/imputation/tefn/core.py | 10 +- pypots/imputation/tefn/data.py | 12 +- pypots/imputation/tefn/model.py | 177 ++++++++++++++--------------- pypots/nn/modules/tefn/backbone.py | 16 +-- pypots/nn/modules/tefn/layers.py | 13 +-- 9 files changed, 118 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 58ff8bfd..ab8b860c 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ The paper references and links are all listed at the bottom of this file. | **Type** | **Algo** | **IMPU** | **FORE** | **CLAS** | **CLUS** | **ANOD** | **Year - Venue** | |:--------------|:---------------------------------------------------------------------------------------------------------------------------------|:--------:|:--------:|:--------:|:--------:|:--------:|:-------------------| | LLM | Time-Series.AI [^36] | ✅ | ✅ | ✅ | ✅ | ✅ | `Later in 2024` | -| Neural Net | TEFN[^39] | ✅ | | | | | `2024 - arXiv` | +| Neural Net | TEFN🧑‍🔧[^39] | ✅ | | | | | `2024 - arXiv` | | Neural Net | TimeMixer[^37] | ✅ | | | | | `2024 - ICLR` | | Neural Net | iTransformer🧑‍🔧[^24] | ✅ | | | | | `2024 - ICLR` | | Neural Net | ModernTCN[^38] | ✅ | | | | | `2024 - ICLR` | @@ -161,7 +161,7 @@ The paper references and links are all listed at the bottom of this file. | Neural Net | GRU-D[^4] | ✅ | | ✅ | | | `2018 - Sci. Rep.` | | Neural Net | TCN🧑‍🔧[^35] | ✅ | | | | | `2018 - arXiv` | | Neural Net | Transformer🧑‍🔧[^2] | ✅ | | | | | `2017 - NeurIPS` | -| Naive | Lerp | ✅ | | | | | | +| Naive | Lerp[^40] | ✅ | | | | | | | Naive | LOCF/NOCB | ✅ | | | | | | | Naive | Mean | ✅ | | | | | | | Naive | Median | ✅ | | | | | | @@ -508,4 +508,4 @@ Time-Series.AI [^39]: Zhan, T., He, Y., Li, Z., & Deng, Y. ( 2024). [Time Evidence Fusion Network: Multi-source View in Long-Term Time Series Forecasting](https://arxiv.org/abs/2405.06419). *arXiv 2024* - +[^40]: [Wikipedia: Linear interpolation](https://en.wikipedia.org/wiki/Linear_interpolation) \ No newline at end of file diff --git a/README_zh.md b/README_zh.md index d43b5e83..1812ae1e 100644 --- a/README_zh.md +++ b/README_zh.md @@ -105,7 +105,7 @@ PyPOTS当前支持多变量POTS数据的插补，预测，分类，聚类以及 | **类型** | **算法** | **插补** | **预测** | **分类** | **聚类** | **异常检测** | **年份 - 刊物** | |:--------------|:---------------------------------------------------------------------------------------------------------------------------------|:------:|:------:|:------:|:------:|:--------:|:-------------------| | LLM | Time-Series.AI [^36] | ✅ | ✅ | ✅ | ✅ | ✅ | `Later in 2024` | -| Neural Net | TEFN[^39] | ✅ | | | | | `2024 - arXiv` | +| Neural Net | TEFN🧑‍🔧[^39] | ✅ | | | | | `2024 - arXiv` | | Neural Net | TimeMixer[^37] | ✅ | | | | | `2024 - ICLR` | | Neural Net | iTransformer🧑‍🔧[^24] | ✅ | | | | | `2024 - ICLR` | | Neural Net | ModernTCN[^38] | ✅ | | | | | `2024 - ICLR` | @@ -142,7 +142,7 @@ PyPOTS当前支持多变量POTS数据的插补，预测，分类，聚类以及 | Neural Net | GRU-D[^4] | ✅ | | ✅ | | | `2018 - Sci. Rep.` | | Neural Net | TCN🧑‍🔧[^35] | ✅ | | | | | `2018 - arXiv` | | Neural Net | Transformer🧑‍🔧[^2] | ✅ | | | | | `2017 - NeurIPS` | -| Naive | Lerp | ✅ | | | | | | +| Naive | Lerp[^40] | ✅ | | | | | | | Naive | LOCF/NOCB | ✅ | | | | | | | Naive | Mean | ✅ | | | | | | | Naive | Median | ✅ | | | | | | @@ -468,3 +468,4 @@ Time-Series.AI [^39]: Zhan, T., He, Y., Li, Z., & Deng, Y. ( 2024). [Time Evidence Fusion Network: Multi-source View in Long-Term Time Series Forecasting](https://arxiv.org/abs/2405.06419). *arXiv 2024* +[^40]: [Wikipedia: Linear interpolation](https://en.wikipedia.org/wiki/Linear_interpolation) \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 113fbb65..c64c9196 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -133,6 +133,8 @@ The paper references are all listed at the bottom of this readme file. +----------------+-----------------------------------------------------------+------+------+------+------+------+-----------------------+ | Type | Algorithm | IMPU | FORE | CLAS | CLUS | ANOD | Year - Venue | +================+===========================================================+======+======+======+======+======+=======================+ +| Neural Net | TEFN🧑‍🔧 :cite:`zhan2024tefn` | ✅ | | | | | ``2024 - arXiv`` | ++----------------+-----------------------------------------------------------+------+------+------+------+------+-----------------------+ | Neural Net | TimeMixer :cite:`wang2024timemixer` | ✅ | | | | | ``2024 - ICLR`` | +----------------+-----------------------------------------------------------+------+------+------+------+------+-----------------------+ | Neural Net | iTransformer🧑‍🔧 :cite:`liu2024itransformer` | ✅ | | | | | ``2024 - ICLR`` | diff --git a/pypots/imputation/__init__.py b/pypots/imputation/__init__.py index 4ae883f8..19a7e2c6 100644 --- a/pypots/imputation/__init__.py +++ b/pypots/imputation/__init__.py @@ -84,5 +84,5 @@ "Mean", "Median", "Lerp", - "TEFN" + "TEFN", ] diff --git a/pypots/imputation/tefn/core.py b/pypots/imputation/tefn/core.py index c6dc6d8b..f71927a6 100644 --- a/pypots/imputation/tefn/core.py +++ b/pypots/imputation/tefn/core.py @@ -14,11 +14,11 @@ class _TEFN(nn.Module): def __init__( - self, - n_steps, - n_features, - n_fod, - apply_nonstationary_norm, + self, + n_steps, + n_features, + n_fod, + apply_nonstationary_norm, ): super().__init__() diff --git a/pypots/imputation/tefn/data.py b/pypots/imputation/tefn/data.py index 3239a6e2..e8aa19a2 100644 --- a/pypots/imputation/tefn/data.py +++ b/pypots/imputation/tefn/data.py @@ -14,11 +14,11 @@ class DatasetForTEFN(DatasetForSAITS): """Actually TEFN uses the same data strategy as SAITS, needs MIT for training.""" def __init__( - self, - data: Union[dict, str], - return_X_ori: bool, - return_y: bool, - file_type: str = "hdf5", - rate: float = 0.2, + self, + data: Union[dict, str], + return_X_ori: bool, + return_y: bool, + file_type: str = "hdf5", + rate: float = 0.2, ): super().__init__(data, return_X_ori, return_y, file_type, rate) diff --git a/pypots/imputation/tefn/model.py b/pypots/imputation/tefn/model.py index c7912164..02024f28 100644 --- a/pypots/imputation/tefn/model.py +++ b/pypots/imputation/tefn/model.py @@ -23,83 +23,83 @@ class TEFN(BaseNNImputer): """The PyTorch implementation of the TEFN model. - TEFN is originally proposed by Zhan et al. in :cite:`zhan2024tefn`. - - Parameters - ---------- - n_steps : - The number of time steps in the time-series data sample. - - n_features : - The number of features in the time-series data sample. - - n_fod : - The number of frame of discernment in the TEFN model. - - apply_nonstationary_norm : - Whether to apply non-stationary normalization to the input data for TimesNet. - Please refer to :cite:`liu2022nonstationary` for details about non-stationary normalization, - which is not the idea of the original TimesNet paper. Hence, we make it optional - and default not to use here. - - batch_size : - The batch size for training and evaluating the model. - - epochs : - The number of epochs for training the model. - - patience : - The patience for the early-stopping mechanism. Given a positive integer, the training process will be - stopped when the model does not perform better after that number of epochs. - Leaving it default as None will disable the early-stopping. - - optimizer : - The optimizer for model training. - If not given, will use a default Adam optimizer. - - num_workers : - The number of subprocesses to use for data loading. - `0` means data loading will be in the main process, i.e. there won't be subprocesses. - - device : - The device for the model to run on. It can be a string, a :class:`torch.device` object, or a list of them. - If not given, will try to use CUDA devices first (will use the default CUDA device if there are multiple), - then CPUs, considering CUDA and CPU are so far the main devices for people to train ML models. - If given a list of devices, e.g. ['cuda:0', 'cuda:1'], or [torch.device('cuda:0'), torch.device('cuda:1')] , - the model will be parallely trained on the multiple devices (so far only support parallel training on CUDA - devices). Other devices like Google TPU and Apple Silicon accelerator MPS may be added in the future. - - saving_path : - The path for automatically saving model checkpoints and tensorboard files (i.e. loss values recorded during - training into a tensorboard file). Will not save if not given. - - model_saving_strategy : - The strategy to save model checkpoints. It has to be one of [None, "best", "better", "all"]. - No model will be saved when it is set as None. - The "best" strategy will only automatically save the best model after the training finished. - The "better" strategy will automatically save the model during training whenever the model performs - better than in previous epochs. - The "all" strategy will save every model after each epoch training. - - verbose : - Whether to print out the training logs during the training process. - """ + TEFN is originally proposed by Zhan et al. in :cite:`zhan2024tefn`. + + Parameters + ---------- + n_steps : + The number of time steps in the time-series data sample. + + n_features : + The number of features in the time-series data sample. + + n_fod : + The number of frame of discernment in the TEFN model. + + apply_nonstationary_norm : + Whether to apply non-stationary normalization to the input data for TimesNet. + Please refer to :cite:`liu2022nonstationary` for details about non-stationary normalization, + which is not the idea of the original TimesNet paper. Hence, we make it optional + and default not to use here. + + batch_size : + The batch size for training and evaluating the model. + + epochs : + The number of epochs for training the model. + + patience : + The patience for the early-stopping mechanism. Given a positive integer, the training process will be + stopped when the model does not perform better after that number of epochs. + Leaving it default as None will disable the early-stopping. + + optimizer : + The optimizer for model training. + If not given, will use a default Adam optimizer. + + num_workers : + The number of subprocesses to use for data loading. + `0` means data loading will be in the main process, i.e. there won't be subprocesses. + + device : + The device for the model to run on. It can be a string, a :class:`torch.device` object, or a list of them. + If not given, will try to use CUDA devices first (will use the default CUDA device if there are multiple), + then CPUs, considering CUDA and CPU are so far the main devices for people to train ML models. + If given a list of devices, e.g. ['cuda:0', 'cuda:1'], or [torch.device('cuda:0'), torch.device('cuda:1')] , + the model will be parallely trained on the multiple devices (so far only support parallel training on CUDA + devices). Other devices like Google TPU and Apple Silicon accelerator MPS may be added in the future. + + saving_path : + The path for automatically saving model checkpoints and tensorboard files (i.e. loss values recorded during + training into a tensorboard file). Will not save if not given. + + model_saving_strategy : + The strategy to save model checkpoints. It has to be one of [None, "best", "better", "all"]. + No model will be saved when it is set as None. + The "best" strategy will only automatically save the best model after the training finished. + The "better" strategy will automatically save the model during training whenever the model performs + better than in previous epochs. + The "all" strategy will save every model after each epoch training. + + verbose : + Whether to print out the training logs during the training process. + """ def __init__( - self, - n_steps: int, - n_features: int, - n_fod: int = 2, - apply_nonstationary_norm: bool = True, - batch_size: int = 32, - epochs: int = 100, - patience: int = None, - optimizer: Optional[Optimizer] = Adam(), - num_workers: int = 0, - device: Optional[Union[str, torch.device, list]] = None, - saving_path: str = None, - model_saving_strategy: Optional[str] = "best", - verbose: bool = True, + self, + n_steps: int, + n_features: int, + n_fod: int = 2, + apply_nonstationary_norm: bool = True, + batch_size: int = 32, + epochs: int = 100, + patience: int = None, + optimizer: Optional[Optimizer] = Adam(), + num_workers: int = 0, + device: Optional[Union[str, torch.device, list]] = None, + saving_path: str = None, + model_saving_strategy: Optional[str] = "best", + verbose: bool = True, ): super().__init__( batch_size, @@ -119,12 +119,7 @@ def __init__( self.n_fod = n_fod # set up the model - self.model = _TEFN( - n_steps, - n_features, - n_fod, - self.apply_nonstationary_norm - ) + self.model = _TEFN(n_steps, n_features, n_fod, self.apply_nonstationary_norm) self._send_model_to_given_device() self._print_model_size() @@ -164,10 +159,10 @@ def _assemble_input_for_testing(self, data: list) -> dict: return inputs def fit( - self, - train_set: Union[dict, str], - val_set: Optional[Union[dict, str]] = None, - file_type: str = "hdf5", + self, + train_set: Union[dict, str], + val_set: Optional[Union[dict, str]] = None, + file_type: str = "hdf5", ) -> None: # Step 1: wrap the input data with classes Dataset and DataLoader training_set = DatasetForTEFN( @@ -202,9 +197,9 @@ def fit( self._auto_save_model_if_necessary(confirm_saving=True) def predict( - self, - test_set: Union[dict, str], - file_type: str = "hdf5", + self, + test_set: Union[dict, str], + file_type: str = "hdf5", ) -> dict: """Make predictions for the input data with the trained model. @@ -260,9 +255,9 @@ def predict( return result_dict def impute( - self, - test_set: Union[dict, str], - file_type: str = "hdf5", + self, + test_set: Union[dict, str], + file_type: str = "hdf5", ) -> np.ndarray: """Impute missing values in the given data with the trained model. diff --git a/pypots/nn/modules/tefn/backbone.py b/pypots/nn/modules/tefn/backbone.py index 78708a62..22dd7c88 100644 --- a/pypots/nn/modules/tefn/backbone.py +++ b/pypots/nn/modules/tefn/backbone.py @@ -1,22 +1,18 @@ """ """ + +# Created by Tianxiang Zhan +# License: BSD-3-Clause + import torch import torch.nn as nn from .layers import EvidenceMachineKernel -# Created by Tianxiang Zhan > -# License: BSD-3-Clause - - class BackboneTEFN(nn.Module): - - def __init__(self, - n_steps, - n_features, - n_fod): + def __init__(self, n_steps, n_features, n_fod): super().__init__() self.n_steps = n_steps @@ -28,5 +24,5 @@ def __init__(self, def forward(self, X) -> torch.Tensor: X = self.T_model(X.permute(0, 2, 1)).permute(0, 2, 1, 3) + self.C_model(X) - X = torch.einsum('btcf->btc', X) + X = torch.einsum("btcf->btc", X) return X diff --git a/pypots/nn/modules/tefn/layers.py b/pypots/nn/modules/tefn/layers.py index f0b8fa0f..5f2e5934 100644 --- a/pypots/nn/modules/tefn/layers.py +++ b/pypots/nn/modules/tefn/layers.py @@ -2,22 +2,21 @@ """ +# Created by Tianxiang Zhan +# License: BSD-3-Clause + import torch import torch.nn as nn -# Created by Tianxiang Zhan > -# License: BSD-3-Clause - - class EvidenceMachineKernel(nn.Module): def __init__(self, C, F): - super(EvidenceMachineKernel, self).__init__() + super().__init__() self.C = C - self.F = 2 ** F + self.F = 2**F self.C_weight = nn.Parameter(torch.randn(self.C, self.F)) self.C_bias = nn.Parameter(torch.randn(self.C, self.F)) def forward(self, x): - x = torch.einsum('btc,cf->btcf', x, self.C_weight) + self.C_bias + x = torch.einsum("btc,cf->btcf", x, self.C_weight) + self.C_bias return x