Upload 5 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,158 @@
 ---
-license: apache-2.0
+license: mit
+datasets:
+- AutonLab/Timeseries-PILE
+metrics:
+- accuracy
+- mse
+- mae
+- f1
+tags:
+- time series
+- forecasting
+- classification
+- anomaly detection
+- imputation
+- transformers
+- pretrained models
+- foundation models
+- time-series
+pipeline_tag: time-series-forecasting
 ---
+# MOMENT-Large
+
+MOMENT is a family of foundation models for general-purpose time-series analysis. The models in this family (1) serve as a building block for diverse **time-series analysis tasks** (e.g., forecasting, classification, anomaly detection, and imputation, etc.), (2) are effective **out-of-the-box**, i.e., with no (or few) task-specific exemplars (enabling e.g., zero-shot forecasting, few-shot classification, etc.), and (3) are **tunable** using in-distribution and task-specific data to improve performance. 
+
+For details on MOMENT models, training data, and experimental results, please refer to the paper [MOMENT: A Family of Open Time-series Foundation Models](https://arxiv.org/pdf/2402.03885.pdf).
+
+MOMENT-1 comes in 3 sizes: [Small](https://huggingface.co/AutonLab/MOMENT-1-small), [Base](https://huggingface.co/AutonLab/MOMENT-1-base), and [Large](https://huggingface.co/AutonLab/MOMENT-1-large). 
+
+# Usage
+
+**Recommended Python Version:** Python 3.11 (support for additional versions is expected soon).
+
+You can install the `momentfm` package using pip:
+```bash
+pip install momentfm
+```
+Alternatively, to install the latest version directly from the GitHub repository:
+```bash
+pip install git+https://github.com/moment-timeseries-foundation-model/moment.git
+```
+
+
+To load the pre-trained model for one of the tasks, use one of the following code snippets:
+
+**Forecasting**
+```python
+from momentfm import MOMENTPipeline
+
+model = MOMENTPipeline.from_pretrained(
+    "AutonLab/MOMENT-1-large", 
+    model_kwargs={
+        'task_name': 'forecasting',
+        'forecast_horizon': 96
+    },
+)
+model.init()
+```
+
+**Classification**
+```python
+from momentfm import MOMENTPipeline
+
+model = MOMENTPipeline.from_pretrained(
+    "AutonLab/MOMENT-1-large", 
+    model_kwargs={
+        'task_name': 'classification',
+        'n_channels': 1,
+        'num_class': 2
+    },
+)
+model.init()
+```
+
+**Anomaly Detection, Imputation, and Pre-training**
+```python
+from momentfm import MOMENTPipeline
+
+model = MOMENTPipeline.from_pretrained(
+    "AutonLab/MOMENT-1-large", 
+    model_kwargs={"task_name": "reconstruction"},
+)
+mode.init()
+```
+
+**Representation Learning**
+```python
+from momentfm import MOMENTPipeline
+
+model = MOMENTPipeline.from_pretrained(
+    "AutonLab/MOMENT-1-large", 
+    model_kwargs={'task_name': 'embedding'},
+)
+```
+
+### Tutorials
+Here is the list of tutorials and reproducibile experiments to get started with MOMENT for various tasks:
+- [Forecasting](https://github.com/moment-timeseries-foundation-model/moment/blob/main/tutorials/forecasting.ipynb)
+- [Classification](https://github.com/moment-timeseries-foundation-model/moment/blob/main/tutorials/classification.ipynb)
+- [Anomaly Detection](https://github.com/moment-timeseries-foundation-model/moment/blob/main/tutorials/anomaly_detection.ipynb)
+- [Imputation](https://github.com/moment-timeseries-foundation-model/moment/blob/main/tutorials/imputation.ipynb)
+- [Representation Learning](https://github.com/moment-timeseries-foundation-model/moment/blob/main/tutorials/representation_learning.ipynb)
+- [Real-world Electrocardiogram (ECG) Case Study](https://github.com/moment-timeseries-foundation-model/moment/blob/main/tutorials/ptbxl_classification.ipynb) -- This tutorial also shows how to fine-tune MOMENT for a real-world ECG classification problem, performing training and inference on multiple GPUs and parameter efficient fine-tuning (PEFT). 
+
+## Model Details
+
+### Model Description
+
+- **Developed by:** [Auton Lab](https://autonlab.org/), [Carnegie Mellon University](https://www.cmu.edu/)
+- **Model type:** Time-series Foundation Model
+- **License:** MIT License
+
+### Model Sources
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** https://github.com/moment-timeseries-foundation-model/ (Pre-training and research code coming out soon!)
+- **Paper:** https://arxiv.org/abs/2402.03885
+- **Demo:** https://github.com/moment-timeseries-foundation-model/moment/tree/main/tutorials
+
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+We train multiple models over many days resulting in significant energy usage and a sizeable carbon footprint. However, we hope that releasing our models will ensure that future time-series modeling efforts are quicker and more efficient, resulting in lower carbon emissions.
+
+We use the Total Graphics Power (TGP) to calculate the total power consumed for training MOMENT models, although the total power consumed by the GPU will likely vary a little based on the GPU utilization while training our model. Our calculations do not account for power demands from other sources of our compute. We use 336.566 Kg C02/MWH as the standard value of CO2 emission per megawatt hour of energy consumed for [Pittsburgh](https://emissionsindex.org/).
+
+- **Hardware Type:** NVIDIA RTX A6000 GPU
+- **GPU Hours:** 404
+- **Compute Region:** Pittsburgh, USA
+- **Carbon Emission (tCO2eq):** 
+
+#### Hardware
+
+All models were trained and evaluated on a computing cluster consisting of 128 AMD EPYC 7502 CPUs, 503 GB of RAM, and 8 NVIDIA RTX A6000 GPUs each with 49 GiB RAM. All MOMENT variants were trained on a single A6000 GPU (with any data or model parallelism).
+
+## Citation
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+If you use MOMENT please cite our paper: 
+
+```bibtex
+@inproceedings{goswami2024moment,
+  title={MOMENT: A Family of Open Time-series Foundation Models},
+  author={Mononito Goswami and Konrad Szafer and Arjun Choudhry and Yifu Cai and Shuo Li and Artur Dubrawski},
+  booktitle={International Conference on Machine Learning},
+  year={2024}
+}
+```
+
+**APA:**
+
+Goswami, M., Szafer, K., Choudhry, A., Cai, Y., Li, S., & Dubrawski, A. (2024). 
+MOMENT: A Family of Open Time-series Foundation Models. In International Conference on Machine Learning. PMLR.

config.json

@@ -0,0 +1,38 @@
+{"task_name": "reconstruction", 
+"model_name": "MOMENT", 
+"transformer_type": "encoder_only", 
+"d_model": null, 
+"seq_len": 512, 
+"patch_len": 8, 
+"patch_stride_len": 8, 
+"device": "cpu", 
+"transformer_backbone": "google/flan-t5-large", 
+"model_kwargs": {},
+"t5_config": {
+    "architectures": [
+      "T5ForConditionalGeneration"
+    ],
+    "d_ff": 2816,
+    "d_kv": 64,
+    "d_model": 1024,
+    "decoder_start_token_id": 0,
+    "dropout_rate": 0.1,
+    "eos_token_id": 1,
+    "feed_forward_proj": "gated-gelu",
+    "initializer_factor": 1.0,
+    "is_encoder_decoder": true,
+    "layer_norm_epsilon": 1e-06,
+    "model_type": "t5",
+    "n_positions": 512,
+    "num_decoder_layers": 24,
+    "num_heads": 16,
+    "num_layers": 24,
+    "output_past": true,
+    "pad_token_id": 0,
+    "relative_attention_max_distance": 128,
+    "relative_attention_num_buckets": 32,
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "vocab_size": 32128
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a56928052ac6f5d09b97c3834bea6ce3aef9f02b513b5fac98954e7377801572
+size 1385501256

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52f5ee65c80790c7f6dab8736243368e193fcc04d2d642d9f69c89dd644bd199
+size 1385575757