Upload folder using huggingface_hub

README.md

@@ -1,3 +1,78 @@
----
-license: cc-by-4.0
----
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: dna_model
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# dna_model
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.0299
+- Accuracy: 0.5324
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0003
+- train_batch_size: 64
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 256
+- total_eval_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 1000
+- num_epochs: 10.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+| Training Loss | Epoch  | Step  | Validation Loss | Accuracy |
+|:-------------:|:------:|:-----:|:---------------:|:--------:|
+| 1.1252        | 0.6908 | 5000  | 1.1206          | 0.4745   |
+| 1.0835        | 1.3816 | 10000 | 1.0814          | 0.4991   |
+| 1.0641        | 2.0724 | 15000 | 1.0639          | 0.5103   |
+| 1.0563        | 2.7632 | 20000 | 1.0547          | 0.5163   |
+| 1.0504        | 3.4540 | 25000 | 1.0486          | 0.5204   |
+| 1.0439        | 4.1448 | 30000 | 1.0439          | 0.5233   |
+| 1.0425        | 4.8356 | 35000 | 1.0407          | 0.5254   |
+| 1.0365        | 5.5264 | 40000 | 1.0380          | 0.5271   |
+| 1.0325        | 6.2172 | 45000 | 1.0361          | 0.5284   |
+| 1.0322        | 6.9080 | 50000 | 1.0341          | 0.5296   |
+| 1.0307        | 7.5988 | 55000 | 1.0328          | 0.5305   |
+| 1.0267        | 8.2896 | 60000 | 1.0316          | 0.5313   |
+| 1.0273        | 8.9804 | 65000 | 1.0306          | 0.5320   |
+| 1.027         | 9.6712 | 70000 | 1.0299          | 0.5324   |
+
+
+### Framework versions
+
+- Transformers 4.52.0.dev0
+- Pytorch 2.3.0+cu121
+- Datasets 3.0.0
+- Tokenizers 0.21.1

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.5323623139821072,
+    "eval_loss": 1.029943823814392,
+    "eval_runtime": 721.9591,
+    "eval_samples": 205861,
+    "eval_samples_per_second": 285.142,
+    "eval_steps_per_second": 8.912,
+    "perplexity": 2.8009084859245172,
+    "total_flos": 9.683106445125485e+18,
+    "train_loss": 1.0569831334908848,
+    "train_runtime": 55085.3166,
+    "train_samples": 1852919,
+    "train_samples_per_second": 336.373,
+    "train_steps_per_second": 1.314
+}

checkpoint-65000/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.52.0.dev0",
+  "use_cache": true,
+  "vocab_size": 5
+}

checkpoint-65000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.52.0.dev0"
+}

checkpoint-65000/merges.txt

@@ -0,0 +1 @@
+#version: 0.2

checkpoint-65000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0b2555f6669e4ccea28c5ca35ca80703169550c0089256aef6f76de2fb02c03
+size 343400064

checkpoint-65000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90b816e703a34fa1d32bb6a8ab69151a6094d3d559265e1abde89cacb4d87f62
+size 686894010

checkpoint-65000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a73a83b1c17f13cda5530062a0df264cb7442b67bef02fa6d887e5c28bad11ae
+size 14960

checkpoint-65000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b193f708cbd330ff0c9344e6b41e847f8fc3944ac5894b34157b1867d2df2b72
+size 14960

checkpoint-65000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38febdeb5d9aabe5b12704ae9ba82c3a576d9abd99c1ad82f0d4f1b455f9ac53
+size 14960

checkpoint-65000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:777f88144775fe8de99b4817ca0fdff9b09cd701f79f6d84d35f15d064018e27
+size 14960

checkpoint-65000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fecf213c35ff38444ea6916370882432e71bfe60a14e22fef3af0107af356e7
+size 988

checkpoint-65000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37b25fb6861357a59b677028c1ca6445fb652f33d7914a60e59280a0be350c36
+size 1064

checkpoint-65000/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-65000/tokenizer.json

@@ -0,0 +1,53 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<|endoftext|>": 0,
+      "A": 1,
+      "C": 2,
+      "G": 3,
+      "T": 4
+    },
+    "merges": []
+  }
+}

checkpoint-65000/tokenizer_config.json

@@ -0,0 +1,21 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-65000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19e8fb7657adc13bdcaf635b1c6fb616dd082a6870cdd6aecd3b669d8cac0873
+size 5304

checkpoint-65000/vocab.json

@@ -0,0 +1 @@
+{"<|endoftext|>":0,"A":1,"C":2,"G":3,"T":4}

checkpoint-70000/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.52.0.dev0",
+  "use_cache": true,
+  "vocab_size": 5
+}

checkpoint-70000/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.52.0.dev0"
+}

checkpoint-70000/merges.txt

@@ -0,0 +1 @@
+#version: 0.2

checkpoint-70000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7694d2d4deca5552713678b7ba3a98015adc907157087d06374e8759397a9704
+size 343400064

checkpoint-70000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:044069d2fe8a5554e5399ab1cbb2bf808bff35ff185a0141569bfcb3bd777b95
+size 686894010

checkpoint-70000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dec82c95e8817f8b4a9d63e77c0f6b5b52a0e039a9397117bc95a18374acfb7
+size 14960

checkpoint-70000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdd996c2bdddb8f92975bb79265833d12d6478a586ba1b97e4281358d5dca6f3
+size 14960

checkpoint-70000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f89b14efe50937d062b0eb38eb7917537ab35011b36cf6050f8736af97eda57c
+size 14960

checkpoint-70000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acbb453cfd80542114994513f8203a5b3b87f5757bf87f3caf54bee3ab7609eb
+size 14960

checkpoint-70000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73c747ac36afcda69f34e348853d519b2859b2dbf2765547178b534ad53d9215
+size 988

checkpoint-70000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac1e915b201c2bf73c557406f586b44f202512a9f35a7541fc5a357a0622ce3f
+size 1064

checkpoint-70000/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-70000/tokenizer.json

@@ -0,0 +1,53 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<|endoftext|>": 0,
+      "A": 1,
+      "C": 2,
+      "G": 3,
+      "T": 4
+    },
+    "merges": []
+  }
+}

checkpoint-70000/tokenizer_config.json

@@ -0,0 +1,21 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-70000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19e8fb7657adc13bdcaf635b1c6fb616dd082a6870cdd6aecd3b669d8cac0873
+size 5304

checkpoint-70000/vocab.json

@@ -0,0 +1 @@
+{"<|endoftext|>":0,"A":1,"C":2,"G":3,"T":4}

checkpoint-72380/config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.52.0.dev0",
+  "use_cache": true,
+  "vocab_size": 5
+}

checkpoint-72380/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.52.0.dev0"
+}

checkpoint-72380/merges.txt

@@ -0,0 +1 @@
+#version: 0.2

checkpoint-72380/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f92899f4db195c8aeb912724ff986896242eb0429b37aee778d302e21091c9de
+size 343400064

checkpoint-72380/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18b7cdf65c9d9b0e5f7cc5a0eefff555fa183b157836316032bf3cb608879251
+size 686894010

checkpoint-72380/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c25515ce626b4abe5a87054259fda00aa4607c6b55105ce503dbf9685168014
+size 14960

checkpoint-72380/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a36cd014a4704101a9bf0bc9cd1ea746333e0872d4713feebc6f28b65046aaf6
+size 14960

checkpoint-72380/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d066faeaef510c6df230bdde787d1a47cc41fa100a5cc9b52c590ddb31b737b0
+size 14960

checkpoint-72380/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12b7e7097bc6610ab93b5b8495ffb37a797cf3694d6cacb86f7a320518ca5b80
+size 14960

checkpoint-72380/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5aa1e803a71b8195096e7e8434c59a48b7e733ca69258660b62b3f2d5128a2e0
+size 988

checkpoint-72380/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:134bc570b6bcd834acbb92457d0f89b92c86b5647b77c08c7ea37da87f1eeeb1
+size 1064

checkpoint-72380/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-72380/tokenizer.json

@@ -0,0 +1,53 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<|endoftext|>": 0,
+      "A": 1,
+      "C": 2,
+      "G": 3,
+      "T": 4
+    },
+    "merges": []
+  }
+}

checkpoint-72380/tokenizer_config.json

@@ -0,0 +1,21 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}