Training in progress, epoch 1

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.4",
+  "use_cache": false,
+  "vocab_size": 128256
+}

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00000-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13fceebe5fc4b5abc6c4c97bb5657655ebd0e6ecd515452174fba1e9fd7e7b83
+size 6917672

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00001-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c036edd5778dd904665ba34ae451890786d5a59170d8edbd7fcad883d8ef0382
+size 6825952

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00002-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23be0816f6776858ad7392838c8bb01856a6b9737043ce9c7162b1c913c8f151
+size 6859424

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00003-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2c5f531ad82f221c205ffdfd5380ffec53a0d36d6e66df498c35541e9831652
+size 6933168

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00004-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83d02dba8d27a67206fcdbf06b137718b70fdf164f24ec804a1ee97565345049
+size 6703448

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00005-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:082933d31f79bb7b3b05d00cfc7350146c30c754d72e61dbcf951f4e8d6591f4
+size 6912016

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00006-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9c2ad7ffa261a201cde6648cf6bbb52335330a6aa6140168da604feb2da5849
+size 6867792

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00007-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f602652d4b781e24fcb795b804b66269a8fcbb943d01a1c5520bdfd726ffff5
+size 7061616

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00008-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ec0db8129bdde35d5421fe4a0006e96f54dfb94589650bcebed5c8b6ac9dfc5
+size 6849160

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00009-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:582e54605c4e0b429b97f717a548c238261441d3afdccb1ff93443cffdfa32f6
+size 6780472

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00010-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c074820939af0bf6d31e57be2d02f98d62422f518618c78093ac375ce384823
+size 6875072

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00011-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4a4665472917c3b8afc2f1ba802bbaccb29728724f1200238b14dbf25222bcd
+size 6973824

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00012-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0653333e6b112c8a3d6648fee97fd900621358bb372261bc3492ccc218aea280
+size 6904480

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00013-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dce31ad7e07c62d7b247dede34138421c5e908ae8317c1c6f1d0d6acf8c909a
+size 6948976

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00014-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25b913f903de5cfae743dce45d43efe9b9ffaf5aec8add388d953a8972510efd
+size 6937328

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00015-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e329498cd850f0f3163c0535fc62f5b02e5a93caea727f7f7bd3c31464f6a189
+size 7158560

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00016-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4cc4255bead8ec94c7e3d245cdeed8e957d1b5e23cf0864e28dd3e7cd41fb69
+size 7046512

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00017-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a89c12b9a140ae6a25276820c5401262f0e3c9f9492a71982170cc63af0bb19d
+size 6867960

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00018-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db1bd66d83dba50d987ef182cf132add642c2137f960fb4126b07c3495950ae9
+size 6904392

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00019-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b97082992fb288254493f688d6866758174c0cce905803a012f2f17b2c5f1a9
+size 7017248

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00020-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cd775ca10234be618f5eb54c623cc6d2cfdd473a36bae72fae7a1b5e7a24407
+size 7085272

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00021-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7a1b6396bc87bda2a2becc9880a165eb01552b1872e85f5cab9baca6360387f
+size 7202112

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00022-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:133490047030ec7e295613f02a6d7767079ceacbb449a57ad5e7b5279e148891
+size 7076888

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00023-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05517673ae62fa233fa2e52be76359e15b017a3bc19cb273d157bff49d162063
+size 7054312

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00024-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af7a7f143ab1d3f4bf8fccdf05d68232a5fbca6b9fdbfa470a3eadcedc3d1950
+size 6994248

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00025-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5800aeb87cdc31b8d377ed5e3bcac2551264b562128ca34bec335d3cc0388c9d
+size 6839208

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00026-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a273ced942be54a02d6d751d8b9a31831903154b2cb5f8e7af8672b82d7c0e8b
+size 6973208

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/data-00027-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:271a44ac87198d71091872779399729e0bd72ec5f773a845f9dc79ef1b6864af
+size 7046528

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/dataset_info.json

@@ -0,0 +1,40 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "input_ids": {
+      "feature": {
+        "dtype": "int32",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "attention_mask": {
+      "feature": {
+        "dtype": "int8",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "labels": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "position_ids": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "length": {
+      "dtype": "int64",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

dataset_prepared/0d17ed7654ff305aa19c77e503fc1265/state.json

@@ -0,0 +1,94 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00028.arrow"
+    },
+    {
+      "filename": "data-00001-of-00028.arrow"
+    },
+    {
+      "filename": "data-00002-of-00028.arrow"
+    },
+    {
+      "filename": "data-00003-of-00028.arrow"
+    },
+    {
+      "filename": "data-00004-of-00028.arrow"
+    },
+    {
+      "filename": "data-00005-of-00028.arrow"
+    },
+    {
+      "filename": "data-00006-of-00028.arrow"
+    },
+    {
+      "filename": "data-00007-of-00028.arrow"
+    },
+    {
+      "filename": "data-00008-of-00028.arrow"
+    },
+    {
+      "filename": "data-00009-of-00028.arrow"
+    },
+    {
+      "filename": "data-00010-of-00028.arrow"
+    },
+    {
+      "filename": "data-00011-of-00028.arrow"
+    },
+    {
+      "filename": "data-00012-of-00028.arrow"
+    },
+    {
+      "filename": "data-00013-of-00028.arrow"
+    },
+    {
+      "filename": "data-00014-of-00028.arrow"
+    },
+    {
+      "filename": "data-00015-of-00028.arrow"
+    },
+    {
+      "filename": "data-00016-of-00028.arrow"
+    },
+    {
+      "filename": "data-00017-of-00028.arrow"
+    },
+    {
+      "filename": "data-00018-of-00028.arrow"
+    },
+    {
+      "filename": "data-00019-of-00028.arrow"
+    },
+    {
+      "filename": "data-00020-of-00028.arrow"
+    },
+    {
+      "filename": "data-00021-of-00028.arrow"
+    },
+    {
+      "filename": "data-00022-of-00028.arrow"
+    },
+    {
+      "filename": "data-00023-of-00028.arrow"
+    },
+    {
+      "filename": "data-00024-of-00028.arrow"
+    },
+    {
+      "filename": "data-00025-of-00028.arrow"
+    },
+    {
+      "filename": "data-00026-of-00028.arrow"
+    },
+    {
+      "filename": "data-00027-of-00028.arrow"
+    }
+  ],
+  "_fingerprint": "d9cfb4a95203cd78",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00000-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed50a45a97932f9b301c5960db14496c7a770c62eaa4353a882bbad1775e8480
+size 4256104

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00001-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:859b8371be178104f881bbfae81745966d5f8b89597ef636bf48d67cc97be6a4
+size 4329376

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00002-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35eb72eed75e16f397e450f301619b9edde042d73c40b4fc0a1194b90cd708a1
+size 4229312

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00003-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0c85a4604ca251367d92caea18c07ebd55351bffa2b86492646bfac4b987627
+size 4195952

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00004-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6c07d894e78863bd12244912cfdd88b87fae6732e64945188edb87683e42f8d
+size 4374608

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00005-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8a93d6aec3f2e84fa94e742506b510ec08c7fe1d9bcab6f253a49b0ac5a87db
+size 4316656

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00006-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71f7e0c89910704e2acf6900077aaf8436d74ea0226065dd0b73d8f6f35c146b
+size 4407544

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00007-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:781c7dc9f7dcbcd5a2cdee471d2ccd340e0bbf6a9e5d2f943a8bd9bacc2875cf
+size 4270040

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00008-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c77ec8363b77c97b2d6727040624dc8367b58196c3a5346daad876f0343196bc
+size 4302048

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00009-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ce9902c943e1ab7b5cfd2bb6c14d2c9f80ad6f55085273a9b72151484fb6926
+size 4334336

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00010-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:661c143c0a4b1b2c4795114882b88d28bf9c3b713ef99b1027b603e7f95deb3c
+size 4297032

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00011-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcb7c50aa9ef171afacf3fb9c7ecb46cf9fcc0e7b4faa0a6d38e57031ec64f33
+size 4312624

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00012-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43d3ee09c7d0094f9a49b1c4e767a1696a56a0b02eda7f1de3de286860051ca4
+size 4274608

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00013-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:342483149fb99f2f5f2a82fa991bb2ba075a5bdd00a77468cf556b8bebd5f29c
+size 4331336

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00014-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:079532d6452e53fa6706ca47f0e52ef125725c59be9ece47cdbd0b59fa043e94
+size 4341456

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00015-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f6512490f51e0d26840e3f6bbe2371709e5709e145fb5a48532a31593e81756
+size 4286616

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00016-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa9756b4cc7bc036161ccd7034a538ec1a771c099c979c6fe9392f7f2bdfe445
+size 4360440

dataset_prepared/9bc662aed65b76546b2d635b3957a343/data-00017-of-00028.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8644aea002ab00f8aa9d3b4299e4a35d3d3b0ed69c7224ce07ad2ccd685c770
+size 4200408