Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,49 @@
----
-license: llama3
----
+---
+base_model:
+- Sao10K/L3-70B-Euryale-v2.1
+- Bllossom/llama-3-Korean-Bllossom-70B
+library_name: transformers
+tags:
+- mergekit
+- merge
+
+---
+# merge
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the SLERP merge method.
+
+### Models Merged
+
+The following models were included in the merge:
+* [Sao10K/L3-70B-Euryale-v2.1](https://huggingface.co/Sao10K/L3-70B-Euryale-v2.1)
+* [Bllossom/llama-3-Korean-Bllossom-70B](https://huggingface.co/Bllossom/llama-3-Korean-Bllossom-70B)
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+
+slices:
+  - sources:
+      - model: Sao10K/L3-70B-Euryale-v2.1
+        layer_range: [0, 70]  # 전체 레이어
+      - model: Bllossom/llama-3-Korean-Bllossom-70B
+        layer_range: [0, 70]  # 전체 레이어
+merge_method: slerp
+base_model: Sao10K/L3-70B-Euryale-v2.1  # 영문 모델을 기반으로
+parameters:
+  t:
+    - filter: self_attn
+      value: [0.5, 0.3, 0.7, 0.9, 1]  # 가중치 조정
+    - filter: mlp
+      value: [1, 0.5, 0.6, 0.4, 0.2]
+    - value: 0.5  # 기본값
+dtype: bfloat16
+
+```

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "Sao10K/L3-70B-Euryale-v2.1",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128009,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 28672,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 64,
+  "num_hidden_layers": 70,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.1",
+  "use_cache": false,
+  "vocab_size": 128256
+}

mergekit_config.yml

@@ -0,0 +1,17 @@
+
+slices:
+  - sources:
+      - model: Sao10K/L3-70B-Euryale-v2.1
+        layer_range: [0, 70]  # 전체 레이어
+      - model: Bllossom/llama-3-Korean-Bllossom-70B
+        layer_range: [0, 70]  # 전체 레이어
+merge_method: slerp
+base_model: Sao10K/L3-70B-Euryale-v2.1  # 영문 모델을 기반으로
+parameters:
+  t:
+    - filter: self_attn
+      value: [0.5, 0.3, 0.7, 0.9, 1]  # 가중치 조정
+    - filter: mlp
+      value: [1, 0.5, 0.6, 0.4, 0.2]
+    - value: 0.5  # 기본값
+dtype: bfloat16

model-00001-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c17a5c2b005f5dc81451d849897d0eb9077f9dfa8c07eaa7a538447b5d9df9b
+size 2101346432

model-00002-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:270cbaf162174e63b40321501569546fa1f3467f8478aaa26cdb3cd6a64a2156
+size 2101346448

model-00003-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31c6342f97ee2eb8d53674457377874bbeeac86c6b8fb85ee3cd98b6ec01e3ae
+size 939540840

model-00004-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f2d58e80164778e99704c739aa0818a8fceb25f0eb3e558b9f1195844d6e70b
+size 771785536

model-00005-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6516cb62c900ed72811bc51087b0ef43a942e2e12c3259bcb2efc01c7feb710
+size 939524360

model-00006-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bdb1ce36dc15696735383770644f949d4ac3d1ba60c34afaa6087a0aa61afd7
+size 771785536

model-00007-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58022fbf4ab692fdb1bb42860b1e415a45c5ddefa4faa7b86b9b24f596dcd9bd
+size 939524360

model-00008-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1940b9bb223d687163625cf06d84c40f783b33b36b6c74bc121206c36fa6a88
+size 771785544

model-00009-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cdbbf8604e073639315d7e096fb9979896ebae7279694240e7789534da44efe
+size 939524360

model-00010-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed2a23a6949f6d9216b21e7e6a7a7be62a76c8cccddcbfe9d8bf2c61cafcb36f
+size 771785544

model-00011-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c7f4ddf28fffa1af2489d36064fe26f9fda78020f5681e3a5215fab218f26da
+size 939524360

model-00012-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36a05249d643abedfa5a81c9f4267a07b63f9a7e053df1f94c4c92a9154945e5
+size 771785544

model-00013-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48dca7e5e2572df80b2c76a6c537d39d7c9b7064b01e3e7f2febfdc8cebb4cb0
+size 939524360

model-00014-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2412a0c0d215bd43bbb1a32ece0648036fb22ee76257d331a87cd87a0839af24
+size 771785544

model-00015-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9de13ed72306503302ee9e44922a574c57670e9629041bcf4aa39c3113b4940
+size 939524360

model-00016-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9561cc3034631e46aa16712fc52819b63203518ea9681c1d1fe7443282ce384b
+size 771785544

model-00017-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8df0b826f429df3a6e388911a6fd917a1b7af9712b5dcca91db9552467051c71
+size 939524360

model-00018-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57712d26663d384e21b9d0c8820af43285188a161938fd76f0af45eb046d7d01
+size 771785544

model-00019-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c88392b924591c67187021c22f03922c6d1c76fe94af2334674cc2a871349870
+size 939524360

model-00020-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5533c44eb873e7b1f588a9441a82c0eaa5cbf984b5bb617cad5d64472f9e307
+size 771785544

model-00021-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ae4f75fb82bb03036de95d34f3fe959cf9a3cd7d453ad1d3ac9f8ee991b599d
+size 939524360

model-00022-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c79975e78c46fdb85af36ca80a8156e4c4e2f8f596983f7516d0712d8e26fa2f
+size 771785544

model-00023-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:919ce33f4cd893f8185ec370c91e8c5a39b7fd9cd4431c61fc9781992bb9edfc
+size 939524360

model-00024-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3462f8d29d482ce39a5aa66a8772c01605768cc611648c7b809fd2357e14a378
+size 771785544

model-00025-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b665018d8825bd986e3ce935ab0c39d5bf75698492f5289f04d78a4f316118b
+size 939524360

model-00026-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b4165145605c494cd4e5a4fa308fa84c71f7e8c15dea4be5f0ffb2f19bbf6c7
+size 771785544

model-00027-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcfc74bc80a6764eb013110c58bd9f0f1abcb3cac042f16523665ef6e4ffe59e
+size 939524360

model-00028-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0624ae666fd4565f32bff784ace74ba98215a5af37fdfabff8c4d88cd7c4092d
+size 771785536

model-00029-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea55f82707b1e34836f0fbc1f0ecdddbc6e2561185bc31b74106642ee4fe074e
+size 939524360

model-00030-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:907df9b2ca8531e952a3fa800e94747ecfaf2be2f189752c5b40c3d3eef84244
+size 771785544

model-00031-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e1619d14c217faf115cc0b89515e15130909a629aa2e69fbdfa54670745a165
+size 939524360

model-00032-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8615124de6eb3c885b5a0057e8948406d531761733f578eba6e5929b1133b6a2
+size 771785544

model-00033-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5ab23c5f13ac44ed6519ebad89790962bb914b286bff361f89a44997d0b0fd4
+size 939524360

model-00034-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9e95484b1616fdae440195f8bca15d83b4353f94f8b52165a2fa0c1fb4d7ead
+size 771785544

model-00035-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4ff06fbe7b1d43fc1568395708fe465d222bcf43d086df1f80a7ac552dce200
+size 939524360

model-00036-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8434942d721fdd937911f29a3b94c8e26d938605f40a55efc10bea25b6168193
+size 771785544

model-00037-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dbefb53e747383aa4f943f8a8ab1fbd25e6ebca54d0635c8f70410811b01fdf
+size 939524360

model-00038-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4839d7ef6aaac0e5248bd8f11a697ed3b8735aed66059b14fe25bda2e2da5882
+size 771785544

model-00039-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e56eabdd8ea2dacf4a124f39a5e23fabd1f633d051428a8d56e6b72e9a1e3c64
+size 939524360

model-00040-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f43d160abc9ffa714953559e8b9870129aaf9ece559563af5948c771cd04f507
+size 771785544

model-00041-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7230e205a1a5b8c92ef2345db1fe0134362487090e1e7692e5ebf91e55ee9643
+size 939524360

model-00042-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b80c5aa35f8365c0c6997e9866f4cb6444c0bbdf3918af0fa1dd34a3cea9707a
+size 771785544

model-00043-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a938e3df835647b5db7b649aecb79c5a7d6c0532bcb3d09943f4951ef33936c9
+size 939524360

model-00044-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7711380558062c81d46970101a4ac6bbfd7ec249f099086100b322ac9b57263c
+size 771785544

model-00045-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7296458946d4dc8da9871106db319a6a0b5ec8313b0afffabb7151904badfe54
+size 939524360

model-00046-of-00142.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a67bee016801613f8e9d04e759e0295eb34c9055004046304f372c5813d4a0b8
+size 771785544