Upload 91 files

Modelfile

@@ -0,0 +1,37 @@
+FROM huihui-ai/DeepSeek-V3-0324-Pruned-Coder-411B/ggml-model-Q4_K_M.gguf
+TEMPLATE """{{- if .System }}{{ .System }}{{ end }}
+{{- range $i, $_ := .Messages }}
+{{- $last := eq (len (slice $.Messages $i)) 1}}
+{{- if eq .Role "user" }}<｜User｜>{{ .Content }}
+{{- else if eq .Role "assistant" }}<｜Assistant｜>{{ .Content }}{{- if not $last }}<｜end▁of▁sentence｜>{{- end }}
+{{- end }}
+{{- if and $last (ne .Role "assistant") }}<｜Assistant｜>{{- end }}
+{{- end }}"""
+PARAMETER stop <｜begin▁of▁sentence｜>
+PARAMETER stop <｜end▁of▁sentence｜>
+PARAMETER stop <｜User｜>
+PARAMETER stop <｜Assistant｜>
+PARAMETER num_gpu 1
+LICENSE """MIT License
+
+Copyright (c) 2023 DeepSeek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+

README.md

@@ -1,3 +1,181 @@
----
-license: apache-2.0
----
+---
+license: mit
+library_name: transformers
+base_model:
+- deepseek-ai/DeepSeek-R1
+tags:
+- deepseek_R1
+- bf16
+- Safetensors
+- custom_code
+- Pruned
+---
+
+# huihui-ai/DeepSeek-V3-0324-Pruned-Coder-411B
+
+
+
+
+This is a pruned version of the [deepseek-ai/DeepSeek-V3-0324](https://huggingface.co/deepseek-ai/DeepSeek-V3-0324), 
+reduced from 256 experts to 160 experts. The pruned model is mainly used for [code](https://huggingface.co/huihui-ai/DeepSeek-V3-0324-Pruned-Coder-411B/blob/main/coding_problems.py) generation.
+
+
+This is a test validation to see if we can prune the model according to professional requirements and still maintain acceptable performance. 
+The model size has been reduced by about 1/3, and no distortion has occurred.
+
+This allows the model to be pruned according to one's needs.
+
+This pruned model has a total parameter is equivalent to 441B.
+
+
+
+
+## Use with transformers
+
+```
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, TextStreamer
+import torch
+import os
+import signal
+
+cpu_count = os.cpu_count()
+print(f"Number of CPU cores in the system: {cpu_count}")
+half_cpu_count = cpu_count // 2
+os.environ["MKL_NUM_THREADS"] = str(half_cpu_count)
+os.environ["OMP_NUM_THREADS"] = str(half_cpu_count)
+torch.set_num_threads(half_cpu_count)
+
+print(f"PyTorch threads: {torch.get_num_threads()}")
+print(f"MKL threads: {os.getenv('MKL_NUM_THREADS')}")
+print(f"OMP threads: {os.getenv('OMP_NUM_THREADS')}")
+
+NEW_MODEL_ID = "huihui-ai/DeepSeek-V3-0324-Pruned-Coder-411B"
+print(f"Load Model {NEW_MODEL_ID} ... ")
+quant_config_4 = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.bfloat16,
+    bnb_4bit_use_double_quant=True,
+    llm_int8_enable_fp32_cpu_offload=True,
+)
+
+# Single RTX 4090
+NUM_TRANS_LAYERS = 61
+
+def create_device_map():
+    device_map = {
+        'model.embed_tokens': 0,
+        'model.norm': 0,
+        'model.rotary_emb': 0,
+        'lm_head': 0
+    }
+    for start, end, gpu_id in [(0, 5, 0)]:
+        for i in range(start, end):
+            device_map[f'model.layers.{i}'] = gpu_id
+    
+    for i in range(5, NUM_TRANS_LAYERS):
+        device_map[f'model.layers.{i}'] = "cpu"
+
+    return device_map
+
+device_map = create_device_map()
+
+model = AutoModelForCausalLM.from_pretrained(
+    NEW_MODEL_ID,
+    device_map=device_map,
+    trust_remote_code=True,
+    quantization_config=quant_config_4,
+    torch_dtype=torch.bfloat16
+)
+tokenizer = AutoTokenizer.from_pretrained(NEW_MODEL_ID, trust_remote_code=True)
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
+tokenizer.pad_token_id = tokenizer.eos_token_id
+
+initial_messages = [{"role": "system", "content": "You are a helpful assistant."}]
+messages = initial_messages.copy()
+
+class CustomTextStreamer(TextStreamer):
+    def __init__(self, tokenizer, skip_prompt=True, skip_special_tokens=True):
+        super().__init__(tokenizer, skip_prompt=skip_prompt, skip_special_tokens=skip_special_tokens)
+        self.generated_text = ""
+        self.stop_flag = False
+
+    def on_finalized_text(self, text: str, stream_end: bool = False):
+        self.generated_text += text
+        print(text, end="", flush=True)
+        if self.stop_flag:
+            raise StopIteration
+
+    def stop_generation(self):
+        self.stop_flag = True
+
+def generate_stream(model, tokenizer, messages, max_new_tokens):
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        tokenize=True,
+        add_generation_prompt=True,
+        return_tensors="pt"
+    )
+    attention_mask = torch.ones_like(input_ids, dtype=torch.long)
+    tokens = input_ids.to(model.device) 
+    attention_mask = attention_mask.to(model.device)
+
+    streamer = CustomTextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+    def signal_handler(sig, frame):
+        streamer.stop_generation()
+        print("\n[Generation stopped by user with Ctrl+C]")
+
+    signal.signal(signal.SIGINT, signal_handler)
+    
+    print("Response: ", end="", flush=True)
+    try:
+        generated_ids = model.generate(
+            tokens,
+            attention_mask=attention_mask,
+            use_cache=False,
+            max_new_tokens=max_new_tokens,
+            do_sample=True,
+            pad_token_id=tokenizer.pad_token_id,
+            streamer=streamer
+        )
+        del generated_ids
+    except StopIteration:
+        print("\n[Stopped by user]")
+
+    del input_ids, attention_mask
+    torch.cuda.empty_cache()
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+    return streamer.generated_text, streamer.stop_flag
+
+while True:
+    user_input = input("User: ").strip()
+    if user_input.lower() == "/exit":
+        print("Exiting chat.")
+        break
+    if user_input.lower() == "/clear":
+        messages = initial_messages.copy()
+        print("Chat history cleared. Starting a new conversation.")
+        continue
+    if not user_input:
+        print("Input cannot be empty. Please enter something.")
+        continue
+    messages.append({"role": "user", "content": user_input})
+    response, stop_flag = generate_stream(model, tokenizer, messages, 8192)
+    if stop_flag:
+        continue
+    messages.append({"role": "assistant", "content": response})
+
+```
+
+### Donation
+
+If you like it, please click 'like' and follow us for more updates.  
+You can follow [x.com/support_huihui](https://x.com/support_huihui) to get the latest model information from huihui.ai.
+
+##### Your donation helps us continue our further development and improvement, a cup of coffee can do it.
+- bitcoin:
+```
+  bc1qqnkhuchxw0zqjh2ku3lu4hq45hc6gy84uk70ge
+```

model-00087-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd5ecc124c0226f8aa7de700537cf75284ef8a39139e37927ebaff8d44dfbe56
+size 4991243024

model-00088-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c14799840b307b44532f50382a62638eab8eb05fb0305035c5df5722e573e55f
+size 4991243096

model-00089-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86bf63bd5c00e34e2c97ed1fd63f96b4e579d010cf807883157e966bf55a5e21
+size 4986098536

model-00090-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02d68203647047e08dc6e037a842090671e27b9d8a03a53c43ace1ba1f865e07
+size 4991243024

model-00091-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:541cba7a7617fec4c5c23a7612938efbb72b0e02415733647b303b2c1a0adadd
+size 4991243112

model-00092-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4516c9d47fe5487e8499a953e3939bddadcef4f115c43671c5c5280db4250977
+size 4986098520

model-00093-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2ec23469f5a7ee17b5ae14a26016a22612ebbb8f65f5031cb0929d633cdcd93
+size 4991243032

model-00094-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecd4cc8a5e4b7f6d603cd4cbe24b082b38e0762079e63b185bc9420cfb768ab5
+size 4991243120

model-00095-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1a619f8f909185e150ca7ed27db115930f85dff2bbffecb56f7c3b13e8fa78f
+size 4986098512

model-00096-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97bd5cd0bacf3eed86184cf83d15e139310d0681908bbfb459c0885a6ec4fa1a
+size 4991243024

model-00097-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:478d2f43d9daf6e63bd283d5d71e45035d5f71ea33304279f5fbc023b8e0e467
+size 4991243136

model-00098-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d05651e0d6a9bb617a9278a65df07e0a4db2ddd2babccf50961c644e5d35f269
+size 4986098496

model-00099-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62c9e927cbb92065362b130bd53aba9d02aa5957467a040598450acc9947e913
+size 4991243024

model-00100-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fab39394ec3e714f24745570237154cc6a6df0cf629fd1748b3d75db8dae229
+size 4991243152

model-00101-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:131c4aa28ade57c358f4d1ec26825077415cb9ad3776190ba02df87273d17aea
+size 4986098480

model-00102-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65a1929754d7a564b42b6044e017e92534ca941e5360b3619562d971568c63e3
+size 4991243032

model-00103-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3687384fb4e40e7d2a949e9601bd2c5ddc26e14cf34db0070760ef8f667b992e
+size 4991243168

model-00104-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3dba8b4788dd86b37fcd398e98080cbaae3bf3c273966c6fcf84776e9cad16
+size 4986098472

model-00105-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bf01790c4638303dc21b949340fd242e0ef71a682b7e1d03e06f2868e84b4e2
+size 4991243024

model-00106-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70944e6b8a83b38a0a29e41960f1c3a4e3095b971b21c01f1133117001b91e2c
+size 4991243176

model-00107-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c97e1b6a55099036a40c23e7d9db6528de7a2f4937f10bd568e2d3cc9739310
+size 4986098448

model-00108-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c0fc9c95a7919349c56eb2b296165deb443919fadabcff44b3945a293ec3a6a
+size 4991243024

model-00109-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14e4a84a9da9af5df756420c7cc48b6979487bee1ba33e0b5905ce124cd9085f
+size 4991243192

model-00110-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31e81915ed6e45d2b7b62904cd01664fe822fec1f14a1eab9ff4a3adba7c08a5
+size 4986098432

model-00111-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3c5a4830ebaa34882407ce649d6bd531a24f91af41724fcebcd32a2655a2ebe
+size 4991243040

model-00112-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e08ebd0883ebd7ce4d9eeaf02111e5175565cdc4f54d8da61aa40801cebabc72
+size 4993537504

model-00113-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eafb3fcec02e51e5691c9e5ee44d99ff47a7913126c999283519d11d00809a5b
+size 4983804112

model-00114-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:607e02f436189b23ad9b135d2be3004226e1dde94ac9999a9c2e5d652b10c6d0
+size 4991243048

model-00115-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02dd7b8bdee8c2e03c547ea2fb3d2249ea15c983e101cdc33b52eeac1499db9a
+size 4809937976

model-00116-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a4190055fd56c86a3424e4d4f601e1144918bbe1353d2b0d105f7cbd045621d
+size 4991242112

model-00117-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efcccbfc1f1df015f0a3ceec8bcdf48adfc6be0fc6b5e37abe80f75751063d00
+size 4991243056

model-00118-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:547ba9466fda14164b6a90fbaa03383e039b927ae37435928aeb3ebfcf29f969
+size 4986098600

model-00119-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c1a19977aecbade99edcc76fe03d6122d939d234963c42e043056ba383dfafa
+size 4991243000

model-00120-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2050bc852bb49cf49f8895edd7d833647285d3d5491be5fe445bdfc39b1d6e5
+size 4991243072

model-00121-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0322837d7e2fc5e1d35573cdfb5fb7dfe0fd8e456e6ff6f1e5797fbd4caa87b7
+size 4986098568

model-00122-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b5fa9a03bcc077ea3d3c584074f39fa9a5136673bfc6b4b5902caaf97243565
+size 4991243016

model-00123-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81cee80fde9cc81ee46bda254792fe791613402bc1469ce30d87996ac4595198
+size 4991243088

model-00124-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9177f75f2ef1cfa8c7e4439533d4e5c5ad5d19849f9c42e0d31b4cb3c7bc82f1
+size 4986098544

model-00125-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc22f94bf712623196933cf846c7518b5d0306934074a0496047f5c2988ee63f
+size 4991243024

model-00126-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d549288e45f32c6b67a85bec973012121c8de0a12b5d47df0598f1880901d627
+size 4991243104

model-00127-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ef3d09d3fc5fbce96f427fcdeb13b55de093cef6f0da832c4b7e59ca8a21459
+size 4986098536

model-00128-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fa0120bdbc73b3f5c8ce7e4e44b5219c722102a6272b6051ff10b798f4e9ae9
+size 4991243024

model-00129-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4f6242c76b6621849d53f23cc461077dff95fd82699167b84fa6edfdec79f91
+size 4991243120

model-00130-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14cf361687630517025d4299722d3b4e851861c70b1122dfc77297b9079b185e
+size 4986098520

model-00131-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63e2b90d75f85f1c4e8df59326e2c1ca69fe4756c076295c87d83d37d3b7a891
+size 4991243032

model-00132-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19ef2fb7c69cba85eb615180b527345af3d6853159f5c77e5dfc7a0db2b868e7
+size 4991243128

model-00133-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58495d2edb9615f0d2ce36e06084f4242342ba00a3cd8e0df11a4d6728e08694
+size 4986098504

model-00134-of-00171.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f3cacec159d3821ff8e52fe323b31b324b8a958c17ca9977d413e83e052b7da
+size 4991243024