Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -3,64 +3,69 @@ library_name: transformers
 pipeline_tag: text-generation
 inference: true
 widget:
-- text: Hello!
-  example_title: Hello world
-  group: Python
+  - text: Hello!
+    example_title: Hello world
+    group: Python
 ---
 
-This model is randomly initialized, using the config from [THUDM/glm-4-9b-chat](https://huggingface.co/THUDM/glm-4-9b-chat) but with smaller size. 
+This tiny model is for debugging. It is randomly initialized with the config adapted from [THUDM/GLM-4-32B-0414](https://huggingface.co/THUDM/GLM-4-32B-0414).
 
-Codes:
+### Example usage:
+
+```python
+from transformers import pipeline
+model_id = "yujiepan/glm-4-tiny-random"
+pipe = pipeline(
+    "text-generation", model=model_id, device="cuda",
+    trust_remote_code=True, max_new_tokens=20,
+)
+print(pipe("Hello World!"))
+```
+
+### Codes to create this repo:
 
 ```python
-import transformers
 import torch
-import os
-from huggingface_hub import create_repo, upload_folder
-import accelerate
 
-source_model_id = 'THUDM/glm-4-9b-chat'
-save_path = '/tmp/yujiepan/glm-4-tiny-random'
-repo_id = 'yujiepan/glm-4-tiny-random'
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    GenerationConfig,
+    pipeline,
+    set_seed,
+)
 
-os.system(f'rm -rf {save_path}')
+source_model_id = "THUDM/GLM-4-32B-0414"
+save_folder = "/tmp/yujiepan/glm-4-tiny-random"
 
-config = transformers.AutoConfig.from_pretrained(
-    source_model_id,
-    trust_remote_code=True,
+tokenizer = AutoTokenizer.from_pretrained(
+    source_model_id, trust_remote_code=True,
 )
-config._name_or_path = source_model_id
-config.hidden_size = 8
-config.ffn_hidden_size = 16
-config.kv_channels = 2
-config.num_attention_heads = 4
-config.multi_query_group_num = 2
-config.num_hidden_layers = 2
-config.num_layers = 2
+tokenizer.save_pretrained(save_folder)
 
-model = transformers.AutoModelForCausalLM.from_config(
+config = AutoConfig.from_pretrained(
+    source_model_id, trust_remote_code=True,
+)
+config.hidden_size = 16
+config.head_dim = 16
+config.intermediate_size = 32
+config.num_attention_heads = 1
+config.num_hidden_layers = 2
+config.num_key_value_heads = 1
+config.tie_word_embeddings = False
+model = AutoModelForCausalLM.from_config(
     config,
+    torch_dtype=torch.bfloat16,
     trust_remote_code=True,
 )
-model.generation_config = transformers.GenerationConfig.from_pretrained(source_model_id)
-model = model.to(torch.bfloat16)
-
-with torch.no_grad():
-    for p in model.parameters():
-        torch.nn.init.normal_(p)
-
-model.save_pretrained(save_path)
-
-tokenizer = transformers.AutoTokenizer.from_pretrained(
-    source_model_id,
-    trust_remote_code=True,
+model.generation_config = GenerationConfig.from_pretrained(
+    source_model_id, trust_remote_code=True,
 )
-tokenizer.save_pretrained(save_path)
-
-output = model.float().generate(torch.tensor([[1, 2, 3]]).long(), max_length=16, do_sample=True)
-
-os.system(f'ls -alh {save_path}')
-# os.system(f'rm -rf {save_path}/model.safetensors')
-create_repo(repo_id, exist_ok=True)
-upload_folder(repo_id=repo_id, folder_path=save_path)
+set_seed(42)
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        torch.nn.init.normal_(p, 0, 0.5)
+        print(name, p.shape)
+model.save_pretrained(save_folder)
 ```

chat_template.jinja

@@ -0,0 +1,41 @@
+[gMASK]<sop>
+{%- if tools -%}
+<|system|>
+# 可用工具
+{% for tool in tools %}
+    {%- set function = tool.function if tool.get("function") else tool %}
+
+## {{ function.name }}
+
+{{ function | tojson(indent=4, ensure_ascii=False) }}
+在调用上述函数时，请使用 Json 格式表示调用的参数。
+{%- endfor %}
+{%- endif -%}
+
+{%- for msg in messages %}
+    {%- if msg.role == 'system' %}
+<|system|>
+{{ msg.content }}
+    {%- endif %}
+{%- endfor %}
+
+{%- for message in messages if message.role != 'system' %}
+    {%- set role = message['role'] %}
+    {%- set content = message['content'] %}
+    {%- set meta = message.get("metadata", "") %}
+
+    {%- if role == 'user' %}
+<|user|>
+{{ content }}
+    {%- elif role == 'assistant' and not meta %}
+<|assistant|>
+{{ content }}
+    {%- elif role == 'assistant' and meta %}
+<|assistant|>{{ meta }}
+{{ content }}
+    {%- elif role == 'observation' %}
+<|observation|>
+{{ content }}
+    {%- endif %}
+{%- endfor %}
+{% if add_generation_prompt %}<|assistant|>{% endif %}

config.json

@@ -1,50 +1,31 @@
 {
-  "_name_or_path": "THUDM/glm-4-9b-chat",
-  "add_bias_linear": false,
-  "add_qkv_bias": true,
-  "apply_query_key_layer_scaling": true,
-  "apply_residual_connection_post_layernorm": false,
   "architectures": [
-    "ChatGLMForConditionalGeneration"
+    "Glm4ForCausalLM"
   ],
+  "attention_bias": false,
   "attention_dropout": 0.0,
-  "attention_softmax_in_fp32": true,
-  "auto_map": {
-    "AutoConfig": "THUDM/glm-4-9b-chat--configuration_chatglm.ChatGLMConfig",
-    "AutoModel": "THUDM/glm-4-9b-chat--modeling_chatglm.ChatGLMForConditionalGeneration",
-    "AutoModelForCausalLM": "THUDM/glm-4-9b-chat--modeling_chatglm.ChatGLMForConditionalGeneration",
-    "AutoModelForSeq2SeqLM": "THUDM/glm-4-9b-chat--modeling_chatglm.ChatGLMForConditionalGeneration",
-    "AutoModelForSequenceClassification": "THUDM/glm-4-9b-chat--modeling_chatglm.ChatGLMForSequenceClassification"
-  },
-  "bias_dropout_fusion": true,
-  "classifier_dropout": null,
   "eos_token_id": [
     151329,
     151336,
     151338
   ],
-  "ffn_hidden_size": 16,
-  "fp32_residual_connection": false,
-  "hidden_dropout": 0.0,
-  "hidden_size": 8,
-  "kv_channels": 2,
-  "layernorm_epsilon": 1.5625e-07,
-  "model_type": "chatglm",
-  "multi_query_attention": true,
-  "multi_query_group_num": 2,
-  "num_attention_heads": 4,
+  "head_dim": 16,
+  "hidden_act": "silu",
+  "hidden_size": 16,
+  "initializer_range": 0.02,
+  "intermediate_size": 32,
+  "max_position_embeddings": 32768,
+  "model_type": "glm4",
+  "num_attention_heads": 1,
   "num_hidden_layers": 2,
-  "num_layers": 2,
-  "original_rope": true,
+  "num_key_value_heads": 1,
   "pad_token_id": 151329,
-  "padded_vocab_size": 151552,
-  "post_layer_norm": true,
-  "rmsnorm": true,
-  "rope_ratio": 500,
-  "seq_length": 131072,
+  "partial_rotary_factor": 0.5,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.38.2",
+  "transformers_version": "4.52.0.dev0",
   "use_cache": true,
   "vocab_size": 151552
 }

generation_config.json

@@ -1,13 +1,10 @@
 {
-  "do_sample": true,
+  "_from_model_config": true,
   "eos_token_id": [
     151329,
     151336,
     151338
   ],
-  "max_length": 128000,
   "pad_token_id": 151329,
-  "temperature": 0.8,
-  "top_p": 0.8,
-  "transformers_version": "4.38.2"
+  "transformers_version": "4.52.0.dev0"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7fa45eed75dcb74cd011870d716fb1a7438ee8246aaf58b9999e025c5b3bfd5f
-size 4854314
+oid sha256:e34426f6099c2c1c6b73631a4cdc6c9f08f744f5a7ebca24d87c1c9b1c71cef2
+size 9712304

special_tokens_map.json

@@ -16,7 +16,7 @@
     "<|end_of_video|>"
   ],
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "<|user|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76ebeac0d8bd7879ead7b43c16b44981f277e47225de2bd7de9ae1a6cc664a8c
+size 19966496

tokenizer_config.json

@@ -129,18 +129,17 @@
     "<|begin_of_video|>",
     "<|end_of_video|>"
   ],
-  "auto_map": {
-    "AutoTokenizer": [
-      "THUDM/glm-4-9b-chat--tokenization_chatglm.ChatGLM4Tokenizer",
-      null
-    ]
-  },
   "clean_up_tokenization_spaces": false,
   "do_lower_case": false,
-  "eos_token": "<|endoftext|>",
+  "eos_token": "<|user|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
   "model_max_length": 128000,
   "pad_token": "<|endoftext|>",
   "padding_side": "left",
   "remove_space": false,
-  "tokenizer_class": "ChatGLM4Tokenizer"
+  "tokenizer_class": "PreTrainedTokenizer"
 }