junnei commited on
Commit
eb143e4
·
verified ·
1 Parent(s): 950a33d

test tokenizer config

Browse files
configuration_gemma3mm.py CHANGED
@@ -1,6 +1,6 @@
1
  from typing import Optional
2
 
3
- from transformers import Gemma3TextConfig
4
  from transformers.configuration_utils import PretrainedConfig
5
  from transformers.modeling_rope_utils import rope_config_validation
6
  from transformers.utils import logging
@@ -203,4 +203,8 @@ class Gemma3MMConfig(PretrainedConfig):
203
  self.audio_token_index = audio_token_index
204
  self.initializer_range = initializer_range
205
 
206
- super().__init__(**kwargs)
 
 
 
 
 
1
  from typing import Optional
2
 
3
+ from transformers import AutoConfig, Gemma3TextConfig
4
  from transformers.configuration_utils import PretrainedConfig
5
  from transformers.modeling_rope_utils import rope_config_validation
6
  from transformers.utils import logging
 
203
  self.audio_token_index = audio_token_index
204
  self.initializer_range = initializer_range
205
 
206
+ super().__init__(**kwargs)
207
+
208
+
209
+ AutoConfig.register("gemma3mm", Gemma3MMConfig)
210
+ Gemma3MMConfig.register_for_auto_class()
tokenizer_config.json CHANGED
@@ -51343,7 +51343,7 @@
51343
  "image_token": "<image_soft_token>",
51344
  "model_max_length": 1000000000000000019884624838656,
51345
  "pad_token": "<pad>",
51346
- "processor_class": "Gemma3Processor",
51347
  "sp_model_kwargs": null,
51348
  "spaces_between_special_tokens": false,
51349
  "tokenizer_class": "GemmaTokenizerFast",
 
51343
  "image_token": "<image_soft_token>",
51344
  "model_max_length": 1000000000000000019884624838656,
51345
  "pad_token": "<pad>",
51346
+ "processor_class": "Gemma3MMProcessor",
51347
  "sp_model_kwargs": null,
51348
  "spaces_between_special_tokens": false,
51349
  "tokenizer_class": "GemmaTokenizerFast",