Upload tokenizer

isoformer_tokenizer.py

@@ -36,7 +36,6 @@ class IsoformerTokenizer(PreTrainedTokenizer):
 
     def __init__(
         self,
-        config,
         **kwargs
     ):
 
@@ -55,9 +54,6 @@ class IsoformerTokenizer(PreTrainedTokenizer):
         # protein_hf_tokenizer.eos_token = None  # Stops the tokenizer adding an EOS/SEP token at the end
         # protein_hf_tokenizer.init_kwargs["eos_token"] = None  # Ensures it doesn't come back when reloading
 
-        self.num_tokens_per_seq_nuctf = config.num_tokens_per_seq_nuctf
-        self.num_tokens_per_seq_nuctf_rna = config.num_tokens_per_seq_nuctf_rna
-        self.num_protein_tokens_per_seq = config.num_protein_tokens_per_seq
         self.dna_tokenizer = dna_hf_tokenizer
         self.rna_tokenizer = rna_hf_tokenizer
         self.protein_tokenizer = protein_hf_tokenizer
@@ -65,12 +61,11 @@ class IsoformerTokenizer(PreTrainedTokenizer):
         self.dna_tokens = open("dna_vocab_list.txt", "r").read() .split("\n")
         self.rna_tokens = open("rna_vocab_list.txt", "r").read() .split("\n")
         self.protein_tokens = open("protein_vocab_list.txt", "r").read() .split("\n")
-        self.config = config
 
         super().__init__(**kwargs)
 
     def __call__(self, dna_input, rna_input, protein_input):
-        dna_output = self.dna_tokenizer(dna_input) #, max_length=196608, padding="max_length")
+        dna_output = self.dna_tokenizer(dna_input)
         rna_output = self.rna_tokenizer(rna_input, max_length=1024, padding="max_length")
         protein_output = self.protein_tokenizer(protein_input, max_length=1024, padding="max_length")
         return dna_output, rna_output, protein_output