Upload 5 files

eval.py

@@ -20,7 +20,6 @@ transformer = keras.models.load_model('models_europarl/en_cs_translator_saved_20
 def read_files(path, lowercase = False):
     with open(path, "r", encoding="utf-8") as f:
         dataset_split = f.read().split("\n")[:-1]
-    #to lowercase, idk why
     if(lowercase):
         dataset_split = [line.lower() for line in dataset_split]
     return dataset_split

preprocess_dataset.py

@@ -1,14 +1,10 @@
 
 import keras_nlp
-import keras
 import tensorflow.data as tf_data
-import pickle
 #hyperparameters
 BATCH_SIZE = 16
 MAX_SEQUENCE_LENGTH = 64
 
-#load tokenizers/en_vocab to list
-
 def read_files(path, lowercase = False):
     with open(path, "r", encoding="utf-8") as f:
         dataset_split = f.read().split("\n")[:-1]

train.py

@@ -2,7 +2,6 @@
 import keras_nlp
 import keras
 import tensorflow.data as tf_data
-import pickle
 from tensorflow.keras.optimizers import Adam
 from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
 import datetime
@@ -13,7 +12,6 @@ EPOCHS = 20
 EMBED_DIM = 256
 INTERMEDIATE_DIM = 2048
 NUM_HEADS = 8
-# TODO probably change dynamically
 MAX_SEQUENCE_LENGTH = 128
 EN_VOCAB_SIZE = 30000
 CS_VOCAB_SIZE = 30000

train_tokenizers.py

@@ -1,8 +1,5 @@
 import keras_nlp
-import keras
 import tensorflow.data as tf_data
-import pickle
-import random
 
 EN_VOCAB_SIZE = 30000
 CS_VOCAB_SIZE = 30000
@@ -20,27 +17,12 @@ def train_word_piece(text_samples, vocab_size, reserved_tokens, save_output_path
 def read_files(path):
     with open(path, "r", encoding="utf-8") as f:
         dataset_split = f.read().split("\n")[:-1]
-    #to lowercase, idk why
     dataset_split = [line.lower() for line in dataset_split]
     return dataset_split
 
-#OPUS cs-en
-# train_cs = read_files('datasets/cs-en/opus.cs-en-train.cs')
-# train_en = read_files('datasets/cs-en/opus.cs-en-train.en')
-
-
-#EUROPARL cs-en
 train_cs = read_files('datasets/europarl/train-cs-en.cs')
 train_en = read_files('datasets/europarl/train-cs-en.en')
-
-
-
-print(train_cs[0])
-print(train_en[0])
-
-
 reserved_tokens = ["[PAD]", "[UNK]", "[START]", "[END]"]
-
 en_vocab = train_word_piece(train_en, EN_VOCAB_SIZE, reserved_tokens, "tokenizers/en_europarl_vocab")
 cs_vocab = train_word_piece(train_cs, CS_VOCAB_SIZE, reserved_tokens, "tokenizers/cs_europarl_vocab")