Upload 5 files

create_dataset_splits.py

@@ -4,13 +4,9 @@ import tensorflow.data as tf_data
 import pickle
 import random
 
-
-
-
 def read_files(path):
     with open(path, "r", encoding="utf-8") as f:
         dataset_split = f.read().split("\n")[:-1]
-    #to lowercase, idk why
     dataset_split = [line.lower() for line in dataset_split]
     return dataset_split
 
@@ -18,8 +14,6 @@ def save_list_to_file(file_path, string_list):
     with open(file_path, 'w') as file:
         file.writelines([f"{string}\n" for string in string_list])
 
-
-#EUROPARL cs-en
 #load files
 cs_file = 'datasets/europarl/europarl-v7.cs-en.cs'
 en_file = 'datasets/europarl/europarl-v7.cs-en.en'

eval.py

@@ -2,18 +2,17 @@
 import keras_nlp
 import keras
 import tensorflow.data as tf_data
-import pickle
 import tensorflow as tf
 from tensorflow.keras.optimizers import Adam
 from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
-import datetime
-import random
 import re
 from sacrebleu.metrics import CHRF
 import time
-# from keras import ops
-#hyperparameters
+
+
 MAX_SEQUENCE_LENGTH = 64
+eval_samples = 100
+
 
 transformer = keras.models.load_model('models_europarl/en_cs_translator_saved_20231209_0046.keras')
 def read_files(path, lowercase = False):
@@ -46,7 +45,6 @@ def next_token_logits(encoder_input_tokens, prompt, predicted_token_index):
 
 
 def greedy_decode(encoder_input_tokens, prompt, end_token_id):
-    
     start_index = 1
     current_prompt = prompt
     for predicted_token_index in range(start_index, MAX_SEQUENCE_LENGTH):
@@ -152,7 +150,7 @@ bleu_metrics = keras_nlp.metrics.Bleu(
     tokenizer = cs_tokenizer
 )
 
-eval_samples = 100
+
 chrf = CHRF() 
 refs = test_cs[:eval_samples]
 translations = []

preprocess_dataset.py

@@ -12,13 +12,10 @@ MAX_SEQUENCE_LENGTH = 64
 def read_files(path, lowercase = False):
     with open(path, "r", encoding="utf-8") as f:
         dataset_split = f.read().split("\n")[:-1]
-    #to lowercase, idk why
     if(lowercase):
         dataset_split = [line.lower() for line in dataset_split]
     return dataset_split
 
-# en_vocab = read_files("tokenizers/en_opus_vocab")
-# cs_vocab = read_files("tokenizers/cs_opus_vocab")
 en_vocab = read_files("tokenizers/en_europarl_vocab")
 cs_vocab = read_files("tokenizers/cs_europarl_vocab")
 
@@ -32,42 +29,28 @@ cs_tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(
 )
 
 
-#opus
-# train_cs_file = 'datasets/cs-en/opus.cs-en-train.cs'
-# train_en_file = 'datasets/cs-en/opus.cs-en-train.en'
-# valid_cs_file = 'datasets/cs-en/opus.cs-en-dev.cs'
-# valid_en_file = 'datasets/cs-en/opus.cs-en-dev.en'
-# test_cs_file = 'datasets/cs-en/opus.cs-en-test.cs'
-# test_en_file = 'datasets/cs-en/opus.cs-en-test.en'
-
 #europarl
 train_cs_file = 'datasets/europarl/train-cs-en.cs'
 train_en_file = 'datasets/europarl/train-cs-en.en'
 valid_cs_file = 'datasets/europarl/valid-cs-en.cs'
 valid_en_file = 'datasets/europarl/valid-cs-en.en'
-test_cs_file = 'datasets/europarl/test-cs-en.cs'
-test_en_file = 'datasets/europarl/test-cs-en.en'
 
 
 train_cs = read_files(train_cs_file, True)
 train_en = read_files(train_en_file, True)
 valid_cs = read_files(valid_cs_file, True)
 valid_en = read_files(valid_en_file, True)
-test_cs = read_files(test_cs_file, True)
-test_en = read_files(test_en_file, True)
 
 def preprocess_batch(en, cs):
     en = en_tokenizer(en)
     cs = cs_tokenizer(cs)
 
-    # Pad `eng` to `MAX_SEQUENCE_LENGTH`.
     en_start_end_packer = keras_nlp.layers.StartEndPacker(
         sequence_length=MAX_SEQUENCE_LENGTH,
         pad_value=en_tokenizer.token_to_id("[PAD]"),
     )
     en = en_start_end_packer(en)
 
-    # Add special tokens (`"[START]"` and `"[END]"`) to `spa` and pad it as well.
     cs_start_end_packer = keras_nlp.layers.StartEndPacker(
         sequence_length=MAX_SEQUENCE_LENGTH + 1,
         start_value=cs_tokenizer.token_to_id("[START]"),

train_tokenizers.py

@@ -4,13 +4,9 @@ import tensorflow.data as tf_data
 import pickle
 import random
 
-
 EN_VOCAB_SIZE = 30000
 CS_VOCAB_SIZE = 30000
 
-
-
-
 def train_word_piece(text_samples, vocab_size, reserved_tokens, save_output_path):
     word_piece_ds = tf_data.Dataset.from_tensor_slices(text_samples)
     vocab = keras_nlp.tokenizers.compute_word_piece_vocabulary(