Update

summarize.py

@@ -99,71 +99,6 @@ def get_results(text, model_selected, num_beams, length_penalty,number_of_senten
             length_penalty=length_penalty,
             no_repeat_ngram_size = 3)[0]['generated_text']
         logger.info('auto-arabic-summarization')
-    
-    elif model_selected == 'BERT2BERT':
-
-        model_name="malmarjeh/bert2bert"
-        preprocessor = ArabertPreprocessor(model_name="")
-
-        tokenizer = AutoTokenizer.from_pretrained(model_name)
-        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
-        pipeline1 = pipeline("text2text-generation",model=model,tokenizer=tokenizer)
-        result = pipeline1(text,
-            pad_token_id= tokenizer.eos_token_id,
-            num_beams=num_beams,
-            repetition_penalty=3.0,
-            max_length=200,
-            length_penalty=length_penalty,
-            no_repeat_ngram_size = 3)[0]['generated_text']
-        logger.info('BERT2BERT')
-    
-    elif model_selected == "xlmroberta2xlmroberta":
-        model_name="ahmeddbahaa/xlmroberta2xlmroberta-finetune-summarization-ar"
-        preprocessor = ArabertPreprocessor(model_name="")
-
-        tokenizer = AutoTokenizer.from_pretrained(model_name)
-        model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
-        pipeline1 = pipeline("text2text-generation",model=model,tokenizer=tokenizer)
-        result = pipeline1(text,
-            pad_token_id= tokenizer.eos_token_id,
-            num_beams=num_beams,
-            repetition_penalty=3.0,
-            max_length=200,
-            length_penalty=length_penalty,
-            no_repeat_ngram_size = 3)[0]['generated_text']
-        logger.info('xlmroberta2xlmroberta')
-    
-    elif model_selected == "nltk_summarizer":
-        # number_of_sentence = 3
-        stopWords = set(nltk.corpus.stopwords.words("arabic") + nltk.corpus.stopwords.words("english"))
-        word_frequencies = {}  
-        for word in nltk.word_tokenize(text):  
-            if word not in stopWords:
-                if word not in punctuation:
-                    if word not in word_frequencies.keys():
-                        word_frequencies[word] = 1
-                    else:
-                        word_frequencies[word] += 1
-
-        maximum_frequncy = max(list(word_frequencies.values()),default=3)
-
-        for word in word_frequencies.keys():  
-            word_frequencies[word] = (word_frequencies[word]/maximum_frequncy)
-
-        sentence_list = nltk.sent_tokenize(text)
-        sentence_scores = {}
-        for sent in sentence_list:
-            for word in nltk.word_tokenize(sent.lower()):
-                if word in word_frequencies.keys():
-                    if len(sent.split(' ')) < 30:
-                        if sent not in sentence_scores.keys():
-                            sentence_scores[sent] = word_frequencies[word]
-                        else:
-                            sentence_scores[sent] += word_frequencies[word]
-
-        summary_sentences = heapq.nlargest(number_of_sentence, sentence_scores, key=sentence_scores.get)
-
-        result = ' '.join(summary_sentences)  
     else:
         result = "الرجاء اختيار نموذج"