using spacy for option generation

app.py

@@ -24,9 +24,6 @@ nlp = spacy.load("en_core_web_sm")
 user_agent = 'QGen/1.0 ([email protected])'
 wiki_wiki = wikipediaapi.Wikipedia(user_agent= user_agent,language='en')
 
-# Load pre-trained word vectors (this may take a while)
-word_vectors = KeyedVectors.load_word2vec_format('vectors/GoogleNews-vectors-negative300.bin', binary=True)
-
 def load_model():
     model_name = "DevBM/t5-large-squad"
     model = T5ForConditionalGeneration.from_pretrained(model_name)
@@ -63,29 +60,23 @@ def extract_keywords(text):
     
     return list(combined_keywords)
 
-# Function to map keywords to sentences with customizable context window size
-def map_keywords_to_sentences(text, keywords, context_window_size):
-    sentences = sent_tokenize(text)
-    keyword_sentence_mapping = {}
-    for keyword in keywords:
-        for i, sentence in enumerate(sentences):
-            if keyword in sentence:
-                # Combine current sentence with surrounding sentences for context
-                start = max(0, i - context_window_size)
-                end = min(len(sentences), i + context_window_size + 1)
-                context = ' '.join(sentences[start:end])
-                if keyword not in keyword_sentence_mapping:
-                    keyword_sentence_mapping[keyword] = context
-                else:
-                    keyword_sentence_mapping[keyword] += ' ' + context
-    return keyword_sentence_mapping
+# Load spaCy model (medium-sized model with word vectors)
+nlp = spacy.load("en_core_web_md")
 
 def get_similar_words(word, n=3):
-    try:
-        similar_words = word_vectors.most_similar(word, topn=n)
-        return [word for word, _ in similar_words]
-    except KeyError:
-        return []
+    # Get the vector for the word
+    word_vector = nlp(word).vector
+    
+    # Find similar words
+    similar_words = []
+    for w in nlp.vocab:
+        if w.has_vector and w.is_lower and w.is_alpha and w.text != word:
+            similarity = nlp(w.text).similarity(nlp(word))
+            similar_words.append((w.text, similarity))
+    
+    # Sort by similarity and return top n
+    similar_words.sort(key=lambda x: x[1], reverse=True)
+    return [word for word, _ in similar_words[:n]]
 
 def get_synonyms(word, n=3):
     synonyms = []
@@ -100,7 +91,7 @@ def get_synonyms(word, n=3):
 def generate_options(answer, context, n=3):
     options = [answer]
     
-    # Try to get similar words based on word embeddings
+    # Try to get similar words based on word vectors
     similar_words = get_similar_words(answer, n)
     options.extend(similar_words)
     
@@ -128,6 +119,24 @@ def generate_options(answer, context, n=3):
     
     return options
 
+# Function to map keywords to sentences with customizable context window size
+def map_keywords_to_sentences(text, keywords, context_window_size):
+    sentences = sent_tokenize(text)
+    keyword_sentence_mapping = {}
+    for keyword in keywords:
+        for i, sentence in enumerate(sentences):
+            if keyword in sentence:
+                # Combine current sentence with surrounding sentences for context
+                start = max(0, i - context_window_size)
+                end = min(len(sentences), i + context_window_size + 1)
+                context = ' '.join(sentences[start:end])
+                if keyword not in keyword_sentence_mapping:
+                    keyword_sentence_mapping[keyword] = context
+                else:
+                    keyword_sentence_mapping[keyword] += ' ' + context
+    return keyword_sentence_mapping
+
+
 # Function to perform entity linking using Wikipedia API
 @lru_cache(maxsize=128)
 def entity_linking(keyword):