used nltk

app.py

@@ -2,8 +2,17 @@ import streamlit as st
 import PyPDF2
 import io
 import os
+import re
+import nltk
+from nltk.corpus import words
 
+# Download the words corpus if not already downloaded
+nltk.download('words')
 
+# Create a set of English words for quick lookup
+english_words_set = set(words.words())
+
+# Your existing mappings
 unicodeatoz = ["ब", "द", "अ", "म", "भ", "ा", "न", "ज", "ष्", "व", "प", "ि", "फ", "ल", "य", "उ", "त्र", "च", "क", "त", "ग", "ख", "ध", "ह", "थ", "श"]
 unicodeAtoZ = ["ब्", "ध", "ऋ", "म्", "भ्", "ँ", "न्", "ज्", "क्ष्", "व्", "प्", "ी", "ः", "ल्", "इ", "ए", "त्त", "च्", "क्", "त्", "ग्", "ख्", "ध्", "ह्", "थ्", "श्"]
 unicode0to9 = ["ण्", "ज्ञ", "द्द", "घ", "द्ध", "छ", "ट", "ठ", "ड", "ढ"]
@@ -81,22 +90,43 @@ def normalizePreeti(preetitxt):
             previoussymbol = ''
     return normalized
 
+def is_english_word(word):
+    # Remove punctuation and convert to lowercase
+    word_clean = re.sub(r'\W+', '', word).lower()
+    return word_clean in english_words_set
+
 def convert(preeti):
     converted = ''
     normalizedpreeti = normalizePreeti(preeti)
-    for index, character in enumerate(normalizedpreeti):
-        try:
-            if ord(character) >= 97 and ord(character) <= 122:
-                converted += unicodeatoz[ord(character) - 97]
-            elif ord(character) >= 65 and ord(character) <= 90:
-                converted += unicodeAtoZ[ord(character) - 65]
-            elif ord(character) >= 48 and ord(character) <= 57:
-                converted += unicode0to9[ord(character) - 48]
+    
+    # Split the text into tokens (words and non-words)
+    tokens = re.findall(r'\w+|\W+', normalizedpreeti)
+    
+    for token in tokens:
+        if re.match(r'\w+', token):
+            # This is a word
+            if is_english_word(token):
+                # English word, skip conversion
+                converted += token
             else:
-                converted += symbolsDict[character]
-        except KeyError:
-            converted += character
-
+                # Convert the word
+                converted_word = ''
+                for index, character in enumerate(token):
+                    try:
+                        if ord(character) >= 97 and ord(character) <= 122:
+                            converted_word += unicodeatoz[ord(character) - 97]
+                        elif ord(character) >= 65 and ord(character) <= 90:
+                            converted_word += unicodeAtoZ[ord(character) - 65]
+                        elif ord(character) >= 48 and ord(character) <= 57:
+                            converted_word += unicode0to9[ord(character) - 48]
+                        else:
+                            converted_word += symbolsDict[character]
+                    except KeyError:
+                        converted_word += character
+                converted += converted_word
+        else:
+            # Non-word token (punctuation, whitespace)
+            converted += token
     return converted
 
 def extract_text_from_pdf(pdf_file):
@@ -149,4 +179,4 @@ def main():
         )
 
 if __name__ == "__main__":
-    main()
+    main()