test: add test cases for my_ghost_writer package

tests/__init__.py

@@ -0,0 +1,4 @@
+from my_ghost_writer.constants import root_folder
+
+
+EVENTS_FOLDER = root_folder / "tests" / "events"

tests/events/cleaned_words.json

@@ -0,0 +1,12 @@
+{
+    "words_tokens": [
+        "Hi", "there", "Mr", "", "Tom", "", "that", "'", "s", "a", "text", "", "I", "'", "m", "in", "need", "of", "",
+        "punctuation", "", "Can", "'", "t", "you", "give", "me", "", "stuff", "", "", "I", "think", "", "or", "I", "hope",
+        "", "also", "", "Do", "you", ""],
+    "offsets_tokens": [
+        [0, 2], [3, 8], [9, 11], [11, 12], [13, 16], [16, 17], [18, 22], [22, 23], [23, 24], [25, 26], [27, 31], [31, 32],
+        [35, 36], [36, 37], [37, 38], [39, 41], [42, 46], [47, 49], [50, 51], [51, 62], [62, 64], [65, 68], [68, 69],
+        [69, 70], [71, 74], [75, 79], [80, 82], [83, 84], [84, 89], [89, 90], [91, 92], [92, 93], [94, 99], [100, 101],
+        [102, 104], [105, 106], [107, 111], [112, 113], [114, 118], [118, 122], [125, 127], [128, 131], [131, 132]
+    ]
+}

tests/events/llm_generated_story_1.txt

@@ -0,0 +1,196 @@
+Once upon a time, in the kingdom of Elwoodia, there lived a young and wise young Princess named Isabella. She loved nothing more than to explore the world around her and experience new things. One day, she set off on an adventure to find out about the mysterious and enchanting forest beyond the village gate. 
+
+As she journeyed through the woods, Isabella stumbled upon a large clearing where she saw a small group of woodland creatures gathered around something. Curious, she went over to take a look. There she noticed a large, red apple lying across the ground. It was so unusual that she had never seen anything like it before; its shape was round but oval, and it was larger than any fruit she'd ever seen.
+
+The creatures surrounding the apple moved aside to let Isabella get a closer look. As she came closer, she saw a tiny fairy perched on top of the apple. The fairy flew to her and spoke in a soft voice.
+
+"Hello, my name is Narnia, and I am from the land of Narnia. What brings you here?" the fairy asked.
+
+Isabella smiled and replied, "I'm exploring the forest and discovered this beautiful place. What's going on here?"
+
+"Oh," the fairy said. "We're having a small gathering of fairies from all over the forest. We are here to celebrate the arrival of our new leader."
+
+Isabella was taken aback by the news. She hadn't realized that there was such a thing as a king for the creatures of the forest. She had heard of kings before, but she'd never met one.
+
+"What will become of the forest now?" she asked.
+
+The fairy looked at her and said, "Our new king will be arriving soon, and he needs someone to help him with his duties. Does anyone know of a young lady who could be a good fit for this role?
+
+Isabella thought about it for a moment. She knew she had no official title, but she was wise and strong. She knew she would be able to fulfill the responsibilities of the new King of the Forest.
+
+As Isabella accepted the invitation to become the next King of the Forest, Narnia smiled. He thanked her and told her what she needed to do to become the next King of the Forest.
+
+Isabella's journey in finding her new home had begun. She found an old tree near the river that seemed like a perfect spot to call her own. With every step, Isabella felt her heart beat in rhythm with her newfound power and responsibility. She felt her connection to the forest growing stronger with each passing day.
+
+One day, Isabella came across a young girl who seemed lost. The girl introduced herself as Laura, and she explained that she had lost her way while walking through the forest. Isabella knew exactly how she felt and offered to take Laura into her home.
+
+The two girls quickly became close, and Isabella soon learned that Laura was also the daughter of the old man who lived in the village. Isabella's heart ached for the girl, and she wanted to make sure she stayed safe.
+
+"If you'd ever need help, just let me know," Isabella said to Laura. "I will be here for you."
+
+The two girls were inseparable ever since they first met. But one day, when Isabella went out to collect wood, she heard a loud noise coming from the village. Isabella ran back to the village to find Laura had been kidnapped by a group of men.
+
+Isabella was determined to save her friend, so she set off on a mission to find them. She kept a close eye on the village and, eventually, found herself face-to-face with the kidnappers. Isabella knew she had to act fast if she wanted to save Laura.
+
+She used her knowledge of the village to her advantage. She was quick in her thinking and could come up with a plan in no time at all. She asked for the men's cooperation and, after the men agreed, Isabella proposed an agreement between them and her.
+
+She would allow the men to keep their freedom in exchange for their loyalty and protection. Isabella hoped this agreement would end the war between them and ensure their safety. The men agreed and, soon after, Isabella's plan to protect Laura was put into action.
+
+Isabella ensured Laura's safety by ensuring that her whereabouts were known to the people she needed. Isabella also made sure that Laura was never alone in the forest. Whenever she went out to collect wood, Isabella made it a habit to check on Laura's safety. 
+
+On one of these occasions, Isabella noticed something strange. While checking on Laura's safety, Isabella noticed that Laura had been eating a piece of fruit every morning before heading out. Isabella quickly realized that this strange behavior was evidence of Laura's mental and emotional well being.
+
+Isabella wondered why Laura seemed to be so attached to the piece of fruit. Was it because she enjoyed eating it? Was it because it provided her with sustenance? Or was it something else entirely? After some careful thought, Isabella concluded that the answer lay in Laura's mind, not her body.
+
+Isabella went to find Laura and asked her about her strange habit. Laura explained that the fruit gave her strength and courage. In other words, it provided her with the ability to fight for what she believed in. Isabella understood now: Laura's actions came from a place of love and loyalty. 
+
+Isabella realized that Laura's situation was in danger if she didn't do something soon. She decided to take matters into her own hands and called for a meeting at the village square the following day. At the meeting, Isabella presented her theory that Laura' behavior originated from her desire to protect the forest and its inhabitants. Everyone agreed that they needed to do whatever it took to save Laura.
+
+The next day, Isabella, Laura, and all the villagers set out to the forest with tools like shovels and shovel-like objects that were necessary to clear the way for new life. They also brought along a few things to combat fatigue, including oranges and bananas. The day went by quickly as the group worked together to clear away the trees. 
+
+At the end of the day, Isabella asked Laura about her eating habits. Laura had grown up eating the same thing every day, but she found that the new nutrition was giving her energy and strength. Isabella realized that Laura had been on a diet which was helping her to stay healthy and energized.
+
+The surprise came when Isabella noticed that Laura's diet was so different from everyone else’s. It seemed that Laura had been eating an apple or a piece of fruit each morning for years, without fail. Isabella couldn't help but marvel at how lucky Laura was to have such dedication to protecting her health.
+
+When the day came to an end, Isabella gave a speech about why she was so passionate about preserving the forest. She talked about the importance of taking care of our bodies, whether through diet or exercise, and reminded everyone about the importance of keeping their daily food intake regular. 
+
+When the dinner hour came, Isabella sat down with Laura to enjoy the meal that she had prepared. As they ate, Isabella noted that Laura's dish was more flavorful and had more variety than the other dishes served by others in the village. This showed Isabella that Laura cared deeply about her health. 
+
+After dinner, the two shared stories. Isabella enjoyed Laura's company and thanked her for being such a great companion. The two made plans to meet again soon, to catch up and share more stories. 
+
+The next day, Isabella felt confident that she had done her part in helping to protect the forest. She was grateful that she and Laura had been able to save the day. 
+
+Isabella, now feeling fulfilled with the work she accomplished, decided to spend her afternoon walking around the village. As she strolled through the streets, she noticed how clean and quiet everything looked. 
+
+She stopped in front of one of the old buildings that housed the local library. It was a brick building, surrounded by trees and grass, but it had an air of sophistication and grandeur. Isabella walked inside, and she instantly noticed something strange; the walls were covered with colorful murals and graffiti. 
+
+It seemed that someone had taken it upon themselves to paint these wonderful images. Isabella marveled at the creativity and skill involved. She felt inspired and wanted to do something similar. 
+
+Meanwhile, across town, Laura was checking out some of the local bookstores. As she perused the shelves, she saw a few books on interior design. She picked them up and began flipping through them, looking for ideas for her home. 
+
+Suddenly, there was a loud thud outside. Isabella rushed out and was shocked at what she saw. In the middle of the village square stood a giant painting of a tree, complete with leaves, branches, and bark. It looked as if someone had carefully painted each detail by hand. 
+
+Isabella recognized the artist right away. It was her neighbor Jake, who had been creating beautiful art pieces all over town for weeks. 
+
+The two women met in the middle of the street, and without saying anything, they both knew what the other was thinking. They both realized that their efforts had brought beauty to the world. 
+
+ Isabella had never thought much about painting or gardening before, but now she was passionate about it. 
+
+Laura smiled, feeling proud of her work. She had found inspiration from the books she'd read, and she felt that she had contributed something to the world. 
+
+As Isabella and Laura talked about their accomplishments, a third woman, named Sarah, walked up and asked what was going on. 
+
+After learning about their projects, Sarah asked if she could help. Isabella agreed since she was an experienced gardener, while Laura offered her knowledge of interior design. 
+
+The three friends spent the next few days working together, planting trees, spreading mulch, and creating murals. They discovered that they were able to accomplish more when they worked together, and that the community appreciated their hard work. 
+
+This project taught them that no matter how small your contribution is, you can still make a difference in the world. 
+
+When the sun began setting, Isabella, Laura, and Sarah sat back and admired the beautiful work they had created. The trees were planted, the walls were covered with colorful art, and the community had come together. Isabella asked Laura and Sarah to be her gardening and interior design experts because she had learned so much. 
+
+Looking around at the trees and the artwork, Laura said, "We created something special here today." 
+
+Sarah nodded, knowing that they had accomplished great things by working together. 
+
+From that day forward, Isabella, Laura, and Sarah stayed connected and used their talents to help the community grow. In fact they had planted 50 trees in only one week, as this was just the beginning of their journey.
+
+Two years later a traveller arrived within Isabella's village asking for help. She explained that her village had been suffering from an epidemic, and needed someone to save them. 
+
+Isabella, Laura, and Sarah remembered what happened two years ago when they worked together on their tree-planting project and agreed to come up with a solution. 
+
+They thought long and hard about how to solve the epidemic problem, then finally came up with a plan. 
+
+The three friends decided to use their skills to build a hospital to cure those who had fallen ill from the epidemic. 
+
+Laura offered to design the hospital's interiors while Isabella would take care of the landscaping. 
+
+Sarah volunteered to coordinate the building and plant flowers in the surrounding area. 
+
+Once the hospital was complete, Laura stepped inside, looked around, and exclaimed ”I can't believe we did it!” 
+
+Isabella nodded, "When you have two people working towards one goal you can accomplish great things. I'm glad our connection has helped us to save the village."
+
+She could not contain her excitement and she smiled from ear to ear.
+
+"We've achieved something amazing," she said. 
+
+"I'm so happy we were able to make such a difference," Laura replied. 
+
+
+The three friends went back home, feeling proud of themselves for helping to save the village.
+
+As they watched the sun set behind the mountains, Isla asked Laura,  "What have you learned from this experience?" 
+
+Laura took a moment to think before she answered. 
+
+"I' m grateful that we had each other and that we worked together like this, with two people in the same room can achieve so much," she said. 
+
+With one last look at the sky, they walked back home with a newfound connection.
+
+Isabella's eyes lit up as she looked around her village. She smiled and pointed out her home. 
+
+She quickly explained to the group how Isabella and her family had been living in the village for generations. The rest of the villagers knew Isabella as a very caring and kind person. They were always willing to help those in need. 
+
+Isabella led her group of close friends inside and told them about her family's history. 
+
+She shared stories of Isabella's grandmother, who had planted wildflowers all over the village. Isabella's grandfather had built bridges between the houses and provided food to those in need. 
+
+At first, everyone was confused and didn't understand what Isabella had just described. But then, Isabella revealed, "We are a village of heroes!" 
+
+The group gasped and stood up. Everyone was taken aback by the announcement. 
+
+As a surprise to the others, Isabella continued, "We're not here to tell stories. We're here to help. I'm asking if anyone wants to join me in planting flowers in the nearby area." 
+
+The group looked at each other and nodded their heads in agreement. They all wanted to help the village and the environment.
+
+After they collected the necessary tools, Isabella and her friends set off on their mission. Isabella and her friends picked wildflowers from the fields and planted them alongside the river that ran near their village. 
+
+They carefully watered and cared for the flowers until they bloomed. With the help of the locals, Isabella and her friends were able to beautify the entire village. 
+
+When Isabella' group returned to the village, they were welcomed with joy and laughter. 
+
+As the days passed by, Isabella noticed something strange growing in the nearby forest. She couldn't quite place what it was, so she decided to take a closer look. After careful investigation, Isabella discovered what was causing the mystery - a small, yet powerful plant called the "magic leaf". 
+
+Isabella knew that this mysterious plant was one of nature's greatest gifts, as it had the ability to provide new life to any living organism around it. She quickly realized that this plant could be the answer to many of the world's problems. 
+
+Excited by the discovery, Isabella set out on her own mission. She gathered a team of scientists, biologists, and botanists to join her in studying the magic leaf and its effects on the environment. 
+
+Together, they observed how the leaf interacted with its surroundings and determined its effects on the local ecosystem. They studied how the leaves reacted when exposed to different conditions and worked together to make sure that the local environment would remain healthy. 
+
+Their hard work paid off - soon the area surrounding the forest flourished with new life. The forest became an oasis of beauty and protection, providing shelter, food, and clean air for those who visited. As the months went by, more and more people began to find the same amazing benefits in their local parks and public spaces. 
+
+Isabella knew that she had stumbled upon something truly special. Her discovery was now being shared with the rest of the world, and Isabella's efforts were celebrated and lauded by many. 
+
+It was a long and difficult journey, but Isabella had made sure that her discoveries would last long after she was gone. She wanted to ensure that future generations would also benefit from the magic leaf. In honor of her legacy, the magic leaf was named after her.
+
+The days seemed to fly by as Isabella and her team continued their mission. The research took them further into uncharted territory, exploring the unknown depths of the local environment and discovering new surprises at every turn. 
+
+The group encountered all kinds of creatures along the way, from tiny insects to massive mammals. The scientists worked tirelessly to protect these animals while still making sure that they weren't harming the environment. 
+
+One day, Isabella and her colleagues came across a rare species of plant that showed incredible promise - it seemed to be immune to the diseases and pests that normally plagued the area. It was a breakthrough, and Isabella knew that this discovery could revolutionize medicine. She had found a way to use the plant to create treatments for illnesses caused by things like AIDS. 
+
+With her newfound knowledge, Isabella set out on a mission to help others suffering from these illnesses. She and her team traveled across the country, working with doctors, medical professionals, and patients to spread awareness about the importance of finding cures for AIDS. They even managed to secure funding from the government to build research labs and support centers where those affected could receive much needed treatment. 
+
+The following years were filled with success. Isabella's efforts towards curing AIDS and other diseases had shown great results in protecting vulnerable populations. The work done by Isabella and her colleagues was creating a healthier, more sustainable future for everyone. 
+
+The following year, Isabella made an unexpected discovery. One of the people she had been helping had recently contracted HIV/AIDS, but miraculously survived and recovered. It turns out that the person had come to her in time to get the new vaccine that the National Institutes of Health (NIH) had just released. 
+
+This experience was a testament to the effectiveness of Isabella’s efforts. With the proper resources and guidance, it is possible to fight back against the dangerous effects of HIV/AIDS and AIDS-related illnesses. And as her mission continued, Isabella looked forward to seeing further progress in the coming years. 
+
+Ryan, Isabella's son, decided one day  that he wanted to follow his mother's path towards helping others. After studying science for many years, Ryan eventually found himself at home, feeling lost. He thought long and hard about what could be done to help those affected by HIV/AIDS, and finally came up with an idea. 
+
+Ryan realized that the key to stopping the spread of HIV was to educate people on how to prevent its transmission. To do this, he began traveling around the country, spreading awareness about the virus and how it spreads. 
+
+Ryan's travels were going well until one day, he met a young girl who had contracted HIV after being exposed to it through unprotected sex. She revealed to him that she never knew much about HIV or AIDS, but that she did know about vaccines – something her doctor had mentioned during a routine checkup. 
+
+The news shook Ryan to his core. He realized that if more people weren't getting vaccinated, then the effects of HIV would continue to spread. 
+
+It seemed that the only way to make things better was to take matters into his own hands. So, he quickly got to work. 
+
+First, he researched existing vaccines and discovered a few of them could potentially protect against HIV as well. He took the research with him and used it to create a vaccine he called the Isabella's son travel (Isst). The Isst vaccine was a special kind – it was created from his travels, research, and knowledge. 
+
+With the help of his parents, Ryan continued to study the Isst vaccine and eventually developed a successful vaccine. It worked so well that the Isst vaccine became the standard for those looking to protect themselves from HIV/AIDS. 
+
+The Isst vaccine was now available to anyone in need of protection, regardless of their destination or origin. Thanks to Ryan's hard work, the spread of HIV/AIDS had slowed significantly, and many people were able to live healthier lives. 
+
+In the end, it was all because of Ryan'a son travel. He had done something extraordinary – following in his mother's footsteps and taking action to change the world.

tests/test_text_parsers.py

@@ -0,0 +1,61 @@
+import json
+import unittest
+
+import nltk
+from nltk.tokenize import wordpunct_tokenize, WordPunctTokenizer
+
+from tests import EVENTS_FOLDER
+
+
+class TestTextParsers(unittest.TestCase):
+    def setUp(self):
+        with open(EVENTS_FOLDER / "llm_generated_story_1.txt", "r") as src_text:
+            text = src_text.read()
+            self.text_split_newline = text.split("\n")
+
+            self.row_words_tokens = []
+            self.row_offsets_tokens = []
+            for row in self.text_split_newline:
+                self.row_words_tokens.append(wordpunct_tokenize(row))
+                self.row_offsets_tokens.append(WordPunctTokenizer().span_tokenize(row))
+
+            nltk.download("punkt")
+            nltk.download('punkt_tab')
+            nltk.download("wordnet")
+            nltk.download("omw-1.4")
+            nltk.download('averaged_perceptron_tagger_eng')
+
+            self.ps = nltk.PorterStemmer()
+            self.wnl = nltk.WordNetLemmatizer()
+
+    def test_clean_word(self):
+        from my_ghost_writer.text_parsers import clean_string
+
+        self.maxDiff = None
+        text_to_clean = """Hi there Mr. Tom, that's a text!\r\r I'm in need of ”punctuation”; """
+        text_to_clean += """Can't you give me "stuff" (I think - or I hope - also)...\n\n Do you?"""
+        words_tokens = wordpunct_tokenize(text_to_clean)
+        offsets_tokens = WordPunctTokenizer().span_tokenize(text_to_clean)
+        output = {
+            "words_tokens": [],
+            "offsets_tokens": []
+        }
+        for word, offsets in zip(words_tokens, offsets_tokens):
+            cleaned_word = clean_string(word)
+            output["words_tokens"].append(cleaned_word)
+            output["offsets_tokens"].append(list(offsets))
+        with open(EVENTS_FOLDER / "cleaned_words.json", "r") as src_json:
+            expected_cleaned_words = json.load(src_json)
+        self.assertDictEqual(output, expected_cleaned_words)
+
+    def test_stemming(self):
+        from my_ghost_writer.text_parsers import get_words_tokens_and_indexes
+        words_stems_dict = get_words_tokens_and_indexes(self.row_words_tokens, self.row_offsets_tokens, self.ps)
+
+        with open(EVENTS_FOLDER / "stem_words.json", "r") as dst_json:
+            expected_words_stems_dict = json.load(dst_json)
+        self.assertDictEqual(words_stems_dict, expected_words_stems_dict)
+
+
+if __name__ == "__main__":
+    unittest.main()