Update scripts/twitter_scraper.py

scripts/twitter_scraper.py

@@ -9,6 +9,8 @@ import tweepy
 import configparser
 import os
 import pandas as pd
+from datetime import datetime, date, timedelta
+
 
 def get_latest_account_tweets(handle):
     try:
@@ -16,13 +18,13 @@ def get_latest_account_tweets(handle):
             config = configparser.ConfigParser()
             config.read("tweepy_auth.ini")
             # Get the authentication details
-            authentication_section = config['AUTHENTICATION']
+            authentication_section = config["AUTHENTICATION"]
             consumer_key = authentication_section["twitter_consumer_key"]
             consumer_secret = authentication_section["twitter_consumer_secret"]
             access_token = authentication_section["twitter_access_token"]
             access_token_secret = authentication_section["twitter_access_token_secret"]
         else:
-            consumer_key = os.environ['twitter_consumer_key']
+            consumer_key = os.environ["twitter_consumer_key"]
             consumer_secret = os.environ["twitter_consumer_secret"]
             access_token = os.environ["twitter_access_token"]
             access_token_secret = os.environ["twitter_access_token_secret"]
@@ -50,7 +52,7 @@ def get_latest_account_tweets(handle):
         df_tweets["handle"] = df_tweets.user.iloc[0]["screen_name"]
 
         return df_tweets
-        
+
     except tweepy.TweepError as e:
         # Handle specific error conditions
         if e.api_code == 63:
@@ -60,7 +62,7 @@ def get_latest_account_tweets(handle):
         else:
             print("Error occurred during API call:", str(e))
         return str(e)
-    
+
     except Exception as e:
         print("An error occurred:", str(e))
         return str(e)
@@ -68,8 +70,8 @@ def get_latest_account_tweets(handle):
 
 
 def get_tweets(
-    query: str,
-) -> list:
+    handle: str,
+):
     """
     Fetches tweets from Twitter based on a given query and returns a list of extracted tweet information.
 
@@ -79,10 +81,25 @@ def get_tweets(
     Returns:
         A list of extracted tweet information.
     """
-    print(f"Fetching tweets with query: {query}")
+    # Get the current date
+    today = datetime.today()
+    two_months_ago = today - timedelta(days=2 * 30)
+
+    start_date = two_months_ago.strftime("%Y-%m-%d")
+    end_date = today.strftime("%Y-%m-%d")
+
+    query = f"from:{handle} since:{start_date} until:{end_date} -filter:replies -filter:retweets"
 
     fetched_tweets = sntwitter.TwitterSearchScraper(query).get_items()
-    return [extract_tweet_info(tweet) for tweet in tqdm(fetched_tweets)]
+    tweets = [extract_tweet_info(tweet) for tweet in tqdm(fetched_tweets)]
+    df_tweets = pd.DataFrame(tweets)
+    df_tweets["full_text"] = df_tweets["content"]
+    df_tweets["clean_text"] = df_tweets["full_text"].apply(
+        lambda r: sentiment.tweet_cleaner(r)
+    )
+    df_tweets["handle"] = df_tweets["username"]
+    df_tweets["created_at"] = df_tweets["date"]
+    return df_tweets
 
 
 def get_replies(username: str, conversation_id: str, max_tweets: int) -> list: