Update scripts/twitter_scraper.py

scripts/twitter_scraper.py

@@ -5,48 +5,66 @@ from tqdm import tqdm
 import requests
 from scripts import sentiment
 
+import tweepy
+import configparser
+import os
+import pandas as pd
+
 def get_latest_account_tweets(handle):
-    import tweepy
-
-    import configparser
-
-    import os
-    if os.path.exists("tweepy_auth.ini"):
-        config = configparser.ConfigParser()
-        config.read("tweepy_auth.ini")
-        # Get the authentication details
-        authentication_section = config['AUTHENTICATION']
-        consumer_key = authentication_section["twitter_consumer_key"]
-        consumer_secret = authentication_section["twitter_consumer_secret"]
-        access_token = authentication_section["twitter_access_token"]
-        access_token_secret = authentication_section["twitter_access_token_secret"]
-    else:
-        consumer_key = os.environ['twitter_consumer_key']
-        consumer_secret = os.environ["twitter_consumer_secret"]
-        access_token = os.environ["twitter_access_token"]
-        access_token_secret = os.environ["twitter_access_token_secret"]
-
-    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
-    auth.set_access_token(access_token, access_token_secret)
-
-    # create the API object
-    api = tweepy.API(auth)
-
-    # load the tweets from a specific user
-    tweets = api.user_timeline(
-        screen_name=handle, count=10000000, tweet_mode="extended"
-    )
+    try:
+        if os.path.exists("tweepy_auth.ini"):
+            config = configparser.ConfigParser()
+            config.read("tweepy_auth.ini")
+            # Get the authentication details
+            authentication_section = config['AUTHENTICATION']
+            consumer_key = authentication_section["twitter_consumer_key"]
+            consumer_secret = authentication_section["twitter_consumer_secret"]
+            access_token = authentication_section["twitter_access_token"]
+            access_token_secret = authentication_section["twitter_access_token_secret"]
+        else:
+            consumer_key = os.environ['twitter_consumer_key']
+            consumer_secret = os.environ["twitter_consumer_secret"]
+            access_token = os.environ["twitter_access_token"]
+            access_token_secret = os.environ["twitter_access_token_secret"]
+
+        auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
+        auth.set_access_token(access_token, access_token_secret)
+
+        # create the API object
+        api = tweepy.API(auth)
+
+        # load the tweets from a specific user
+        tweets = api.user_timeline(
+            screen_name=handle, count=10000000, tweet_mode="extended"
+        )
+
+        df_tweets = pd.DataFrame(data=[t._json for t in tweets])
+        df_tweets["created_at"] = pd.to_datetime(df_tweets["created_at"])
+        df_tweets = df_tweets.sort_values("created_at")
+
+        # print the tweet texts
+        tweets_txt = []
+        for tweet in tweets:
+            tweets_txt.append(sentiment.tweet_cleaner(tweet.full_text))
+        df_tweets["clean_text"] = tweets_txt
+        df_tweets["handle"] = df_tweets.user.iloc[0]["screen_name"]
+
+        return df_tweets
+
+    except tweepy.TweepError as e:
+        # Handle specific error conditions
+        if e.api_code == 63:
+            print("User has been suspended.")
+        elif e.api_code == 88:
+            print("Rate limit exceeded. Please try again later.")
+        else:
+            print("Error occurred during API call:", str(e))
+    
+    except Exception as e:
+        print("An error occurred:", str(e))
+
+    return None
 
-    df_tweets = pd.DataFrame(data=[t._json for t in tweets])
-    df_tweets["created_at"] = pd.DataFrame(df_tweets["created_at"])
-    df_tweets = df_tweets.sort_values("created_at")
-    # print the tweet texts
-    tweets_txt = []
-    for tweet in tweets:
-        tweets_txt.append(sentiment.tweet_cleaner(tweet.full_text))
-    df_tweets["clean_text"] = tweets_txt
-    df_tweets["handle"] = df_tweets.user.iloc[0]["screen_name"]
-    return df_tweets
 
 def get_tweets(
     query: str,