import requests from bs4 import BeautifulSoup def get_url_content(url: str) -> str: """ Retrieve the content of a URL. :param url: The URL to retrieve content from. :return: The content of the URL as a string. """ try: response = requests.get( url, headers={ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" } ) if response.status_code != 200: print(f"Failed to retrieve content from {url}. Status code: {response.status_code} - {response.reason}") return "" # parse the html content using BeautifulSoup parser = BeautifulSoup(response.text, 'html.parser') # extract text from the parsed HTML return parser.text.strip() if parser.text else "" except Exception as e: print(f"An error occurred while retrieving content from {url}: {e}") return ""