update

requirements.txt

@@ -1,30 +1,144 @@
-tqdm==4.41.0
-transformers==3.0.2
-faiss_gpu==1.7.2
-syllables==0.1.0
-scipy==1.3.1
-nltk==3.2.5
-lxml==4.6.3
-requests==2.22.0
-Flask_Cors==3.0.8
+aiofiles==23.2.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.0.1
+annotated-types==0.5.0
+anyio==3.7.1
+anykeystore==0.2
+apex==0.9.10.dev0
+async-timeout==4.0.3
+asynctest==0.13.0
+attrs==24.2.0
+beautifulsoup4==4.9.3
+blessings==1.7
+blis==0.7.11
+boilerpipe3==1.1
+catalogue==2.0.10
+certifi @ file:///croot/certifi_1671487769961/work/certifi
+charade==1.0.3
+chardet==3.0.4
+charset-normalizer==3.4.3
+click==7.1.2
+cryptacular==1.6.2
+cycler==0.11.0
+cymem==2.0.11
+defusedxml==0.7.1
+exceptiongroup==1.3.0
+faiss-gpu==1.6.3
+fastapi==0.103.2
+ffmpy==0.3.2
+filelock==3.0.12
+Flask==2.0.0
+Flask-Cors==3.0.8
+frozenlist==1.3.3
+fsspec==2023.1.0
+future==1.0.0
+gdown==4.7.3
 gpustat==0.6.0
+gradio==3.34.0
+gradio_client==0.2.6
+greenlet==3.1.1
+h11==0.14.0
 h5py==2.10.0
-packaging==20.3
+httpcore==0.17.3
+httpx==0.24.1
+huggingface-hub==0.16.4
+hupper==1.12.1
+idna==2.8
+importlib-metadata==6.7.0
+importlib-resources==5.12.0
+itsdangerous==2.1.2
+Jinja2==3.0.3
+joblib==1.3.2
+JPype1-py3==0.5.5.4
+jsonschema==4.17.3
+kiwisolver==1.4.5
+linkify-it-py==2.0.3
+lxml==4.6.3
+markdown-it-py==2.2.0
+MarkupSafe==2.1.5
+matplotlib==3.3.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.5
+murmurhash==1.0.13
+nltk==3.2.5
+numpy==1.17.4
+nvidia-ml-py3==7.352.0
+oauthlib==3.2.2
+orjson==3.9.7
+packaging==24.0
 pandas==1.0.3
-spacy==3.0.3
+PasteDeploy==3.1.0
+pathy==0.10.3
+pbkdf2==1.3
+Pillow==9.5.0
+pkgutil_resolve_name==1.3.10
+plaster==1.1.2
+plaster-pastedeploy==1.0.1
+preshed==3.0.10
+psutil==7.0.0
+pydantic==1.10.22
+pydantic_core==2.14.6
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.4
+pyramid==2.0.2
+pyramid-mailer==0.15.1
+pyrsistent==0.19.3
+PySocks==1.7.1
+python-dateutil==2.8.1
+python-multipart==0.0.8
+python3-openid==3.2.0
+pytz==2025.2
+PyYAML==6.0.1
+regex==2024.4.16
+repoze.sendmail==4.4.1
+requests==2.22.0
+requests-oauthlib==2.0.0
+sacremoses==0.0.53
+scikit-learn==0.24.1
+scipy==1.3.1
+semantic-version==2.10.0
 sentence-transformers==0.3.0
-numpy==1.17.4
-matplotlib==3.3.1
-filelock==3.0.12
-boilerpipe3==1.1
-Unidecode==1.1.1
-Flask==2.0.0
-apex
-beautifulsoup4==4.9.3
-python_dateutil==2.8.1
-scikit_learn==0.24.1
+sentencepiece==0.2.0
+six==1.17.0
+smart-open==6.4.0
+sniffio==1.3.1
+soupsieve==2.4.1
+spacy==3.0.3
+spacy-legacy==3.0.12
+SQLAlchemy==1.4.54
+srsly==2.4.8
+starlette==0.27.0
+syllables==0.1.0
 tantivy==0.13.2
-huggingface_hub==0.16.4
+thinc==8.0.17
+threadpoolctl==3.1.0
+tokenizers==0.8.1rc1
+toolz==0.12.1
 torch==1.6.0+cu101
 torchvision==0.7.0+cu101
-gradio==3.34.0
+tqdm==4.67.1
+transaction==4.0
+transformers==3.0.2
+translationstring==1.4
+typer==0.3.2
+typing_extensions==4.7.1
+uc-micro-py==1.0.3
+Unidecode==1.1.1
+urllib3==1.25.11
+uvicorn==0.22.0
+velruse==1.1.1
+venusian==3.1.1
+wasabi==0.10.1
+WebOb==1.8.9
+websockets==11.0.3
+Werkzeug==2.2.3
+WTForms==3.0.1
+wtforms-recaptcha==0.3.2
+yarl==1.9.4
+zipp==3.15.0
+zope.deprecation==5.0
+zope.interface==6.4.post2
+zope.sqlalchemy==3.1

web_search.py

@@ -1,13 +1,64 @@
+import random
 import re
 
 from pprint import pprint
 from urllib.request import Request, urlopen
 from html.parser import HTMLParser
 from bs4 import BeautifulSoup
-from urllib.parse import quote_plus
+from urllib.parse import quote_plus, parse_qs
+
+import requests, lxml, json
+import logging
+
+import pdb
+
+def USER_AGENT():
+    uastrings = [
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0',
+        'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1',
+    ]
+    ua = random.choice(uastrings)
+    logging.info(f'Using user agent: {ua}')
+    return ua
+
+
+def brave_search(query: str, pages=1) -> list:
+    params = {
+        'q': quote_plus(query),
+        'source': 'web',
+        'tf': 'at',
+    }
+    headers = {
+        'User-Agent': USER_AGENT()
+    }
 
+    urls = []
+    for page in range(pages):
+        params['offset'] = page
+        html = requests.get('https://search.brave.com/search', headers=headers, params=params)
+        soup = BeautifulSoup(html.text, 'lxml')
+
+        for result in soup.select('.snippet'):
+            link = result.select_one('.result-header').get('href')
+            """
+            sitelinks_container = result.select('.deep-results-buttons .deep-link')
+            sitelinks = None
+
+            if sitelinks_container:
+                sitelinks = []
+                for sitelink in sitelinks_container:
+                    sitelinks.append({
+                        'title': sitelink.get_text().strip(),
+                        'link': sitelink.get('href')
+                    })
+            """
+            urls.append(link)
+    return urls
 
-def bing_search(query: str, pages_number=1) -> list:
+
+def bing_search(query: str, pages=1) -> list:
     """
     Gets web results from Bing
     :param query: query to search
@@ -15,7 +66,7 @@ def bing_search(query: str, pages_number=1) -> list:
     :return: a list of links in ranked order
     """
     urls = []
-    for page in range(pages_number):
+    for page in range(pages):
         first = page * 10 + 1
         address = "https://www.bing.com/search?q=" + quote_plus(query) + '&first=' + str(first)
         data = get_html(address)
@@ -27,6 +78,9 @@ def bing_search(query: str, pages_number=1) -> list:
 
 
 def duckduckgo_search(query: str, pages=1):
+    """
+    NOT WORKING; LIKELY BLOCKED
+    """
     urls = []
     start_index = 0
     for page in range(pages):
@@ -36,7 +90,10 @@ def duckduckgo_search(query: str, pages=1):
         links = soup.findAll('a', {'class': 'result__a'})
         urls.extend([link['href'] for link in links])
         start_index = len(urls)
-
+    try:
+        urls = [parse_qs(l.split('/')[-1][5:])[''][0] for l in urls]
+    except:
+        logging.warn(f'Parsing failed for {len(urls)} urls')
     return urls
 
 
@@ -58,8 +115,10 @@ def get_html(url: str) -> str:
     :return: html source code
     """
     try:
-        custom_user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
-        req = Request(url, headers={"User-Agent": custom_user_agent})
+        headers = {
+            'User-Agent': USER_AGENT()
+        }
+        req = Request(url, headers=headers)
         page = urlopen(req, timeout=3)
         return str(page.read())
     except: