Spaces:
Running
Running
Rename helper_functions_api.py to utils.py
Browse files- helper_functions_api.py +0 -268
- utils.py +181 -0
helper_functions_api.py
DELETED
@@ -1,268 +0,0 @@
|
|
1 |
-
# !pip install mistune
|
2 |
-
import mistune
|
3 |
-
from mistune.plugins.table import table
|
4 |
-
from jinja2 import Template
|
5 |
-
import re
|
6 |
-
import os
|
7 |
-
import hrequests
|
8 |
-
from fastapi_cache.decorator import cache
|
9 |
-
|
10 |
-
def md_to_html(md_text):
|
11 |
-
renderer = mistune.HTMLRenderer()
|
12 |
-
markdown_renderer = mistune.Markdown(renderer, plugins=[table])
|
13 |
-
html_content = markdown_renderer(md_text)
|
14 |
-
return html_content.replace('\n', '')
|
15 |
-
|
16 |
-
####------------------------------ OPTIONAL--> User id and persistant data storage-------------------------------------####
|
17 |
-
from datetime import datetime
|
18 |
-
import psycopg2
|
19 |
-
|
20 |
-
from dotenv import load_dotenv, find_dotenv
|
21 |
-
|
22 |
-
# Load environment variables from .env file
|
23 |
-
load_dotenv("keys.env")
|
24 |
-
|
25 |
-
TOGETHER_API_KEY = os.getenv('TOGETHER_API_KEY')
|
26 |
-
BRAVE_API_KEY = os.getenv('BRAVE_API_KEY')
|
27 |
-
GROQ_API_KEY = os.getenv("GROQ_API_KEY")
|
28 |
-
HELICON_API_KEY = os.getenv("HELICON_API_KEY")
|
29 |
-
SUPABASE_USER = os.environ['SUPABASE_USER']
|
30 |
-
SUPABASE_PASSWORD = os.environ['SUPABASE_PASSWORD']
|
31 |
-
|
32 |
-
def insert_data(user_id, user_query, subtopic_query, response, html_report):
|
33 |
-
# Connect to your database
|
34 |
-
conn = psycopg2.connect(
|
35 |
-
dbname="postgres",
|
36 |
-
user=SUPABASE_USER,
|
37 |
-
password=SUPABASE_PASSWORD,
|
38 |
-
host="aws-0-us-west-1.pooler.supabase.com",
|
39 |
-
port="5432"
|
40 |
-
)
|
41 |
-
cur = conn.cursor()
|
42 |
-
insert_query = """
|
43 |
-
INSERT INTO research_pro_chat_v2 (user_id, user_query, subtopic_query, response, html_report, created_at)
|
44 |
-
VALUES (%s, %s, %s, %s, %s, %s);
|
45 |
-
"""
|
46 |
-
cur.execute(insert_query, (user_id,user_query, subtopic_query, response, html_report, datetime.now()))
|
47 |
-
conn.commit()
|
48 |
-
cur.close()
|
49 |
-
conn.close()
|
50 |
-
|
51 |
-
####-----------------------------------------------------END----------------------------------------------------------####
|
52 |
-
|
53 |
-
|
54 |
-
import ast
|
55 |
-
from fpdf import FPDF
|
56 |
-
import re
|
57 |
-
import pandas as pd
|
58 |
-
import nltk
|
59 |
-
import requests
|
60 |
-
import json
|
61 |
-
from retry import retry
|
62 |
-
from concurrent.futures import ThreadPoolExecutor, as_completed
|
63 |
-
from bs4 import BeautifulSoup
|
64 |
-
from nltk.corpus import stopwords
|
65 |
-
from nltk.tokenize import word_tokenize
|
66 |
-
from brave import Brave
|
67 |
-
from fuzzy_json import loads
|
68 |
-
from half_json.core import JSONFixer
|
69 |
-
from openai import OpenAI
|
70 |
-
from together import Together
|
71 |
-
from urllib.parse import urlparse
|
72 |
-
import trafilatura
|
73 |
-
|
74 |
-
llm_default_small = "meta-llama/Llama-3-8b-chat-hf"
|
75 |
-
llm_default_medium = "meta-llama/Llama-3-70b-chat-hf"
|
76 |
-
|
77 |
-
SysPromptData = """You are expert in information extraction from the given context.
|
78 |
-
Steps to follow:
|
79 |
-
1. Check if relevant factual data regarding <USER QUERY> is present in the <SCRAPED DATA>.
|
80 |
-
- IF YES, extract the maximum relevant factual information related to <USER QUERY> from the <SCRAPED DATA>.
|
81 |
-
- IF NO, then return "N/A"
|
82 |
-
|
83 |
-
Rules to follow:
|
84 |
-
- Return N/A if information is not present in the scraped data.
|
85 |
-
- FORGET EVERYTHING YOU KNOW, Only output information that is present in the scraped data, DO NOT MAKE UP INFORMATION
|
86 |
-
"""
|
87 |
-
SysPromptDefault = "You are an expert AI, complete the given task. Do not add any additional comments."
|
88 |
-
SysPromptSearch = """You are a search query generator, create a concise Google search query, focusing only on the main topic and omitting additional redundant details, include year if necessory, 2024, Do not add any additional comments. OUTPUT ONLY THE SEARCH QUERY
|
89 |
-
#Additional instructions:
|
90 |
-
##Use the following search operators if necessory
|
91 |
-
OR #to cover multiple topics
|
92 |
-
* #wildcard to match any word or phrase
|
93 |
-
AND #to include specific topics."""
|
94 |
-
|
95 |
-
import tiktoken # Used to limit tokens
|
96 |
-
encoding = tiktoken.encoding_for_model("gpt-3.5-turbo") # Instead of Llama3 using available option/ replace if found anything better
|
97 |
-
|
98 |
-
def limit_tokens(input_string, token_limit=7500):
|
99 |
-
"""
|
100 |
-
Limit tokens sent to the model
|
101 |
-
"""
|
102 |
-
return encoding.decode(encoding.encode(input_string)[:token_limit])
|
103 |
-
|
104 |
-
together_client = OpenAI(
|
105 |
-
api_key=TOGETHER_API_KEY,
|
106 |
-
base_url="https://together.hconeai.com/v1",
|
107 |
-
default_headers={ "Helicone-Auth": f"Bearer {HELICON_API_KEY}"})
|
108 |
-
|
109 |
-
groq_client = OpenAI(
|
110 |
-
api_key=GROQ_API_KEY,
|
111 |
-
base_url="https://groq.hconeai.com/openai/v1",
|
112 |
-
default_headers={ "Helicone-Auth": f"Bearer {HELICON_API_KEY}"})
|
113 |
-
|
114 |
-
# Groq model names
|
115 |
-
llm_default_small = "llama3-8b-8192"
|
116 |
-
llm_default_medium = "llama3-70b-8192"
|
117 |
-
|
118 |
-
# Together Model names (fallback)
|
119 |
-
llm_fallback_small = "meta-llama/Llama-3-8b-chat-hf"
|
120 |
-
llm_fallback_medium = "meta-llama/Llama-3-70b-chat-hf"
|
121 |
-
|
122 |
-
### ------END OF LLM CONFIG-------- ###
|
123 |
-
|
124 |
-
def together_response(message, model = llm_default_small, SysPrompt = SysPromptDefault, temperature=0.2, frequency_penalty =0.1, max_tokens= 2000):
|
125 |
-
|
126 |
-
messages=[{"role": "system", "content": SysPrompt},{"role": "user", "content": message}]
|
127 |
-
params = {
|
128 |
-
"model": model,
|
129 |
-
"messages": messages,
|
130 |
-
"temperature": temperature,
|
131 |
-
"frequency_penalty": frequency_penalty,
|
132 |
-
"max_tokens": max_tokens
|
133 |
-
}
|
134 |
-
try:
|
135 |
-
response = groq_client.chat.completions.create(**params)
|
136 |
-
return response.choices[0].message.content
|
137 |
-
|
138 |
-
except Exception as e:
|
139 |
-
print(f"Error calling GROQ API: {e}")
|
140 |
-
params["model"] = llm_fallback_small if model == llm_default_small else llm_fallback_medium
|
141 |
-
response = together_client.chat.completions.create(**params)
|
142 |
-
return response.choices[0].message.content
|
143 |
-
|
144 |
-
def json_from_text(text):
|
145 |
-
"""
|
146 |
-
Extracts JSON from text using regex and fuzzy JSON loading.
|
147 |
-
"""
|
148 |
-
try:
|
149 |
-
return json.loads(text)
|
150 |
-
except:
|
151 |
-
match = re.search(r'\{[\s\S]*\}', text)
|
152 |
-
if match:
|
153 |
-
json_out = match.group(0)
|
154 |
-
else:
|
155 |
-
json_out = text
|
156 |
-
# Use Fuzzy JSON loading
|
157 |
-
return loads(json_out)
|
158 |
-
|
159 |
-
def remove_stopwords(text):
|
160 |
-
stop_words = set(stopwords.words('english'))
|
161 |
-
words = word_tokenize(text)
|
162 |
-
filtered_text = [word for word in words if word.lower() not in stop_words]
|
163 |
-
return ' '.join(filtered_text)
|
164 |
-
|
165 |
-
def rephrase_content(data_format, content, query):
|
166 |
-
|
167 |
-
if data_format == "Structured data":
|
168 |
-
return together_response(f"""
|
169 |
-
<SCRAPED DATA>{content}</SCRAPED DATA>
|
170 |
-
extract the maximum relevant factual information covering all aspects of <USER QUERY>{query}</USER QUERY> ONLY IF AVAILABLE in the scraped data.""",
|
171 |
-
SysPrompt=SysPromptData,
|
172 |
-
max_tokens=900,
|
173 |
-
)
|
174 |
-
elif data_format == "Quantitative data":
|
175 |
-
return together_response(
|
176 |
-
f"return only the numerical or quantitative data regarding the query: {{{query}}} structured into .md tables, using the scraped context:{{{limit_tokens(content,token_limit=1000)}}}",
|
177 |
-
SysPrompt=SysPromptData,
|
178 |
-
max_tokens=500,
|
179 |
-
)
|
180 |
-
else:
|
181 |
-
return together_response(
|
182 |
-
f"return only the factual information regarding the query: {{{query}}} using the scraped context:{{{limit_tokens(content,token_limit=1000)}}}",
|
183 |
-
SysPrompt=SysPromptData,
|
184 |
-
max_tokens=500,
|
185 |
-
)
|
186 |
-
|
187 |
-
|
188 |
-
def fetch_content(url):
|
189 |
-
try:
|
190 |
-
response = hrequests.get(url)
|
191 |
-
if response.status_code == 200:
|
192 |
-
return response.text
|
193 |
-
except Exception as e:
|
194 |
-
print(f"Error fetching page content for {url}: {e}")
|
195 |
-
return None
|
196 |
-
|
197 |
-
def extract_main_content(html):
|
198 |
-
extracted = trafilatura.extract(
|
199 |
-
html,
|
200 |
-
output_format="markdown",
|
201 |
-
target_language="en",
|
202 |
-
include_tables=True,
|
203 |
-
include_images=False,
|
204 |
-
include_links=False,
|
205 |
-
deduplicate=True,
|
206 |
-
)
|
207 |
-
|
208 |
-
if extracted:
|
209 |
-
return trafilatura.utils.sanitize(extracted)
|
210 |
-
else:
|
211 |
-
return ""
|
212 |
-
|
213 |
-
|
214 |
-
def process_content(data_format, url, query):
|
215 |
-
html_content = fetch_content(url)
|
216 |
-
if html_content:
|
217 |
-
content = extract_main_content(html_content)
|
218 |
-
if content:
|
219 |
-
rephrased_content = rephrase_content(
|
220 |
-
data_format=data_format,
|
221 |
-
content=limit_tokens(remove_stopwords(content), token_limit=4000),
|
222 |
-
query=query,
|
223 |
-
)
|
224 |
-
return rephrased_content, url
|
225 |
-
return "", url
|
226 |
-
|
227 |
-
def fetch_and_extract_content(data_format, urls, query):
|
228 |
-
with ThreadPoolExecutor(max_workers=len(urls)) as executor:
|
229 |
-
future_to_url = {
|
230 |
-
executor.submit(process_content, data_format, url, query): url
|
231 |
-
for url in urls
|
232 |
-
}
|
233 |
-
all_text_with_urls = [future.result() for future in as_completed(future_to_url)]
|
234 |
-
|
235 |
-
return all_text_with_urls
|
236 |
-
|
237 |
-
@cache(expire=604800)
|
238 |
-
def search_brave(query, num_results=5):
|
239 |
-
"""Fetch search results from Brave's API."""
|
240 |
-
|
241 |
-
cleaned_query = query #re.sub(r'[^a-zA-Z0-9]+', '', query)
|
242 |
-
search_query = together_response(cleaned_query, model=llm_default_small, SysPrompt=SysPromptSearch, max_tokens = 25).strip()
|
243 |
-
cleaned_search_query = re.sub(r'[^\w\s]', '', search_query).strip() #re.sub(r'[^a-zA-Z0-9*]+', '', search_query)
|
244 |
-
|
245 |
-
url = "https://api.search.brave.com/res/v1/web/search"
|
246 |
-
headers = {
|
247 |
-
"Accept": "application/json",
|
248 |
-
"Accept-Encoding": "gzip",
|
249 |
-
"X-Subscription-Token": BRAVE_API_KEY
|
250 |
-
}
|
251 |
-
params = {"q": cleaned_search_query}
|
252 |
-
|
253 |
-
response = requests.get(url, headers=headers, params=params)
|
254 |
-
|
255 |
-
if response.status_code == 200:
|
256 |
-
result = response.json() # Return the JSON response if successful
|
257 |
-
return [item["url"] for item in result["web"]["results"]][:num_results],cleaned_search_query
|
258 |
-
else:
|
259 |
-
return [],cleaned_search_query # Return error code if not successful
|
260 |
-
|
261 |
-
# #@retry(tries=3, delay=0.25)
|
262 |
-
# def search_brave(query, num_results=5):
|
263 |
-
# cleaned_query = query #re.sub(r'[^a-zA-Z0-9]+', '', query)
|
264 |
-
# search_query = together_response(cleaned_query, model=llm_default_small, SysPrompt=SysPromptSearch, max_tokens = 25).strip()
|
265 |
-
# cleaned_search_query = re.sub(r'[^\w\s]', '', search_query).strip() #re.sub(r'[^a-zA-Z0-9*]+', '', search_query)
|
266 |
-
# brave = Brave(BRAVE_API_KEY)
|
267 |
-
# search_results = brave.search(q=cleaned_search_query, count=num_results)
|
268 |
-
# return [url.__str__() for url in search_results.urls],cleaned_search_query
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
utils.py
ADDED
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
imoprt os
|
2 |
+
OR_API_KEY = os.getenv("OR_API_KEY")
|
3 |
+
# HELICON_API_KEY = os.getenv("HELICON_API_KEY")
|
4 |
+
# SUPABASE_USER = os.environ['SUPABASE_USER']
|
5 |
+
# SUPABASE_PASSWORD = os.environ['SUPABASE_PASSWORD']
|
6 |
+
|
7 |
+
import fitz # PyMuPDF for PDFs
|
8 |
+
import docx # python-docx for Word documents
|
9 |
+
import requests
|
10 |
+
from bs4 import BeautifulSoup
|
11 |
+
import json
|
12 |
+
import re
|
13 |
+
import sys
|
14 |
+
from pydantic import HttpUrl
|
15 |
+
|
16 |
+
SysPromptOpt = """
|
17 |
+
**You are a professional resume optimization expert. You have been optimizing resumes for over 20 years, helping thousands of job seekers secure their dream jobs across various industries. Your expertise lies in aligning resumes perfectly with job descriptions to highlight relevant skills, experience, and accomplishments.**
|
18 |
+
|
19 |
+
**Objective:**
|
20 |
+
Optimize the user's resume by tailoring it to a specific job description. Ensure that the final resume showcases the most relevant skills, experience, and achievements in a way that aligns perfectly with the requirements and preferences outlined in the job description. The goal is to significantly increase the user's chances of landing an interview for this position.
|
21 |
+
|
22 |
+
**Steps:**
|
23 |
+
|
24 |
+
1. **Analyze the Job Description:**
|
25 |
+
- Carefully read the provided job description.
|
26 |
+
- Identify and list the key skills, qualifications, and experiences the employer is seeking.
|
27 |
+
- Highlight any specific keywords, phrases, or technical skills that are emphasized.
|
28 |
+
|
29 |
+
2. **Review the User's Resume:**
|
30 |
+
- Examine the user's current resume thoroughly.
|
31 |
+
- Identify and list the skills, experiences, and accomplishments that are most relevant to the job description.
|
32 |
+
- Note any gaps or areas that could be better aligned with the job description.
|
33 |
+
|
34 |
+
3. **Optimize the Resume:**
|
35 |
+
- Rewrite sections of the resume to better match the job description, incorporating relevant keywords and phrases.
|
36 |
+
- Emphasize the user’s most pertinent skills and experiences in a way that demonstrates their suitability for the role.
|
37 |
+
- Ensure the resume is formatted professionally, with clear and concise language, and free of any errors.
|
38 |
+
|
39 |
+
4. **Finalize and Review:**
|
40 |
+
- Review the optimized resume for coherence and alignment with the job description.
|
41 |
+
- Make any necessary adjustments to improve clarity and impact.
|
42 |
+
- Ensure the final resume is compelling and effectively tailored to the job description.
|
43 |
+
|
44 |
+
ONLY OUTPUT THE FOLLOWING IN THE FOLLOWING FORMAT:
|
45 |
+
<optimized_resume>.....</optimized_resume>
|
46 |
+
<changes_made>......</changes_made>
|
47 |
+
"""
|
48 |
+
|
49 |
+
OPENROUTER_API_KEY = "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" #os.environ["OPENROUTER_API_KEY"]
|
50 |
+
import json
|
51 |
+
from openai import OpenAI
|
52 |
+
|
53 |
+
|
54 |
+
or_client = OpenAI(api_key=OPENROUTER_API_KEY, base_url = "https://openrouter.ai/api/v1")
|
55 |
+
|
56 |
+
def together_response(messages,model="openai/gpt-4o-mini",SysPrompt=SysPromptOpt):
|
57 |
+
response = or_client.chat.completions.create(
|
58 |
+
model=model,
|
59 |
+
messages=messages,
|
60 |
+
max_tokens=4096,
|
61 |
+
)
|
62 |
+
|
63 |
+
response_message = response.choices[0].message.content
|
64 |
+
|
65 |
+
return response_message
|
66 |
+
|
67 |
+
# Function to optimize the resume using OpenAI
|
68 |
+
def optimize_resume_api(resume, job_desc):
|
69 |
+
prompt = f"Optimize the following Resume:\n ## RESUME:{resume}\n\n for the:## JOB DESCRIPTION:{job_desc}\n\n"
|
70 |
+
messages = [{"role": "assistant", "content": SysPromptOpt},
|
71 |
+
{"role": "user", "content": prompt}]
|
72 |
+
response = together_response(messages, model = "openai/gpt-4o-mini")
|
73 |
+
return response
|
74 |
+
|
75 |
+
# Function to extract text from a Word document
|
76 |
+
def extract_text_from_word(docx_file):
|
77 |
+
doc = docx.Document(docx_file)
|
78 |
+
return "\n".join([para.text for para in doc.paragraphs])
|
79 |
+
|
80 |
+
# Function to extract text from a PDF file
|
81 |
+
def extract_text_from_pdf(pdf_file):
|
82 |
+
doc = fitz.open(stream=pdf_file.read(), filetype="pdf")
|
83 |
+
text = ""
|
84 |
+
for page in doc:
|
85 |
+
text += page.get_text()
|
86 |
+
return text
|
87 |
+
|
88 |
+
# Function to read text from a plain text file
|
89 |
+
def read_text_file(text_file):
|
90 |
+
return text_file.read().decode("utf-8")
|
91 |
+
|
92 |
+
# Function to scrape a website and extract content
|
93 |
+
def scrape_website(url):
|
94 |
+
response = requests.post(
|
95 |
+
'https://pvanand-web-scraping.hf.space/scrape',
|
96 |
+
headers={
|
97 |
+
'accept': 'application/json',
|
98 |
+
'Content-Type': 'application/json'
|
99 |
+
},
|
100 |
+
json={'url': url}
|
101 |
+
)
|
102 |
+
return response.json()
|
103 |
+
|
104 |
+
# Function to extract main content from HTML
|
105 |
+
def extract_main_content(html):
|
106 |
+
if html:
|
107 |
+
soup = BeautifulSoup(html, 'lxml')
|
108 |
+
plain_text = soup.get_text(separator=" ", strip=True)
|
109 |
+
return clean_paragraph(plain_text)
|
110 |
+
return ""
|
111 |
+
|
112 |
+
# Function to extract JSON from script tags in HTML
|
113 |
+
def extract_json_from_script(html):
|
114 |
+
if html:
|
115 |
+
soup = BeautifulSoup(html, 'lxml')
|
116 |
+
script_tags = soup.find_all('script')
|
117 |
+
|
118 |
+
json_objects = []
|
119 |
+
|
120 |
+
for tag in script_tags:
|
121 |
+
if tag.string and '{' in tag.string and '}' in tag.string:
|
122 |
+
script_content = tag.string.strip()
|
123 |
+
|
124 |
+
json_like_content = re.search(r'\{.*\}', script_content, re.DOTALL)
|
125 |
+
if json_like_content:
|
126 |
+
json_text = json_like_content.group(0)
|
127 |
+
try:
|
128 |
+
data = json.loads(json_text)
|
129 |
+
json_objects.append(data)
|
130 |
+
except json.JSONDecodeError:
|
131 |
+
continue # Skip this script if JSON parsing fails
|
132 |
+
|
133 |
+
if json_objects:
|
134 |
+
return json.dumps(json_objects, indent=4) # Pretty-print the JSON data
|
135 |
+
|
136 |
+
return "No suitable JSON script tag found."
|
137 |
+
return "No HTML content."
|
138 |
+
|
139 |
+
# Function to fetch and parse job description from a link
|
140 |
+
def fetch_job_description(link: HttpUrl):
|
141 |
+
# Convert the Url object to a string
|
142 |
+
link_str = str(link)
|
143 |
+
response = scrape_website(link_str)["content"]
|
144 |
+
job_desc = extract_main_content(response)
|
145 |
+
if len(job_desc) < 100:
|
146 |
+
job_desc = extract_main_content(extract_json_from_script(response))
|
147 |
+
return job_desc
|
148 |
+
|
149 |
+
# Function to read file content based on file type
|
150 |
+
def read_file(file):
|
151 |
+
if not file or not file.filename:
|
152 |
+
raise ValueError("No file uploaded or filename is empty")
|
153 |
+
|
154 |
+
if file.filename.endswith(".pdf"):
|
155 |
+
return extract_text_from_pdf(file.file)
|
156 |
+
elif file.filename.endswith(".docx"):
|
157 |
+
return extract_text_from_word(file.file)
|
158 |
+
elif file.filename.endswith(".txt"):
|
159 |
+
return read_text_file(file.file)
|
160 |
+
else:
|
161 |
+
raise ValueError(f"Unsupported file type: {file.filename}")
|
162 |
+
|
163 |
+
def clean_paragraph(paragraph, n=4):
|
164 |
+
"""
|
165 |
+
Clean a paragraph of text by removing words that occur more than n times.
|
166 |
+
"""
|
167 |
+
# Split the paragraph into words
|
168 |
+
words = re.findall(r'\b\w+\b', paragraph)
|
169 |
+
|
170 |
+
# Create a dictionary to count the occurrences of each word
|
171 |
+
word_count = {}
|
172 |
+
for word in words:
|
173 |
+
word_count[word] = word_count.get(word, 0) + 1
|
174 |
+
|
175 |
+
# Filter out words that occur more than n times
|
176 |
+
cleaned_words = [word for word in words if word_count[word] <= n]
|
177 |
+
|
178 |
+
# Join the cleaned words back into a paragraph
|
179 |
+
cleaned_paragraph = ' '.join(cleaned_words)
|
180 |
+
|
181 |
+
return cleaned_paragraph
|