import openai import pandas as pd import numpy as np import faiss import os from dotenv import load_dotenv from sklearn.metrics.pairwise import cosine_similarity load_dotenv() # take environment variables from .env. api_key = os.getenv('OPENAI_API_KEY') #print(api_key) from openai import OpenAI client = OpenAI() def get_openai_embedding(text, model="text-embedding-3-small"): text = text.replace("\n", " ") return client.embeddings.create(input = [text], model=model).data[0].embedding def save_openai_embeddings(csv_file, model='text-embedding-3-small'): # Load the CSV file df = pd.read_csv(csv_file) # Save the embeddings df['ada_embeddings'] = df.utterance.apply(lambda x: get_openai_embedding(x, model='text-embedding-3-small')) df.to_csv('embeddings/openai_embeddings.csv', index=False) print(f"Embeddings saved to embeddings/openai_embeddings.csv.") # get and save the embeddings for Intent cleared data #save_openai_embeddings(r'C:\Users\ZZ029K826\Documents\GitHub\LLM_Intent_Recognition\data\Pager_Intents_cleaned.csv') def load_openai_embeddings(csv_file): # Load the CSV file df = pd.read_csv(csv_file) # Extract the embeddings embeddings = df['ada_embeddings'].tolist() return embeddings # Function to calculate similarity between user input and precomputed embeddings def calculate_openai_similarity(user_text, df, top_n=5): # Get embedding for the user input text user_embedding = np.array(get_openai_embedding(user_text, model='text-embedding-3-small')).reshape(1, -1) # Calculate cosine similarity between user input and all precomputed embeddings df['similarity'] = df['ada_embedding'].apply(lambda x: cosine_similarity([x], user_embedding)[0][0]) # Sort by similarity score (descending) and return the top_n most similar top_matches = df.sort_values(by='similarity', ascending=False).head(top_n) return top_matches[['combined', 'similarity']] def get_openai_similarity(user_text, df, top_n=5): # Get embedding for the user input text user_embedding = np.array(get_openai_embedding(user_text, model='text-embedding-3-small')).reshape(1, -1) # Calculate cosine similarity between user input and all precomputed embeddings df['similarity'] = df['ada_embedding'].apply(lambda x: cosine_similarity([x], user_embedding)[0][0]) # Sort by similarity score (descending) and return the top_n most similar top_matches = df.sort_values(by='similarity', ascending=False).head(top_n) return top_matches[['combined', 'similarity']]