File size: 1,142 Bytes
5ecde30 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 |
import os
import faiss
import numpy as np
from pathlib import Path
import sys
# Add the project directory to the path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from import_Path import BASE_DIR
def save_faiss_embeddings_index(embeddings, file_name):
# Ensure embeddings are in float32 format
if not isinstance(embeddings, np.ndarray):
embeddings = embeddings.numpy()
embeddings = embeddings.astype('float32')
# Create the directory if it doesn't exist
#os.makedirs(os.path.dirname(file_name), exist_ok=True)
# Create a FAISS index
index = faiss.IndexFlatL2(embeddings.shape[1]) # L2 distance
index.add(embeddings)
# Save the FAISS index
# old faiss.write_index(index, file_name)
index_path = BASE_DIR / "embeddings" / file_name
faiss.write_index(index, str(index_path))
def load_faiss_index(index_path):
index = faiss.read_index(index_path)
return index
def normalize_embeddings(embeddings):
# Normalize embeddings
embeddings = embeddings / np.linalg.norm(embeddings, axis=1)[:, None]
return embeddings |