Spaces:
Sleeping
Sleeping
replacing hardcoded values with constants
Browse files
rag_app/knowledge_base/build_vector_store.py
CHANGED
@@ -9,9 +9,10 @@ from rag_app.utils.generate_summary import generate_description, generate_keywor
|
|
9 |
import time
|
10 |
import os
|
11 |
|
|
|
|
|
12 |
def build_vector_store(
|
13 |
docs: list,
|
14 |
-
db_path: str,
|
15 |
embedding_model: str,
|
16 |
new_db:bool=False,
|
17 |
chunk_size:int=500,
|
@@ -21,15 +22,16 @@ def build_vector_store(
|
|
21 |
|
22 |
"""
|
23 |
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
|
31 |
#load chunks into vector store
|
32 |
print(f'Loading chunks into faiss vector store ...')
|
|
|
33 |
st = time.time()
|
34 |
if new_db:
|
35 |
db_faiss = FAISS.from_documents(chunks, embeddings)
|
@@ -37,15 +39,18 @@ def build_vector_store(
|
|
37 |
else:
|
38 |
db_faiss = FAISS.add_documents(chunks, embeddings)
|
39 |
bm25_retriever = BM25Retriever.add_documents(chunks)
|
|
|
40 |
db_faiss.save_local(FAISS_INDEX_PATH)
|
41 |
et = time.time() - st
|
42 |
print(f'Time taken: {et} seconds.')
|
43 |
|
44 |
print(f'Loading chunks into chroma vector store ...')
|
|
|
45 |
st = time.time()
|
46 |
persist_directory='./vectorstore/chroma-insurance-agent-1500'
|
47 |
db_chroma = Chroma.from_documents(chunks, embeddings, persist_directory=persist_directory)
|
48 |
et = time.time() - st
|
|
|
49 |
print(f'Time taken: {et} seconds.')
|
50 |
result = f"built vectore store at {FAISS_INDEX_PATH}"
|
51 |
return result
|
|
|
9 |
import time
|
10 |
import os
|
11 |
|
12 |
+
from config import FAISS_INDEX_PATH
|
13 |
+
|
14 |
def build_vector_store(
|
15 |
docs: list,
|
|
|
16 |
embedding_model: str,
|
17 |
new_db:bool=False,
|
18 |
chunk_size:int=500,
|
|
|
22 |
|
23 |
"""
|
24 |
|
25 |
+
embeddings,chunks = create_embeddings(
|
26 |
+
docs,
|
27 |
+
chunk_size,
|
28 |
+
chunk_overlap,
|
29 |
+
embedding_model
|
30 |
+
)
|
31 |
|
32 |
#load chunks into vector store
|
33 |
print(f'Loading chunks into faiss vector store ...')
|
34 |
+
|
35 |
st = time.time()
|
36 |
if new_db:
|
37 |
db_faiss = FAISS.from_documents(chunks, embeddings)
|
|
|
39 |
else:
|
40 |
db_faiss = FAISS.add_documents(chunks, embeddings)
|
41 |
bm25_retriever = BM25Retriever.add_documents(chunks)
|
42 |
+
|
43 |
db_faiss.save_local(FAISS_INDEX_PATH)
|
44 |
et = time.time() - st
|
45 |
print(f'Time taken: {et} seconds.')
|
46 |
|
47 |
print(f'Loading chunks into chroma vector store ...')
|
48 |
+
|
49 |
st = time.time()
|
50 |
persist_directory='./vectorstore/chroma-insurance-agent-1500'
|
51 |
db_chroma = Chroma.from_documents(chunks, embeddings, persist_directory=persist_directory)
|
52 |
et = time.time() - st
|
53 |
+
|
54 |
print(f'Time taken: {et} seconds.')
|
55 |
result = f"built vectore store at {FAISS_INDEX_PATH}"
|
56 |
return result
|