isayahc commited on
Commit
1e4bae2
·
unverified ·
1 Parent(s): 5439651

replacing hardcoded values with constants

Browse files
rag_app/knowledge_base/build_vector_store.py CHANGED
@@ -9,9 +9,10 @@ from rag_app.utils.generate_summary import generate_description, generate_keywor
9
  import time
10
  import os
11
 
 
 
12
  def build_vector_store(
13
  docs: list,
14
- db_path: str,
15
  embedding_model: str,
16
  new_db:bool=False,
17
  chunk_size:int=500,
@@ -21,15 +22,16 @@ def build_vector_store(
21
 
22
  """
23
 
24
- if db_path is None:
25
- FAISS_INDEX_PATH = os.getenv("FAISS_INDEX_PATH")
26
- else:
27
- FAISS_INDEX_PATH = db_path
28
-
29
- embeddings,chunks = create_embeddings(docs, chunk_size, chunk_overlap, embedding_model)
30
 
31
  #load chunks into vector store
32
  print(f'Loading chunks into faiss vector store ...')
 
33
  st = time.time()
34
  if new_db:
35
  db_faiss = FAISS.from_documents(chunks, embeddings)
@@ -37,15 +39,18 @@ def build_vector_store(
37
  else:
38
  db_faiss = FAISS.add_documents(chunks, embeddings)
39
  bm25_retriever = BM25Retriever.add_documents(chunks)
 
40
  db_faiss.save_local(FAISS_INDEX_PATH)
41
  et = time.time() - st
42
  print(f'Time taken: {et} seconds.')
43
 
44
  print(f'Loading chunks into chroma vector store ...')
 
45
  st = time.time()
46
  persist_directory='./vectorstore/chroma-insurance-agent-1500'
47
  db_chroma = Chroma.from_documents(chunks, embeddings, persist_directory=persist_directory)
48
  et = time.time() - st
 
49
  print(f'Time taken: {et} seconds.')
50
  result = f"built vectore store at {FAISS_INDEX_PATH}"
51
  return result
 
9
  import time
10
  import os
11
 
12
+ from config import FAISS_INDEX_PATH
13
+
14
  def build_vector_store(
15
  docs: list,
 
16
  embedding_model: str,
17
  new_db:bool=False,
18
  chunk_size:int=500,
 
22
 
23
  """
24
 
25
+ embeddings,chunks = create_embeddings(
26
+ docs,
27
+ chunk_size,
28
+ chunk_overlap,
29
+ embedding_model
30
+ )
31
 
32
  #load chunks into vector store
33
  print(f'Loading chunks into faiss vector store ...')
34
+
35
  st = time.time()
36
  if new_db:
37
  db_faiss = FAISS.from_documents(chunks, embeddings)
 
39
  else:
40
  db_faiss = FAISS.add_documents(chunks, embeddings)
41
  bm25_retriever = BM25Retriever.add_documents(chunks)
42
+
43
  db_faiss.save_local(FAISS_INDEX_PATH)
44
  et = time.time() - st
45
  print(f'Time taken: {et} seconds.')
46
 
47
  print(f'Loading chunks into chroma vector store ...')
48
+
49
  st = time.time()
50
  persist_directory='./vectorstore/chroma-insurance-agent-1500'
51
  db_chroma = Chroma.from_documents(chunks, embeddings, persist_directory=persist_directory)
52
  et = time.time() - st
53
+
54
  print(f'Time taken: {et} seconds.')
55
  result = f"built vectore store at {FAISS_INDEX_PATH}"
56
  return result