Update auditqa/process_chunks.py

auditqa/process_chunks.py

@@ -67,12 +67,21 @@ def load_chunks():
     # placeholder for collection
     qdrant_collections = {}
     print("embeddings started")
-    qdrant_collections['reportsFeb2025'] = Qdrant.from_documents(
-                chunks_list,
-                embeddings,
-                path="/data/local_qdrant",
-                collection_name='reportsFeb2025',
-            )
+    batch_size = 10000  # Adjust this value based on your system's memory capacity
+    for i in range(0, len(docs), batch_size):
+        batch_docs = chunks_list[i:i+batch_size]
+        qdrant = Qdrant.from_documents(
+            batch_docs, embeddings,
+            path="/data/local_qdrant",
+            collection_name='reportsFeb2025',
+        )
+        
+    #qdrant_collections['reportsFeb2025'] = Qdrant.from_documents(
+    #            chunks_list,
+    #            embeddings,
+    #            path="/data/local_qdrant",
+    #            collection_name='reportsFeb2025',
+    #        )
     print(qdrant_collections)
     print("vector embeddings done")
     return qdrant_collections