Update auditqa/doc_process.py

auditqa/doc_process.py

@@ -45,17 +45,18 @@ def process_pdf():
     categories = list(files.keys())
     # iterate through 'source'
     for category in categories:
-        print(category)
+        print("documents splitting in source:",category)
         all_documents[category] = []
         subtypes = list(files[category].keys())
         # iterate through 'subtype' within the source
         # example source/category == 'District', has subtypes which is district names
         for subtype in subtypes:
-            print(subtype)
+            print("document splitting for subtype:",subtype)
             for file in files[category][subtype]:
 
                 # create the chunks
                 doc_processed = text_splitter.split_documents(docs[file])
+                print("chunks in subtype:",subtype, "are:",len(doc_processed))
 
                 # add metadata information 
                 for doc in doc_processed:
@@ -69,6 +70,7 @@ def process_pdf():
     # convert list of list to flat list
     for key, docs_processed in all_documents.items():
         docs_processed = [item for sublist in docs_processed for item in sublist]
+        print("length of chunks in source:",source, "are:",len(docs_processed)
         all_documents[key] = docs_processed
     all_documents['allreports'] = [sublist for key,sublist in all_documents.items()]
     all_documents['allreports'] = [item for sublist in all_documents['allreports'] for item in sublist]
@@ -90,7 +92,7 @@ def process_pdf():
             location=":memory:", 
             collection_name=file,
         )
-    
+    print(qdrant_collections)
     print("vector embeddings done")
     return qdrant_collections