Update auditqa/doc_process.py

auditqa/doc_process.py

@@ -37,7 +37,15 @@ def process_pdf():
     all_documents = {}
     categories = list(files.keys())
     for category in categories:
-        all_documents[category] = {}
+        all_documents[category] = []
+        subtypes = list(files[category].keys())
+        for subtype in subtypes:
+            for file in files[category][subtype]:
+                doc_processed = text_splitter.split_documents(docs[file])
+                for doc in doc_processed:
+                    doc.metadata["source"] = category
+                    doc.metadata["subtype"] = subtype
+                    doc.metadata["year"] = file[-4:]
 
-    print(all_documents)
+                all_documents[category].append(doc_processed)