Update auditqa/doc_process.py

auditqa/doc_process.py

@@ -15,4 +15,18 @@ def process_pdf():
             docs[file] = PyMuPDFLoader(value).load()
         except Exception as e:
             print("Exception: ", e)
+
+    
+    # text splitter based on the tokenizer of a model of your choosing
+    # to make texts fit exactly a transformer's context window size
+    # langchain text splitters: https://python.langchain.com/docs/modules/data_connection/document_transformers/
+    chunk_size = 256
+    text_splitter = RecursiveCharacterTextSplitter.from_huggingface_tokenizer(
+            AutoTokenizer.from_pretrained("BAAI/bge-small-en-v1.5"),
+            chunk_size=chunk_size,
+            chunk_overlap=int(chunk_size / 10),
+            add_start_index=True,
+            strip_whitespace=True,
+            separators=["\n\n", "\n", ".", " ", ""],
+    )