Spaces:
Running
on
T4
Running
on
T4
Update auditqa/process_chunks.py
Browse files
auditqa/process_chunks.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
import glob
|
|
|
|
| 2 |
import os
|
| 3 |
from langchain.text_splitter import RecursiveCharacterTextSplitter, SentenceTransformersTokenTextSplitter
|
| 4 |
from transformers import AutoTokenizer
|
|
|
|
| 1 |
import glob
|
| 2 |
+
import json
|
| 3 |
import os
|
| 4 |
from langchain.text_splitter import RecursiveCharacterTextSplitter, SentenceTransformersTokenTextSplitter
|
| 5 |
from transformers import AutoTokenizer
|