Update app.py

app.py

@@ -28,9 +28,10 @@ dataset = load_dataset("Namitg02/Test", split='train', streaming=False)
 #Returns a list of dictionaries, each representing a row in the dataset.
 print(dataset[1])
 dataset.features
+length = len(dataset)
 #Itemdetails = dataset.items()
 #print(Itemdetails)
-splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=25) # ["\n\n", "\n", " ", ""])
+#splitter = RecursiveCharacterTextSplitter(chunk_size=150, chunk_overlap=25) # ["\n\n", "\n", " ", ""])
 
 #docs = splitter.create_documents(str(dataset))
 
@@ -46,8 +47,16 @@ embedding_model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
 
 #doc_func = lambda x: x.text
 #dataset = list(map(doc_func, dataset))
-embeddings = embedding_model.encode(dataset[0]["text"])
+
+def embedder(dataset):
+    embeddings = embedding_model.encode(dataset[0]["text"])
+    dataset = dataset.add_column('embeddings', embeddings)
+    return dataset
+updated_dataset = dataset.map(embedder)
+dataset['text'][:length]
+
 print(embeddings)
+
 #def embedder(dataset[i]):
 #    return embedding_model.encode(dataset[i])
     
@@ -57,7 +66,7 @@ print(dataset[2])
 #embeddings = embedding_model.encode(dataset)
 
 #embeddings = embedding_model.embed_documents(docs)
-dataset = dataset.add_column('embeddings', embeddings)
+
 embedding_dim = embedding_model.get_sentence_embedding_dimension()
 print(dataset[1])
 #data = FAISS.from_embeddings(embed, embedding_model)