Update app.py

app.py

@@ -19,15 +19,46 @@ tokenizer = AutoTokenizer.from_pretrained(llm_model)
 
 #import numpy as np
 
-datasetiter = load_dataset("Namitg02/Test", split='train', streaming=False)
-dataset = to_map_style_dataset(datasetiter)
+from torch.utils.data import Dataset, IterableDataset
+
+class MyIterableDataset(IterableDataset):
+    def __init__(self, iterable):
+        super().__init__()
+        self.iterable = iterable
+
+    def __iter__(self):
+        return iter(self.iterable)
+
+class MapStyleDataset(Dataset):
+    def __init__(self, iterable):
+        super().__init__()
+        self.data = list(iterable)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        return self.data[idx]
+
+# Create an iterable
+iterable = "Namitg02/Test"
+
+# Convert the iterable to a MapStyle dataset
+map_style_dataset = MapStyleDataset(iterable)
+
+# Create a DataLoader for the MapStyle dataset
+data_loader = torch.utils.data.DataLoader(map_style_dataset, batch_size=2)
+
+
+#datasetiter = load_dataset("Namitg02/Test", split='train', streaming=False)
+#dataset = to_map_style_dataset(datasetiter)
 
 
 #dataset = load_dataset("not-lain/wikipedia",revision = "embedded")
 #dataset = load_dataset("epfl-llm/guidelines", split='train')
 #Returns a list of dictionaries, each representing a row in the dataset.
-print(dataset[1])
-length = len(dataset)
+print(map_style_dataset[1])
+length = len(map_style_dataset)
 
 #Itemdetails = dataset.items()
 #print(Itemdetails)
@@ -39,18 +70,18 @@ embedding_model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
 #doc_func = lambda x: x.text
 #dataset = list(map(doc_func, dataset))
 
-def embedder(dataset):
-    embeddings = embedding_model.encode(dataset["text"])
-    dataset = dataset.add_column('embeddings', embeddings)
-    return dataset
-updated_dataset = dataset.map(embedder)
+def embedder(map_style_dataset):
+    embeddings = embedding_model.encode(map_style_dataset["text"])
+    map_style_dataset = map_style_dataset.add_column('embeddings', embeddings)
+    return map_style_dataset
+updated_dataset = map_style_dataset.map(embedder)
 dataset['text'][:length]
 
 #print(embeddings)
 
 print(updated_dataset[1])
 print(updated_dataset[2])
-print(dataset[1])
+print(map_style_dataset[1])
 
 embedding_dim = embedding_model.get_sentence_embedding_dimension()
 #data = FAISS.from_embeddings(embed, embedding_model)