Update app.py

app.py

@@ -21,49 +21,30 @@ tokenizer = AutoTokenizer.from_pretrained(llm_model)
 # pulling tokeinzer for text generation model
 
 dataset = load_dataset("Namitg02/Test", split='train', streaming=False)
-#dataset = load_dataset("not-lain/wikipedia",revision = "embedded")
 #dataset = load_dataset("epfl-llm/guidelines", split='train')
 #Returns a list of dictionaries, each representing a row in the dataset.
-#print(dataset[1])
 length = len(dataset)
 
-#Itemdetails = dataset.items()
-#print(Itemdetails)
-
 embedding_model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
-#embedding_model = HuggingFaceEmbeddings(model_name = "mixedbread-ai/mxbai-embed-large-v1")
 #all-MiniLM-L6-v2, BAAI/bge-base-en-v1.5,infgrad/stella-base-en-v2, BAAI/bge-large-en-v1.5 working with default dimensions
 
 df = pd.DataFrame(dataset)
-print(df.iloc[[1]])
-
+#print(df.iloc[[1]])
+print(check1)
 df['embeddings'] = df['text'].apply(lambda x: embedding_model.encode(x))
+# add_embeddings as a new column
+
+print(check1a)
 print(df.iloc[[1]])
 dataset = Dataset.from_pandas(df)
-print(dataset[1])
-print(dataset[2])
+print(check1b)
 
-#doc_func = lambda x: x.text
-#dataset = list(map(doc_func, dataset))
-
-#def embedder(dataset):
-#    embeddings = embedding_model.encode(dataset["text"])
-#    dataset = dataset.add_column('embeddings', embeddings)
-#    return dataset
-#updated_dataset = dataset.map(embedder)
 #dataset['text'][:length]
 
-#print(embeddings)
-
-print(dataset[1])
+print(dataset[1c])
 
 embedding_dim = embedding_model.get_sentence_embedding_dimension()
-#data = FAISS.from_embeddings(embed, embedding_model)
-#data = FAISS.from_texts(docs, embedding_model)
-
-# Returns a FAISS wrapper vector store. Input is a list of strings. from_documents method used documents to Return VectorStore
-# add_embeddings
-#data = dataset["clean_text"]
+# Returns dimensions of embedidngs
 data = dataset
 
 #print(data)
@@ -75,7 +56,7 @@ m = 32  # hnsw parameter. Higher is more accurate but takes more time to index (
 data.add_faiss_index("embeddings")
 # adds an index column for the embeddings
 
-print("check1")
+print("check1d")
 #question = "How can I reverse Diabetes?"
 
 SYS_PROMPT = """You are an assistant for answering questions.