Update app.py

app.py

@@ -11,6 +11,19 @@ from transformers import AutoTokenizer
 from transformers import AutoModelForCausalLM
 from transformers import TextIteratorStreamer
 from threading import Thread
+from torchtext.data import to_map_style_dataset
+
+llm_model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
+tokenizer = AutoTokenizer.from_pretrained(llm_model)
+# pulling tokeinzer for text generation model
+
+#import numpy as np
+
+datasetiter = load_dataset("Namitg02/Test", split='train', streaming=False)
+#dataset = np.array(list(datasetiter))
+
+#dataset = np.dataset(np.array(list(datasetiter)))
+dataset = to_map_style_dataset(datasetiter)
 
 llm_model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 tokenizer = AutoTokenizer.from_pretrained(llm_model)