Update app.py

app.py

@@ -1,4 +1,4 @@
-from fastapi import FastAPI
+"""from fastapi import FastAPI
 from datasets import load_dataset
 from transformers import AutoTokenizer
 
@@ -32,4 +32,33 @@ def summarize():
 
     tokenized_billsum = billsum.map(preprocess_function, batched=True)
     
-    return tokenized_billsum 
+    return tokenized_billsum """
+
+from fastapi import FastAPI
+from datasets import load_dataset
+from transformers import AutoTokenizer
+
+app = FastAPI()
+
+# Load dataset and tokenizer
+billsum = load_dataset("billsum", split="train[:1%]")  # Load a small sample
+tokenizer = AutoTokenizer.from_pretrained("t5-small")
+prefix = "summarize: "  # Example prefix for text generation
+
+def preprocess_function(examples):
+    inputs = [prefix + doc for doc in examples["text"]]
+    model_inputs = tokenizer(inputs, max_length=1024, truncation=True)
+
+    labels = tokenizer(text_target=examples["summary"], max_length=128, truncation=True)
+    model_inputs["labels"] = labels["input_ids"]
+
+    return model_inputs
+
+@app.get("/tokenized")
+def get_tokenized_data():
+    tokenized_billsum = billsum.map(preprocess_function, batched=True)
+
+    # Convert to list of dictionaries
+    json_serializable_output = tokenized_billsum.to_pandas().to_dict(orient="records")
+
+    return {"tokenized_data": json_serializable_output}  # Ensure JSON format