Add appy_chat_template process

app.py

@@ -10,7 +10,7 @@ import sys
 import torch
 
 
-from spanish_medica_llm import run_training, run_training_process, run_finnetuning_process
+from spanish_medica_llm import run_training, run_training_process, run_finnetuning_process, generate_response
 
 import gradio as gr
 
@@ -31,13 +31,15 @@ def generate(prompt):
     image = pipe(prompt).images[0]  
     return(image)
     
-def evaluate_model():
+def evaluate_model(input):
     #from diffusers import StableDiffusionPipeline
     
     #pipe = StableDiffusionPipeline.from_pretrained("./output_model", torch_dtype=torch.float16)
     #pipe = pipe.to("cuda")
     #image = pipe(prompt).images[0]  
-    return(f"Evaluate Model {os.environ.get('HF_LLM_MODEL_ID')} from dataset {os.environ.get('HF_LLM_DATASET_ID')}")
+    output = generate_response(input)
+    return output
+
 
 
 
@@ -66,7 +68,7 @@ with gr.Blocks() as demo:
     with gr.Row():
         inp = gr.Textbox(placeholder="What is your name?")
         out = gr.Textbox()
-        
+
     btn_response = gr.Button("Generate Response")
     btn_response.click(fn=generate_model, inputs=inp, outputs=out)
     btn_train = gr.Button("Train Model")
@@ -74,7 +76,7 @@ with gr.Blocks() as demo:
     btn_finnetuning = gr.Button("Finnetuning Model")
     btn_finnetuning.click(fn=finnetuning_model, inputs=[], outputs=out)
     btn_evaluate = gr.Button("Evaluate Model")
-    btn_evaluate.click(fn=evaluate_model, inputs=[], outputs=out)
+    btn_evaluate.click(fn=evaluate_model, inputs=inp, outputs=out)
     btn_stop = gr.Button("Stop Model")
     btn_stop.click(fn=stop_model, inputs=[], outputs=out)
 

spanish_medica_llm.py

@@ -19,7 +19,8 @@ from transformers import (
     BitsAndBytesConfig, 
     DataCollatorForLanguageModeling,
     TrainingArguments,
-    Trainer
+    Trainer,
+    GenerationConfig
 )
 
 from accelerate import FullyShardedDataParallelPlugin, Accelerator
@@ -711,6 +712,41 @@ def run_finnetuning_process():
     print('Dataset in One ')
     print (train_dataset[5])
     configAndRunFineTuning(base_model,train_dataset, eval_dataset, tokenizer)
-def generate_response(query):
 
+def generate_response(query):
+        max_new_tokens=256
+        temperature=0.1
+        top_p=0.75
+        top_k=40
+        num_beams=2
+
+        tokenizer = loadSpanishTokenizer()
+        model =  loadBaseModel(HUB_MODEL_ID)
+
+        system = f"[INST]\nYou are a helpful coding assistant.[/INST]\n"
+        prompt = f"{system}\n{query}\n \n"
+        print(prompt)
+        inputs = tokenizer(prompt, return_tensors="pt")
+        input_ids = inputs["input_ids"].to("cuda")
+        attention_mask = inputs["attention_mask"].to("cuda")
+        generation_config = GenerationConfig(
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            num_beams=num_beams,       
+        )
+        with torch.no_grad():
+            generation_output = model.generate(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                generation_config=generation_config,
+                return_dict_in_generate=True,
+                #output_scores=True,
+                max_new_tokens=max_new_tokens,
+                early_stopping=True
+            )
+        s = generation_output.sequences[0]
+        output = tokenizer.decode(s, skip_special_tokens=True)
+        return output
+        # return output.split("<|assistant|>")[1]