Update app.py

app.py

@@ -1,6 +1,14 @@
+#INFERENCE NLP+EMOTION DETECTION CV+TTS
+
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 import torch
+from deepface import DeepFace
+import threading
+import time
+from parler_tts import ParlerTTSForConditionalGeneration
+import soundfile as sf
+import numpy as np
 
 # Model setup
 model_name = "IniNLP247/Kenko-mental-health-llama-3-model"
@@ -14,7 +22,7 @@ if tokenizer.pad_token is None:
 
 model = AutoModelForCausalLM.from_pretrained(
     model_name,
-    torch_dtype=torch.float16,  # Changed from 'dtype' to 'torch_dtype'
+    load_in_8bit=True,
     device_map="auto"
 )
 
@@ -24,7 +32,7 @@ pipe = pipeline(
     model=model,
     tokenizer=tokenizer,
     return_full_text=False,
-    max_new_tokens=300,
+    max_new_tokens=1024,
     temperature=0.7,
     top_p=0.9,
     repetition_penalty=1.1,
@@ -33,16 +41,93 @@ pipe = pipeline(
 
 print("✅ Model loaded successfully!")
 
+#Loading of TTS
+print("Loading Parler TTS Model...")
+tts_device = "cuda:0" if torch.cuda.is_available() else "cpu"
+tts_model = ParlerTTSForConditionalGeneration.from_pretrained("parler-tts/parler-tts-mini-v1", torch_dtype=torch.float16).to(tts_device)
+tts_tokenizer = AutoTokenizer.from_pretrained("parler-tts/parler-tts-mini-v1")
+print("✅ Parler TTS Model loaded successfully!")
+
+
+# Global variable to store current emotion state
+current_emotion_state = {
+    "dominant": "neutral",
+    "confidence": 0.0,
+    "all_emotions": {},
+    "last_update": None
+}
+
+def analyze_emotion(image):
+    """Analyze emotion from webcam image"""
+    global current_emotion_state
+
+    try:
+        if image is None:
+            return {}
+
+        result = DeepFace.analyze(
+            img_path=image,
+            actions=['emotion'],
+            enforce_detection=False,
+            detector_backend='opencv'
+        )
+
+        if isinstance(result, list):
+            emotions = result[0]['emotion']
+            dominant = result[0]['dominant_emotion']
+        else:
+            emotions = result['emotion']
+            dominant = result['dominant_emotion']
+
+        # Update global emotion state
+        current_emotion_state = {
+            "dominant": dominant,
+            "confidence": emotions[dominant],
+            "all_emotions": emotions,
+            "last_update": time.time()
+        }
+
+        # Format for display - REMOVE the % symbol and keep as numbers
+        output = {}
+        for emotion, score in sorted(emotions.items(), key=lambda x: x[1], reverse=True):
+            output[emotion.capitalize()] = score  # Just the number, no formatting
+
+        return output
+
+    except Exception as e:
+        print(f"Emotion analysis error: {str(e)}")
+        return {}
+
+def get_emotion_context():
+    """Get current emotion as context string for the model"""
+    if current_emotion_state["last_update"] is None:
+        return ""
+
+    # Check if emotion data is recent (within last 60 seconds)
+    if time.time() - current_emotion_state["last_update"] > 60:
+        return ""
+
+    dominant = current_emotion_state["dominant"]
+    confidence = current_emotion_state["confidence"]
+
+    emotion_context = f"\n[User's Current Detected Emotion: {dominant} ({confidence:.1f}% confidence)]"
+    return emotion_context
+
 def chat_with_kenko(message, history):
-    """Chat function for Gradio interface"""
+    """Chat function for Gradio interface with emotion awareness"""
+
     # Build conversation context
     conversation = ""
     for user_msg, bot_msg in history:
         conversation += f"User: {user_msg}\nKenko: {bot_msg}\n\n"
 
-    # Create prompt in instruction format
+    # Get emotion context
+    emotion_context = get_emotion_context()
+
+    # Create prompt in instruction format with emotion awareness
     prompt = f"""### Instruction:
 You are Kenko, a compassionate mental health therapist. Provide empathetic, helpful, and professional responses to support the user's mental wellbeing.
+{emotion_context}
 
 {conversation}User: {message}
 
@@ -56,11 +141,58 @@ You are Kenko, a compassionate mental health therapist. Provide empathetic, help
     except Exception as e:
         return f"I'm sorry, I'm having trouble processing your message right now. Error: {str(e)}"
 
+def generate_tts(text):
+    try:
+        # Limit text severely for testing
+        text = text[:200]  # Even shorter for testing
+
+        print(f"[TTS] Starting generation for {len(text)} chars: '{text[:50]}...'")
+
+        description = "A calm, empathetic voice speaking at a moderate pace."
+
+        input_ids = tts_tokenizer(description, return_tensors="pt").input_ids.to(tts_device)
+        prompt_input_ids = tts_tokenizer(text, return_tensors="pt").input_ids.to(tts_device)
+
+        print(f"[TTS] Tokenization complete. Generating audio...")
+
+        # Use proper generation parameters for Parler TTS
+        generation = tts_model.generate(
+            input_ids=input_ids,
+            prompt_input_ids=prompt_input_ids,
+            do_sample=True,
+            temperature=1.0,
+            min_new_tokens=10,
+            max_new_tokens=500  # Use max_new_tokens instead of max_length
+        )
+
+        print(f"[TTS] Generation complete. Processing audio...")
+
+        audio_arr = generation.cpu().numpy().squeeze()
+
+        print(f"[TTS] Audio array shape: {audio_arr.shape}")
+
+        return (tts_model.config.sampling_rate, audio_arr)
+
+    except Exception as e:
+        print(f"❌ TTS generation error: {str(e)}")
+        import traceback
+        traceback.print_exc()
+        return None
+
+print(f"TTS Model Device: {tts_model.device}")
+print(f"TTS Device Variable: {tts_device}")
+
 # Custom CSS for a calming interface
 css = """
 .gradio-container {
     font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
 }
+.emotion-box {
+    border: 2px solid #4CAF50;
+    border-radius: 10px;
+    padding: 10px;
+    margin: 10px 0;
+}
 """
 
 # Create Gradio interface
@@ -71,35 +203,63 @@ with gr.Blocks(
 ) as demo:
 
     gr.Markdown("""
-    # 🧠💚 Kenko - Your Mental Health Assistant
+    # 🧠💚 Kenko - Your Emotion-Aware Mental Health Assistant
 
-    Welcome! I'm Kenko, an AI mental health therapist here to provide support, guidance, and a listening ear.
-    Feel free to share what's on your mind - I'm here to help you through whatever you're experiencing.
+    Welcome! I'm Kenko, an AI mental health therapist enhanced with real-time emotion detection.
+    Allow webcam access to enable emotion-aware responses that adapt to how you're feeling.
 
     *Please remember: I'm an AI assistant and cannot replace professional mental health care. In crisis situations, please contact emergency services or a mental health professional.*
     """)
 
-    chatbot = gr.Chatbot(
-        height=500,
-        show_label=False,
-        container=True,
-        bubble_full_width=False,
-        avatar_images=("👤", "🧠")
-    )
-
     with gr.Row():
-        msg = gr.Textbox(
-            placeholder="Share what's on your mind... (press Enter to send)",
-            container=False,
-            scale=7,
-            lines=2,
-            max_lines=4
-        )
-        send_btn = gr.Button("Send 💬", scale=1, variant="primary")
+        # Left column: Chat interface
+        with gr.Column(scale=2):
+            chatbot = gr.Chatbot(
+                height=500,
+                show_label=False,
+                container=True,
+                bubble_full_width=False,
+                avatar_images=("👤", "🧠")
+            )
 
-    with gr.Row():
-        clear_btn = gr.Button("🗑️ Clear Chat", scale=1, variant="secondary")
-        examples_btn = gr.Button("💡 Example Topics", scale=1, variant="secondary")
+            audio_output = gr.Audio(
+                label="Kenko's Voice Response",
+                autoplay=True,
+                show_label=True
+            )
+
+            with gr.Row():
+                msg = gr.Textbox(
+                    placeholder="Share what's on your mind... (press Enter to send)",
+                    container=False,
+                    scale=7,
+                    lines=2,
+                    max_lines=4
+                )
+                send_btn = gr.Button("Send 💬", scale=1, variant="primary")
+
+            with gr.Row():
+                clear_btn = gr.Button("🗑️ Clear Chat", scale=1, variant="secondary")
+                examples_btn = gr.Button("💡 Example Topics", scale=1, variant="secondary")
+
+        # Right column: Emotion detection
+        with gr.Column(scale=1):
+            gr.Markdown("### 📸 Emotion Detection")
+            gr.Markdown("*Your emotional state helps me provide more personalized support*")
+
+            webcam_input = gr.Image(
+                sources=["webcam"],
+                type="numpy",
+                streaming=True,
+                label="Live Webcam Feed"
+            )
+
+            emotion_output = gr.Label(
+                num_top_classes=7,
+                label="Detected Emotions"
+            )
+
+            emotion_status = gr.Markdown("*Waiting for emotion data...*")
 
     # Example prompts
     with gr.Row(visible=False) as examples_row:
@@ -121,37 +281,85 @@ with gr.Blocks(
     with gr.Accordion("ℹ️ About Kenko", open=False):
         gr.Markdown("""
         **What I can help with:**
-        - Active listening and emotional support
+        - Active listening and emotional support (now emotion-aware!)
         - Coping strategies and stress management techniques
         - Guidance on anxiety, depression, and mood concerns
         - Relationship and communication advice
         - Mindfulness and self-care suggestions
         - Building healthy habits and routines
 
+        **Emotion Detection Feature:**
+        - Real-time facial emotion analysis
+        - Adapts responses based on your current emotional state
+        - Updates automatically every 30 seconds
+        - Completely optional - works without webcam too
+
         **Important Notes:**
         - I'm an AI trained to provide mental health support
         - For immediate crisis support, contact emergency services (911) or crisis hotlines
         - Consider professional therapy for ongoing mental health needs
         - I don't diagnose conditions or prescribe medications
 
-        **Privacy:** Your conversations are not stored or shared.
+        **Privacy:** Your conversations and emotion data are not stored or shared.
         """)
 
     def respond(message, chat_history):
         if not message.strip():
-            return "", chat_history
+            return "", chat_history, None
+
+        import time
 
+        start = time.time()
         bot_response = chat_with_kenko(message, chat_history)
+        text_time = time.time() - start
+        print(f"Text Generation Time: {text_time:.2f} seconds: {len(bot_response)} characters")
         chat_history.append((message, bot_response))
-        return "", chat_history
+
+        tts_start = time.time()
+        print(f"Generating TTS for: '{bot_response[:100]}...'")
+        audio = generate_tts(bot_response)
+        tts_time = time.time() - tts_start
+        print(f"TTS Generation Time: {tts_time:.2f} seconds")
+        print(f"TOTAL TIME: {time.time() - start:.2f}s")
+
+        return "", chat_history, audio
 
     def toggle_examples():
         return gr.Row(visible=True)
 
-    msg.submit(respond, [msg, chatbot], [msg, chatbot])
-    send_btn.click(respond, [msg, chatbot], [msg, chatbot])
-    clear_btn.click(lambda: [], outputs=chatbot)
+    def update_emotion_status():
+        """Update emotion status text"""
+        if current_emotion_state["last_update"] is None:
+            return "*Waiting for emotion data...*"
+
+        elapsed = time.time() - current_emotion_state["last_update"]
+        if elapsed > 60:
+            return "*Emotion data outdated - please ensure webcam is active*"
+
+        dominant = current_emotion_state["dominant"]
+        confidence = current_emotion_state["confidence"]
+        return f"**Current Emotion:** {dominant.capitalize()} ({confidence:.1f}% confidence)\n*Last updated: {int(elapsed)}s ago*"
+
+    # Event handlers
+    submit = msg.submit(fn=respond, inputs=[msg, chatbot], outputs=[msg, chatbot, audio_output])
+    send = send_btn.click(fn=respond, inputs=[msg, chatbot], outputs=[msg, chatbot, audio_output])
+    clear_btn.click(lambda: [], None, outputs=[chatbot, audio_output])
     examples_btn.click(toggle_examples, outputs=examples_row)
 
+    # Emotion detection with streaming (analyzes continuously)
+    webcam_input.stream(
+        analyze_emotion,
+        inputs=webcam_input,
+        outputs=emotion_output,
+        time_limit=30,  # Analyze every 30 seconds
+        stream_every=30  # Update interval
+    )
+
+    timer = gr.Timer(value=5)  # Update every 5 seconds
+    timer.tick(
+        fn=update_emotion_status,
+        outputs=emotion_status
+    )
+    
 if __name__ == "__main__":
-    demo.launch()  # Simplified for HF Spaces - no need for server config
+    demo.launch()