upd

backend_service.py

@@ -7,6 +7,8 @@ Provides OpenAI-compatible chat completion endpoints
 import os
 os.environ.setdefault("HF_HOME", "/tmp/.cache/huggingface")
 os.environ.setdefault("TRANSFORMERS_CACHE", "/tmp/.cache/huggingface")
+# Define Hugging Face auth token from environment
+hf_token = os.environ.get("HF_TOKEN")
 import asyncio
 import logging
 import time
@@ -191,7 +193,16 @@ async def lifespan(app: FastAPI):
         # Initialize tokenizer for better text handling
         if transformers_available and AutoTokenizer:
             try:
-                tokenizer = AutoTokenizer.from_pretrained(current_model)  # type: ignore
+                # Load tokenizer, using auth token if provided
+                if hf_token:
+                    tokenizer = AutoTokenizer.from_pretrained(
+                        current_model,
+                        use_auth_token=hf_token
+                    )  # type: ignore
+                else:
+                    tokenizer = AutoTokenizer.from_pretrained(
+                        current_model
+                    )  # type: ignore
                 logger.info("✅ Tokenizer loaded successfully")
             except Exception as e:
                 logger.warning(f"⚠️ Could not load tokenizer: {e}")
@@ -469,33 +480,49 @@ async def list_models():
     
     return ModelsResponse(data=models)
 
-@app.post("/v1/chat/completions")
+        
+        # Clean up the response
+        response_text = response_text.strip() if response_text else "No response generated."
+        
+        # Create OpenAI-compatible response
+        response = ChatCompletionResponse(
+            id=f"chatcmpl-{int(time.time())}",
+            created=int(time.time()),
+            model=request.model,
+            choices=[
+                ChatCompletionChoice(
+                    index=0,
+                    message=ChatMessage(role="assistant", content=response_text),
+                    finish_reason="stop"
+                )
+            ]
+        )
+        
+        return response
+        
+    except Exception as e:
+        logger.error(f"Error in chat completion: {e}")
+        raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
+@app.post("/api/response")
+@app.post("/v1/chat/completions", response_model=ChatCompletionResponse)
 async def create_chat_completion(
     request: ChatCompletionRequest,
     client: InferenceClient = Depends(get_inference_client)
-):
-    """Create a chat completion (OpenAI-compatible) with multimodal support"""
+) -> ChatCompletionResponse:
+    """Create a chat completion (OpenAI-compatible) with multimodal support."""
     try:
-        # Validate request
         if not request.messages:
             raise HTTPException(status_code=400, detail="Messages cannot be empty")
-        
-        # Check if this is a multimodal request (contains images)
         is_multimodal = has_images(request.messages)
-        
         if is_multimodal:
-            # Handle multimodal request with image-text pipeline
             if not image_text_pipeline:
                 raise HTTPException(status_code=503, detail="Image processing not available")
-            
             response_text = await generate_multimodal_response(request.messages, request)
         else:
-            # Handle text-only request with existing logic
             prompt = convert_messages_to_prompt(request.messages)
             logger.info(f"Generated prompt: {prompt[:200]}...")
-            
             if request.stream:
-                # Return streaming response
                 return StreamingResponse(
                     generate_streaming_response(client, prompt, request),
                     media_type="text/plain",
@@ -504,37 +531,26 @@ async def create_chat_completion(
                         "Connection": "keep-alive",
                         "Content-Type": "text/plain; charset=utf-8"
                     }
-                )
-            else:
-                # Generate non-streaming response
-                response_text = await asyncio.to_thread(
-                    generate_response_safe,
-                    client,
-                    prompt,
-                    request.max_tokens or 512,
-                    request.temperature or 0.7,
-                    request.top_p or 0.95
-                )
-        
-        # Clean up the response
+                )  # type: ignore
+            response_text = await asyncio.to_thread(
+                generate_response_safe,
+                client,
+                prompt,
+                request.max_tokens or 512,
+                request.temperature or 0.7,
+                request.top_p or 0.95
+            )
         response_text = response_text.strip() if response_text else "No response generated."
-        
-        # Create OpenAI-compatible response
-        response = ChatCompletionResponse(
+        return ChatCompletionResponse(
             id=f"chatcmpl-{int(time.time())}",
             created=int(time.time()),
             model=request.model,
-            choices=[
-                ChatCompletionChoice(
-                    index=0,
-                    message=ChatMessage(role="assistant", content=response_text),
-                    finish_reason="stop"
-                )
-            ]
+            choices=[ChatCompletionChoice(
+                index=0,
+                message=ChatMessage(role="assistant", content=response_text),
+                finish_reason="stop"
+            )]
         )
-        
-        return response
-        
     except Exception as e:
         logger.error(f"Error in chat completion: {e}")
         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
@@ -548,17 +564,14 @@ async def create_completion(
     try:
         if not request.prompt:
             raise HTTPException(status_code=400, detail="Prompt cannot be empty")
-        
-        # Generate response
         response_text = await asyncio.to_thread(
             generate_response_safe,
             client,
             request.prompt,
             request.max_tokens or 512,
             request.temperature or 0.7,
-            0.95  # default top_p
+            0.95
         )
-        
         return {
             "id": f"cmpl-{int(time.time())}",
             "object": "text_completion",
@@ -570,57 +583,21 @@ async def create_completion(
                 "finish_reason": "stop"
             }]
         }
-        
     except Exception as e:
         logger.error(f"Error in completion: {e}")
         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
 @app.post("/api/response")
-async def api_response(request: Request):
+async def api_response(request: Request) -> JSONResponse:
     """Endpoint to receive and send responses via API."""
     try:
         data = await request.json()
         message = data.get("message", "No message provided")
-        response: dict[str, str] = {
+        return JSONResponse(content={
             "status": "success",
             "received_message": message,
             "response_message": f"You sent: {message}"
-        }
-        return JSONResponse(content=response)
+        })
     except Exception as e:
         logger.error(f"Error processing API response: {e}")
         raise HTTPException(status_code=500, detail="Internal server error")
-
-@app.exception_handler(Exception)
-async def global_exception_handler(request: Any, exc: Exception) -> JSONResponse:
-    """Global exception handler"""
-    logger.error(f"Unhandled exception: {exc}")
-    return JSONResponse(
-        status_code=500,
-        content={"detail": f"Internal server error: {str(exc)}"}
-    )
-
-if __name__ == "__main__":
-    import argparse
-    
-    parser = argparse.ArgumentParser(description="AI Backend Service")
-    parser.add_argument("--host", default="0.0.0.0", help="Host to bind to")
-    parser.add_argument("--port", type=int, default=8000, help="Port to bind to")
-    parser.add_argument("--model", default=current_model, help="HuggingFace model to use")
-    parser.add_argument("--reload", action="store_true", help="Enable auto-reload for development")
-    
-    args = parser.parse_args()
-    
-    if args.model != current_model:
-        current_model = args.model
-        logger.info(f"Using model: {current_model}")
-    
-    logger.info(f"🚀 Starting AI Backend Service on {args.host}:{args.port}")
-    
-    uvicorn.run(
-        "backend_service:app",
-        host=args.host,
-        port=args.port,
-        reload=args.reload,
-        log_level="info"
-    )