VoiceAPI / src /api.py
Harshil748's picture
Add voice cloning service and update API for voice generation
24d1041
"""
REST API Server for Multi-lingual TTS
FastAPI-based server with OpenAPI documentation
Hackathon API Specification:
- GET /Get_Inference with text, lang, speaker_wav parameters
"""
import os
import io
import time
import logging
import tempfile
import uuid
from typing import Optional, List, Dict
from pathlib import Path
import numpy as np
from fastapi import (
FastAPI,
HTTPException,
Query,
Response,
BackgroundTasks,
UploadFile,
File,
Form,
)
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse, FileResponse, JSONResponse
from pydantic import BaseModel, Field
import soundfile as sf
from .engine import TTSEngine, TTSOutput
from .config import (
LANGUAGE_CONFIGS,
get_available_languages,
get_available_voices,
STYLE_PRESETS,
)
from .elevenlabs_service import ElevenLabsService
# Language mapping for XTTS voice cloning
XTTS_LANG_MAP = {
"english": "en",
"hindi": "hi",
"bengali": "bn",
"gujarati": "gu",
"marathi": "mr",
"telugu": "te",
"kannada": "kn",
}
# Language name to voice key mapping (for hackathon API)
LANG_TO_VOICE = {
"hindi": "hi_female",
"bengali": "bn_female",
"marathi": "mr_female",
"telugu": "te_female",
"kannada": "kn_female",
"bhojpuri": "bho_female",
"chhattisgarhi": "hne_female",
"maithili": "mai_female",
"magahi": "mag_female",
"english": "en_female",
"gujarati": "gu_mms",
}
# Setup logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Initialize FastAPI app
app = FastAPI(
title="Voice Tech for All - Multi-lingual TTS API",
description="""
A multi-lingual Text-to-Speech API supporting 10+ Indian languages.
## Features
- 10 Indian languages with male/female voices
- Real-time speech synthesis
- Text normalization for Indian languages
- Speed control
- Multiple audio formats (WAV, MP3)
## Supported Languages
Hindi, Bengali, Marathi, Telugu, Kannada, Bhojpuri,
Chhattisgarhi, Maithili, Magahi, English
## Use Case
Built for an LLM-based healthcare assistant for pregnant mothers
in low-income communities.
""",
version="1.0.0",
contact={
"name": "Harshil PAtel",
"url": "https://harshilpatel.me/#contact",
},
license_info={
"name": "CC BY 4.0",
"url": "https://creativecommons.org/licenses/by/4.0/",
},
)
# CORS middleware
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Initialize TTS Engine (lazy loading)
_engine: Optional[TTSEngine] = None
_elevenlabs = ElevenLabsService()
# In-memory session voice cache for temporary cloned voice IDs
_voice_session_cache: Dict[str, str] = {}
ALLOWED_AUDIO_TYPES = {
"audio/wav",
"audio/x-wav",
"audio/mpeg",
"audio/mp3",
}
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", str(10 * 1024 * 1024)))
def get_engine() -> TTSEngine:
"""Get or create TTS engine instance"""
global _engine
if _engine is None:
_engine = TTSEngine(device="auto")
return _engine
# Request/Response Models
class SynthesizeRequest(BaseModel):
"""Request body for text synthesis"""
text: str = Field(
..., description="Text to synthesize", min_length=1, max_length=5000
)
voice: str = Field(
"hi_male", description="Voice key (e.g., hi_male, bn_female, gu_mms)"
)
speed: float = Field(1.0, description="Speech speed (0.5-2.0)", ge=0.5, le=2.0)
pitch: float = Field(1.0, description="Pitch multiplier (0.5-2.0)", ge=0.5, le=2.0)
energy: float = Field(1.0, description="Energy/volume (0.5-2.0)", ge=0.5, le=2.0)
style: Optional[str] = Field(
None, description="Style preset (happy, sad, calm, excited, etc.)"
)
normalize: bool = Field(True, description="Apply text normalization")
class Config:
schema_extra = {
"example": {
"text": "નમસ્તે, હું તમારી કેવી રીતે મદદ કરી શકું?",
"voice": "gu_mms",
"speed": 1.0,
"pitch": 1.0,
"energy": 1.0,
"style": "calm",
"normalize": True,
}
}
class SynthesizeResponse(BaseModel):
"""Response metadata for synthesis"""
success: bool
duration: float
sample_rate: int
voice: str
text: str
inference_time: float
class CloneResponse(BaseModel):
"""Response metadata for voice cloning"""
success: bool
duration: float
sample_rate: int
inference_time: float
language: str
def _validate_audio_upload(upload: UploadFile, raw_bytes: bytes) -> None:
if upload is None:
raise HTTPException(status_code=400, detail="speaker_wav is required")
filename = (upload.filename or "").lower()
if not filename.endswith((".wav", ".mp3")):
raise HTTPException(
status_code=400, detail="Only .wav or .mp3 files are supported"
)
content_type = upload.content_type or ""
if content_type and content_type not in ALLOWED_AUDIO_TYPES:
raise HTTPException(
status_code=400, detail=f"Unsupported content type: {content_type}"
)
if len(raw_bytes) == 0:
raise HTTPException(status_code=400, detail="Uploaded audio file is empty")
if len(raw_bytes) > MAX_UPLOAD_BYTES:
raise HTTPException(
status_code=400,
detail=f"Audio file too large. Max allowed is {MAX_UPLOAD_BYTES // (1024 * 1024)} MB",
)
class VoiceInfo(BaseModel):
"""Information about a voice"""
key: str
name: str
language_code: str
gender: str
loaded: bool
downloaded: bool
model_type: str = "vits"
class HealthResponse(BaseModel):
"""Health check response"""
status: str
device: str
loaded_voices: List[str]
available_voices: int
style_presets: List[str]
# API Endpoints
@app.get("/", response_class=JSONResponse)
async def root():
"""API root - welcome message"""
return {
"message": "Voice Tech for All - Multi-lingual TTS API",
"docs": "/docs",
"health": "/health",
"synthesize": "/synthesize",
}
@app.get("/health", response_model=HealthResponse)
async def health_check():
"""Health check endpoint"""
engine = get_engine()
return HealthResponse(
status="healthy",
device=str(engine.device),
loaded_voices=engine.get_loaded_voices(),
available_voices=len(LANGUAGE_CONFIGS),
style_presets=list(STYLE_PRESETS.keys()),
)
@app.get("/voices", response_model=List[VoiceInfo])
async def list_voices():
"""List all available voices"""
engine = get_engine()
voices = engine.get_available_voices()
return [
VoiceInfo(
key=key,
name=info["name"],
language_code=info["code"],
gender=info["gender"],
loaded=info["loaded"],
downloaded=info["downloaded"],
model_type=info.get("type", "vits"),
)
for key, info in voices.items()
]
@app.get("/styles")
async def list_styles():
"""List available style presets for prosody control"""
return {
"presets": STYLE_PRESETS,
"description": {
"speed": "Speech rate multiplier (0.5-2.0)",
"pitch": "Pitch multiplier (0.5-2.0), >1 = higher",
"energy": "Volume/energy multiplier (0.5-2.0)",
},
}
@app.get("/languages")
async def list_languages():
"""List supported languages"""
return get_available_languages()
@app.post("/synthesize", response_class=Response)
async def synthesize_audio(request: SynthesizeRequest):
"""
Synthesize speech from text
Returns WAV audio file directly
"""
engine = get_engine()
# Validate voice
if request.voice not in LANGUAGE_CONFIGS:
raise HTTPException(
status_code=400,
detail=f"Unknown voice: {request.voice}. Use /voices to see available options.",
)
try:
start_time = time.time()
# Synthesize
output = engine.synthesize(
text=request.text,
voice=request.voice,
speed=request.speed,
pitch=request.pitch,
energy=request.energy,
style=request.style,
normalize_text=request.normalize,
)
inference_time = time.time() - start_time
# Convert to WAV bytes
buffer = io.BytesIO()
sf.write(buffer, output.audio, output.sample_rate, format="WAV")
buffer.seek(0)
# Return audio with metadata headers
return Response(
content=buffer.read(),
media_type="audio/wav",
headers={
"X-Duration": str(output.duration),
"X-Sample-Rate": str(output.sample_rate),
"X-Voice": output.voice,
"X-Style": output.style or "default",
"X-Inference-Time": str(inference_time),
},
)
except Exception as e:
logger.error(f"Synthesis error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.post("/synthesize/stream")
async def synthesize_stream(request: SynthesizeRequest):
"""
Synthesize speech and stream the audio
Returns streaming WAV audio
"""
engine = get_engine()
if request.voice not in LANGUAGE_CONFIGS:
raise HTTPException(status_code=400, detail=f"Unknown voice: {request.voice}")
try:
output = engine.synthesize(
text=request.text,
voice=request.voice,
speed=request.speed,
pitch=request.pitch,
energy=request.energy,
style=request.style,
normalize_text=request.normalize,
)
# Create streaming response
buffer = io.BytesIO()
sf.write(buffer, output.audio, output.sample_rate, format="WAV")
buffer.seek(0)
return StreamingResponse(
buffer,
media_type="audio/wav",
headers={"Content-Disposition": "attachment; filename=speech.wav"},
)
except Exception as e:
logger.error(f"Streaming error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.post("/generate", response_class=StreamingResponse)
async def generate_with_cloned_voice(
text: str = Form(...),
lang: str = Form("english"),
session_id: Optional[str] = Form(None),
speaker_wav: Optional[UploadFile] = File(None),
):
"""
Production-ready temporary voice cloning + speech generation using ElevenLabs.
"""
lang_lower = (lang or "english").lower().strip()
local_session_id = session_id or uuid.uuid4().hex
fallback_voice_id = os.getenv("ELEVENLABS_FALLBACK_VOICE_ID")
voice_id = _voice_session_cache.get(local_session_id)
if voice_id is None:
if speaker_wav is None:
raise HTTPException(
status_code=400,
detail="speaker_wav is required for first request in a session",
)
audio_bytes = await speaker_wav.read()
_validate_audio_upload(speaker_wav, audio_bytes)
# Re-wrap file for downstream service after validation read
clone_file = UploadFile(
filename=speaker_wav.filename,
file=io.BytesIO(audio_bytes),
headers=speaker_wav.headers,
)
try:
voice_id = await _elevenlabs.clone_voice(clone_file)
_voice_session_cache[local_session_id] = voice_id
logger.info(
"Cloned ElevenLabs voice for session=%s voice_id=%s",
local_session_id,
voice_id,
)
except HTTPException as exc:
if fallback_voice_id:
logger.warning(
"Clone failed for session=%s, using fallback voice",
local_session_id,
)
voice_id = fallback_voice_id
else:
raise exc
try:
audio_bytes = _elevenlabs.generate_speech(
text=text, voice_id=voice_id, language=lang_lower
)
except HTTPException as exc:
if fallback_voice_id and voice_id != fallback_voice_id:
logger.warning(
"TTS failed for voice=%s, retrying with fallback voice", voice_id
)
audio_bytes = _elevenlabs.generate_speech(
text=text,
voice_id=fallback_voice_id,
language=lang_lower,
)
else:
raise exc
headers = {
"Content-Disposition": "attachment; filename=generated.mp3",
"X-Session-Id": local_session_id,
"X-Voice-Id": voice_id,
"X-Provider": "elevenlabs",
}
return StreamingResponse(
io.BytesIO(audio_bytes), media_type="audio/mpeg", headers=headers
)
@app.post("/clone", response_class=Response)
async def clone_voice(
text: str = Query(..., description="Text to synthesize with cloned voice"),
lang: str = Query(
"english",
description="Language name (english, hindi, bengali, gujarati, marathi, telugu, kannada)",
),
speed: float = Query(1.0, description="Speech speed", ge=0.5, le=2.0),
pitch: float = Query(1.0, description="Pitch", ge=0.5, le=2.0),
energy: float = Query(1.0, description="Energy", ge=0.5, le=2.0),
style: Optional[str] = Query(None, description="Style preset"),
session_id: Optional[str] = Query(
None, description="Session key to reuse cloned voice"
),
speaker_wav: UploadFile = File(
..., description="Reference speaker WAV (3-15 seconds recommended)"
),
):
"""
Backward-compatible clone endpoint using ElevenLabs voice cloning.
"""
lang_lower = lang.lower().strip()
local_session_id = session_id or uuid.uuid4().hex
fallback_voice_id = os.getenv("ELEVENLABS_FALLBACK_VOICE_ID")
if lang_lower not in XTTS_LANG_MAP:
supported = ", ".join(sorted(XTTS_LANG_MAP.keys()))
raise HTTPException(
status_code=400,
detail=f"Unsupported clone language: {lang}. Supported: {supported}",
)
voice_id = _voice_session_cache.get(local_session_id)
if voice_id is None:
audio_bytes = await speaker_wav.read()
_validate_audio_upload(speaker_wav, audio_bytes)
clone_file = UploadFile(
filename=speaker_wav.filename,
file=io.BytesIO(audio_bytes),
headers=speaker_wav.headers,
)
try:
voice_id = await _elevenlabs.clone_voice(clone_file)
_voice_session_cache[local_session_id] = voice_id
except HTTPException as exc:
if fallback_voice_id:
voice_id = fallback_voice_id
else:
raise exc
try:
audio_bytes = _elevenlabs.generate_speech(
text=text,
voice_id=voice_id,
language=lang_lower,
)
except HTTPException as exc:
if fallback_voice_id and voice_id != fallback_voice_id:
audio_bytes = _elevenlabs.generate_speech(
text=text,
voice_id=fallback_voice_id,
language=lang_lower,
)
else:
raise exc
return Response(
content=audio_bytes,
media_type="audio/mpeg",
headers={
"Content-Disposition": "attachment; filename=cloned_output.mp3",
"X-Language": lang_lower,
"X-Voice": voice_id,
"X-Session-Id": local_session_id,
"X-Provider": "elevenlabs",
},
)
@app.get("/synthesize/get")
async def synthesize_get(
text: str = Query(
..., description="Text to synthesize", min_length=1, max_length=1000
),
voice: str = Query("hi_male", description="Voice key"),
speed: float = Query(1.0, description="Speech speed", ge=0.5, le=2.0),
pitch: float = Query(1.0, description="Pitch", ge=0.5, le=2.0),
energy: float = Query(1.0, description="Energy", ge=0.5, le=2.0),
style: Optional[str] = Query(None, description="Style preset"),
):
"""
GET endpoint for simple synthesis
Useful for testing and simple integrations
"""
request = SynthesizeRequest(
text=text, voice=voice, speed=speed, pitch=pitch, energy=energy, style=style
)
return await synthesize_audio(request)
@app.api_route("/Get_Inference", methods=["GET", "POST"])
async def get_inference(
text: str = Query(
...,
description="The input text to be converted into speech. For English, text must be lowercase.",
),
lang: str = Query(
...,
description="Language of input text. Supported: bhojpuri, bengali, english, gujarati, hindi, chhattisgarhi, kannada, magahi, maithili, marathi, telugu",
),
speaker_wav: UploadFile = File(
...,
description="A reference WAV file representing the speaker's voice (mandatory per hackathon spec).",
),
):
"""
Hackathon API - Generate speech audio from text
This endpoint follows the Voice Tech for All hackathon specification.
Supports both GET and POST methods with multipart form data.
Parameters:
- text: Input text to synthesize (query param)
- lang: Language (query param) - bhojpuri, bengali, english, gujarati, hindi, chhattisgarhi, kannada, magahi, maithili, marathi, telugu
- speaker_wav: Reference WAV file (multipart file upload, mandatory)
Returns:
- 200 OK: WAV audio file as streaming response
"""
engine = get_engine()
# Normalize language name
lang_lower = lang.lower().strip()
# Enforce lowercase for English text (per spec)
if lang_lower == "english":
text = text.lower()
# Map language to voice
if lang_lower not in LANG_TO_VOICE:
supported = list(LANG_TO_VOICE.keys())
raise HTTPException(
status_code=400,
detail=f"Unsupported language: {lang}. Supported languages: {', '.join(supported)}",
)
voice = LANG_TO_VOICE[lang_lower]
# Read speaker_wav (mandatory per spec)
# Note: Current VITS models don't support voice cloning, but we accept the file
# for API compatibility and validation. In future, this could be used for voice adaptation.
try:
speaker_audio_bytes = await speaker_wav.read()
logger.info(
f"Received speaker reference WAV: {len(speaker_audio_bytes)} bytes, filename: {speaker_wav.filename}"
)
# Validate it's a valid audio file (basic check)
if len(speaker_audio_bytes) < 44: # Minimum WAV header size
raise HTTPException(
status_code=400,
detail="Invalid speaker_wav: file too small to be a valid WAV",
)
except HTTPException:
raise
except Exception as e:
logger.error(f"Could not read speaker_wav: {e}")
raise HTTPException(
status_code=400, detail=f"Failed to read speaker_wav file: {str(e)}"
)
try:
# Synthesize audio
output = engine.synthesize(
text=text,
voice=voice,
speed=1.0,
normalize_text=True,
)
# Convert to WAV bytes
buffer = io.BytesIO()
sf.write(buffer, output.audio, output.sample_rate, format="WAV")
buffer.seek(0)
# Return as streaming response (per spec)
return StreamingResponse(
buffer,
media_type="audio/wav",
headers={
"Content-Disposition": "attachment; filename=output.wav",
"X-Duration": str(output.duration),
"X-Sample-Rate": str(output.sample_rate),
"X-Language": lang,
"X-Voice": voice,
},
)
except Exception as e:
logger.error(f"Synthesis error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.post("/preload")
async def preload_voice(voice: str):
"""Preload a voice model into memory"""
engine = get_engine()
if voice not in LANGUAGE_CONFIGS:
raise HTTPException(status_code=400, detail=f"Unknown voice: {voice}")
try:
engine.load_voice(voice)
return {"message": f"Voice {voice} loaded successfully"}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/unload")
async def unload_voice(voice: str):
"""Unload a voice model from memory"""
engine = get_engine()
engine.unload_voice(voice)
return {"message": f"Voice {voice} unloaded"}
@app.post("/batch")
async def batch_synthesize(
texts: List[str], voice: str = "hi_male", speed: float = 1.0
):
"""
Synthesize multiple texts
Returns a list of base64-encoded audio
"""
import base64
engine = get_engine()
if voice not in LANGUAGE_CONFIGS:
raise HTTPException(status_code=400, detail=f"Unknown voice: {voice}")
results = []
for text in texts:
output = engine.synthesize(text, voice, speed)
buffer = io.BytesIO()
sf.write(buffer, output.audio, output.sample_rate, format="WAV")
buffer.seek(0)
results.append(
{
"text": text,
"audio_base64": base64.b64encode(buffer.read()).decode(),
"duration": output.duration,
}
)
return results
# Startup/Shutdown events
@app.on_event("startup")
async def startup_event():
"""Initialize on startup"""
logger.info("Starting TTS API server...")
# Optionally preload default voice
# get_engine().load_voice("hi_male")
@app.on_event("shutdown")
async def shutdown_event():
"""Cleanup on shutdown"""
logger.info("Shutting down TTS API server...")
def start_server(host: str = "0.0.0.0", port: int = 8000, reload: bool = False):
"""Start the API server"""
import uvicorn
uvicorn.run("src.api:app", host=host, port=port, reload=reload, log_level="info")
if __name__ == "__main__":
start_server()