api.py

"""
STEP 6 — FASTAPI BACKEND  (v7 — three-model system, multi-instrument)
======================================================================
Complete rewrite of api.py to use the v7 model system.

Key changes from original api.py:
  - Three models loaded: Stage1, Stage2, 3-class (was single XGBRegressor)
  - All v7 asymmetric gates applied (strict BUY, loose SELL)
  - New /signal endpoint returns full decision trail + gate results
  - New /analyse endpoint accepts uploaded CSV for any instrument
  - CSV upload runs full pipeline: indicators → Chronos → models → signal
  - Auto-detects column names (handles any OHLCV CSV format)
  - Backtest results returned alongside live signal
  - CORS enabled for frontend integration
  - Live data via Twelve Data API (free tier: 800 calls/day)

Usage:
  pip install fastapi uvicorn python-multipart python-dotenv requests
  uvicorn api:app --reload --port 8080

Endpoints:
  GET  /                      → status
  GET  /signal                → live signal from data/nifty_5m.csv
  POST /analyse               → upload any OHLCV CSV, get signal + backtest
  GET  /instruments            → list of supported instruments
  GET  /live-analyse/{symbol}  → fetch live data + full analysis
  GET  /price/{symbol}         → quick current price lookup
"""

import io
import os
import time
import traceback
import numpy as np
import pandas as pd
import torch
import requests as http_requests
import xgboost as xgb
from fastapi import FastAPI, File, UploadFile, Form, Query
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from ta.momentum import RSIIndicator
from ta.trend import ADXIndicator, MACD
from ta.volatility import AverageTrueRange
from chronos import Chronos2Pipeline
from dotenv import load_dotenv
from yahooquery import Ticker


load_dotenv()

# ─── APP ──────────────────────────────────────────────────────────────────────
app = FastAPI(title="MPC Quant AI Engine", version="7.0")

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_methods=["*"],
    allow_headers=["*"],
)

# Serve frontend
if os.path.exists("frontend"):
    app.mount("/quant", StaticFiles(directory="frontend", html=True), name="quant")

# ─── CONFIG ───────────────────────────────────────────────────────────────────
DEFAULT_DATA_FILE = "data/nifty_5m.csv"
DEFAULT_DATE_FMT  = "%d-%m-%Y %H:%M"
CONTEXT_LENGTH    = 512
PREDICTION_LENGTH = 6

STAGE1_CONF  = 0.70
STAGE2_CONF  = 0.60
MODEL3_CONF  = 0.45
ADX_MIN      = 20

# BUY gates — strict
BUY_ADX_MIN    = 28
BUY_EMA50_SIDE = True
BUY_EMA_CROSS  = True
BUY_MACD_CROSS = True
BUY_CHRONOS    = True

# SELL gates — loose
SELL_EMA200_MAX = -0.5
SELL_ADX_MIN    = 20
SELL_EMA50_SIDE = True

LABEL_MAP   = {0:"SELL", 1:"NO TRADE", 2:"BUY"}
STAGE1_COLS = [
    "RSI","ATR","ADX","EMA20","EMA50","EMA200","EMA200_DISTANCE",
    "MACD","MACD_SIGNAL",
    "CHRONOS_RETURN","CHRONOS_SPREAD","CHRONOS_Q25","CHRONOS_Q75","CHRONOS_AGREE",
    "EMA_CROSS","EMA200_SIDE","MACD_CROSS","RSI_ZONE","ADX_TREND","ATR_NORM",
    "HOUR","IS_OPEN_NOISE","IS_CLOSE_NOISE",
]
REGIME_COLS = ["REGIME","REGIME_STRONG","EMA_ALIGN","TREND_BARS","EMA50_SIDE"]
STAGE2_COLS = STAGE1_COLS + REGIME_COLS

# ─── LOAD MODELS ON STARTUP ───────────────────────────────────────────────────
print("Loading Chronos-2...")
torch.set_num_threads(1)  # Reduce memory overhead on small instances
chronos_pipeline = Chronos2Pipeline.from_pretrained(
    "autogluon/chronos-2", device_map="cpu", dtype=torch.float32)
print("Chronos-2 loaded.")

print("Loading XGBoost models...")
stage1 = xgb.XGBClassifier(); stage1.load_model("models/nifty_stage1_tradeable.json")
stage2 = xgb.XGBClassifier(); stage2.load_model("models/nifty_stage2_direction.json")
model3 = xgb.XGBClassifier(); model3.load_model("models/nifty_xgboost_v2.json")
print("All models loaded. System ready.")


# ══════════════════════════════════════════════════════════════════════════════
# SHARED UTILITIES
# ══════════════════════════════════════════════════════════════════════════════

def detect_columns(df):
    """Auto-detect OHLCV column names regardless of case/spacing."""
    mapping = {}
    cols_lower = {c.strip().lower(): c for c in df.columns}
    for standard, candidates in {
        "date":   ["date","datetime","time","timestamp","Date","DateTime"],
        "open":   ["open","Open","OPEN","o"],
        "high":   ["high","High","HIGH","h"],
        "low":    ["low","Low","LOW","l"],
        "close":  ["close","Close","CLOSE","c","ltp","LTP"],
        "volume": ["volume","Volume","VOLUME","vol","Vol"],
    }.items():
        for cand in candidates:
            if cand.lower() in cols_lower:
                mapping[standard] = cols_lower[cand.lower()]
                break
    return mapping


def detect_interval(df):
    """Detect the timeframe interval from the 'date' column of a DataFrame."""
    if "date" not in df.columns or len(df) < 2:
        return "5min"
    try:
        # Sort values just in case they are out of order
        dates = pd.to_datetime(df["date"]).sort_values()
        diff = dates.iloc[1] - dates.iloc[0]
        minutes = diff.total_seconds() / 60
        if minutes <= 2:
            return "1min"
        elif minutes <= 8:
            return "5min"
        elif minutes <= 20:
            return "15min"
        elif minutes <= 90:
            return "1h"
        elif minutes <= 300:
            return "4h"
        else:
            return "1day"
    except Exception:
        return "5min"


def clean_and_parse(df, col_map):
    """Rename columns to standard names and clean flat candles."""
    df = df.rename(columns={v: k for k, v in col_map.items()})
    for col in ["open","high","low","close"]:
        if col in df.columns:
            df[col] = pd.to_numeric(df[col].astype(str).str.replace(",",""), errors="coerce")
    df = df.dropna(subset=["open","high","low","close"]).reset_index(drop=True)
    flat = ((df["open"]==df["high"]) & (df["high"]==df["low"]) & (df["low"]==df["close"]))
    df   = df[~flat].copy().reset_index(drop=True)
    # Parse date — try multiple formats
    if "date" in df.columns:
        for fmt in ["%d-%m-%Y %H:%M", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M",
                    "%d/%m/%Y %H:%M", "%m/%d/%Y %H:%M", None]:
            try:
                df["date"] = pd.to_datetime(df["date"], format=fmt)
                break
            except Exception:
                continue
    return df


def add_indicators(df):
    df["RSI"] = RSIIndicator(close=df["close"], window=14).rsi()
    df["ATR"] = AverageTrueRange(
        high=df["high"], low=df["low"], close=df["close"], window=14
    ).average_true_range()
    df["ADX"] = ADXIndicator(
        high=df["high"], low=df["low"], close=df["close"], window=14
    ).adx()
    df["EMA20"]  = df["close"].ewm(span=20,  adjust=False).mean()
    df["EMA50"]  = df["close"].ewm(span=50,  adjust=False).mean()
    df["EMA200"] = df["close"].ewm(span=200, adjust=False).mean()
    m = MACD(close=df["close"])
    df["MACD"]        = m.macd()
    df["MACD_SIGNAL"] = m.macd_signal()
    df["EMA200_DISTANCE"] = (df["close"] - df["EMA200"]) / df["EMA200"] * 100
    return df.dropna().reset_index(drop=True)


def run_chronos(close_series):
    history = close_series.tail(CONTEXT_LENGTH).values.astype("float32")
    cp      = float(history[-1])
    inputs  = torch.tensor(history).reshape(1,1,-1)
    fc      = chronos_pipeline.predict(inputs=inputs, prediction_length=PREDICTION_LENGTH)
    qt      = fc[0]
    q25,q50,q75 = qt[0,5,:].numpy(), qt[0,10,:].numpy(), qt[0,15,:].numpy()
    r50 = (float(q50[-1])-cp)/cp*100
    r25 = (float(q25[-1])-cp)/cp*100
    r75 = (float(q75[-1])-cp)/cp*100
    diffs = np.diff(q50)
    agree = (float((diffs>0).sum()) if r50>0 else float((diffs<0).sum()))/len(diffs) if r50!=0 else 0.0
    return {
        "CHRONOS_RETURN":r50, "CHRONOS_SPREAD":r75-r25,
        "CHRONOS_Q25":r25,    "CHRONOS_Q75":r75,
        "CHRONOS_AGREE":agree,
        "predicted_price":float(q50[-1]),
        "current_price":cp,
    }


def get_regime(df, idx):
    row   = df.iloc[idx]
    count = 0
    for j in range(max(0,idx-50), idx+1):
        r = df.iloc[j]
        count = max(count+1,1) if r["close"]>r["EMA50"] else min(count-1,-1)
    bs = int(row["EMA20"]>row["EMA50"] and row["EMA50"]>row["EMA200"])
    bb = int(row["EMA20"]<row["EMA50"] and row["EMA50"]<row["EMA200"])
    return {
        "REGIME":        float(np.sign(row["close"]-row["EMA200"])),
        "REGIME_STRONG": int(abs(row["EMA200_DISTANCE"])>1.0),
        "EMA_ALIGN":     bs-bb,
        "TREND_BARS":    int(np.clip(count,-50,50)),
        "EMA50_SIDE":    float(np.sign(row["close"]-row["EMA50"])),
    }


def build_features(df, idx, cf):
    row  = df.iloc[idx]
    h    = row["date"].hour if "date" in df.columns else 12
    m    = row["date"].minute if "date" in df.columns else 0
    reg  = get_regime(df, idx)
    feat = {
        "RSI":float(row["RSI"]), "ATR":float(row["ATR"]), "ADX":float(row["ADX"]),
        "EMA20":float(row["EMA20"]), "EMA50":float(row["EMA50"]), "EMA200":float(row["EMA200"]),
        "EMA200_DISTANCE":float(row["EMA200_DISTANCE"]),
        "MACD":float(row["MACD"]), "MACD_SIGNAL":float(row["MACD_SIGNAL"]),
        "CHRONOS_RETURN":cf["CHRONOS_RETURN"], "CHRONOS_SPREAD":cf["CHRONOS_SPREAD"],
        "CHRONOS_Q25":cf["CHRONOS_Q25"], "CHRONOS_Q75":cf["CHRONOS_Q75"],
        "CHRONOS_AGREE":cf["CHRONOS_AGREE"],
        "EMA_CROSS":    float(np.sign(row["EMA20"]-row["EMA50"])),
        "EMA200_SIDE":  float(np.sign(row["close"]-row["EMA200"])),
        "MACD_CROSS":   float(np.sign(row["MACD"]-row["MACD_SIGNAL"])),
        "RSI_ZONE":     0 if row["RSI"]<30 else (2 if row["RSI"]>70 else 1),
        "ADX_TREND":    int(row["ADX"]>20),
        "ATR_NORM":     float(row["ATR"]/row["close"])*100,
        "HOUR":h, "IS_OPEN_NOISE":int(h==9 and m<=30), "IS_CLOSE_NOISE":int(h==15),
    }
    feat.update(reg)
    return feat, reg


def predict_and_gate(feat, ema200_dist, chronos_ret):
    """Full three-model prediction + v7 asymmetric gates."""
    X1   = pd.DataFrame([{c:feat[c] for c in STAGE1_COLS}])
    s1_p = float(stage1.predict_proba(X1)[0][1])
    m3p  = model3.predict_proba(X1)[0]
    m3sig = LABEL_MAP[int(np.argmax(m3p))]

    if s1_p < STAGE1_CONF:
        return "NO TRADE", s1_p, None, m3sig, m3p, "Stage 1 below threshold"

    X2   = pd.DataFrame([{c:feat[c] for c in STAGE2_COLS}])
    s2_p = float(stage2.predict_proba(X2)[0][1])

    if   s2_p >= STAGE2_CONF:       raw,dc = "BUY",      s2_p
    elif s2_p <= (1-STAGE2_CONF):   raw,dc = "SELL",     1-s2_p
    else:                            return "NO TRADE", s1_p, None, m3sig, m3p, "Stage 2 uncertain"

    # 3-class confirmation
    if m3sig != raw and float(m3p.max()) >= MODEL3_CONF:
        return "NO TRADE", s1_p, dc, m3sig, m3p, f"3-class disagreement ({m3sig} vs {raw})"

    ema50_side  = feat.get("EMA50_SIDE",      0)
    ema_cross   = feat.get("EMA_CROSS",        0)
    macd_cross  = feat.get("MACD_CROSS",       0)
    adx         = feat.get("ADX",              0)

    if raw == "BUY":
        if BUY_EMA50_SIDE  and ema50_side <= 0:  return "NO TRADE",s1_p,dc,m3sig,m3p,"BUY GATE: price below EMA50"
        if adx < BUY_ADX_MIN:                    return "NO TRADE",s1_p,dc,m3sig,m3p,f"BUY GATE: ADX={adx:.1f}<{BUY_ADX_MIN}"
        if BUY_EMA_CROSS   and ema_cross  <= 0:  return "NO TRADE",s1_p,dc,m3sig,m3p,"BUY GATE: EMA20 not > EMA50"
        if BUY_MACD_CROSS  and macd_cross <= 0:  return "NO TRADE",s1_p,dc,m3sig,m3p,"BUY GATE: MACD not > Signal"
        if BUY_CHRONOS     and chronos_ret <= 0: return "NO TRADE",s1_p,dc,m3sig,m3p,f"BUY GATE: Chronos={chronos_ret:+.4f}%"

    if raw == "SELL":
        if SELL_EMA50_SIDE and ema50_side >= 0:  return "NO TRADE",s1_p,dc,m3sig,m3p,"SELL GATE: price above EMA50"
        if ema200_dist > SELL_EMA200_MAX:        return "NO TRADE",s1_p,dc,m3sig,m3p,f"SELL GATE: EMA200 dist={ema200_dist:+.2f}%"
        if adx < SELL_ADX_MIN:                  return "NO TRADE",s1_p,dc,m3sig,m3p,f"SELL GATE: ADX={adx:.1f}<{SELL_ADX_MIN}"

    return raw, s1_p, dc, m3sig, m3p, "PASSED ALL GATES"


def build_signal_response(df, cf, instrument_name="unknown", interval="5min"):
    """Run full pipeline on prepared df and return signal dict."""
    latest_idx  = len(df) - 1
    latest      = df.iloc[latest_idx]
    feat, regime = build_features(df, latest_idx, cf)
    ema200_dist  = float(latest["EMA200_DISTANCE"])
    chronos_ret  = cf["CHRONOS_RETURN"]

    signal, s1_p, dir_conf, m3sig, m3p, gate_note = predict_and_gate(
        feat, ema200_dist, chronos_ret)

    # Time filter — only applied to low intraday intervals (1min, 5min, 15min)
    if interval in ["1min", "5min", "15min"] and "date" in df.columns:
        h = latest["date"].hour
        m_min = latest["date"].minute
        if h==9 and m_min<=30: signal="NO TRADE"; gate_note="TIME: open noise"
        if h==15:               signal="NO TRADE"; gate_note="TIME: close noise"

    # Calculate ATR-based levels for live recommendation
    latest_close = float(latest["close"])
    latest_atr   = float(latest["ATR"]) if "ATR" in latest else 0.0
    
    target_price = None
    stop_loss    = None
    
    if signal == "BUY":
        target_price = round(latest_close + 4.0 * latest_atr, 4)
        stop_loss    = round(latest_close - 4.5 * latest_atr, 4)
    elif signal == "SELL":
        target_price = round(latest_close - 4.0 * latest_atr, 4)
        stop_loss    = round(latest_close + 4.5 * latest_atr, 4)

    # Trend description
    if   regime["EMA_ALIGN"]>0:  trend = "FULL BULL"
    elif regime["EMA_ALIGN"]<0:  trend = "FULL BEAR"
    elif regime["EMA50_SIDE"]>0: trend = "MIXED — above EMA50"
    else:                         trend = "MIXED — below EMA50"

    return {
        "signal":           signal,
        "instrument":       instrument_name,
        "stage1_tradeable": s1_p >= STAGE1_CONF,
        "stage1_conf":      round(s1_p, 4),
        "stage2_direction": LABEL_MAP.get(2 if (dir_conf or 0)>0.5 else 0, "NO TRADE") if dir_conf else "N/A",
        "stage2_conf":      round(dir_conf, 4) if dir_conf else 0.0,
        "model3_signal":    m3sig,
        "model3_conf":      round(float(m3p.max()), 4),
        "prob_sell":        round(float(m3p[0]), 4),
        "prob_no_trade":    round(float(m3p[1]), 4),
        "prob_buy":         round(float(m3p[2]), 4),
        "gate_result":      gate_note,
        "regime":           trend,
        "trend_bars":       regime["TREND_BARS"],
        "ema50_side":       "ABOVE" if regime["EMA50_SIDE"]>0 else "BELOW",
        "ema200_side":      "ABOVE" if regime["REGIME"]>0 else "BELOW",
        "current_price":    round(cf["current_price"], 4),
        "forecast_price":   round(cf["predicted_price"], 4),
        "predicted_price":  round(cf["predicted_price"], 4),
        "target_price_atr": target_price,
        "stop_loss_atr":    stop_loss,
        "chronos_return_pct": round(chronos_ret, 4),
        "chronos_return":   round(chronos_ret, 4),
        "chronos_spread":   round(cf["CHRONOS_SPREAD"], 4),
        "chronos_agree":    round(cf["CHRONOS_AGREE"], 2),
        "adx":              round(float(latest["ADX"]), 2),
        "rsi":              round(float(latest["RSI"]), 2),
        "ema200_distance":  round(ema200_dist, 4),
        "data_rows":        len(df),
        "timestamp":        str(latest.get("date", "N/A")),
    }


def quick_backtest(df, instrument_name, interval="5min"):
    """
    Run a quick backtest on the uploaded data.
    Uses simplified signal generation (no Chronos per-bar — too slow).
    Uses rule-based signals from indicators for backtest only.
    Returns summary stats.
    """
    ATR_TRAIL = 4.5
    ATR_TGT   = 4.0
    MAX_BARS  = 12
    
    # Scale slippage dynamically based on asset price (0.002% of price)
    median_price = float(df["close"].median()) if len(df) > 0 else 100.0
    SLIP = median_price * 0.00002

    # Simple rule-based signals for backtest (no per-bar Chronos)
    df2 = df.copy().reset_index(drop=True)
    df2["EMA_CROSS"]  = np.sign(df2["EMA20"]  - df2["EMA50"])
    df2["MACD_CROSS"] = np.sign(df2["MACD"]   - df2["MACD_SIGNAL"])
    df2["EMA50_SIDE"] = np.sign(df2["close"]  - df2["EMA50"])
    df2["EMA200_SIDE"]= np.sign(df2["close"]  - df2["EMA200"])

    signals = []
    for i in range(len(df2)):
        row = df2.iloc[i]
        adx = float(row.get("ADX", 0))
        ema50 = float(row.get("EMA50_SIDE", 0))
        ema_x = float(row.get("EMA_CROSS",  0))
        macd_x= float(row.get("MACD_CROSS", 0))
        e200  = float(row.get("EMA200_DISTANCE", 0))

        # Time filter — only applied to low intraday intervals (1min, 5min, 15min)
        if interval in ["1min", "5min", "15min"] and "date" in df2.columns:
            h = row["date"].hour
            mn= row["date"].minute
            if (h==9 and mn<=30) or h==15:
                signals.append("NO TRADE"); continue

        # BUY: strict
        if (ema50>0 and adx>=BUY_ADX_MIN and ema_x>0 and macd_x>0):
            signals.append("BUY")
        # SELL: loose
        elif (ema50<0 and e200<SELL_EMA200_MAX and adx>=SELL_ADX_MIN):
            signals.append("SELL")
        else:
            signals.append("NO TRADE")

    df2["SIGNAL"] = signals
    trades  = []
    equity  = [100.0]
    in_trade= False
    entry_bar=entry_price=trade_dir=None
    trail_stop=target_price=peak_high=trough_low=None

    for i in range(len(df2)-1):
        row      = df2.iloc[i]
        next_row = df2.iloc[i+1]

        if in_trade:
            bars_held = i - entry_bar
            close = float(next_row["close"])
            high  = float(next_row.get("high", close))
            low   = float(next_row.get("low",  close))
            atr   = float(row.get("ATR", 1)) or 1.0
            ep=er=None

            if trade_dir=="BUY":
                peak_high  = max(peak_high, high)
                trail_stop = peak_high - ATR_TRAIL * atr
                if   low  <= trail_stop:   ep=max(trail_stop-SLIP,low); er="CHANDELIER"
                elif high >= target_price: ep=target_price-SLIP;        er="TARGET"
                elif bars_held>=MAX_BARS:  ep=close-SLIP;               er="TIME"
            else:
                trough_low = min(trough_low, low)
                trail_stop = trough_low + ATR_TRAIL * atr
                if   high >= trail_stop:   ep=min(trail_stop+SLIP,high); er="CHANDELIER"
                elif low  <= target_price: ep=target_price+SLIP;         er="TARGET"
                elif bars_held>=MAX_BARS:  ep=close+SLIP;                er="TIME"

            if ep is not None:
                pnl = ((ep-entry_price)/entry_price*100 if trade_dir=="BUY"
                       else (entry_price-ep)/entry_price*100)
                equity.append(equity[-1]*(1+pnl/100))
                trades.append({"direction":trade_dir,"pnl_pct":round(pnl,4),
                                "exit_reason":er,"win":pnl>0,"bars_held":bars_held})
                in_trade=False; entry_bar=entry_price=trade_dir=None
                trail_stop=target_price=peak_high=trough_low=None
            else:
                equity.append(equity[-1])
        
        if not in_trade:
            sig = row["SIGNAL"]
            if sig in ("BUY","SELL"):
                atr = float(row.get("ATR",0))
                if atr<=0: equity.append(equity[-1]); continue
                ep  = float(next_row.get("open", next_row["close"]))
                ep += SLIP if sig=="BUY" else -SLIP
                if sig=="BUY":
                    peak_high=ep; trail_stop=ep-ATR_TRAIL*atr; target_price=ep+ATR_TGT*atr; trough_low=None
                else:
                    trough_low=ep; trail_stop=ep+ATR_TRAIL*atr; target_price=ep-ATR_TGT*atr; peak_high=None
                entry_bar=i; entry_price=ep; trade_dir=sig; in_trade=True
                equity.append(equity[-1])
            else:
                equity.append(equity[-1])

    if not trades:
        return {"total_trades":0,"message":"No trades generated — check data length and format"}

    tr   = pd.DataFrame(trades)
    wins = tr[tr["win"]==True]; losses = tr[tr["win"]==False]
    pf   = (wins["pnl_pct"].sum()/abs(losses["pnl_pct"].sum())
            if abs(losses["pnl_pct"].sum())>0 else 0)
    eq   = np.array(equity)
    peak = np.maximum.accumulate(eq)
    dd   = float(((eq-peak)/peak).min())*100
    wr   = float(tr["win"].mean())*100
    avg_w= float(wins["pnl_pct"].mean()) if len(wins)>0 else 0
    avg_l= float(losses["pnl_pct"].mean()) if len(losses)>0 else 0
    rr   = abs(avg_w/avg_l) if avg_l!=0 else 0
    ret  = float(eq[-1]-100)

    by_exit = {}
    for reason in tr["exit_reason"].unique():
        rr_df = tr[tr["exit_reason"]==reason]
        by_exit[reason] = {
            "count": int(len(rr_df)),
            "win_rate": round(rr_df["win"].mean()*100, 1),
            "avg_pnl":  round(rr_df["pnl_pct"].mean(), 4),
        }

    return {
        "total_trades":   int(len(tr)),
        "buy_trades":     int((tr["direction"]=="BUY").sum()),
        "sell_trades":    int((tr["direction"]=="SELL").sum()),
        "win_rate":       round(wr, 1),
        "avg_win":        round(avg_w, 4),
        "avg_loss":       round(avg_l, 4),
        "reward_risk":    round(rr, 2),
        "profit_factor":  round(pf, 2),
        "total_return":   round(ret, 2),
        "max_drawdown":   round(dd, 2),
        "exit_breakdown": by_exit,
        "equity_curve":   [round(e,4) for e in equity[::10]],  # every 10th point
    }


# ══════════════════════════════════════════════════════════════════════════════
# ENDPOINTS
# ══════════════════════════════════════════════════════════════════════════════

@app.get("/")
def home():
    return {
        "status":  "running",
        "system":  "MPC Quant AI Engine v7",
        "models":  ["Stage1","Stage2","3-Class"],
        "endpoints": ["/signal", "/analyse", "/instruments", "/live-analyse/{symbol}", "/price/{symbol}", "/quant"],
        "live_data": "Twelve Data API",
        "api_key_set": bool(os.getenv("TWELVEDATA_API_KEY", "").strip() and os.getenv("TWELVEDATA_API_KEY") != "your_api_key_here"),
    }


@app.get("/signal")
def get_signal():
    """Live signal from the default NIFTY data file."""
    try:
        df = pd.read_csv(DEFAULT_DATA_FILE)
        col_map = detect_columns(df)
        df = clean_and_parse(df, col_map)
        df = add_indicators(df)
        cf = run_chronos(df["close"])
        interval = detect_interval(df)
        return build_signal_response(df, cf, instrument_name="NIFTY 50", interval=interval)
    except Exception as e:
        return {"error": str(e), "signal": "ERROR"}


@app.post("/analyse")
async def analyse_csv(
    file: UploadFile = File(...),
    instrument_name: str = Form(default=""),
):
    """
    Upload any OHLCV CSV and receive:
      1. Live signal (from latest bar)
      2. Quick backtest summary
      3. Full indicator values
    
    Accepts any column naming convention.
    Auto-detects date format.
    Works for NIFTY, Stocks, Gold, Crypto, Forex — any instrument.
    """
    try:
        # Read uploaded file
        contents = await file.read()
        df_raw   = pd.read_csv(io.StringIO(contents.decode("utf-8", errors="replace")))

        if len(df_raw) < 250:
            return {"error": f"Need at least 250 bars for indicators. Got {len(df_raw)}."}

        # Detect and standardise columns
        col_map = detect_columns(df_raw)
        required = ["open","high","low","close"]
        missing  = [r for r in required if r not in col_map]
        if missing:
            return {
                "error": f"Could not find columns: {missing}. "
                         f"Found: {list(df_raw.columns)}. "
                         f"CSV must have open, high, low, close columns."
            }

        df = clean_and_parse(df_raw, col_map)
        df = df.sort_values("date").reset_index(drop=True) if "date" in df.columns else df

        if len(df) < 250:
            return {"error": f"After cleaning, only {len(df)} valid rows. Need 250+."}

        # Auto-detect instrument name from filename if not provided
        name = instrument_name.strip() or file.filename.replace(".csv","").replace("_"," ")

        # Run indicators
        df = add_indicators(df)

        # Run Chronos on latest bars
        cf = run_chronos(df["close"])

        # Detect interval
        interval = detect_interval(df)

        # Get live signal
        signal_data = build_signal_response(df, cf, instrument_name=name, interval=interval)

        # Run quick backtest
        backtest_data = quick_backtest(df, name, interval=interval)

        # Data summary
        data_summary = {
            "instrument":  name,
            "filename":    file.filename,
            "total_rows":  len(df),
            "date_start":  str(df["date"].iloc[0])  if "date" in df.columns else "N/A",
            "date_end":    str(df["date"].iloc[-1]) if "date" in df.columns else "N/A",
            "price_range": {
                "low":  round(float(df["close"].min()), 4),
                "high": round(float(df["close"].max()), 4),
                "current": round(float(df["close"].iloc[-1]), 4),
            },
            "avg_atr_pct": round(float((df["ATR"]/df["close"]*100).median()), 4),
            "columns_detected": col_map,
        }

        return {
            "status":       "success",
            "data_summary": data_summary,
            "signal":       signal_data,
            "backtest":     backtest_data,
        }

    except Exception as e:
        return {"error": str(e), "detail": traceback.format_exc()}


# ══════════════════════════════════════════════════════════════════════════════
# TWELVE DATA — LIVE DATA INTEGRATION
# ══════════════════════════════════════════════════════════════════════════════

TWELVEDATA_BASE = "https://api.twelvedata.com"

INSTRUMENT_CATALOG = [
    # ── Indian Indices ──
    {"symbol": "NIFTY 50",   "exchange": "NSE", "name": "NIFTY 50",          "category": "Indices",      "currency": "INR", "flag": "🇮🇳"},
    {"symbol": "SENSEX",     "exchange": "BSE", "name": "SENSEX",            "category": "Indices",      "currency": "INR", "flag": "🇮🇳"},
    # ── Indian Stocks ──
    {"symbol": "RELIANCE",   "exchange": "NSE", "name": "Reliance Industries","category": "Indian Stocks","currency": "INR", "flag": "🇮🇳"},
    {"symbol": "TCS",        "exchange": "NSE", "name": "TCS",               "category": "Indian Stocks","currency": "INR", "flag": "🇮🇳"},
    {"symbol": "HDFCBANK",   "exchange": "NSE", "name": "HDFC Bank",          "category": "Indian Stocks","currency": "INR", "flag": "🇮🇳"},
    {"symbol": "INFY",       "exchange": "NSE", "name": "Infosys",            "category": "Indian Stocks","currency": "INR", "flag": "🇮🇳"},
    {"symbol": "ICICIBANK",  "exchange": "NSE", "name": "ICICI Bank",         "category": "Indian Stocks","currency": "INR", "flag": "🇮🇳"},
    {"symbol": "SBIN",       "exchange": "NSE", "name": "SBI",                "category": "Indian Stocks","currency": "INR", "flag": "🇮🇳"},
    # ── US Stocks ──
    {"symbol": "AAPL",       "exchange": "",    "name": "Apple",              "category": "US Stocks",    "currency": "USD", "flag": "🇺🇸"},
    {"symbol": "TSLA",       "exchange": "",    "name": "Tesla",              "category": "US Stocks",    "currency": "USD", "flag": "🇺🇸"},
    {"symbol": "MSFT",       "exchange": "",    "name": "Microsoft",          "category": "US Stocks",    "currency": "USD", "flag": "🇺🇸"},
    {"symbol": "GOOGL",      "exchange": "",    "name": "Alphabet (Google)",  "category": "US Stocks",    "currency": "USD", "flag": "🇺🇸"},
    {"symbol": "AMZN",       "exchange": "",    "name": "Amazon",             "category": "US Stocks",    "currency": "USD", "flag": "🇺🇸"},
    {"symbol": "NVDA",       "exchange": "",    "name": "NVIDIA",             "category": "US Stocks",    "currency": "USD", "flag": "🇺🇸"},
    {"symbol": "META",       "exchange": "",    "name": "Meta (Facebook)",    "category": "US Stocks",    "currency": "USD", "flag": "🇺🇸"},
    # ── Crypto ──
    {"symbol": "BTC/USD",    "exchange": "",    "name": "Bitcoin",            "category": "Crypto",       "currency": "USD", "flag": "₿"},
    {"symbol": "ETH/USD",    "exchange": "",    "name": "Ethereum",           "category": "Crypto",       "currency": "USD", "flag": "Ξ"},
    {"symbol": "SOL/USD",    "exchange": "",    "name": "Solana",             "category": "Crypto",       "currency": "USD", "flag": "◎"},
    {"symbol": "XRP/USD",    "exchange": "",    "name": "XRP",                "category": "Crypto",       "currency": "USD", "flag": "✕"},
    # ── Forex ──
    {"symbol": "EUR/USD",    "exchange": "",    "name": "Euro / Dollar",      "category": "Forex",        "currency": "USD", "flag": "🇪🇺"},
    {"symbol": "GBP/USD",    "exchange": "",    "name": "Pound / Dollar",     "category": "Forex",        "currency": "USD", "flag": "🇬🇧"},
    {"symbol": "USD/JPY",    "exchange": "",    "name": "Dollar / Yen",       "category": "Forex",        "currency": "JPY", "flag": "🇯🇵"},
    {"symbol": "USD/INR",    "exchange": "",    "name": "Dollar / Rupee",     "category": "Forex",        "currency": "INR", "flag": "🇮🇳"},
    # ── Commodities ──
    {"symbol": "XAU/USD",    "exchange": "",    "name": "Gold",               "category": "Commodities",  "currency": "USD", "flag": "🥇"},
    {"symbol": "XAG/USD",    "exchange": "",    "name": "Silver",             "category": "Commodities",  "currency": "USD", "flag": "🥈"},
]


# Symbol mappings from Frontend symbols to Yahoo Finance symbols
SYMBOL_MAP = {
    "NIFTY 50": "^NSEI",
    "NIFTY50": "^NSEI",
    "SENSEX": "^BSESN",
    "RELIANCE": "RELIANCE.NS",
    "TCS": "TCS.NS",
    "HDFCBANK": "HDFCBANK.NS",
    "INFY": "INFY.NS",
    "ICICIBANK": "ICICIBANK.NS",
    "SBIN": "SBIN.NS",
    "AAPL": "AAPL",
    "TSLA": "TSLA",
    "MSFT": "MSFT",
    "GOOGL": "GOOGL",
    "AMZN": "AMZN",
    "NVDA": "NVDA",
    "META": "META",
    "BTC/USD": "BTC-USD",
    "ETH/USD": "ETH-USD",
    "SOL/USD": "SOL-USD",
    "XRP/USD": "XRP-USD",
    "EUR/USD": "EURUSD=X",
    "GBP/USD": "GBPUSD=X",
    "USD/JPY": "USDJPY=X",
    "USD/INR": "USDINR=X",
    "XAU/USD": "GC=F",
    "XAG/USD": "SI=F",
}


def fetch_live_data(symbol: str, exchange: str = "", interval: str = "5min",
                    outputsize: int = 5000) -> pd.DataFrame:
    """
    Fetch OHLCV time-series from Yahoo Finance via yahooquery.
    Returns a pandas DataFrame with columns: date, open, high, low, close, volume.
    """
    yf_symbol = SYMBOL_MAP.get(symbol, symbol)
    if not yf_symbol:
        yf_symbol = symbol

    # Map frontend intervals to yahooquery intervals
    intervals_map = {
        "1min": "1m",
        "5min": "5m",
        "15min": "15m",
        "1h": "1h",
        "4h": "1h",  # We download 1h and resample to 4h
        "1day": "1d",
    }
    yq_interval = intervals_map.get(interval, "5m")

    # Select history period based on interval to ensure we get 250+ valid bars
    if yq_interval == "1m":
        period = "5d"
    elif yq_interval in ["5m", "15m"]:
        period = "1mo"
    elif yq_interval == "1h":
        # For 4h we resample from 1h, so we need 4x the data length (approx 1 year of 1h data)
        period = "1y" if interval == "4h" else "3mo"
    else:
        period = "2y"  # 1day interval

    t = Ticker(yf_symbol)
    df = t.history(period=period, interval=yq_interval)

    if isinstance(df, dict):
        error_msg = df.get(yf_symbol, {}).get("description", str(df))
        raise ValueError(f"Yahoo Finance error: {error_msg}")

    if df is None or df.empty:
        raise ValueError(f"No data returned for symbol '{symbol}' (Yahoo Symbol: '{yf_symbol}')")

    # Reset MultiIndex to convert index level 1 to column 'date'
    df = df.reset_index()
    if 'symbol' in df.columns:
        df = df.drop(columns=['symbol'])

    # Standardize columns
    df = df.rename(columns={"date": "date"})
    for col in ["open", "high", "low", "close", "volume"]:
        if col in df.columns:
            df[col] = pd.to_numeric(df[col], errors="coerce")

    df = df.dropna(subset=["open", "high", "low", "close"]).reset_index(drop=True)

    # Sort by date
    df["date"] = pd.to_datetime(df["date"])
    df = df.sort_values("date").reset_index(drop=True)

    # Resample 1h data to 4h if requested
    if interval == "4h":
        df.set_index("date", inplace=True)
        df = df.resample("4h").agg({
            "open": "first",
            "high": "max",
            "low": "min",
            "close": "last",
            "volume": "sum"
        }).dropna().reset_index()

    # Remove flat candles
    flat = ((df["open"]==df["high"]) & (df["high"]==df["low"]) & (df["low"]==df["close"]))
    df = df[~flat].copy().reset_index(drop=True)

    return df


def fetch_current_price(symbol: str, exchange: str = ""):
    """Quick price lookup via yahooquery — 1 call, no API keys."""
    yf_symbol = SYMBOL_MAP.get(symbol, symbol)
    try:
        t = Ticker(yf_symbol)
        price_dict = t.price
        if isinstance(price_dict, dict) and yf_symbol in price_dict:
            details = price_dict[yf_symbol]
            if isinstance(details, dict):
                price = details.get("regularMarketPrice")
                if price is not None:
                    return float(price)
    except Exception:
        pass
    return None


# ── Live data endpoints ───────────────────────────────────────────────────────

@app.get("/instruments")
def list_instruments():
    """Return the catalog of supported instruments with categories."""
    return {
        "api_ready":   True,
        "instruments": INSTRUMENT_CATALOG,
        "categories":  sorted(set(i["category"] for i in INSTRUMENT_CATALOG)),
        "timeframes":  ["1min","5min","15min","1h","4h","1day"],
    }


@app.get("/search")
def search_tickers(q: str = Query(...)):
    """Search Yahoo Finance for symbols/tickers matching a query."""
    try:
        from yahooquery import search
        res = search(q)
        quotes = res.get("quotes", [])
        results = []
        for quote in quotes:
            symbol = quote.get("symbol")
            if not symbol:
                continue
            name = quote.get("longname") or quote.get("shortname") or symbol
            exchange = quote.get("exchDisp") or quote.get("exchange") or ""
            quote_type = quote.get("typeDisp") or quote.get("quoteType") or "Equity"
            
            # Map type to category
            category = "Equity"
            if quote_type.lower() in ["cryptocurrency", "crypto"]:
                category = "Crypto"
            elif quote_type.lower() in ["currency", "forex"]:
                category = "Forex"
            elif quote_type.lower() in ["commodity", "future"]:
                category = "Commodities"
            elif quote_type.lower() in ["index"]:
                category = "Indices"
            elif quote_type.lower() in ["etf"]:
                category = "ETF"
            else:
                category = "Equity"
                
            # Determine visual flag
            flag = "📈"
            if category == "Crypto":
                flag = "₿"
            elif category == "Forex":
                flag = "💱"
            elif category == "Commodities":
                flag = "🥇"
            elif category == "Indices":
                flag = "📊"
            elif exchange == "NSE" or exchange == "BSE":
                flag = "🇮🇳"
            elif exchange in ["NASDAQ", "NYSE", "AMEX"]:
                flag = "🇺🇸"
                
            results.append({
                "symbol": symbol,
                "exchange": exchange,
                "name": name,
                "category": category,
                "currency": quote.get("currency", "USD"),
                "flag": flag
            })
        return {"results": results}
    except Exception as e:
        return {"error": str(e), "results": []}


@app.get("/price/{symbol:path}")
def get_price(symbol: str, exchange: str = Query(default="")):
    """Quick current-price lookup for a symbol."""
    price = fetch_current_price(symbol, exchange)
    if price is None:
        return {"error": "Could not fetch price. Check symbol."}
    return {"symbol": symbol, "price": price}


@app.get("/live-analyse/{symbol:path}")
def live_analyse(symbol: str, exchange: str = Query(default=""),
                 name: str = Query(default=""),
                 interval: str = Query(default="5min")):
    """
    Fetch live OHLCV data from Yahoo Finance and run the full analysis pipeline:
      indicators → Chronos forecast → 3 XGBoost models → signal + backtest.
    Returns the same JSON structure as POST /analyse.
    Supports intervals: 1min, 5min, 15min, 1h, 4h, 1day.
    """
    valid_intervals = ["1min","5min","15min","1h","4h","1day"]
    if interval not in valid_intervals:
        return {"error": f"Invalid interval '{interval}'. Use one of: {valid_intervals}"}

    try:
        # Resolve from catalog if no exchange provided
        if not exchange and not name:
            for inst in INSTRUMENT_CATALOG:
                if inst["symbol"].upper() == symbol.upper():
                    exchange = inst.get("exchange", "")
                    name     = inst.get("name", symbol)
                    break

        display_name = name.strip() or symbol

        # Fetch live data
        df = fetch_live_data(symbol, exchange=exchange, interval=interval, outputsize=5000)

        if len(df) < 250:
            return {"error": f"Only {len(df)} bars returned for {interval} timeframe. "
                              f"Need 250+ for indicators. Try a shorter timeframe."}

        # Run full pipeline (same as /analyse)
        df = add_indicators(df)
        cf = run_chronos(df["close"])
        signal_data   = build_signal_response(df, cf, instrument_name=display_name, interval=interval)
        backtest_data = quick_backtest(df, display_name, interval=interval)

        data_summary = {
            "instrument":  display_name,
            "filename":    f"live:{symbol}",
            "interval":    interval,
            "total_rows":  len(df),
            "date_start":  str(df["date"].iloc[0])  if "date" in df.columns else "N/A",
            "date_end":    str(df["date"].iloc[-1]) if "date" in df.columns else "N/A",
            "price_range": {
                "low":     round(float(df["close"].min()), 4),
                "high":    round(float(df["close"].max()), 4),
                "current": round(float(df["close"].iloc[-1]), 4),
            },
            "avg_atr_pct": round(float((df["ATR"]/df["close"]*100).median()), 4),
            "source":      "Yahoo Finance (live)",
        }

        return {
            "status":       "success",
            "data_summary": data_summary,
            "signal":       signal_data,
            "backtest":     backtest_data,
        }

    except ValueError as e:
        return {"error": str(e)}
    except Exception as e:
        return {"error": str(e), "detail": traceback.format_exc()}