added code

.dockerignore

@@ -0,0 +1,62 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+logs/
+
+# Git
+.git/
+.gitignore
+
+# Environment files (will be set via GCP Secrets Manager or env vars)
+.env
+.env.local
+
+# Data files (if too large, consider mounting as volume or using GCS)
+# Uncomment if you want to exclude data files
+# data/*.csv
+
+# Reports (optional - uncomment if you don't need reports in container)
+# reports/
+
+# Test files
+test_*.py
+*_test.py
+
+# Jupyter notebooks
+*.ipynb
+*.ipynb_checkpoints/
+
+# Documentation
+*.md
+README.md
+docs/
+
+# Docker files
+Dockerfile*
+docker-compose*.yml
+.dockerignore

.env

@@ -0,0 +1,2 @@
+GEMINI_API_KEY=AIzaSyBMFn9f05ghcNH3Sl-LdQdHW6MlqCtPYn0
+GROQ_API_KEY=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

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+"reports/" 

DEPLOYMENT.md

@@ -0,0 +1,308 @@
+# GCP Deployment Guide for Phishing Detection API
+
+This guide explains how to deploy the Phishing Detection API to Google Cloud Platform (GCP) using Docker and Cloud Run.
+
+## Prerequisites
+
+1. **Google Cloud Platform Account**: Ensure you have a GCP account and billing enabled
+2. **gcloud CLI**: Install the [Google Cloud SDK](https://cloud.google.com/sdk/docs/install)
+3. **Docker**: Install [Docker Desktop](https://www.docker.com/products/docker-desktop) or Docker Engine
+4. **GROQ API Key**: Obtain your API key from [Groq](https://console.groq.com/)
+
+## Quick Start
+
+### Option 1: Using Cloud Build (Recommended)
+
+1. **Set your GCP project**:
+   ```bash
+   gcloud config set project YOUR_PROJECT_ID
+   ```
+
+2. **Run the deployment script**:
+   ```bash
+   # For Linux/Mac
+   chmod +x deploy.sh
+   ./deploy.sh YOUR_PROJECT_ID us-central1
+
+   # For Windows PowerShell
+   .\deploy.ps1 -ProjectId YOUR_PROJECT_ID -Region us-central1
+   ```
+
+   The script will:
+   - Enable required APIs
+   - Create the GROQ_API_KEY secret in Secret Manager
+   - Build the Docker image
+   - Deploy to Cloud Run
+
+### Option 2: Manual Deployment
+
+#### Step 1: Set Up GCP Project
+
+```bash
+# Set your project
+gcloud config set project YOUR_PROJECT_ID
+
+# Enable required APIs
+gcloud services enable cloudbuild.googleapis.com
+gcloud services enable run.googleapis.com
+gcloud services enable containerregistry.googleapis.com
+gcloud services enable secretmanager.googleapis.com
+```
+
+#### Step 2: Create Secret for GROQ API Key
+
+```bash
+# Create the secret
+echo -n "your-groq-api-key" | gcloud secrets create GROQ_API_KEY \
+    --data-file=- \
+    --replication-policy="automatic"
+
+# Grant Cloud Run access to the secret
+PROJECT_NUMBER=$(gcloud projects describe YOUR_PROJECT_ID --format="value(projectNumber)")
+gcloud secrets add-iam-policy-binding GROQ_API_KEY \
+    --member="serviceAccount:$PROJECT_NUMBER-compute@developer.gserviceaccount.com" \
+    --role="roles/secretmanager.secretAccessor"
+```
+
+#### Step 3: Build and Deploy
+
+```bash
+# Using Cloud Build (recommended)
+gcloud builds submit --config=cloudbuild.yaml
+
+# Or build locally and push
+docker build -t gcr.io/YOUR_PROJECT_ID/phishing-detection-api:latest .
+docker push gcr.io/YOUR_PROJECT_ID/phishing-detection-api:latest
+
+# Deploy to Cloud Run
+gcloud run deploy phishing-detection-api \
+    --image gcr.io/YOUR_PROJECT_ID/phishing-detection-api:latest \
+    --region us-central1 \
+    --platform managed \
+    --allow-unauthenticated \
+    --memory 4Gi \
+    --cpu 2 \
+    --timeout 300 \
+    --max-instances 10 \
+    --set-env-vars PYTHONUNBUFFERED=1 \
+    --set-secrets GROQ_API_KEY=GROQ_API_KEY:latest
+```
+
+## Configuration Options
+
+### Cloud Run Settings
+
+The deployment uses these default settings (configured in `cloudbuild.yaml`):
+
+- **Memory**: 4GB (required for ML models)
+- **CPU**: 2 vCPUs
+- **Timeout**: 300 seconds (5 minutes)
+- **Max Instances**: 10
+- **Region**: us-central1 (change in cloudbuild.yaml or deploy command)
+
+### Adjusting Resources
+
+If you need more resources, modify `cloudbuild.yaml`:
+
+```yaml
+- '--memory'
+- '8Gi'  # Increase memory for larger models
+- '--cpu'
+- '4'    # Increase CPU for faster inference
+```
+
+Or deploy with custom settings:
+
+```bash
+gcloud run deploy phishing-detection-api \
+    --image gcr.io/YOUR_PROJECT_ID/phishing-detection-api:latest \
+    --memory 8Gi \
+    --cpu 4 \
+    --timeout 600 \
+    --max-instances 20
+```
+
+## Verifying Deployment
+
+1. **Check service status**:
+   ```bash
+   gcloud run services describe phishing-detection-api --region us-central1
+   ```
+
+2. **Get service URL**:
+   ```bash
+   SERVICE_URL=$(gcloud run services describe phishing-detection-api \
+       --region us-central1 \
+       --format="value(status.url)")
+   echo $SERVICE_URL
+   ```
+
+3. **Test health endpoint**:
+   ```bash
+   curl $SERVICE_URL/health
+   ```
+
+4. **View logs**:
+   ```bash
+   gcloud run services logs read phishing-detection-api --region us-central1
+   ```
+
+## API Endpoints
+
+Once deployed, your service will have these endpoints:
+
+- **Root**: `GET /` - API information
+- **Health Check**: `GET /health` - Service health and model status
+- **Prediction**: `POST /predict` - Main prediction endpoint
+- **API Docs**: `GET /docs` - Interactive API documentation (Swagger UI)
+
+## Testing the API
+
+### Using curl:
+
+```bash
+curl -X POST "$SERVICE_URL/predict" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "sender": "test@example.com",
+    "subject": "Urgent Action Required",
+    "text": "Please verify your account at http://suspicious-site.com",
+    "metadata": {}
+  }'
+```
+
+### Using Python:
+
+```python
+import requests
+
+url = "YOUR_SERVICE_URL/predict"
+payload = {
+    "sender": "test@example.com",
+    "subject": "Urgent Action Required",
+    "text": "Please verify your account at http://suspicious-site.com",
+    "metadata": {}
+}
+
+response = requests.post(url, json=payload)
+print(response.json())
+```
+
+## Monitoring and Logging
+
+### View Logs in Console
+
+1. Go to [Cloud Run Console](https://console.cloud.google.com/run)
+2. Click on your service: `phishing-detection-api`
+3. Navigate to the "Logs" tab
+
+### View Logs via CLI
+
+```bash
+gcloud run services logs read phishing-detection-api --region us-central1 --limit 50
+```
+
+### Set Up Alerts
+
+1. Go to [Cloud Monitoring](https://console.cloud.google.com/monitoring)
+2. Create alerts for:
+   - Error rate
+   - Request latency
+   - Memory usage
+   - CPU utilization
+
+## Troubleshooting
+
+### Container fails to start
+
+- Check logs: `gcloud run services logs read phishing-detection-api --region us-central1`
+- Verify models are present in the container
+- Check memory limits (may need to increase)
+
+### Models not loading
+
+- Ensure all model files are included in the Docker image
+- Check model paths in `config.py`
+- Verify model files exist in `models/`, `finetuned_bert/`, and `Message_model/final_semantic_model/`
+
+### GROQ API errors
+
+- Verify the secret is correctly set: `gcloud secrets versions access latest --secret="GROQ_API_KEY"`
+- Check IAM permissions for the Cloud Run service account
+- Verify the API key is valid
+
+### High memory usage
+
+- Increase memory allocation in Cloud Run settings
+- Consider using model quantization
+- Check for memory leaks in the application
+
+## Cost Optimization
+
+1. **Set minimum instances to 0**: Scales to zero when not in use
+   ```bash
+   gcloud run services update phishing-detection-api \
+       --min-instances 0 \
+       --max-instances 10
+   ```
+
+2. **Use appropriate instance sizes**: Start with smaller instances and scale up if needed
+
+3. **Enable request concurrency**: Reduce number of instances needed
+   ```bash
+   gcloud run services update phishing-detection-api \
+       --concurrency 10
+   ```
+
+## Security Considerations
+
+1. **Authentication**: Currently deployed as public. Consider adding authentication:
+   ```bash
+   gcloud run services update phishing-detection-api \
+       --no-allow-unauthenticated
+   ```
+
+2. **API Keys**: Store sensitive keys in Secret Manager (already configured)
+
+3. **VPC**: Consider deploying in a VPC for additional network isolation
+
+## Updating the Service
+
+To update after code changes:
+
+```bash
+# Rebuild and deploy
+gcloud builds submit --config=cloudbuild.yaml
+
+# Or manually
+docker build -t gcr.io/YOUR_PROJECT_ID/phishing-detection-api:latest .
+docker push gcr.io/YOUR_PROJECT_ID/phishing-detection-api:latest
+gcloud run deploy phishing-detection-api \
+    --image gcr.io/YOUR_PROJECT_ID/phishing-detection-api:latest \
+    --region us-central1
+```
+
+## Local Testing with Docker
+
+Before deploying to GCP, test locally:
+
+```bash
+# Build the image
+docker build -t phishing-detection-api:local .
+
+# Run with environment variable
+docker run -p 8000:8000 \
+    -e GROQ_API_KEY=your-api-key \
+    phishing-detection-api:local
+
+# Test
+curl http://localhost:8000/health
+```
+
+## Support
+
+For issues or questions:
+1. Check the logs first
+2. Verify all prerequisites are met
+3. Ensure models are properly loaded
+4. Review the API documentation at `/docs` endpoint

Dockerfile

@@ -0,0 +1,38 @@
+# Use Python 3.10 slim image as base
+FROM python:3.10-slim
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies for SSL, whois, and other required tools
+RUN apt-get update && apt-get install -y \
+    gcc \
+    g++ \
+    curl \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/* \
+    && curl --version
+
+# Copy requirements first for better caching
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+# Copy the entire Model_sprint_2 directory
+COPY . .
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Expose the port the app runs on
+EXPOSE 8000
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+# Run the application using uvicorn
+CMD ["python", "-m", "uvicorn", "app1:app", "--host", "0.0.0.0", "--port", "8000"]

Message_model/final_semantic_model/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "[MASK]": 128000
+}

Message_model/final_semantic_model/config.json

@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "DebertaV2ForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "dtype": "float32",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "transformers_version": "4.57.1",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

Message_model/final_semantic_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a960912824e3268b5fdf0217b8b5d7ea5a3817a6437c01b5be36e5aa94808745
+size 737719272

Message_model/final_semantic_model/special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Message_model/final_semantic_model/spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

Message_model/final_semantic_model/tokenizer_config.json

@@ -0,0 +1,59 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128000": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

Message_model/final_semantic_model/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14bca844639ea9d701739f1946f783acd435eceecb6fb1e12773a8a645269dd2
+size 5777

Message_model/predict.py

@@ -0,0 +1,150 @@
+import torch
+import numpy as np
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from scipy.special import softmax
+import os
+
+class PhishingPredictor:
+    def __init__(self, model_path="final_semantic_model"):
+        self.model_path = model_path
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        
+        if not os.path.exists(model_path):
+            raise FileNotFoundError(f"Model not found at {model_path}. Please run train.py first.")
+        
+        print(f"Loading model from {model_path}...")
+        self.tokenizer = AutoTokenizer.from_pretrained(model_path)
+        self.model = AutoModelForSequenceClassification.from_pretrained(
+            model_path,
+            use_safetensors=True
+        )
+        self.model.to(self.device)
+        self.model.eval()
+        print(f"Model loaded successfully on {self.device}")
+    
+    def predict(self, text):
+        if not text or not text.strip():
+            return {
+                "text": text,
+                "phishing_probability": 0.0,
+                "prediction": "ham",
+                "confidence": "low"
+            }
+        
+        inputs = self.tokenizer(
+            text,
+            truncation=True,
+            padding=True,
+            max_length=128,
+            return_tensors="pt"
+        )
+        
+        inputs = {key: value.to(self.device) for key, value in inputs.items()}
+        
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+            logits = outputs.logits.cpu().numpy()
+        
+        probabilities = softmax(logits, axis=1)[0]
+        
+        phishing_prob = float(probabilities[1])
+        
+        prediction = "phishing" if phishing_prob > 0.5 else "ham"
+        
+        confidence_score = max(phishing_prob, 1 - phishing_prob)
+        if confidence_score > 0.8:
+            confidence = "high"
+        elif confidence_score > 0.6:
+            confidence = "medium"
+        else:
+            confidence = "low"
+        
+        return {
+            "text": text,
+            "phishing_probability": round(phishing_prob, 4),
+            "ham_probability": round(float(probabilities[0]), 4),
+            "prediction": prediction,
+            "confidence": confidence,
+            "confidence_score": round(confidence_score, 4)
+        }
+    
+    def predict_batch(self, texts):
+        results = []
+        for text in texts:
+            results.append(self.predict(text))
+        return results
+
+
+def main():
+    try:
+        predictor = PhishingPredictor()
+        
+        print("\n" + "="*60)
+        print("SMS PHISHING DETECTION SYSTEM")
+        print("="*60)
+        print("Enter SMS messages to analyze (type 'quit' to exit)")
+        print("Type 'batch' to analyze multiple messages at once")
+        print("-"*60)
+        
+        while True:
+            user_input = input("\nEnter SMS message: ").strip()
+            
+            if user_input.lower() in ['quit', 'exit', 'q']:
+                print("Goodbye!")
+                break
+            
+            elif user_input.lower() == 'batch':
+                print("\nBatch mode - Enter messages (empty line to finish):")
+                messages = []
+                while True:
+                    msg = input(f"Message {len(messages) + 1}: ").strip()
+                    if not msg:
+                        break
+                    messages.append(msg)
+                
+                if messages:
+                    results = predictor.predict_batch(messages)
+                    print(f"\n{'='*60}")
+                    print("BATCH RESULTS")
+                    print(f"{'='*60}")
+                    for i, result in enumerate(results, 1):
+                        print(f"\nMessage {i}: {result['text'][:50]}...")
+                        print(f"Prediction: {result['prediction'].upper()}")
+                        print(f"Phishing Probability: {result['phishing_probability']:.1%}")
+                        print(f"Confidence: {result['confidence'].upper()}")
+                        print("-" * 40)
+                else:
+                    print("No messages entered.")
+            
+            elif user_input:
+                result = predictor.predict(user_input)
+                
+                print(f"\n{'='*60}")
+                print("PREDICTION RESULT")
+                print(f"{'='*60}")
+                print(f"Message: {result['text']}")
+                print(f"Prediction: {result['prediction'].upper()}")
+                print(f"Phishing Probability: {result['phishing_probability']:.1%}")
+                print(f"Ham Probability: {result['ham_probability']:.1%}")
+                print(f"Confidence: {result['confidence'].upper()} ({result['confidence_score']:.1%})")
+                
+                prob = result['phishing_probability']
+                if prob > 0.7:
+                    print("🚨 HIGH RISK - Likely phishing!")
+                elif prob > 0.3:
+                    print("⚠️  MEDIUM RISK - Be cautious")
+                else:
+                    print("✅ LOW RISK - Appears legitimate")
+            
+            else:
+                print("Please enter a message or 'quit' to exit.")
+    
+    except FileNotFoundError as e:
+        print(f"Error: {e}")
+        print("Please run train.py first to create the model.")
+    except Exception as e:
+        print(f"An error occurred: {e}")
+
+
+if __name__ == "__main__":
+    main()

Message_model/train.py

@@ -0,0 +1,218 @@
+import pandas as pd
+import torch
+import numpy as np
+import os
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import (
+    accuracy_score, 
+    f1_score, 
+    classification_report, 
+    confusion_matrix
+)
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    Trainer,
+    TrainingArguments
+)
+from transformers.trainer_utils import get_last_checkpoint
+from scipy.special import softmax
+
+# --- 1. Check for CUDA (GPU) ---
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f"--- 1. Using device: {device} ---")
+if device == "cpu":
+    print("--- WARNING: CUDA not available. Training will run on CPU and will be very slow. ---")
+print("---------------------------------")
+# --- End CUDA Check ---
+
+MODEL_NAME = "microsoft/deberta-v3-base"
+FINAL_MODEL_DIR = "final_semantic_model"
+REPORT_DIR = "evaluation_report"
+CHECKPOINT_DIR = "training_checkpoints"
+
+os.makedirs(FINAL_MODEL_DIR, exist_ok=True)
+os.makedirs(REPORT_DIR, exist_ok=True)
+os.makedirs(CHECKPOINT_DIR, exist_ok=True)
+
+print("--- 2. Loading and splitting dataset ---")
+try:
+    df = pd.read_csv("dataset.csv")
+except FileNotFoundError:
+    print("Error: dataset.csv not found.")
+    print("Please make sure the file is in the same directory as this script.")
+    df = pd.DataFrame(columns=['ext_type', 'text'])
+    exit()
+
+df.rename(columns={"ext_type": "label"}, inplace=True)
+df['label'] = df['label'].map({'spam': 1, 'ham': 0})
+df.dropna(subset=['label', 'text'], inplace=True)
+df['label'] = df['label'].astype(int)
+
+if len(df['label'].unique()) < 2:
+    print("Error: The dataset must contain both 'ham' (0) and 'spam' (1) labels.")
+    print(f"Found labels: {df['label'].unique()}")
+    print("Please update dataset.csv with examples for both classes.")
+    exit()
+
+train_df, temp_df = train_test_split(df, test_size=0.3, random_state=42, stratify=df['label'])
+val_df, test_df = train_test_split(temp_df, test_size=0.5, random_state=42, stratify=temp_df['label'])
+
+print(f"Total examples: {len(df)}")
+print(f"Training examples: {len(train_df)}")
+print(f"Validation examples: {len(val_df)}")
+print(f"Test examples: {len(test_df)}")
+print("---------------------------------")
+
+
+print("--- 3. Loading model and tokenizer ---")
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+model = AutoModelForSequenceClassification.from_pretrained(
+    MODEL_NAME, 
+    num_labels=2,
+    use_safetensors=True  # Use secure safetensors format to avoid torch.load error
+)
+print("---------------------------------")
+
+
+class PhishingDataset(torch.utils.data.Dataset):
+    def __init__(self, texts, labels, tokenizer):
+        self.encodings = tokenizer(texts, truncation=True, padding=True, max_length=128)
+        self.labels = labels
+
+    def __getitem__(self, idx):
+        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
+        item['labels'] = torch.tensor(self.labels[idx])
+        return item
+
+    def __len__(self):
+        return len(self.labels)
+
+train_dataset = PhishingDataset(train_df['text'].tolist(), train_df['label'].tolist(), tokenizer)
+val_dataset = PhishingDataset(val_df['text'].tolist(), val_df['label'].tolist(), tokenizer)
+test_dataset = PhishingDataset(test_df['text'].tolist(), test_df['label'].tolist(), tokenizer)
+
+
+def compute_metrics(pred):
+    labels = pred.label_ids
+    preds = pred.predictions.argmax(-1)
+    f1 = f1_score(labels, preds, average="weighted")
+    acc = accuracy_score(labels, preds)
+    return {"accuracy": acc, "f1": f1}
+
+
+print("--- 4. Starting model training ---")
+training_args = TrainingArguments(
+    output_dir=CHECKPOINT_DIR,
+    num_train_epochs=3,
+    per_device_train_batch_size=32,
+    per_device_eval_batch_size=32,
+    warmup_steps=50,
+    weight_decay=0.01,
+    logging_dir='./logs',
+    logging_steps=10,
+    eval_strategy="steps",
+    eval_steps=10,
+    save_strategy="steps",
+    save_steps=10,
+    load_best_model_at_end=True,
+    metric_for_best_model="f1",
+    save_total_limit=2,
+    no_cuda=(device == "cpu"), 
+    save_safetensors=True # This saves new checkpoints securely
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=train_dataset,
+    eval_dataset=val_dataset,
+    compute_metrics=compute_metrics,
+)
+
+# This logic automatically detects if a checkpoint exists
+last_checkpoint = get_last_checkpoint(CHECKPOINT_DIR)
+if last_checkpoint:
+    print(f"--- Resuming training from: {last_checkpoint} ---")
+else:
+    print("--- No checkpoint found. Starting training from scratch. ---")
+
+# Pass the found checkpoint (or None) to the trainer
+trainer.train(resume_from_checkpoint=last_checkpoint)
+
+print("--- Training finished ---")
+print("---------------------------------")
+
+
+print(f"--- 5. Saving best model to {FINAL_MODEL_DIR} ---")
+trainer.save_model(FINAL_MODEL_DIR)
+tokenizer.save_pretrained(FINAL_MODEL_DIR)
+print("--- Model saved ---")
+print("---------------------------------")
+
+
+print(f"--- 6. Generating report on TEST set ---")
+model_for_eval = AutoModelForSequenceClassification.from_pretrained(
+    FINAL_MODEL_DIR,
+    use_safetensors=True
+)
+eval_tokenizer = AutoTokenizer.from_pretrained(FINAL_MODEL_DIR)
+
+eval_trainer = Trainer(model=model_for_eval, args=training_args)
+
+predictions = eval_trainer.predict(test_dataset)
+
+y_true = predictions.label_ids
+y_pred_logits = predictions.predictions
+y_pred_probs = softmax(y_pred_logits, axis=1)
+y_pred_labels = np.argmax(y_pred_logits, axis=1)
+
+print("--- Generating Classification Report ---")
+report = classification_report(y_true, y_pred_labels, target_names=["Ham (0)", "Phishing (1)"])
+report_path = os.path.join(REPORT_DIR, "classification_report.txt")
+
+with open(report_path, "w") as f:
+    f.write("--- Semantic Model Classification Report ---\n\n")
+    f.write(report)
+
+print(report)
+print(f"Classification report saved to {report_path}")
+
+print("--- Generating Confusion Matrix ---")
+cm = confusion_matrix(y_true, y_pred_labels)
+cm_path = os.path.join(REPORT_DIR, "confusion_matrix.png")
+
+plt.figure(figsize=(8, 6))
+sns.heatmap(cm, annot=True, fmt="d", cmap="Blues",
+            xticklabels=["Ham (0)", "Phishing (1)"],
+            yticklabels=["Ham (0)", "Phishing (1)"])
+plt.title("Confusion Matrix for Semantic Model")
+plt.xlabel("Predicted Label")
+plt.ylabel("True Label")
+plt.savefig(cm_path)
+plt.close()
+print(f"Confusion matrix saved to {cm_path}")
+
+print("--- Generating Probability Scatterplot ---")
+prob_df = pd.DataFrame({
+    'true_label': y_true,
+    'predicted_phishing_prob': y_pred_probs[:, 1]
+})
+prob_path = os.path.join(REPORT_DIR, "probability_scatterplot.png")
+
+plt.figure(figsize=(10, 6))
+sns.stripplot(data=prob_df, x='true_label', y='predicted_phishing_prob', jitter=0.2, alpha=0.7)
+plt.title("Model Confidence: Predicted Phishing Probability vs. True Label")
+plt.xlabel("True Label")
+plt.ylabel("Predicted Phishing Probability")
+plt.xticks([0, 1], ["Ham (0)", "Phishing (1)"])
+plt.axhline(0.5, color='r', linestyle='--', label='Decision Boundary (0.5)')
+plt.legend()
+plt.savefig(prob_path)
+plt.close()
+print(f"Probability scatterplot saved to {prob_path}")
+
+print("---------------------------------")
+print(f"--- Evaluation Complete. Reports saved to {REPORT_DIR} ---")

README.md

@@ -0,0 +1,11 @@
+---
+title: AEGIS SECURE API
+emoji: 🔥
+colorFrom: indigo
+colorTo: gray
+sdk: docker
+pinned: false
+short_description: This is the main model api for the project aegis secure.
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

README_DEPLOYMENT.md

@@ -0,0 +1,125 @@
+# Quick Start: Deploy Phishing Detection API to GCP
+
+## Prerequisites Checklist
+
+- [ ] Google Cloud Platform account with billing enabled
+- [ ] gcloud CLI installed and authenticated (`gcloud auth login`)
+- [ ] Docker installed and running
+- [ ] GROQ API key (get from https://console.groq.com/)
+
+## Fastest Deployment (One Command)
+
+### For Linux/Mac:
+```bash
+chmod +x deploy.sh
+./deploy.sh YOUR_PROJECT_ID us-central1
+```
+
+### For Windows PowerShell:
+```powershell
+.\deploy.ps1 -ProjectId YOUR_PROJECT_ID -Region us-central1
+```
+
+**Replace `YOUR_PROJECT_ID` with your actual GCP project ID.**
+
+The script will:
+1. ✅ Enable required GCP APIs
+2. ✅ Create GROQ_API_KEY secret
+3. ✅ Build Docker image
+4. ✅ Deploy to Cloud Run
+5. ✅ Display your service URL
+
+## Manual Deployment Steps
+
+If you prefer manual steps, see [DEPLOYMENT.md](./DEPLOYMENT.md) for detailed instructions.
+
+## Files Created for Deployment
+
+- **Dockerfile** - Container configuration for the application
+- **.dockerignore** - Files to exclude from Docker build
+- **cloudbuild.yaml** - GCP Cloud Build configuration
+- **deploy.sh** - Automated deployment script (Linux/Mac)
+- **deploy.ps1** - Automated deployment script (Windows)
+- **docker-compose.yml** - For local testing with Docker
+- **DEPLOYMENT.md** - Comprehensive deployment guide
+
+## Testing Locally First (Optional)
+
+Before deploying to GCP, test locally:
+
+```bash
+# Build the image
+docker build -t phishing-api:local .
+
+# Run locally
+docker run -p 8000:8000 \
+    -e GROQ_API_KEY=your-api-key \
+    phishing-api:local
+
+# Test in another terminal
+curl http://localhost:8000/health
+```
+
+Or use Docker Compose:
+
+```bash
+# Set your API key in environment
+export GROQ_API_KEY=your-api-key  # Linux/Mac
+# or
+$env:GROQ_API_KEY="your-api-key"  # Windows PowerShell
+
+# Run
+docker-compose up
+```
+
+## After Deployment
+
+Your service will be available at:
+- **API**: `https://phishing-detection-api-[hash]-uc.a.run.app`
+- **Health Check**: `https://phishing-detection-api-[hash]-uc.a.run.app/health`
+- **API Docs**: `https://phishing-detection-api-[hash]-uc.a.run.app/docs`
+
+## Quick Test
+
+```bash
+# Get your service URL
+SERVICE_URL=$(gcloud run services describe phishing-detection-api \
+    --region us-central1 \
+    --format="value(status.url)")
+
+# Test health endpoint
+curl $SERVICE_URL/health
+
+# Test prediction endpoint
+curl -X POST "$SERVICE_URL/predict" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "sender": "test@example.com",
+    "subject": "Urgent Action Required",
+    "text": "Please verify your account at http://suspicious-site.com",
+    "metadata": {}
+  }'
+```
+
+## Troubleshooting
+
+1. **Build fails**: Check that all model files are in the correct directories
+2. **Service won't start**: Check logs with `gcloud run services logs read phishing-detection-api --region us-central1`
+3. **GROQ API errors**: Verify secret is set correctly: `gcloud secrets versions access latest --secret="GROQ_API_KEY"`
+4. **Memory issues**: Increase memory in cloudbuild.yaml or deployment command
+
+For detailed troubleshooting, see [DEPLOYMENT.md](./DEPLOYMENT.md#troubleshooting).
+
+## Next Steps
+
+- Monitor your service in [Cloud Run Console](https://console.cloud.google.com/run)
+- Set up alerts in [Cloud Monitoring](https://console.cloud.google.com/monitoring)
+- Review API documentation at `/docs` endpoint
+- Scale resources if needed (see DEPLOYMENT.md)
+
+## Support
+
+For issues:
+1. Check logs: `gcloud run services logs read phishing-detection-api --region us-central1`
+2. Review [DEPLOYMENT.md](./DEPLOYMENT.md) for detailed information
+3. Verify all prerequisites are met

api_keys.py

@@ -0,0 +1,92 @@
+import os
+import sys
+from typing import List, Optional
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from dotenv import load_dotenv
+import logging
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Load environment variables from .env file
+env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.env')
+load_dotenv(dotenv_path=env_path)
+
+# Debug: Print environment variables
+logger.info(f"Current working directory: {os.getcwd()}")
+logger.info(f"Loading .env from: {env_path}")
+logger.info(f"GEMINI_API_KEY: {'*' * 8 + os.getenv('GEMINI_API_KEY', '')[-4:] if os.getenv('GEMINI_API_KEY') else 'Not set'}")
+logger.info(f"GEMINI_API_KEYS: {'*' * 8 + os.getenv('GEMINI_API_KEYS', '')[-4:] if os.getenv('GEMINI_API_KEYS') else 'Not set'}")
+
+@dataclass
+class APIKey:
+    key: str
+    last_used: Optional[datetime] = None
+    is_available: bool = True
+    rate_limit_reset: Optional[datetime] = None
+
+class APIKeyManager:
+    _instance = None
+    _lock = asyncio.Lock()
+    
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(APIKeyManager, cls).__new__(cls)
+            cls._instance._initialize()
+        return cls._instance
+    
+    def _initialize(self):
+        self.keys: List[APIKey] = []
+        self._current_index = 0
+        self._load_api_keys()
+    
+    def _load_api_keys(self):
+        # Try to load from GEMINI_API_KEY first
+        single_key = os.getenv('GEMINI_API_KEY', '').strip()
+        if single_key:
+            single_key = single_key.strip('"\'')
+            self.keys = [APIKey(key=single_key)]
+            logger.info(f"Loaded 1 API key from GEMINI_API_KEY")
+            return
+            
+        # Fall back to GEMINI_API_KEYS if GEMINI_API_KEY is not set
+        api_keys_str = os.getenv('GEMINI_API_KEYS', '').strip()
+        if api_keys_str:
+            keys = [key.strip().strip('"\'') for key in api_keys_str.split(',') if key.strip()]
+            self.keys = [APIKey(key=key) for key in keys]
+            logger.info(f"Loaded {len(keys)} API keys from GEMINI_API_KEYS")
+            return
+            
+        logger.warning("No API keys found in environment variables")
+    
+    def get_available_key(self) -> Optional[str]:
+        """Get an available API key, considering rate limits."""
+        now = datetime.utcnow()
+        
+        for key_obj in self.keys:
+            if not key_obj.is_available:
+                if key_obj.rate_limit_reset and now >= key_obj.rate_limit_reset:
+                    key_obj.is_available = True
+                    key_obj.rate_limit_reset = None
+                else:
+                    continue
+            
+            key_obj.last_used = now
+            return key_obj.key
+        
+        return None
+    
+    def mark_key_unavailable(self, key: str, retry_after_seconds: int = 60):
+        """Mark a key as unavailable due to rate limiting."""
+        for key_obj in self.keys:
+            if key_obj.key == key:
+                key_obj.is_available = False
+                key_obj.rate_limit_reset = datetime.utcnow() + timedelta(seconds=retry_after_seconds)
+                logger.warning(f"Rate limit hit for API key. Will retry after {retry_after_seconds} seconds")
+                return
+        logger.warning(f"Tried to mark unknown API key as unavailable")
+
+api_key_manager = APIKeyManager()

app.py

@@ -0,0 +1,566 @@
+import os
+import re
+import json
+import time
+import sys
+import asyncio
+from typing import List, Dict, Optional
+from urllib.parse import urlparse
+import socket
+import httpx
+
+import joblib
+import torch
+import numpy as np
+import pandas as pd
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+import google.generativeai as genai
+from dotenv import load_dotenv
+
+import config
+from models import get_ml_models, get_dl_models, FinetunedBERT
+from feature_extraction import process_row
+
+load_dotenv()
+sys.path.append(os.path.join(config.BASE_DIR, 'Message_model'))
+from predict import PhishingPredictor
+
+app = FastAPI(
+    title="Phishing Detection API",
+    description="Advanced phishing detection system using multiple ML/DL models and Gemini AI",
+    version="1.0.0"
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+class MessageInput(BaseModel):
+    text: str
+    metadata: Optional[Dict] = {}
+
+class PredictionResponse(BaseModel):
+    confidence: float
+    reasoning: str
+    highlighted_text: str
+    final_decision: str
+    suggestion: str
+
+ml_models = {}
+dl_models = {}
+bert_model = None
+semantic_model = None
+gemini_model = None
+
+MODEL_BOUNDARIES = {
+    'logistic': 0.5,
+    'svm': 0.5,
+    'xgboost': 0.5,
+    'attention_blstm': 0.5,
+    'rcnn': 0.5,
+    'bert': 0.5,
+    'semantic': 0.5
+}
+
+def load_models():
+    global ml_models, dl_models, bert_model, semantic_model, gemini_model
+    
+    print("Loading models...")
+    
+    models_dir = config.MODELS_DIR
+    for model_name in ['logistic', 'svm', 'xgboost']:
+        model_path = os.path.join(models_dir, f'{model_name}.joblib')
+        if os.path.exists(model_path):
+            ml_models[model_name] = joblib.load(model_path)
+            print(f"✓ Loaded {model_name} model")
+        else:
+            print(f"⚠ Warning: {model_name} model not found at {model_path}")
+    
+    for model_name in ['attention_blstm', 'rcnn']:
+        model_path = os.path.join(models_dir, f'{model_name}.pt')
+        if os.path.exists(model_path):
+            model_template = get_dl_models(input_dim=len(config.NUMERICAL_FEATURES))
+            dl_models[model_name] = model_template[model_name]
+            dl_models[model_name].load_state_dict(torch.load(model_path, map_location='cpu'))
+            dl_models[model_name].eval()
+            print(f"✓ Loaded {model_name} model")
+        else:
+            print(f"⚠ Warning: {model_name} model not found at {model_path}")
+    
+    bert_path = os.path.join(config.BASE_DIR, 'finetuned_bert')
+    if os.path.exists(bert_path):
+        try:
+            bert_model = FinetunedBERT(bert_path)
+            print("✓ Loaded BERT model")
+        except Exception as e:
+            print(f"⚠ Warning: Could not load BERT model: {e}")
+    
+    semantic_model_path = os.path.join(config.BASE_DIR, 'Message_model', 'final_semantic_model')
+    if os.path.exists(semantic_model_path) and os.listdir(semantic_model_path):
+        try:
+            semantic_model = PhishingPredictor(model_path=semantic_model_path)
+            print("✓ Loaded semantic model")
+        except Exception as e:
+            print(f"⚠ Warning: Could not load semantic model: {e}")
+    else:
+        checkpoint_path = os.path.join(config.BASE_DIR, 'Message_model', 'training_checkpoints', 'checkpoint-30')
+        if os.path.exists(checkpoint_path):
+            try:
+                semantic_model = PhishingPredictor(model_path=checkpoint_path)
+                print("✓ Loaded semantic model from checkpoint")
+            except Exception as e:
+                print(f"⚠ Warning: Could not load semantic model from checkpoint: {e}")
+    
+    gemini_api_key = os.environ.get('GEMINI_API_KEY')
+    if gemini_api_key:
+        genai.configure(api_key=gemini_api_key)
+        gemini_model = genai.GenerativeModel('gemini-2.0-flash')
+        print("✓ Initialized Gemini API")
+    else:
+        print("⚠ Warning: GEMINI_API_KEY not set. Set it as environment variable.")
+        print("   Example: export GEMINI_API_KEY='your-api-key-here'")
+
+def parse_message(text: str) -> tuple:
+    url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+|(?:www\.)?[a-zA-Z0-9-]+\.[a-z]{2,12}\b(?:/[^\s]*)?'
+    urls = re.findall(url_pattern, text)
+    cleaned_text = re.sub(url_pattern, '', text)
+    cleaned_text = ' '.join(cleaned_text.lower().split())
+    cleaned_text = re.sub(r'[^a-z0-9\s.,!?-]', '', cleaned_text)
+    cleaned_text = re.sub(r'([.,!?])+', r'\1', cleaned_text)
+    cleaned_text = ' '.join(cleaned_text.split())
+    return urls, cleaned_text
+
+async def extract_url_features(urls: List[str]) -> pd.DataFrame:
+    if not urls:
+        return pd.DataFrame()
+    
+    df = pd.DataFrame({'url': urls})
+    whois_cache = {}
+    ssl_cache = {}
+    
+    tasks = []
+    for _, row in df.iterrows():
+        tasks.append(asyncio.to_thread(process_row, row, whois_cache, ssl_cache))
+    
+    feature_list = await asyncio.gather(*tasks)
+    features_df = pd.DataFrame(feature_list)
+    result_df = pd.concat([df, features_df], axis=1)
+    return result_df
+
+def custom_boundary(raw_score: float, boundary: float) -> float:
+    return (raw_score - boundary) * 100
+
+def get_model_predictions(features_df: pd.DataFrame, message_text: str) -> Dict:
+    predictions = {}
+    
+    numerical_features = config.NUMERICAL_FEATURES
+    categorical_features = config.CATEGORICAL_FEATURES
+    
+    try:
+        X = features_df[numerical_features + categorical_features]
+    except KeyError as e:
+        print(f"Error: Missing columns in features_df. {e}")
+        print(f"Available columns: {features_df.columns.tolist()}")
+        X = pd.DataFrame(columns=numerical_features + categorical_features)
+    
+    if not X.empty:
+        X.loc[:, numerical_features] = X.loc[:, numerical_features].fillna(-1)
+        X.loc[:, categorical_features] = X.loc[:, categorical_features].fillna('N/A')
+
+        for model_name, model in ml_models.items():
+            try:
+                all_probas = model.predict_proba(X)[:, 1]  
+                raw_score = np.max(all_probas)          
+                
+                scaled_score = custom_boundary(raw_score, MODEL_BOUNDARIES[model_name])
+                predictions[model_name] = {
+                    'raw_score': float(raw_score),
+                    'scaled_score': float(scaled_score)
+                }
+            except Exception as e:
+                print(f"Error with {model_name} (Prediction Step): {e}") 
+        
+        X_numerical = X[numerical_features].values 
+        
+        for model_name, model in dl_models.items():
+            try:
+                X_tensor = torch.tensor(X_numerical, dtype=torch.float32)
+                with torch.no_grad():
+                    all_scores = model(X_tensor)
+                    raw_score = torch.max(all_scores).item()
+                    
+                scaled_score = custom_boundary(raw_score, MODEL_BOUNDARIES[model_name])
+                predictions[model_name] = {
+                    'raw_score': float(raw_score),
+                    'scaled_score': float(scaled_score)
+                }
+            except Exception as e:
+                print(f"Error with {model_name}: {e}")
+    
+    if bert_model and len(features_df) > 0:
+        try:
+            urls = features_df['url'].tolist()
+            raw_scores = bert_model.predict_proba(urls)
+            avg_raw_score = np.mean([score[1] for score in raw_scores])
+            scaled_score = custom_boundary(avg_raw_score, MODEL_BOUNDARIES['bert'])
+            predictions['bert'] = {
+                'raw_score': float(avg_raw_score),
+                'scaled_score': float(scaled_score)
+            }
+        except Exception as e:
+            print(f"Error with BERT: {e}")
+    
+    if semantic_model and message_text:
+        try:
+            result = semantic_model.predict(message_text)
+            raw_score = result['phishing_probability']
+            scaled_score = custom_boundary(raw_score, MODEL_BOUNDARIES['semantic'])
+            predictions['semantic'] = {
+                'raw_score': float(raw_score),
+                'scaled_score': float(scaled_score),
+                'confidence': result['confidence']
+            }
+        except Exception as e:
+            print(f"Error with semantic model: {e}")
+    
+    return predictions
+
+async def get_network_features_for_gemini(urls: List[str]) -> str:
+    if not urls:
+        return "No URLs to analyze for network features."
+    
+    results = []
+    async with httpx.AsyncClient() as client:
+        for i, url_str in enumerate(urls[:3]): 
+            try:
+                hostname = urlparse(url_str).hostname
+                if not hostname:
+                    results.append(f"\nURL {i+1} ({url_str}): Invalid URL, no hostname.")
+                    continue
+                
+                try:
+                    ip_address = await asyncio.to_thread(socket.gethostbyname, hostname)
+                except socket.gaierror:
+                    results.append(f"\nURL {i+1} ({hostname}): Could not resolve domain to IP.")
+                    continue
+                
+                try:
+                    geo_url = f"http://ip-api.com/json/{ip_address}?fields=status,message,country,city,isp,org,as"
+                    response = await client.get(geo_url, timeout=3.0)
+                    response.raise_for_status()
+                    data = response.json()
+                    
+                    if data.get('status') == 'success':
+                        geo_info = (
+                            f"   • IP Address: {ip_address}\n"
+                            f"   • Location: {data.get('city', 'N/A')}, {data.get('country', 'N/A')}\n"
+                            f"   • ISP: {data.get('isp', 'N/A')}\n"
+                            f"   • Organization: {data.get('org', 'N/A')}\n"
+                            f"   • ASN: {data.get('as', 'N/A')}"
+                        )
+                        results.append(f"\nURL {i+1} ({hostname}):\n{geo_info}")
+                    else:
+                        results.append(f"\nURL {i+1} ({hostname}):\n   • IP Address: {ip_address}\n   • Geo-Data: API lookup failed ({data.get('message')})")
+                
+                except httpx.RequestError as e:
+                    results.append(f"\nURL {i+1} ({hostname}):\n   • IP Address: {ip_address}\n   • Geo-Data: Network error while fetching IP info ({str(e)})")
+                
+            except Exception as e:
+                results.append(f"\nURL {i+1} ({url_str}): Error processing URL ({str(e)})")
+    
+    if not results:
+        return "No valid hostnames found in URLs to analyze."
+
+    return "\n".join(results)
+
+
+async def get_gemini_final_decision(urls: List[str], features_df: pd.DataFrame, 
+                                    message_text: str, predictions: Dict, 
+                                    original_text: str) -> Dict:
+    
+    if not gemini_model:
+        avg_scaled_score = np.mean([p['scaled_score'] for p in predictions.values()]) if predictions else 0
+        confidence = min(100, max(0, 50 + abs(avg_scaled_score)))
+        
+        return {
+            "confidence": round(confidence, 2),
+            "reasoning": "Gemini API not available. Using average model scores.",
+            "highlighted_text": original_text,
+            "final_decision": "phishing" if avg_scaled_score > 0 else "legitimate",
+            "suggestion": "Do not interact with this message. Delete it immediately and report it to your IT department." if avg_scaled_score > 0 else "This message appears safe, but remain cautious with any links or attachments."
+        }
+    
+    url_features_summary = "No URLs detected in message"
+    has_urls = len(features_df) > 0
+    
+    if has_urls:
+        feature_summary_parts = []
+        for idx, row in features_df.iterrows():
+            url = row.get('url', 'Unknown')
+            feature_summary_parts.append(f"\nURL {idx+1}: {url}")
+            feature_summary_parts.append(f"   • Length: {row.get('url_length', 'N/A')} chars")
+            feature_summary_parts.append(f"   • Dots in URL: {row.get('count_dot', 'N/A')}")
+            feature_summary_parts.append(f"   • Special characters: {row.get('count_special_chars', 'N/A')}")
+            feature_summary_parts.append(f"   • Domain age: {row.get('domain_age_days', 'N/A')} days")
+            feature_summary_parts.append(f"   • SSL certificate valid: {row.get('cert_has_valid_hostname', 'N/A')}")
+            feature_summary_parts.append(f"   • Uses HTTPS: {row.get('https', 'N/A')}")
+        url_features_summary = "\n".join(feature_summary_parts)
+
+    network_features_summary = await get_network_features_for_gemini(urls)
+    
+    model_predictions_summary = []
+    for model_name, pred_data in predictions.items():
+        scaled = pred_data['scaled_score']
+        raw = pred_data['raw_score']
+        model_predictions_summary.append(
+            f"   • {model_name.upper()}: scaled_score={scaled:.2f} (raw={raw:.3f})"
+        )
+    model_scores_text = "\n".join(model_predictions_summary)
+    
+    MAX_TEXT_LEN = 3000
+    if len(original_text) > MAX_TEXT_LEN:
+        truncated_original_text = original_text[:MAX_TEXT_LEN] + "\n... [TRUNCATED]"
+    else:
+        truncated_original_text = original_text
+
+    if len(message_text) > MAX_TEXT_LEN:
+        truncated_message_text = message_text[:MAX_TEXT_LEN] + "\n... [TRUNCATED]"
+    else:
+        truncated_message_text = message_text
+
+    context = f"""You are a security model that must decide if a message is phishing or legitimate.
+
+Use all evidence below:
+- URL/network data (trust NETWORK_GEO more than URL_FEATURES when they disagree; domain_age = -1 means unknown).
+- Model scores (scaled_score > 0 → more phishing, < 0 → more legitimate).
+- Message content (urgency, threats, credential/OTP/payment requests, impersonation).
+
+If strong phishing signals exist, prefer "phishing". If everything matches a normal, known service/organization and content is routine, prefer "legitimate".
+
+Return only this JSON object:
+{{
+  "confidence": <float 0-100>,
+  "reasoning": "<brief explanation referring to key evidence>",
+  "highlighted_text": "<full original message with suspicious spans wrapped in $$...$$>",
+  "final_decision": "phishing" or "legitimate",
+  "suggestion": "<practical advice for the user on what to do>"
+}}
+
+MESSAGE_ORIGINAL:
+{truncated_original_text}
+
+MESSAGE_CLEANED:
+{truncated_message_text}
+
+URLS:
+{', '.join(urls) if urls else 'None'}
+
+URL_FEATURES:
+{url_features_summary}
+
+NETWORK_GEO:
+{network_features_summary}
+
+MODEL_SCORES (scaled_score > 0 phishing, < 0 legitimate):
+{model_scores_text}
+"""
+
+    try:
+        generation_config = {
+            'temperature': 0.2,
+            'top_p': 0.85,
+            'top_k': 40,
+            'max_output_tokens': 8192,
+            'response_mime_type': 'application/json'
+        }
+
+        safety_settings = {
+            "HARM_CATEGORY_HARASSMENT": "BLOCK_NONE",
+            "HARM_CATEGORY_HATE_SPEECH": "BLOCK_NONE",
+            "HARM_CATEGORY_SEXUALLY_EXPLICIT": "BLOCK_NONE",
+            "HARM_CATEGORY_DANGEROUS_CONTENT": "BLOCK_NONE",
+        }
+
+        max_retries = 3
+        retry_delay = 2
+        
+        for attempt in range(max_retries):
+            try:
+                response = await gemini_model.generate_content_async(
+                    context,
+                    generation_config=generation_config,
+                    safety_settings=safety_settings
+                )
+                if not response.candidates or not response.candidates[0].content.parts:
+                   raise ValueError(f"No content returned. Finish Reason: {response.candidates[0].finish_reason}")
+                break
+            
+            except Exception as retry_error:
+                print(f"Gemini API attempt {attempt + 1} failed: {retry_error}")
+                if attempt < max_retries - 1:
+                    print(f"Retrying in {retry_delay}s...")
+                    await asyncio.sleep(retry_delay) 
+                    retry_delay *= 2
+                else:
+                    raise retry_error
+        
+        response_text = response.text.strip()
+        
+        if '```json' in response_text:
+            response_text = response_text.split('```json')[1].split('```')[0].strip()
+        elif response_text.startswith('```') and response_text.endswith('```'):
+             response_text = response_text[3:-3].strip()
+
+        if not response_text.startswith('{'):
+            json_match = re.search(r'\{(?:[^{}]|(?:\{[^{}]*\}))*\}', response_text, re.DOTALL)
+            if json_match:
+                response_text = json_match.group(0)
+            else:
+                raise ValueError(f"Could not find JSON in Gemini response: {response_text[:200]}")
+        
+        result = json.loads(response_text)
+        
+        required_fields = ['confidence', 'reasoning', 'highlighted_text', 'final_decision', 'suggestion']
+        if not all(field in result for field in required_fields):
+            raise ValueError(f"Missing required fields. Got: {list(result.keys())}")
+        
+        result['confidence'] = float(result['confidence'])
+        if not 0 <= result['confidence'] <= 100:
+            result['confidence'] = max(0, min(100, result['confidence']))
+        
+        if result['final_decision'].lower() not in ['phishing', 'legitimate']:
+            result['final_decision'] = 'phishing' if result['confidence'] >= 50 else 'legitimate'
+        else:
+            result['final_decision'] = result['final_decision'].lower()
+        
+        if not result['highlighted_text'].strip() or '...' in result['highlighted_text']:
+            result['highlighted_text'] = original_text
+        
+        if not result.get('suggestion', '').strip():
+            if result['final_decision'] == 'phishing':
+                result['suggestion'] = "Do not interact with this message. Delete it immediately and report it as phishing."
+            else:
+                result['suggestion'] = "This message appears safe, but always verify sender identity before taking any action."
+        
+        return result
+    
+    except json.JSONDecodeError as e:
+        print(f"JSON parsing error: {e}")
+        print(f"Response text that failed parsing: {response_text[:500]}")
+        
+        avg_scaled_score = np.mean([p['scaled_score'] for p in predictions.values()]) if predictions else 0
+        confidence = min(100, max(0, 50 + abs(avg_scaled_score)))
+        
+        return {
+            "confidence": round(confidence, 2),
+            "reasoning": f"Gemini response parsing failed. Fallback: Based on model average (score: {avg_scaled_score:.2f}), message appears {'legitimate' if avg_scaled_score <= 0 else 'suspicious'}.",
+            "highlighted_text": original_text,
+            "final_decision": "phishing" if avg_scaled_score > 0 else "legitimate",
+            "suggestion": "Do not interact with this message. Delete it immediately and be cautious." if avg_scaled_score > 0 else "Exercise caution. Verify the sender before taking any action."
+        }
+    
+    except Exception as e:
+        print(f"Error with Gemini API: {e}") 
+        
+        avg_scaled_score = np.mean([p['scaled_score'] for p in predictions.values()]) if predictions else 0
+        confidence = min(100, max(0, 50 + abs(avg_scaled_score)))
+        
+        return {
+            "confidence": round(confidence, 2),
+            "reasoning": f"Gemini API error: {str(e)}. Fallback decision based on {len(predictions)} model predictions (average score: {avg_scaled_score:.2f}).",
+            "highlighted_text": original_text,
+            "final_decision": "phishing" if avg_scaled_score > 0 else "legitimate",
+            "suggestion": "Treat this message with caution. Delete it if suspicious, or verify the sender through official channels before taking action." if avg_scaled_score > 0 else "This message appears safe based on models, but always verify sender identity before clicking links or providing information."
+        }
+
+@app.on_event("startup")
+async def startup_event():
+    load_models()
+    print("\n" + "="*60)
+    print("Phishing Detection API is ready!")
+    print("="*60)
+    print("API Documentation: http://localhost:8000/docs")
+    print("="*60 + "\n")
+
+@app.get("/")
+async def root():
+    return {
+        "message": "Phishing Detection API",
+        "version": "1.0.0",
+        "endpoints": {
+            "predict": "/predict (POST)",
+            "health": "/health (GET)",
+            "docs": "/docs (GET)"
+        }
+    }
+
+@app.get("/health")
+async def health_check():
+    models_loaded = {
+        "ml_models": list(ml_models.keys()),
+        "dl_models": list(dl_models.keys()),
+        "bert_model": bert_model is not None,
+        "semantic_model": semantic_model is not None,
+        "gemini_model": gemini_model is not None
+    }
+    
+    return {
+        "status": "healthy",
+        "models_loaded": models_loaded
+    }
+
+@app.post("/predict", response_model=PredictionResponse)
+async def predict(message_input: MessageInput):
+    try:
+        original_text = message_input.text
+        
+        if not original_text or not original_text.strip():
+            raise HTTPException(status_code=400, detail="Message text cannot be empty")
+        
+        urls, cleaned_text = parse_message(original_text)
+        
+        features_df = pd.DataFrame()
+        if urls:
+            features_df = await extract_url_features(urls)
+        
+        predictions = {}
+        if len(features_df) > 0 or (cleaned_text and semantic_model):
+            predictions = await asyncio.to_thread(get_model_predictions, features_df, cleaned_text)
+        
+        if not predictions:
+            if not urls and not cleaned_text:
+                detail = "Message text is empty after cleaning."
+            elif not urls and not semantic_model:
+                detail = "No URLs provided and semantic model is not loaded."
+            elif not any([ml_models, dl_models, bert_model, semantic_model]):
+                 detail = "No models available for prediction. Please ensure models are trained and loaded."
+            else:
+                detail = "Could not generate predictions. Models may be missing or feature extraction failed."
+            
+            raise HTTPException(
+                status_code=500, 
+                detail=detail
+            )
+        
+        final_result = await get_gemini_final_decision(
+            urls, features_df, cleaned_text, predictions, original_text
+        )
+        
+        return PredictionResponse(**final_result)
+    
+    except HTTPException:
+        raise
+    except Exception as e:
+        import traceback
+        print(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

app1.py

@@ -0,0 +1,639 @@
+import os
+import re
+import json
+import time
+import sys
+import asyncio
+from typing import List, Dict, Optional
+from urllib.parse import urlparse
+import socket
+import httpx
+
+import joblib
+import torch
+import numpy as np
+import pandas as pd
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from groq import AsyncGroq
+from dotenv import load_dotenv
+from bs4 import BeautifulSoup
+
+import config
+from models import get_ml_models, get_dl_models, FinetunedBERT
+from feature_extraction import process_row
+
+load_dotenv()
+sys.path.append(os.path.join(config.BASE_DIR, 'Message_model'))
+from predict import PhishingPredictor
+
+app = FastAPI(
+    title="Phishing Detection API",
+    description="Advanced phishing detection system using multiple ML/DL models and Groq",
+    version="1.0.0"
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+class MessageInput(BaseModel):
+    sender: Optional[str] = ""
+    subject: Optional[str] = ""
+    text: str
+    metadata: Optional[Dict] = {}
+
+class PredictionResponse(BaseModel):
+    confidence: float
+    reasoning: str
+    highlighted_text: str
+    final_decision: str
+    suggestion: str
+
+ml_models = {}
+dl_models = {}
+bert_model = None
+semantic_model = None
+groq_async_client = None
+
+MODEL_BOUNDARIES = {
+    'logistic': 0.5,
+    'svm': 0.5,
+    'xgboost': 0.5,
+    'attention_blstm': 0.5,
+    'rcnn': 0.5,
+    'bert': 0.5,
+    'semantic': 0.5
+}
+
+def load_models():
+    global ml_models, dl_models, bert_model, semantic_model, groq_async_client
+    
+    print("Loading models...")
+    
+    models_dir = config.MODELS_DIR
+    for model_name in ['logistic', 'svm', 'xgboost']:
+        model_path = os.path.join(models_dir, f'{model_name}.joblib')
+        if os.path.exists(model_path):
+            ml_models[model_name] = joblib.load(model_path)
+            print(f"✓ Loaded {model_name} model")
+        else:
+            print(f"⚠ Warning: {model_name} model not found at {model_path}")
+    
+    for model_name in ['attention_blstm', 'rcnn']:
+        model_path = os.path.join(models_dir, f'{model_name}.pt')
+        if os.path.exists(model_path):
+            model_template = get_dl_models(input_dim=len(config.NUMERICAL_FEATURES))
+            dl_models[model_name] = model_template[model_name]
+            dl_models[model_name].load_state_dict(torch.load(model_path, map_location='cpu'))
+            dl_models[model_name].eval()
+            print(f"✓ Loaded {model_name} model")
+        else:
+            print(f"⚠ Warning: {model_name} model not found at {model_path}")
+    
+    bert_path = os.path.join(config.BASE_DIR, 'finetuned_bert')
+    if os.path.exists(bert_path):
+        try:
+            bert_model = FinetunedBERT(bert_path)
+            print("✓ Loaded BERT model")
+        except Exception as e:
+            print(f"⚠ Warning: Could not load BERT model: {e}")
+    
+    semantic_model_path = os.path.join(config.BASE_DIR, 'Message_model', 'final_semantic_model')
+    if os.path.exists(semantic_model_path) and os.listdir(semantic_model_path):
+        try:
+            semantic_model = PhishingPredictor(model_path=semantic_model_path)
+            print("✓ Loaded semantic model")
+        except Exception as e:
+            print(f"⚠ Warning: Could not load semantic model: {e}")
+    else:
+        checkpoint_path = os.path.join(config.BASE_DIR, 'Message_model', 'training_checkpoints', 'checkpoint-30')
+        if os.path.exists(checkpoint_path):
+            try:
+                semantic_model = PhishingPredictor(model_path=checkpoint_path)
+                print("✓ Loaded semantic model from checkpoint")
+            except Exception as e:
+                print(f"⚠ Warning: Could not load semantic model from checkpoint: {e}")
+    
+    groq_api_key = os.environ.get('GROQ_API_KEY')
+    if groq_api_key:
+        groq_async_client = AsyncGroq(api_key=groq_api_key)
+        print("✓ Initialized Groq API Client")
+    else:
+        print("⚠ Warning: GROQ_API_KEY not set. Set it as environment variable.")
+        print("   Example: export GROQ_API_KEY='your-api-key-here'")
+
+def parse_message(text: str) -> tuple:
+    url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+|(?:www\.)?[a-zA-Z0-9-]+\.[a-z]{2,12}\b(?:/[^\s]*)?'
+    urls = re.findall(url_pattern, text)
+    cleaned_text = re.sub(url_pattern, '', text)
+    cleaned_text = ' '.join(cleaned_text.lower().split())
+    cleaned_text = re.sub(r'[^a-z0-9\s.,!?-]', '', cleaned_text)
+    cleaned_text = re.sub(r'([.,!?])+', r'\1', cleaned_text)
+    cleaned_text = ' '.join(cleaned_text.split())
+    return urls, cleaned_text
+
+async def extract_url_features(urls: List[str]) -> pd.DataFrame:
+    if not urls:
+        return pd.DataFrame()
+    
+    df = pd.DataFrame({'url': urls})
+    whois_cache = {}
+    ssl_cache = {}
+    
+    tasks = []
+    for _, row in df.iterrows():
+        tasks.append(asyncio.to_thread(process_row, row, whois_cache, ssl_cache))
+    
+    feature_list = await asyncio.gather(*tasks)
+    features_df = pd.DataFrame(feature_list)
+    result_df = pd.concat([df, features_df], axis=1)
+    return result_df
+
+def custom_boundary(raw_score: float, boundary: float) -> float:
+    return (raw_score - boundary) * 100
+
+def get_model_predictions(features_df: pd.DataFrame, message_text: str) -> Dict:
+    predictions = {}
+    
+    numerical_features = config.NUMERICAL_FEATURES
+    categorical_features = config.CATEGORICAL_FEATURES
+    
+    try:
+        X = features_df[numerical_features + categorical_features]
+    except KeyError as e:
+        print(f"Error: Missing columns in features_df. {e}")
+        print(f"Available columns: {features_df.columns.tolist()}")
+        X = pd.DataFrame(columns=numerical_features + categorical_features)
+    
+    if not X.empty:
+        X.loc[:, numerical_features] = X.loc[:, numerical_features].fillna(-1)
+        X.loc[:, categorical_features] = X.loc[:, categorical_features].fillna('N/A')
+
+        for model_name, model in ml_models.items():
+            try:
+                all_probas = model.predict_proba(X)[:, 1]  
+                raw_score = np.max(all_probas)      
+                
+                scaled_score = custom_boundary(raw_score, MODEL_BOUNDARIES[model_name])
+                predictions[model_name] = {
+                    'raw_score': float(raw_score),
+                    'scaled_score': float(scaled_score)
+                }
+            except Exception as e:
+                print(f"Error with {model_name} (Prediction Step): {e}") 
+        
+        X_numerical = X[numerical_features].values 
+        
+        for model_name, model in dl_models.items():
+            try:
+                X_tensor = torch.tensor(X_numerical, dtype=torch.float32)
+                with torch.no_grad():
+                    all_scores = model(X_tensor)
+                    raw_score = torch.max(all_scores).item()
+                    
+                scaled_score = custom_boundary(raw_score, MODEL_BOUNDARIES[model_name])
+                predictions[model_name] = {
+                    'raw_score': float(raw_score),
+                    'scaled_score': float(scaled_score)
+                }
+            except Exception as e:
+                print(f"Error with {model_name}: {e}")
+    
+    if bert_model and len(features_df) > 0:
+        try:
+            urls = features_df['url'].tolist()
+            raw_scores = bert_model.predict_proba(urls)
+            avg_raw_score = np.mean([score[1] for score in raw_scores])
+            scaled_score = custom_boundary(avg_raw_score, MODEL_BOUNDARIES['bert'])
+            predictions['bert'] = {
+                'raw_score': float(avg_raw_score),
+                'scaled_score': float(scaled_score)
+            }
+        except Exception as e:
+            print(f"Error with BERT: {e}")
+    
+    if semantic_model and message_text:
+        try:
+            result = semantic_model.predict(message_text)
+            raw_score = result['phishing_probability']
+            scaled_score = custom_boundary(raw_score, MODEL_BOUNDARIES['semantic'])
+            predictions['semantic'] = {
+                'raw_score': float(raw_score),
+                'scaled_score': float(scaled_score),
+                'confidence': result['confidence']
+            }
+        except Exception as e:
+            print(f"Error with semantic model: {e}")
+    
+    return predictions
+
+async def get_network_features_for_gemini(urls: List[str]) -> str:
+    if not urls:
+        return "No URLs to analyze for network features."
+    
+    results = []
+    async with httpx.AsyncClient() as client:
+        for i, url_str in enumerate(urls[:3]): 
+            try:
+                hostname = urlparse(url_str).hostname
+                if not hostname:
+                    results.append(f"\nURL {i+1} ({url_str}): Invalid URL, no hostname.")
+                    continue
+                
+                try:
+                    ip_address = await asyncio.to_thread(socket.gethostbyname, hostname)
+                except socket.gaierror:
+                    results.append(f"\nURL {i+1} ({hostname}): Could not resolve domain to IP.")
+                    continue
+                
+                try:
+                    geo_url = f"http://ip-api.com/json/{ip_address}?fields=status,message,country,city,isp,org,as"
+                    response = await client.get(geo_url, timeout=3.0)
+                    response.raise_for_status()
+                    data = response.json()
+                    
+                    if data.get('status') == 'success':
+                        geo_info = (
+                            f"   • IP Address: {ip_address}\n"
+                            f"   • Location: {data.get('city', 'N/A')}, {data.get('country', 'N/A')}\n"
+                            f"   • ISP: {data.get('isp', 'N/A')}\n"
+                            f"   • Organization: {data.get('org', 'N/A')}\n"
+                            f"   • ASN: {data.get('as', 'N/A')}"
+                        )
+                        results.append(f"\nURL {i+1} ({hostname}):\n{geo_info}")
+                    else:
+                        results.append(f"\nURL {i+1} ({hostname}):\n   • IP Address: {ip_address}\n   • Geo-Data: API lookup failed ({data.get('message')})")
+                
+                except httpx.RequestError as e:
+                    results.append(f"\nURL {i+1} ({hostname}):\n   • IP Address: {ip_address}\n   • Geo-Data: Network error while fetching IP info ({str(e)})")
+                
+            except Exception as e:
+                results.append(f"\nURL {i+1} ({url_str}): Error processing URL ({str(e)})")
+    
+    if not results:
+        return "No valid hostnames found in URLs to analyze."
+
+    return "\n".join(results)
+
+SYSTEM_PROMPT = """You are the FINAL JUDGE in a phishing detection system. Your role is critical: analyze ALL available evidence and make the ultimate decision.
+
+IMPORTANT INSTRUCTIONS:
+1. You have FULL AUTHORITY to override model predictions if evidence suggests they're wrong.
+2. **TRUST THE 'INDEPENDENT NETWORK & GEO-DATA' OVER 'URL FEATURES'.** The ML model features (like `domain_age: -1`) can be wrong due to lookup failures. The 'INDEPENDENT' data is a real-time check.
+3. If 'INDEPENDENT' data shows a legitimate organization (e.g., "Cloudflare", "Google", "Codeforces") for a known domain, but the models score it as phishing (due to `domain_age: -1`), you **should override** and classify as 'legitimate'.
+4. Your confidence score is DIRECTIONAL (0-100):
+    - Scores > 50.0 mean 'phishing'.
+    - Scores < 50.0 mean 'legitimate'.
+    - 50.0 is neutral.
+    - The magnitude indicates certainty (e.g., 95.0 is 'very confident phishing'; 5.0 is 'very confident legitimate').
+    - Your confidence score MUST match your 'final_decision'.
+5. BE WARY OF FALSE POSITIVES. Legitimate messages (bank alerts, contest notifications) can seem urgent.
+
+PRIORITY GUIDANCE (Use this logic):
+- IF URLs are present: Focus heavily on URL features.
+  - Examine 'URL FEATURES' for patterns (e.g., domain_age: -1 or 0, high special_chars).
+  - **CRITICAL:** Cross-reference this with the 'INDEPENDENT NETWORK & GEO-DATA'. This real-time data (IP, Location, ISP) is your ground truth.
+  - **If `domain_age` is -1, it's a lookup failure.** IGNORE IT and trust the 'INDEPENDENT NETWORK & GEO-DATA' to see if the domain is real (e.g., 'codeforces.com' with a valid IP).
+  - Then supplement with message content analysis.
+- IF NO URLs are present: Focus entirely on message content and semantics.
+  - Analyze language patterns, urgency tactics, and social engineering techniques
+  - Look for credential requests, financial solicitations, or threats
+  - Evaluate the semantic model's assessment heavily
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+FEW-SHOT EXAMPLES FOR GUIDANCE:
+
+Example 1 - Clear Phishing:
+Message: "URGENT! Click: http://paypa1-secure.xyz/verify"
+URL Features: domain_age: 5
+Network Data: IP: 123.45.67.89, Location: Russia, ISP: Shady-Host
+Model Scores: All positive
+Correct Decision: {{
+  "confidence": 95.0,
+  "reasoning": "Classic phishing. Misspelled domain, new age, and network data points to a suspicious ISP in Russia.",
+  "highlighted_text": "URGENT! Click: $$http://paypa1-secure.xyz/verify$$",
+  "final_decision": "phishing",
+  "suggestion": "Do NOT click. Delete immediately."
+}}
+
+Example 2 - Legitimate (False Positive Case):
+Message: "Hi, join Codeforces Round 184. ... Unsubscribe: https://codeforces.com/unsubscribe/..."
+URL Features: domain_age: -1 (This is a lookup failure!)
+Network Data: URL (codeforces.com): IP: 104.22.6.109, Location: San Francisco, USA, ISP: Cloudflare, Inc.
+Model Scores: Mixed (some positive due to domain_age: -1)
+Correct Decision: {{
+  "confidence": 10.0,
+  "reasoning": "OVERRIDING models. The 'URL FEATURES' show a 'domain_age: -1' which is a clear lookup error that confused the models. The 'INDEPENDENT NETWORK & GEO-DATA' confirms the domain 'codeforces.com' is real and hosted on Cloudflare, a legitimate provider. The message content is a standard, safe notification.",
+  "highlighted_text": "Hi, join Codeforces Round 184. ... Unsubscribe: https://codeforces.com/unsubscribe/...",
+  "final_decision": "legitimate",
+  "suggestion": "This message is safe. It is a legitimate notification from Codeforces."
+}}
+
+Example 3 - Legitimate (Long Formal Text):
+Message: "TATA MOTORS PASSENGER VEHICLES LIMITED... GENERAL GUIDANCE NOTE... [TRUNCATED]"
+URL Features: domain_age: 8414
+Network Data: URL (cars.tatamotors.com): IP: 23.209.113.12, Location: Boardman, USA, ISP: Akamai Technologies
+Model Scores: All negative
+Correct Decision: {{
+  "confidence": 5.0,
+  "reasoning": "This is a legitimate corporate communication. The text, although truncated, is clearly a formal guidance note for shareholders. The network data confirms 'cars.tatamotors.com' is hosted on Akamai, a major CDN used by large corporations. The models correctly identify this as safe.",
+  "highlighted_text": "TATA MOTORS PASSENGER VEHICLES LIMITED... GENERAL GUIDANCE NOTE... [TRUNCATED]",
+  "final_decision": "legitimate",
+  "suggestion": "This message is a legitimate corporate communication and appears safe."
+}}
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+YOUR ANALYSIS TASK:
+Analyze the message data provided by the user (in the 'user' message) following the steps and logic outlined above.
+
+**CRITICAL for `highlighted_text`:** You MUST return the *entire original message*. Only wrap the specific words/URLs that are suspicious with `$$...$$`. If nothing is suspicious (i.e., `final_decision` is 'legitimate'), return the original message with NO `$$` markers.
+
+OUTPUT FORMAT (respond with ONLY this JSON, no markdown, no explanation):
+{{
+  "confidence": <float (0-100, directional score where >50 is phishing)>,
+  "reasoning": "<your detailed analysis explaining why this is/isn't phishing, mentioning why you trust/override models>",
+  "highlighted_text": "<THE FULL, ENTIRE original message with suspicious parts marked as $$suspicious text$$>",
+  "final_decision": "phishing" or "legitimate",
+  "suggestion": "<specific, actionable advice for the user on how to handle this message - what to do or not do>"
+}}"""
+
+async def get_groq_final_decision(urls: List[str], features_df: pd.DataFrame, 
+                                  message_text: str, predictions: Dict, 
+                                  original_text: str,
+                                  sender: Optional[str] = "",
+                                  subject: Optional[str] = "") -> Dict:
+    
+    if not groq_async_client:
+        avg_scaled_score = np.mean([p['scaled_score'] for p in predictions.values()]) if predictions else 0
+        confidence = min(100, max(0, 50 + avg_scaled_score))
+        final_decision = "phishing" if confidence > 50 else "legitimate"
+        
+        return {
+            "confidence": round(confidence, 2),
+            "reasoning": f"Groq API not available. Using average model scores. (Avg Scaled Score: {avg_scaled_score:.2f})",
+            "highlighted_text": original_text,
+            "final_decision": final_decision,
+            "suggestion": "Do not interact with this message. Delete it immediately and report it to your IT department." if final_decision == "phishing" else "This message appears safe, but remain cautious with any links or attachments."
+        }
+    
+    url_features_summary = "No URLs detected in message"
+    if len(features_df) > 0:
+        feature_summary_parts = []
+        for idx, row in features_df.iterrows():
+            url = row.get('url', 'Unknown')
+            feature_summary_parts.append(f"\nURL {idx+1}: {url}")
+            feature_summary_parts.append(f"   • Length: {row.get('url_length', 'N/A')} chars")
+            feature_summary_parts.append(f"   • Dots in URL: {row.get('count_dot', 'N/A')}")
+            feature_summary_parts.append(f"   • Special characters: {row.get('count_special_chars', 'N/A')}")
+            feature_summary_parts.append(f"   • Domain age: {row.get('domain_age_days', 'N/A')} days")
+            feature_summary_parts.append(f"   • SSL certificate valid: {row.get('cert_has_valid_hostname', 'N/A')}")
+            feature_summary_parts.append(f"   • Uses HTTPS: {row.get('https', 'N/A')}")
+        url_features_summary = "\n".join(feature_summary_parts)
+
+    network_features_summary = await get_network_features_for_gemini(urls)
+    
+    model_predictions_summary = []
+    for model_name, pred_data in predictions.items():
+        scaled = pred_data['scaled_score']
+        raw = pred_data['raw_score']
+        model_predictions_summary.append(
+            f"   • {model_name.upper()}: scaled_score={scaled:.2f} (raw={raw:.3f})"
+        )
+    model_scores_text = "\n".join(model_predictions_summary)
+    
+    MAX_TEXT_LEN = 3000
+    if len(original_text) > MAX_TEXT_LEN:
+        truncated_original_text = original_text[:MAX_TEXT_LEN] + "\n... [TRUNCATED]"
+    else:
+        truncated_original_text = original_text
+
+    if len(message_text) > MAX_TEXT_LEN:
+        truncated_message_text = message_text[:MAX_TEXT_LEN] + "\n... [TRUNCATED]"
+    else:
+        truncated_message_text = message_text
+
+    user_prompt = f"""MESSAGE DATA:
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Sender: {sender if sender else 'N/A'}
+Subject: {subject if subject else 'N/A'}
+
+Original Message (Parsed from HTML):
+{truncated_original_text}
+
+Cleaned Text (for models):
+{truncated_message_text}
+
+URLs Found: {', '.join(urls) if urls else 'None'}
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+URL FEATURES (from ML models):
+{url_features_summary}
+
+INDEPENDENT NETWORK & GEO-DATA (for Gemini analysis only):
+{network_features_summary}
+
+MODEL PREDICTIONS:
+(Positive scaled scores → phishing, Negative → legitimate. Range: -50 to +50)
+{model_scores_text}
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Please analyze this data and provide your JSON response."""
+    
+    try:
+        max_retries = 3
+        retry_delay = 2
+        
+        for attempt in range(max_retries):
+            try:
+                chat_completion = await groq_async_client.chat.completions.create(
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": SYSTEM_PROMPT,
+                        },
+                        {
+                            "role": "user",
+                            "content": user_prompt,
+                        }
+                    ],
+                    model="meta-llama/llama-4-scout-17b-16e-instruct",
+                    temperature=0.2,
+                    max_tokens=4096, 
+                    top_p=0.85,
+                    response_format={"type": "json_object"},
+                )
+                
+                response_text = chat_completion.choices[0].message.content
+                break
+
+            except Exception as retry_error:
+                print(f"Groq API attempt {attempt + 1} failed: {retry_error}")
+                if attempt < max_retries - 1:
+                    print(f"Retrying in {retry_delay}s...")
+                    await asyncio.sleep(retry_delay) 
+                    retry_delay *= 2
+                else:
+                    raise retry_error
+
+        result = json.loads(response_text)
+        
+        required_fields = ['confidence', 'reasoning', 'highlighted_text', 'final_decision', 'suggestion']
+        if not all(field in result for field in required_fields):
+            raise ValueError(f"Missing required fields. Got: {list(result.keys())}")
+        
+        result['confidence'] = float(result['confidence'])
+        if not 0 <= result['confidence'] <= 100:
+            result['confidence'] = max(0, min(100, result['confidence']))
+        
+        if result['final_decision'].lower() not in ['phishing', 'legitimate']:
+            result['final_decision'] = 'phishing' if result['confidence'] > 50 else 'legitimate'
+        else:
+            result['final_decision'] = result['final_decision'].lower()
+        
+        if result['final_decision'] == 'phishing' and result['confidence'] < 50:
+            print(f"Warning: Groq decision 'phishing' mismatches confidence {result['confidence']}. Adjusting confidence.")
+            result['confidence'] = 51.0
+        elif result['final_decision'] == 'legitimate' and result['confidence'] > 50:
+            print(f"Warning: Groq decision 'legitimate' mismatches confidence {result['confidence']}. Adjusting confidence.")
+            result['confidence'] = 49.0
+            
+        if not result['highlighted_text'].strip() or '...' in result['highlighted_text'] or 'TRUNCATED' in result['highlighted_text']:
+            print("Warning: Groq returned empty or truncated 'highlighted_text'. Falling back to original_text.")
+            result['highlighted_text'] = original_text
+        
+        if not result.get('suggestion', '').strip():
+            if result['final_decision'] == 'phishing':
+                result['suggestion'] = "Do not interact with this message. Delete it immediately and report it as phishing."
+            else:
+                result['suggestion'] = "This message appears safe, but always verify sender identity before taking any action."
+        
+        return result
+    
+    except json.JSONDecodeError as e:
+        print(f"JSON parsing error: {e}")
+        print(f"Response text that failed parsing: {response_text[:500]}")
+        
+        avg_scaled_score = np.mean([p['scaled_score'] for p in predictions.values()]) if predictions else 0
+        confidence = min(100, max(0, 50 + avg_scaled_score))
+        final_decision = "phishing" if confidence > 50 else "legitimate"
+        
+        return {
+            "confidence": round(confidence, 2),
+            "reasoning": f"Groq response parsing failed. Fallback: Based on model average (directional score: {confidence:.2f}), message appears {'suspicious' if final_decision == 'phishing' else 'legitimate'}.",
+            "highlighted_text": original_text,
+            "final_decision": final_decision,
+            "suggestion": "Do not interact with this message. Delete it immediately and be cautious." if final_decision == 'phishing' else "Exercise caution. Verify the sender before taking any action."
+        }
+    
+    except Exception as e:
+        print(f"Error with Groq API: {e}")
+        
+        avg_scaled_score = np.mean([p['scaled_score'] for p in predictions.values()]) if predictions else 0
+        confidence = min(100, max(0, 50 + avg_scaled_score))
+        final_decision = "phishing" if confidence > 50 else "legitimate"
+        
+        return {
+            "confidence": round(confidence, 2),
+            "reasoning": f"Groq API error: {str(e)}. Fallback decision based on {len(predictions)} model predictions (average directional score: {confidence:.2f}).",
+            "highlighted_text": original_text,
+            "final_decision": final_decision,
+            "suggestion": "Treat this message with caution. Delete it if suspicious, or verify the sender through official channels before taking action." if final_decision == 'phishing' else "This message appears safe based on models, but always verify sender identity before clicking links or providing information."
+        }
+
+@app.on_event("startup")
+async def startup_event():
+    load_models()
+    print("\n" + "="*60)
+    print("Phishing Detection API is ready!")
+    print("="*60)
+    print("API Documentation: http://localhost:8000/docs")
+    print("="*60 + "\n")
+
+@app.get("/")
+async def root():
+    return {
+        "message": "Phishing Detection API",
+        "version": "1.0.0",
+        "endpoints": {
+            "predict": "/predict (POST)",
+            "health": "/health (GET)",
+            "docs": "/docs (GET)"
+        }
+    }
+
+@app.get("/health")
+async def health_check():
+    models_loaded = {
+        "ml_models": list(ml_models.keys()),
+        "dl_models": list(dl_models.keys()),
+        "bert_model": bert_model is not None,
+        "semantic_model": semantic_model is not None,
+        "groq_client": groq_async_client is not None
+    }
+    
+    return {
+        "status": "healthy",
+        "models_loaded": models_loaded
+    }
+
+@app.post("/predict", response_model=PredictionResponse)
+async def predict(message_input: MessageInput):
+    try:
+        html_body = message_input.text
+        sender = message_input.sender
+        subject = message_input.subject
+        
+        soup = BeautifulSoup(html_body, 'html.parser')
+        original_text = soup.get_text(separator=' ', strip=True)
+        
+        if not original_text or not original_text.strip():
+            raise HTTPException(status_code=400, detail="Message text (after HTML parsing) cannot be empty")
+        
+        urls, cleaned_text = parse_message(original_text)
+        
+        features_df = pd.DataFrame()
+        if urls:
+            features_df = await extract_url_features(urls)
+        
+        predictions = {}
+        if len(features_df) > 0 or (cleaned_text and semantic_model):
+            predictions = await asyncio.to_thread(get_model_predictions, features_df, cleaned_text)
+        
+        if not predictions:
+            if not urls and not cleaned_text:
+                detail = "Message text is empty after cleaning."
+            elif not urls and not semantic_model:
+                detail = "No URLs provided and semantic model is not loaded."
+            elif not any([ml_models, dl_models, bert_model, semantic_model]):
+                    detail = "No models available for prediction. Please ensure models are trained and loaded."
+            else:
+                detail = "Could not generate predictions. Models may be missing or feature extraction failed."
+            
+            raise HTTPException(
+                status_code=500, 
+                detail=detail
+            )
+        
+        final_result = await get_groq_final_decision(
+            urls, features_df, cleaned_text, predictions, original_text,
+            sender, subject
+        )
+        
+        return PredictionResponse(**final_result)
+    
+    except HTTPException:
+        raise
+    except Exception as e:
+        import traceback
+        print(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

app2.py

@@ -0,0 +1,658 @@
+import os
+import re
+import json
+import time
+import sys
+import asyncio
+from typing import List, Dict, Optional
+from urllib.parse import urlparse
+import socket
+import httpx
+
+import joblib
+import torch
+import numpy as np
+import pandas as pd
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from groq import AsyncGroq
+from dotenv import load_dotenv
+
+# --- Make sure 'config.py' and 'models.py' are in the same directory or accessible
+import config
+from models import get_ml_models, get_dl_models, FinetunedBERT
+from feature_extraction import process_row
+
+load_dotenv()
+sys.path.append(os.path.join(config.BASE_DIR, 'Message_model'))
+from predict import PhishingPredictor
+
+app = FastAPI(
+    title="Phishing Detection API",
+    description="Advanced phishing detection system using multiple ML/DL models and Groq",
+    version="1.0.0"
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Allows all origins
+    allow_credentials=True,
+    allow_methods=["*"],  # Allows all methods
+    allow_headers=["*"],  # Allows all headers
+)
+
+# --- Pydantic Models ---
+
+class MessageInput(BaseModel):
+    text: str
+    metadata: Optional[Dict] = {}
+
+class PredictionResponse(BaseModel):
+    confidence: float
+    reasoning: str
+    highlighted_text: str
+    final_decision: str
+    suggestion: str
+
+# --- Global Variables ---
+
+ml_models = {}
+dl_models = {}
+bert_model = None
+semantic_model = None
+groq_async_client = None
+
+MODEL_BOUNDARIES = {
+    'logistic': 0.5,
+    'svm': 0.5,
+    'xgboost': 0.5,
+    'attention_blstm': 0.5,
+    'rcnn': 0.5,
+    'bert': 0.5,
+    'semantic': 0.5
+}
+
+# --- Model Loading ---
+
+def load_models():
+    global ml_models, dl_models, bert_model, semantic_model, groq_async_client
+    
+    print("Loading models...")
+    
+    models_dir = config.MODELS_DIR
+    for model_name in ['logistic', 'svm', 'xgboost']:
+        model_path = os.path.join(models_dir, f'{model_name}.joblib')
+        if os.path.exists(model_path):
+            ml_models[model_name] = joblib.load(model_path)
+            print(f"✓ Loaded {model_name} model")
+        else:
+            print(f"⚠ Warning: {model_name} model not found at {model_path}")
+    
+    for model_name in ['attention_blstm', 'rcnn']:
+        model_path = os.path.join(models_dir, f'{model_name}.pt')
+        if os.path.exists(model_path):
+            model_template = get_dl_models(input_dim=len(config.NUMERICAL_FEATURES))
+            dl_models[model_name] = model_template[model_name]
+            dl_models[model_name].load_state_dict(torch.load(model_path, map_location='cpu'))
+            dl_models[model_name].eval()
+            print(f"✓ Loaded {model_name} model")
+        else:
+            print(f"⚠ Warning: {model_name} model not found at {model_path}")
+    
+    bert_path = os.path.join(config.BASE_DIR, 'finetuned_bert')
+    if os.path.exists(bert_path):
+        try:
+            bert_model = FinetunedBERT(bert_path)
+            print("✓ Loaded BERT model")
+        except Exception as e:
+            print(f"⚠ Warning: Could not load BERT model: {e}")
+    
+    semantic_model_path = os.path.join(config.BASE_DIR, 'Message_model', 'final_semantic_model')
+    if os.path.exists(semantic_model_path) and os.listdir(semantic_model_path):
+        try:
+            semantic_model = PhishingPredictor(model_path=semantic_model_path)
+            print("✓ Loaded semantic model")
+        except Exception as e:
+            print(f"⚠ Warning: Could not load semantic model: {e}")
+    else:
+        checkpoint_path = os.path.join(config.BASE_DIR, 'Message_model', 'training_checkpoints', 'checkpoint-30')
+        if os.path.exists(checkpoint_path):
+            try:
+                semantic_model = PhishingPredictor(model_path=checkpoint_path)
+                print("✓ Loaded semantic model from checkpoint")
+            except Exception as e:
+                print(f"⚠ Warning: Could not load semantic model from checkpoint: {e}")
+    
+    groq_api_key = os.environ.get('GROQ_API_KEY')
+    if groq_api_key:
+        groq_async_client = AsyncGroq(api_key=groq_api_key)
+        print("✓ Initialized Groq API Client")
+    else:
+        print("⚠ Warning: GROQ_API_KEY not set. Set it as environment variable.")
+        print("   Example: export GROQ_API_KEY='your-api-key-here'")
+
+# --- Feature Extraction & Prediction Logic ---
+
+def parse_message(text: str) -> tuple:
+    url_pattern = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+|(?:www\.)?[a-zA-Z0-9-]+\.[a-z]{2,12}\b(?:/[^\s]*)?'
+    urls = re.findall(url_pattern, text)
+    cleaned_text = re.sub(url_pattern, '', text)
+    cleaned_text = ' '.join(cleaned_text.lower().split())
+    cleaned_text = re.sub(r'[^a-z0-9\s.,!?-]', '', cleaned_text)
+    cleaned_text = re.sub(r'([.,!?])+', r'\1', cleaned_text)
+    cleaned_text = ' '.join(cleaned_text.split())
+    return urls, cleaned_text
+
+async def extract_url_features(urls: List[str]) -> pd.DataFrame:
+    if not urls:
+        return pd.DataFrame()
+    
+    df = pd.DataFrame({'url': urls})
+    whois_cache = {}
+    ssl_cache = {}
+    
+    tasks = []
+    for _, row in df.iterrows():
+        tasks.append(asyncio.to_thread(process_row, row, whois_cache, ssl_cache))
+    
+    feature_list = await asyncio.gather(*tasks)
+    features_df = pd.DataFrame(feature_list)
+    result_df = pd.concat([df, features_df], axis=1)
+    return result_df
+
+def custom_boundary(raw_score: float, boundary: float) -> float:
+    # --- MODIFIED: This now returns a score from -50 to +50 ---
+    return (raw_score - boundary) * 100
+
+def get_model_predictions(features_df: pd.DataFrame, message_text: str) -> Dict:
+    predictions = {}
+    
+    numerical_features = config.NUMERICAL_FEATURES
+    categorical_features = config.CATEGORICAL_FEATURES
+    
+    try:
+        X = features_df[numerical_features + categorical_features]
+    except KeyError as e:
+        print(f"Error: Missing columns in features_df. {e}")
+        print(f"Available columns: {features_df.columns.tolist()}")
+        X = pd.DataFrame(columns=numerical_features + categorical_features)
+    
+    if not X.empty:
+        X.loc[:, numerical_features] = X.loc[:, numerical_features].fillna(-1)
+        X.loc[:, categorical_features] = X.loc[:, categorical_features].fillna('N/A')
+
+        for model_name, model in ml_models.items():
+            try:
+                all_probas = model.predict_proba(X)[:, 1]  
+                raw_score = np.max(all_probas)      
+                
+                # --- MODIFIED: 'scaled_score' is now from -50 (legit) to +50 (phishing) ---
+                scaled_score = custom_boundary(raw_score, MODEL_BOUNDARIES[model_name])
+                predictions[model_name] = {
+                    'raw_score': float(raw_score),
+                    'scaled_score': float(scaled_score)
+                }
+            except Exception as e:
+                print(f"Error with {model_name} (Prediction Step): {e}") 
+        
+        X_numerical = X[numerical_features].values 
+        
+        for model_name, model in dl_models.items():
+            try:
+                X_tensor = torch.tensor(X_numerical, dtype=torch.float32)
+                with torch.no_grad():
+                    all_scores = model(X_tensor)
+                    raw_score = torch.max(all_scores).item()
+                    
+                scaled_score = custom_boundary(raw_score, MODEL_BOUNDARIES[model_name])
+                predictions[model_name] = {
+                    'raw_score': float(raw_score),
+                    'scaled_score': float(scaled_score)
+                }
+            except Exception as e:
+                print(f"Error with {model_name}: {e}")
+    
+    if bert_model and len(features_df) > 0:
+        try:
+            urls = features_df['url'].tolist()
+            raw_scores = bert_model.predict_proba(urls)
+            avg_raw_score = np.mean([score[1] for score in raw_scores])
+            scaled_score = custom_boundary(avg_raw_score, MODEL_BOUNDARIES['bert'])
+            predictions['bert'] = {
+                'raw_score': float(avg_raw_score),
+                'scaled_score': float(scaled_score)
+            }
+        except Exception as e:
+            print(f"Error with BERT: {e}")
+    
+    if semantic_model and message_text:
+        try:
+            result = semantic_model.predict(message_text)
+            raw_score = result['phishing_probability']
+            scaled_score = custom_boundary(raw_score, MODEL_BOUNDARIES['semantic'])
+            predictions['semantic'] = {
+                'raw_score': float(raw_score),
+                'scaled_score': float(scaled_score),
+                'confidence': result['confidence'] # Note: this is the semantic model's own confidence
+            }
+        except Exception as e:
+            print(f"Error with semantic model: {e}")
+    
+    return predictions
+
+# --- Groq/LLM Final Decision Logic ---
+
+async def get_network_features_for_gemini(urls: List[str]) -> str:
+    """
+    Fetches real-time IP, Geo, and ISP data for URLs.
+    This runs independently and is ONLY used to inform the LLM prompt.
+    """
+    if not urls:
+        return "No URLs to analyze for network features."
+    
+    results = []
+    async with httpx.AsyncClient() as client:
+        for i, url_str in enumerate(urls[:3]): 
+            try:
+                hostname = urlparse(url_str).hostname
+                if not hostname:
+                    results.append(f"\nURL {i+1} ({url_str}): Invalid URL, no hostname.")
+                    continue
+                
+                try:
+                    ip_address = await asyncio.to_thread(socket.gethostbyname, hostname)
+                except socket.gaierror:
+                    results.append(f"\nURL {i+1} ({hostname}): Could not resolve domain to IP.")
+                    continue
+                
+                try:
+                    geo_url = f"http://ip-api.com/json/{ip_address}?fields=status,message,country,city,isp,org,as"
+                    response = await client.get(geo_url, timeout=3.0)
+                    response.raise_for_status()
+                    data = response.json()
+                    
+                    if data.get('status') == 'success':
+                        geo_info = (
+                            f"   • IP Address: {ip_address}\n"
+                            f"   • Location: {data.get('city', 'N/A')}, {data.get('country', 'N/A')}\n"
+                            f"   • ISP: {data.get('isp', 'N/A')}\n"
+                            f"   • Organization: {data.get('org', 'N/A')}\n"
+                            f"   • ASN: {data.get('as', 'N/A')}"
+                        )
+                        results.append(f"\nURL {i+1} ({hostname}):\n{geo_info}")
+                    else:
+                        results.append(f"\nURL {i+1} ({hostname}):\n   • IP Address: {ip_address}\n   • Geo-Data: API lookup failed ({data.get('message')})")
+                
+                except httpx.RequestError as e:
+                    results.append(f"\nURL {i+1} ({hostname}):\n   • IP Address: {ip_address}\n   • Geo-Data: Network error while fetching IP info ({str(e)})")
+                
+            except Exception as e:
+                results.append(f"\nURL {i+1} ({url_str}): Error processing URL ({str(e)})")
+    
+    if not results:
+        return "No valid hostnames found in URLs to analyze."
+
+    return "\n".join(results)
+
+# --- CORRECTED: Static system prompt with fixed examples ---
+# This contains all the instructions, few-shot examples, and output format.
+SYSTEM_PROMPT = """You are the FINAL JUDGE in a phishing detection system. Your role is critical: analyze ALL available evidence and make the ultimate decision.
+
+IMPORTANT INSTRUCTIONS:
+1. You have FULL AUTHORITY to override model predictions if evidence suggests they're wrong.
+2. **TRUST THE 'INDEPENDENT NETWORK & GEO-DATA' OVER 'URL FEATURES'.** The ML model features (like `domain_age: -1`) can be wrong due to lookup failures. The 'INDEPENDENT' data is a real-time check.
+3. If 'INDEPENDENT' data shows a legitimate organization (e.g., "Cloudflare", "Google", "Codeforces") for a known domain, but the models score it as phishing (due to `domain_age: -1`), you **should override** and classify as 'legitimate'.
+4. Your confidence score is DIRECTIONAL (0-100):
+   - Scores > 50.0 mean 'phishing'.
+   - Scores < 50.0 mean 'legitimate'.
+   - 50.0 is neutral.
+   - The magnitude indicates certainty (e.g., 95.0 is 'very confident phishing'; 5.0 is 'very confident legitimate').
+   - Your confidence score MUST match your 'final_decision'.
+5. BE WARY OF FALSE POSITIVES. Legitimate messages (bank alerts, contest notifications) can seem urgent.
+
+PRIORITY GUIDANCE (Use this logic):
+- IF URLs are present: Focus heavily on URL features.
+  - Examine 'URL FEATURES' for patterns (e.g., domain_age: -1 or 0, high special_chars).
+  - **CRITICAL:** Cross-reference this with the 'INDEPENDENT NETWORK & GEO-DATA'. This real-time data (IP, Location, ISP) is your ground truth.
+  - **If `domain_age` is -1, it's a lookup failure.** IGNORE IT and trust the 'INDEPENDENT NETWORK & GEO-DATA' to see if the domain is real (e.g., 'codeforces.com' with a valid IP).
+  - Then supplement with message content analysis.
+- IF NO URLs are present: Focus entirely on message content and semantics.
+  - Analyze language patterns, urgency tactics, and social engineering techniques
+  - Look for credential requests, financial solicitations, or threats
+  - Evaluate the semantic model's assessment heavily
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+FEW-SHOT EXAMPLES FOR GUIDANCE:
+
+Example 1 - Clear Phishing:
+Message: "URGENT! Click: http://paypa1-secure.xyz/verify"
+URL Features: domain_age: 5
+Network Data: IP: 123.45.67.89, Location: Russia, ISP: Shady-Host
+Model Scores: All positive
+Correct Decision: {{
+  "confidence": 95.0,
+  "reasoning": "Classic phishing. Misspelled domain, new age, and network data points to a suspicious ISP in Russia.",
+  "highlighted_text": "URGENT! Click: $$http://paypa1-secure.xyz/verify$$",
+  "final_decision": "phishing",
+  "suggestion": "Do NOT click. Delete immediately."
+}}
+
+Example 2 - Legitimate (False Positive Case):
+Message: "Hi, join Codeforces Round 184. ... Unsubscribe: https://codeforces.com/unsubscribe/..."
+URL Features: domain_age: -1 (This is a lookup failure!)
+Network Data: URL (codeforces.com): IP: 104.22.6.109, Location: San Francisco, USA, ISP: Cloudflare, Inc.
+Model Scores: Mixed (some positive due to domain_age: -1)
+Correct Decision: {{
+  "confidence": 10.0,
+  "reasoning": "OVERRIDING models. The 'URL FEATURES' show a 'domain_age: -1' which is a clear lookup error that confused the models. The 'INDEPENDENT NETWORK & GEO-DATA' confirms the domain 'codeforces.com' is real and hosted on Cloudflare, a legitimate provider. The message content is a standard, safe notification.",
+  "highlighted_text": "Hi, join Codeforces Round 184. ... Unsubscribe: https://codeforces.com/unsubscribe/...",
+  "final_decision": "legitimate",
+  "suggestion": "This message is safe. It is a legitimate notification from Codeforces."
+}}
+
+Example 3 - Legitimate (Long Formal Text):
+Message: "TATA MOTORS PASSENGER VEHICLES LIMITED... GENERAL GUIDANCE NOTE... [TRUNCATED]"
+URL Features: domain_age: 8414
+Network Data: URL (cars.tatamotors.com): IP: 23.209.113.12, Location: Boardman, USA, ISP: Akamai Technologies
+Model Scores: All negative
+Correct Decision: {{
+  "confidence": 5.0,
+  "reasoning": "This is a legitimate corporate communication. The text, although truncated, is clearly a formal guidance note for shareholders. The network data confirms 'cars.tatamotors.com' is hosted on Akamai, a major CDN used by large corporations. The models correctly identify this as safe.",
+  "highlighted_text": "TATA MOTORS PASSENGER VEHICLES LIMITED... GENERAL GUIDANCE NOTE... [TRUNCATED]",
+  "final_decision": "legitimate",
+  "suggestion": "This message is a legitimate corporate communication and appears safe."
+}}
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+YOUR ANALYSIS TASK:
+Analyze the message data provided by the user (in the 'user' message) following the steps and logic outlined above.
+
+**CRITICAL for `highlighted_text`:** You MUST return the *entire original message*. Only wrap the specific words/URLs that are suspicious with `$$...$$`. If nothing is suspicious (i.e., `final_decision` is 'legitimate'), return the original message with NO `$$` markers.
+
+OUTPUT FORMAT (respond with ONLY this JSON, no markdown, no explanation):
+{{
+  "confidence": <float (0-100, directional score where >50 is phishing)>,
+  "reasoning": "<your detailed analysis explaining why this is/isn't phishing, mentioning why you trust/override models>",
+  "highlighted_text": "<THE FULL, ENTIRE original message with suspicious parts marked as $$suspicious text$$>",
+  "final_decision": "phishing" or "legitimate",
+  "suggestion": "<specific, actionable advice for the user on how to handle this message - what to do or not do>"
+}}"""
+
+
+async def get_groq_final_decision(urls: List[str], features_df: pd.DataFrame, 
+                                  message_text: str, predictions: Dict, 
+                                  original_text: str) -> Dict:
+    
+    if not groq_async_client:
+        # --- MODIFIED: Fallback logic for confidence score ---
+        # avg_scaled_score is from -50 (legit) to +50 (phishing)
+        avg_scaled_score = np.mean([p['scaled_score'] for p in predictions.values()]) if predictions else 0
+        # We add 50 to shift the range to 0-100
+        confidence = min(100, max(0, 50 + avg_scaled_score))
+        final_decision = "phishing" if confidence > 50 else "legitimate"
+        
+        return {
+            "confidence": round(confidence, 2),
+            "reasoning": f"Groq API not available. Using average model scores. (Avg Scaled Score: {avg_scaled_score:.2f})",
+            "highlighted_text": original_text,
+            "final_decision": final_decision,
+            "suggestion": "Do not interact with this message. Delete it immediately and report it to your IT department." if final_decision == "phishing" else "This message appears safe, but remain cautious with any links or attachments."
+        }
+    
+    url_features_summary = "No URLs detected in message"
+    if len(features_df) > 0:
+        feature_summary_parts = []
+        for idx, row in features_df.iterrows():
+            url = row.get('url', 'Unknown')
+            feature_summary_parts.append(f"\nURL {idx+1}: {url}")
+            feature_summary_parts.append(f"   • Length: {row.get('url_length', 'N/A')} chars")
+            feature_summary_parts.append(f"   • Dots in URL: {row.get('count_dot', 'N/A')}")
+            feature_summary_parts.append(f"   • Special characters: {row.get('count_special_chars', 'N/A')}")
+            feature_summary_parts.append(f"   • Domain age: {row.get('domain_age_days', 'N/A')} days")
+            feature_summary_parts.append(f"   • SSL certificate valid: {row.get('cert_has_valid_hostname', 'N/A')}")
+            feature_summary_parts.append(f"   • Uses HTTPS: {row.get('https', 'N/A')}")
+        url_features_summary = "\n".join(feature_summary_parts)
+
+    network_features_summary = await get_network_features_for_gemini(urls)
+    
+    model_predictions_summary = []
+    for model_name, pred_data in predictions.items():
+        scaled = pred_data['scaled_score'] # This is now -50 to +50
+        raw = pred_data['raw_score']
+        model_predictions_summary.append(
+            f"   • {model_name.upper()}: scaled_score={scaled:.2f} (raw={raw:.3f})"
+        )
+    model_scores_text = "\n".join(model_predictions_summary)
+    
+    MAX_TEXT_LEN = 3000
+    if len(original_text) > MAX_TEXT_LEN:
+        truncated_original_text = original_text[:MAX_TEXT_LEN] + "\n... [TRUNCATED]"
+    else:
+        truncated_original_text = original_text
+
+    if len(message_text) > MAX_TEXT_LEN:
+        truncated_message_text = message_text[:MAX_TEXT_LEN] + "\n... [TRUNCATED]"
+    else:
+        truncated_message_text = message_text
+
+    # --- NEW: User prompt only contains dynamic data ---
+    user_prompt = f"""MESSAGE DATA:
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Original Message:
+{truncated_original_text}
+
+Cleaned Text:
+{truncated_message_text}
+
+URLs Found: {', '.join(urls) if urls else 'None'}
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+URL FEATURES (from ML models):
+{url_features_summary}
+
+INDEPENDENT NETWORK & GEO-DATA (for Gemini analysis only):
+{network_features_summary}
+
+MODEL PREDICTIONS:
+(Positive scaled scores → phishing, Negative → legitimate. Range: -50 to +50)
+{model_scores_text}
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Please analyze this data and provide your JSON response."""
+    
+    try:
+        max_retries = 3
+        retry_delay = 2
+        
+        for attempt in range(max_retries):
+            try:
+                # --- MODIFIED: API call now uses system and user roles ---
+                chat_completion = await groq_async_client.chat.completions.create(
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": SYSTEM_PROMPT,
+                        },
+                        {
+                            "role": "user",
+                            "content": user_prompt,
+                        }
+                    ],
+                    model="meta-llama/llama-4-scout-17b-16e-instruct", # Using 8B for speed, can be 70b
+                    temperature=0.2,
+                    max_tokens=4096, 
+                    top_p=0.85,
+                    response_format={"type": "json_object"},
+                )
+                
+                response_text = chat_completion.choices[0].message.content
+                break # Success
+
+            except Exception as retry_error:
+                print(f"Groq API attempt {attempt + 1} failed: {retry_error}")
+                if attempt < max_retries - 1:
+                    print(f"Retrying in {retry_delay}s...")
+                    await asyncio.sleep(retry_delay) 
+                    retry_delay *= 2
+                else:
+                    raise retry_error # Raise the final error
+
+        result = json.loads(response_text)
+        
+        required_fields = ['confidence', 'reasoning', 'highlighted_text', 'final_decision', 'suggestion']
+        if not all(field in result for field in required_fields):
+            raise ValueError(f"Missing required fields. Got: {list(result.keys())}")
+        
+        result['confidence'] = float(result['confidence'])
+        if not 0 <= result['confidence'] <= 100:
+            result['confidence'] = max(0, min(100, result['confidence']))
+        
+        if result['final_decision'].lower() not in ['phishing', 'legitimate']:
+            # --- MODIFIED: Decision is based on the directional confidence score ---
+            result['final_decision'] = 'phishing' if result['confidence'] > 50 else 'legitimate'
+        else:
+            result['final_decision'] = result['final_decision'].lower()
+        
+        # --- MODIFIED: Check that confidence and decision match ---
+        if result['final_decision'] == 'phishing' and result['confidence'] < 50:
+            print(f"Warning: Groq decision 'phishing' mismatches confidence {result['confidence']}. Adjusting confidence.")
+            result['confidence'] = 51.0 # Set to a default phishing score
+        elif result['final_decision'] == 'legitimate' and result['confidence'] > 50:
+            print(f"Warning: Groq decision 'legitimate' mismatches confidence {result['confidence']}. Adjusting confidence.")
+            result['confidence'] = 49.0 # Set to a default legitimate score
+            
+        # --- Fallback for empty or truncated highlighted_text ---
+        if not result['highlighted_text'].strip() or '...' in result['highlighted_text'] or 'TRUNCATED' in result['highlighted_text']:
+            print("Warning: Groq returned empty or truncated 'highlighted_text'. Falling back to original_text.")
+            result['highlighted_text'] = original_text
+        
+        if not result.get('suggestion', '').strip():
+            if result['final_decision'] == 'phishing':
+                result['suggestion'] = "Do not interact with this message. Delete it immediately and report it as phishing."
+            else:
+                result['suggestion'] = "This message appears safe, but always verify sender identity before taking any action."
+        
+        return result
+    
+    except json.JSONDecodeError as e:
+        print(f"JSON parsing error: {e}")
+        print(f"Response text that failed parsing: {response_text[:500]}")
+        
+        # --- MODIFIED: Fallback logic for confidence score ---
+        avg_scaled_score = np.mean([p['scaled_score'] for p in predictions.values()]) if predictions else 0
+        confidence = min(100, max(0, 50 + avg_scaled_score))
+        final_decision = "phishing" if confidence > 50 else "legitimate"
+        
+        return {
+            "confidence": round(confidence, 2),
+            "reasoning": f"Groq response parsing failed. Fallback: Based on model average (directional score: {confidence:.2f}), message appears {'suspicious' if final_decision == 'phishing' else 'legitimate'}.",
+            "highlighted_text": original_text,
+            "final_decision": final_decision,
+            "suggestion": "Do not interact with this message. Delete it immediately and be cautious." if final_decision == 'phishing' else "Exercise caution. Verify the sender before taking any action."
+        }
+    
+    except Exception as e:
+        print(f"Error with Groq API: {e}")
+        
+        # --- MODIFIED: Fallback logic for confidence score ---
+        avg_scaled_score = np.mean([p['scaled_score'] for p in predictions.values()]) if predictions else 0
+        confidence = min(100, max(0, 50 + avg_scaled_score))
+        final_decision = "phishing" if confidence > 50 else "legitimate"
+        
+        return {
+            "confidence": round(confidence, 2),
+            "reasoning": f"Groq API error: {str(e)}. Fallback decision based on {len(predictions)} model predictions (average directional score: {confidence:.2f}).",
+            "highlighted_text": original_text,
+            "final_decision": final_decision,
+            "suggestion": "Treat this message with caution. Delete it if suspicious, or verify the sender through official channels before taking action." if final_decision == 'phishing' else "This message appears safe based on models, but always verify sender identity before clicking links or providing information."
+        }
+
+# --- FastAPI Endpoints ---
+
+@app.on_event("startup")
+async def startup_event():
+    load_models()
+    print("\n" + "="*60)
+    print("Phishing Detection API is ready!")
+    print("="*60)
+    print("API Documentation: http://localhost:8000/docs")
+    print("="*60 + "\n")
+
+@app.get("/")
+async def root():
+    return {
+        "message": "Phishing Detection API",
+        "version": "1.0.0",
+        "endpoints": {
+            "predict": "/predict (POST)",
+            "health": "/health (GET)",
+            "docs": "/docs (GET)"
+        }
+    }
+
+@app.get("/health")
+async def health_check():
+    models_loaded = {
+        "ml_models": list(ml_models.keys()),
+        "dl_models": list(dl_models.keys()),
+        "bert_model": bert_model is not None,
+        "semantic_model": semantic_model is not None,
+        "groq_client": groq_async_client is not None
+    }
+    
+    return {
+        "status": "healthy",
+        "models_loaded": models_loaded
+    }
+
+@app.post("/predict", response_model=PredictionResponse)
+async def predict(message_input: MessageInput):
+    try:
+        original_text = message_input.text
+        
+        if not original_text or not original_text.strip():
+            raise HTTPException(status_code=400, detail="Message text cannot be empty")
+        
+        urls, cleaned_text = parse_message(original_text)
+        
+        features_df = pd.DataFrame()
+        if urls:
+            features_df = await extract_url_features(urls)
+        
+        predictions = {}
+        if len(features_df) > 0 or (cleaned_text and semantic_model):
+            # --- MODIFIED: Run this in a thread to avoid blocking ---
+            predictions = await asyncio.to_thread(get_model_predictions, features_df, cleaned_text)
+        
+        if not predictions:
+            if not urls and not cleaned_text:
+                detail = "Message text is empty after cleaning."
+            elif not urls and not semantic_model:
+                detail = "No URLs provided and semantic model is not loaded."
+            elif not any([ml_models, dl_models, bert_model, semantic_model]):
+                 detail = "No models available for prediction. Please ensure models are trained and loaded."
+            else:
+                detail = "Could not generate predictions. Models may be missing or feature extraction failed."
+            
+            raise HTTPException(
+                status_code=500, 
+                detail=detail
+            )
+        
+        final_result = await get_groq_final_decision(
+            urls, features_df, cleaned_text, predictions, original_text
+        )
+        
+        return PredictionResponse(**final_result)
+    
+    except HTTPException:
+        raise
+    except Exception as e:
+        import traceback
+        print(traceback.format_exc())
+        raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

cloudbuild.yaml

@@ -0,0 +1,59 @@
+steps:
+  # Build the Docker image
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'build'
+      - '-t'
+      - 'gcr.io/$PROJECT_ID/phishing-detection-api:$SHORT_SHA'
+      - '-t'
+      - 'gcr.io/$PROJECT_ID/phishing-detection-api:latest'
+      - '.'
+    dir: '.'
+
+  # Push the Docker image to Container Registry
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'push'
+      - 'gcr.io/$PROJECT_ID/phishing-detection-api:$SHORT_SHA'
+
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'push'
+      - 'gcr.io/$PROJECT_ID/phishing-detection-api:latest'
+
+  # Deploy to Cloud Run
+  - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
+    entrypoint: gcloud
+    args:
+      - 'run'
+      - 'deploy'
+      - 'phishing-detection-api'
+      - '--image'
+      - 'gcr.io/$PROJECT_ID/phishing-detection-api:$SHORT_SHA'
+      - '--region'
+      - 'us-central1'
+      - '--platform'
+      - 'managed'
+      - '--allow-unauthenticated'
+      - '--memory'
+      - '4Gi'
+      - '--cpu'
+      - '2'
+      - '--timeout'
+      - '300'
+      - '--max-instances'
+      - '10'
+      - '--set-env-vars'
+      - 'PYTHONUNBUFFERED=1'
+      - '--set-secrets'
+      - 'GROQ_API_KEY=GROQ_API_KEY:latest'
+
+images:
+  - 'gcr.io/$PROJECT_ID/phishing-detection-api:$SHORT_SHA'
+  - 'gcr.io/$PROJECT_ID/phishing-detection-api:latest'
+
+options:
+  machineType: 'E2_HIGHCPU_8'
+  logging: CLOUD_LOGGING_ONLY
+
+timeout: '1200s'

config.py

@@ -0,0 +1,73 @@
+import os
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+DATA_DIR = os.path.join(BASE_DIR, "data")
+MODELS_DIR = os.path.join(BASE_DIR, "models")
+REPORTS_DIR = os.path.join(BASE_DIR, "reports")
+
+TRAIN_SAMPLE_FRACTION = 0.5
+ENGINEERED_TRAIN_FILE = os.path.join(BASE_DIR, "engineered_features.csv")
+
+REPORT_SAMPLE_SIZE = 1500
+ENGINEERED_TEST_FILE = os.path.join(BASE_DIR, "engineered_features_test.csv")
+
+LEXICAL_FEATURES = [
+    'url_length',
+    'hostname_length',
+    'path_length',
+    'query_length',
+    'fragment_length',
+    'count_dot',
+    'count_hyphen',
+    'count_underscore',
+    'count_slash',
+    'count_at',
+    'count_equals',
+    'count_percent',
+    'count_digits',
+    'count_letters',
+    'count_special_chars',
+    'has_ip_address',
+    'has_http',
+    'has_https',
+]
+
+WHOIS_FEATURES = [
+    'domain_age_days',
+    'domain_lifespan_days',
+    'days_since_domain_update',
+    'registrar_name',
+]
+
+SSL_FEATURES = [
+    'cert_age_days',
+    'cert_validity_days',
+    'cert_issuer_cn',
+    'cert_subject_cn',
+    'ssl_protocol_version',
+    'cert_has_valid_hostname',
+]
+
+ALL_FEATURE_COLUMNS = (
+    LEXICAL_FEATURES +
+    WHOIS_FEATURES +
+    SSL_FEATURES
+)
+
+CATEGORICAL_FEATURES = [
+    'registrar_name',
+    'cert_issuer_cn',
+    'cert_subject_cn',
+    'ssl_protocol_version'
+]
+
+NUMERICAL_FEATURES = [
+    col for col in ALL_FEATURE_COLUMNS if col not in CATEGORICAL_FEATURES
+]
+
+ML_MODEL_RANDOM_STATE = 42
+ML_TEST_SIZE = 0.2
+
+DL_EPOCHS = 50
+DL_BATCH_SIZE = 64
+DL_LEARNING_RATE = 0.001

data_pipeline.py

@@ -0,0 +1,80 @@
+import os
+import glob
+import pandas as pd
+import config
+from feature_extraction import extract_features_from_dataframe
+
+def load_and_sample_raw_data(data_dir, fraction=0.1, random_state=42):
+    raw_data_files = glob.glob(os.path.join(data_dir, "*.csv"))
+    
+    if not raw_data_files:
+        print(f"Error: No .csv files found in '{data_dir}'.")
+        print("Please add your raw data files (e.g., phishing.csv, legit.csv) to the /data/ folder.")
+        return pd.DataFrame()
+
+    print(f"Found {len(raw_data_files)} raw data files.")
+    
+    all_samples = []
+    for file_path in raw_data_files:
+        try:
+            print(f"Loading and sampling {os.path.basename(file_path)}...")
+            df = pd.read_csv(file_path, on_bad_lines='skip')
+            
+            if 'label' not in df.columns or 'url' not in df.columns:
+                print(f"Warning: Skipping {file_path}. Must contain 'label' and 'url' columns.")
+                continue
+            
+            sample_df = df.sample(frac=fraction, random_state=random_state)
+            all_samples.append(sample_df)
+            
+        except Exception as e:
+            print(f"Error processing {file_path}: {e}")
+
+    if not all_samples:
+        print("Error: No valid data could be loaded.")
+        return pd.DataFrame()
+        
+    combined_df = pd.concat(all_samples, ignore_index=True)
+    combined_df = combined_df.sample(frac=0.1, random_state=random_state).reset_index(drop=True)
+    
+    print(f"Total raw training data prepared: {len(combined_df)} samples.")
+    return combined_df
+
+def main():
+    print("--- Starting Data Pipeline ---")
+    
+    raw_df = load_and_sample_raw_data(
+        data_dir=config.DATA_DIR,
+        fraction=config.TRAIN_SAMPLE_FRACTION
+    )
+    
+    if raw_df.empty:
+        print("Data pipeline failed. Exiting.")
+        return
+
+    engineered_df = extract_features_from_dataframe(raw_df)
+    
+    engineered_df.to_csv(config.ENGINEERED_TRAIN_FILE, index=False)
+    
+    print(f"\n--- Data Pipeline Complete ---")
+    print(f"Engineered training set saved to: {config.ENGINEERED_TRAIN_FILE}")
+    print(f"Total features: {len(config.ALL_FEATURE_COLUMNS)}")
+
+if __name__ == "__main__":
+    os.makedirs(config.DATA_DIR, exist_ok=True)
+    if not glob.glob(os.path.join(config.DATA_DIR, "*.csv")):
+        print("Creating dummy data files...")
+        dummy_phish = pd.DataFrame({
+            'label': [1, 1],
+            'url': ['facebook.com.login-support.ru', 'myetherwallets.kr/wallet']
+        })
+        dummy_phish.to_csv(os.path.join(config.DATA_DIR, 'phishing_data_1.csv'), index=False)
+        
+        dummy_legit = pd.DataFrame({
+            'label': [0, 0],
+            'url': ['google.com', 'https://www.millect.com/Plans']
+        })
+        dummy_legit.to_csv(os.path.join(config.DATA_DIR, 'legit_data_1.csv'), index=False)
+        print(f"Dummy files created in {config.DATA_DIR}. Please replace them with your real data.")
+        
+    main()

deploy.ps1

@@ -0,0 +1,90 @@
+# GCP Deployment Script for Phishing Detection API (PowerShell)
+# This script helps deploy the application to Google Cloud Run
+
+param(
+    [Parameter(Mandatory=$true)]
+    [string]$ProjectId,
+    
+    [Parameter(Mandatory=$false)]
+    [string]$Region = "us-central1"
+)
+
+$ServiceName = "phishing-detection-api"
+
+Write-Host "==========================================" -ForegroundColor Cyan
+Write-Host "Phishing Detection API - GCP Deployment" -ForegroundColor Cyan
+Write-Host "==========================================" -ForegroundColor Cyan
+Write-Host "Project ID: $ProjectId"
+Write-Host "Region: $Region"
+Write-Host "Service Name: $ServiceName"
+Write-Host "==========================================" -ForegroundColor Cyan
+
+# Check if gcloud is installed
+try {
+    $null = Get-Command gcloud -ErrorAction Stop
+} catch {
+    Write-Host "Error: gcloud CLI is not installed. Please install it first." -ForegroundColor Red
+    exit 1
+}
+
+# Check if docker is installed
+try {
+    $null = Get-Command docker -ErrorAction Stop
+} catch {
+    Write-Host "Error: Docker is not installed. Please install it first." -ForegroundColor Red
+    exit 1
+}
+
+# Set the project
+Write-Host "Setting GCP project..." -ForegroundColor Yellow
+gcloud config set project $ProjectId
+
+# Enable required APIs
+Write-Host "Enabling required APIs..." -ForegroundColor Yellow
+gcloud services enable cloudbuild.googleapis.com
+gcloud services enable run.googleapis.com
+gcloud services enable containerregistry.googleapis.com
+
+# Check if GROQ_API_KEY secret exists, if not create it
+Write-Host "Checking for GROQ_API_KEY secret..." -ForegroundColor Yellow
+$secretExists = gcloud secrets describe GROQ_API_KEY --project=$ProjectId 2>&1
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "GROQ_API_KEY secret not found. Creating it..." -ForegroundColor Yellow
+    $GroqKey = Read-Host "Enter your GROQ_API_KEY" -AsSecureString
+    $GroqKeyPlain = [Runtime.InteropServices.Marshal]::PtrToStringAuto(
+        [Runtime.InteropServices.Marshal]::SecureStringToBSTR($GroqKey)
+    )
+    
+    echo $GroqKeyPlain | gcloud secrets create GROQ_API_KEY `
+        --data-file=- `
+        --replication-policy="automatic" `
+        --project=$ProjectId
+    
+    # Grant Cloud Run service account access to the secret
+    $ProjectNumber = (gcloud projects describe $ProjectId --format="value(projectNumber)")
+    gcloud secrets add-iam-policy-binding GROQ_API_KEY `
+        --member="serviceAccount:$ProjectNumber-compute@developer.gserviceaccount.com" `
+        --role="roles/secretmanager.secretAccessor" `
+        --project=$ProjectId
+} else {
+    Write-Host "GROQ_API_KEY secret already exists." -ForegroundColor Green
+}
+
+# Build and deploy using Cloud Build
+Write-Host "Building and deploying using Cloud Build..." -ForegroundColor Yellow
+gcloud builds submit --config=cloudbuild.yaml --project=$ProjectId
+
+# Get the service URL
+Write-Host "Deployment complete!" -ForegroundColor Green
+Write-Host "Getting service URL..." -ForegroundColor Yellow
+$ServiceUrl = (gcloud run services describe $ServiceName `
+    --region=$Region `
+    --format="value(status.url)" `
+    --project=$ProjectId)
+
+Write-Host "==========================================" -ForegroundColor Green
+Write-Host "Deployment Successful!" -ForegroundColor Green
+Write-Host "Service URL: $ServiceUrl"
+Write-Host "Health Check: $ServiceUrl/health"
+Write-Host "API Docs: $ServiceUrl/docs"
+Write-Host "==========================================" -ForegroundColor Green

deploy.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# GCP Deployment Script for Phishing Detection API
+# This script helps deploy the application to Google Cloud Run
+
+set -e
+
+PROJECT_ID=${1:-"your-project-id"}
+REGION=${2:-"us-central1"}
+SERVICE_NAME="phishing-detection-api"
+
+echo "=========================================="
+echo "Phishing Detection API - GCP Deployment"
+echo "=========================================="
+echo "Project ID: $PROJECT_ID"
+echo "Region: $REGION"
+echo "Service Name: $SERVICE_NAME"
+echo "=========================================="
+
+# Check if gcloud is installed
+if ! command -v gcloud &> /dev/null; then
+    echo "Error: gcloud CLI is not installed. Please install it first."
+    exit 1
+fi
+
+# Check if docker is installed
+if ! command -v docker &> /dev/null; then
+    echo "Error: Docker is not installed. Please install it first."
+    exit 1
+fi
+
+# Set the project
+echo "Setting GCP project..."
+gcloud config set project $PROJECT_ID
+
+# Enable required APIs
+echo "Enabling required APIs..."
+gcloud services enable cloudbuild.googleapis.com
+gcloud services enable run.googleapis.com
+gcloud services enable containerregistry.googleapis.com
+
+# Check if GROQ_API_KEY secret exists, if not create it
+echo "Checking for GROQ_API_KEY secret..."
+if ! gcloud secrets describe GROQ_API_KEY --project=$PROJECT_ID &> /dev/null; then
+    echo "GROQ_API_KEY secret not found. Creating it..."
+    read -sp "Enter your GROQ_API_KEY: " GROQ_KEY
+    echo
+    echo -n "$GROQ_KEY" | gcloud secrets create GROQ_API_KEY \
+        --data-file=- \
+        --replication-policy="automatic" \
+        --project=$PROJECT_ID
+    
+    # Grant Cloud Run service account access to the secret
+    PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
+    gcloud secrets add-iam-policy-binding GROQ_API_KEY \
+        --member="serviceAccount:$PROJECT_NUMBER-compute@developer.gserviceaccount.com" \
+        --role="roles/secretmanager.secretAccessor" \
+        --project=$PROJECT_ID
+else
+    echo "GROQ_API_KEY secret already exists."
+fi
+
+# Build and deploy using Cloud Build
+echo "Building and deploying using Cloud Build..."
+gcloud builds submit --config=cloudbuild.yaml --project=$PROJECT_ID
+
+# Get the service URL
+echo "Deployment complete!"
+echo "Getting service URL..."
+SERVICE_URL=$(gcloud run services describe $SERVICE_NAME \
+    --region=$REGION \
+    --format="value(status.url)" \
+    --project=$PROJECT_ID)
+
+echo "=========================================="
+echo "Deployment Successful!"
+echo "Service URL: $SERVICE_URL"
+echo "Health Check: $SERVICE_URL/health"
+echo "API Docs: $SERVICE_URL/docs"
+echo "=========================================="

docker-compose.yml

@@ -0,0 +1,27 @@
+version: '3.8'
+
+services:
+  phishing-api:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: phishing-detection-api
+    ports:
+      - "8000:8000"
+    environment:
+      - PYTHONUNBUFFERED=1
+      - PYTHONDONTWRITEBYTECODE=1
+      # Add your GROQ_API_KEY here or use .env file
+      - GROQ_API_KEY=${GROQ_API_KEY:-your-api-key-here}
+    volumes:
+      # Optional: Mount models directory if you want to update models without rebuilding
+      # - ./models:/app/models:ro
+      # - ./finetuned_bert:/app/finetuned_bert:ro
+      # - ./Message_model:/app/Message_model:ro
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s

feature_extraction.py

@@ -0,0 +1,198 @@
+import whois
+import tldextract
+import ssl
+import socket
+import pandas as pd
+from datetime import datetime
+from urllib.parse import urlparse
+from OpenSSL import SSL
+import re
+import sys
+from tqdm import tqdm
+
+def get_lexical_features(url):
+    features = {}
+    try:
+        if not (url.startswith('http://') or url.startswith('https://')):
+             url = 'http://' + url
+        parsed_url = urlparse(url)
+        hostname = parsed_url.hostname if parsed_url.hostname else ''
+        path = parsed_url.path
+        query = parsed_url.query
+        fragment = parsed_url.fragment
+    except Exception:
+        hostname = ''
+        path = ''
+        query = ''
+        fragment = ''
+        url = ''
+
+    features['url_length'] = len(url)
+    features['hostname_length'] = len(hostname)
+    features['path_length'] = len(path)
+    features['query_length'] = len(query)
+    features['fragment_length'] = len(fragment)
+    features['count_dot'] = url.count('.')
+    features['count_hyphen'] = url.count('-')
+    features['count_underscore'] = url.count('_')
+    features['count_slash'] = url.count('/')
+    features['count_at'] = url.count('@')
+    features['count_equals'] = url.count('=')
+    features['count_percent'] = url.count('%')
+    features['count_digits'] = sum(c.isdigit() for c in url)
+    features['count_letters'] = sum(c.isalpha() for c in url)
+    features['count_special_chars'] = len(re.findall(r'[^a-zA-Z0-9\s]', url))
+
+    ip_regex = r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"
+    features['has_ip_address'] = 1 if re.match(ip_regex, hostname) else 0
+    features['has_http'] = 1 if 'http:' in url else 0
+    features['has_https'] = 1 if 'https:' in url else 0
+    
+    return features
+
+def get_whois_features(registrable_domain, whois_cache):
+    if registrable_domain in whois_cache:
+        return whois_cache[registrable_domain]
+
+    features = {
+        'domain_age_days': -1,
+        'domain_lifespan_days': -1,
+        'days_since_domain_update': -1,
+        'registrar_name': 'N/A',
+    }
+
+    try:
+        w = whois.whois(registrable_domain)
+        if not w.creation_date:
+            whois_cache[registrable_domain] = features
+            return features
+
+        creation_date = w.creation_date[0] if isinstance(w.creation_date, list) else w.creation_date
+        expiration_date = w.expiration_date[0] if isinstance(w.expiration_date, list) else w.expiration_date
+        updated_date = w.updated_date[0] if isinstance(w.updated_date, list) else w.updated_date
+
+        if creation_date:
+            features['domain_age_days'] = (datetime.now() - creation_date).days
+        
+        if creation_date and expiration_date:
+            features['domain_lifespan_days'] = (expiration_date - creation_date).days
+            
+        if updated_date:
+            features['days_since_domain_update'] = (datetime.now() - updated_date).days
+
+        if w.registrar:
+            features['registrar_name'] = str(w.registrar).split(' ')[0].replace(',', '').replace('"', '')
+
+    except Exception as e:
+        pass
+
+    whois_cache[registrable_domain] = features
+    return features
+
+def get_ssl_features(hostname, ssl_cache):
+    if hostname in ssl_cache:
+        return ssl_cache[hostname]
+
+    features = {
+        'cert_age_days': -1,
+        'cert_validity_days': -1,
+        'cert_issuer_cn': 'N/A',
+        'cert_subject_cn': 'N/A',
+        'ssl_protocol_version': 'N/A',
+        'cert_has_valid_hostname': 0,
+    }
+
+    try:
+        context = SSL.Context(SSL.SSLv23_METHOD)
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(2)
+        
+        ssl_sock = SSL.Connection(context, sock)
+        ssl_sock.set_tlsext_host_name(hostname.encode('utf-8'))
+        ssl_sock.connect((hostname, 443))
+        ssl_sock.do_handshake()
+        
+        cert = ssl_sock.get_peer_certificate()
+        
+        features['ssl_protocol_version'] = ssl_sock.get_protocol_version_name()
+        not_before_str = cert.get_notBefore().decode('utf-8')
+        not_before_date = datetime.strptime(not_before_str, '%Y%m%d%H%M%SZ')
+        features['cert_age_days'] = (datetime.now() - not_before_date).days
+        not_after_str = cert.get_notAfter().decode('utf-8')
+        not_after_date = datetime.strptime(not_after_str, '%Y%m%d%H%M%SZ')
+        features['cert_validity_days'] = (not_after_date - not_before_date).days
+        issuer_components = dict(cert.get_issuer().get_components())
+        subject_components = dict(cert.get_subject().get_components())
+        
+        features['cert_issuer_cn'] = issuer_components.get(b'CN', b'N/A').decode('utf-8')
+        features['cert_subject_cn'] = subject_components.get(b'CN', b'N/A').decode('utf-8')
+        if features['cert_subject_cn'] == hostname or f"*.{tldextract.extract(hostname).registered_domain}" == features['cert_subject_cn']:
+            features['cert_has_valid_hostname'] = 1
+
+        ssl_sock.close()
+        sock.close()
+
+    except SSL.Error as e:
+        features['cert_issuer_cn'] = 'SSL_ERROR'
+    except socket.timeout:
+        features['cert_issuer_cn'] = 'TIMEOUT'
+    except socket.error:
+        features['cert_issuer_cn'] = 'CONN_FAILED'
+    except Exception as e:
+        features['cert_issuer_cn'] = 'SSL_UNKNOWN_ERROR'
+        pass
+
+    ssl_cache[hostname] = features
+    return features
+
+def process_row(row, whois_cache, ssl_cache):
+    url = row['url']
+    try:
+        if not (url.startswith('http://') or url.startswith('https://')):
+             url_for_parse = 'http://' + url
+        else:
+             url_for_parse = url
+             
+        parsed_url = urlparse(url_for_parse)
+        hostname = parsed_url.hostname if parsed_url.hostname else ''
+        
+        ext = tldextract.extract(url_for_parse)
+        registrable_domain = f"{ext.domain}.{ext.suffix}"
+        
+    except Exception:
+        hostname = ''
+        registrable_domain = ''
+    lexical_data = get_lexical_features(url)
+    
+    whois_data = {}
+    if registrable_domain:
+        whois_data = get_whois_features(registrable_domain, whois_cache)
+        
+    ssl_data = {}
+    if hostname:
+        ssl_data = get_ssl_features(hostname, ssl_cache)
+    all_features = {**lexical_data, **whois_data, **ssl_data}
+    
+    return pd.Series(all_features)
+
+
+def extract_features_from_dataframe(df):
+    if 'url' not in df.columns:
+        raise ValueError("DataFrame must contain a 'url' column.")
+    whois_cache = {}
+    ssl_cache = {}
+    
+    print("Starting feature extraction... This may take a very long time.")
+    try:
+       
+        tqdm.pandas(desc="Extracting features")
+        feature_df = df.progress_apply(process_row, args=(whois_cache, ssl_cache), axis=1)
+    except ImportError:
+        print("tqdm not found. Running without progress bar. `pip install tqdm` to see progress.")
+        feature_df = df.apply(process_row, args=(whois_cache, ssl_cache), axis=1)
+
+    print("Feature extraction complete.")
+    
+    final_df = pd.concat([df, feature_df], axis=1)
+    
+    return final_df

models.py

@@ -0,0 +1,181 @@
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+from xgboost import XGBClassifier
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset
+from transformers import BertTokenizer, BertForSequenceClassification
+import config
+import os
+
+def get_ml_models():
+    models = {
+        'logistic': LogisticRegression(
+            max_iter=1000, 
+            random_state=config.ML_MODEL_RANDOM_STATE
+        ),
+        'svm': SVC(
+            probability=True,
+            random_state=config.ML_MODEL_RANDOM_STATE
+        ),
+        'xgboost': XGBClassifier(
+            n_estimators=100,
+            random_state=config.ML_MODEL_RANDOM_STATE,
+            use_label_encoder=False,
+            eval_metric='logloss'
+        )
+    }
+    return models
+
+class Attention_BLSTM(nn.Module):
+    def __init__(self, input_dim, hidden_dim=128, num_layers=2, dropout=0.3):
+        super(Attention_BLSTM, self).__init__()
+        self.hidden_dim = hidden_dim
+        self.num_layers = num_layers
+        
+        self.lstm = nn.LSTM(
+            input_dim, 
+            hidden_dim, 
+            num_layers, 
+            batch_first=True, 
+            bidirectional=True,
+            dropout=dropout if num_layers > 1 else 0
+        )
+        
+        self.attention = nn.Linear(hidden_dim * 2, 1)
+        self.fc = nn.Linear(hidden_dim * 2, 64)
+        self.relu = nn.ReLU()
+        self.dropout = nn.Dropout(dropout)
+        self.output = nn.Linear(64, 1)
+        self.sigmoid = nn.Sigmoid()
+    
+    def forward(self, x):
+        if len(x.shape) == 2:
+            x = x.unsqueeze(1)
+        
+        lstm_out, _ = self.lstm(x)
+        
+        attention_weights = torch.softmax(self.attention(lstm_out), dim=1)
+        context_vector = torch.sum(attention_weights * lstm_out, dim=1)
+        
+        out = self.fc(context_vector)
+        out = self.relu(out)
+        out = self.dropout(out)
+        out = self.output(out)
+        out = self.sigmoid(out)
+        
+        return out
+
+class RCNN(nn.Module):
+    def __init__(self, input_dim, embed_dim=64, num_filters=100, filter_sizes=[3, 4, 5], dropout=0.5):
+        super(RCNN, self).__init__()
+        
+        self.lstm = nn.LSTM(1, embed_dim // 2, batch_first=True, bidirectional=True)
+        
+        self.convs = nn.ModuleList([
+            nn.Conv1d(embed_dim, num_filters, kernel_size=fs)
+            for fs in filter_sizes
+        ])
+        
+        self.fc = nn.Linear(len(filter_sizes) * num_filters, 64)
+        self.relu = nn.ReLU()
+        self.dropout = nn.Dropout(dropout)
+        self.output = nn.Linear(64, 1)
+        self.sigmoid = nn.Sigmoid()
+    
+    def forward(self, x):
+        batch_size = x.size(0)
+        seq_len = x.size(1)
+        
+        x = x.unsqueeze(2)
+        
+        lstm_out, _ = self.lstm(x)
+        
+        lstm_out = lstm_out.permute(0, 2, 1)
+        
+        conv_outs = [torch.relu(conv(lstm_out)) for conv in self.convs]
+        
+        pooled = [torch.max_pool1d(conv_out, conv_out.size(2)).squeeze(2) for conv_out in conv_outs]
+        
+        cat = torch.cat(pooled, dim=1)
+        
+        out = self.fc(cat)
+        out = self.relu(out)
+        out = self.dropout(out)
+        out = self.output(out)
+        out = self.sigmoid(out)
+        
+        return out
+
+class FinetunedBERT:
+    def __init__(self, model_path=None):
+        if model_path is None:
+            model_path = os.path.join(config.BASE_DIR, 'finetuned_bert')
+        
+        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+        self.model = BertForSequenceClassification.from_pretrained(
+            model_path,
+            num_labels=2,
+            local_files_only=True
+        )
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.model.to(self.device)
+        self.model.eval()
+    
+    def predict(self, urls):
+        if isinstance(urls, str):
+            urls = [urls]
+        
+        encodings = self.tokenizer(
+            urls,
+            padding=True,
+            truncation=True,
+            max_length=512,
+            return_tensors='pt'
+        )
+        
+        encodings = {key: val.to(self.device) for key, val in encodings.items()}
+        
+        with torch.no_grad():
+            outputs = self.model(**encodings)
+            predictions = torch.argmax(outputs.logits, dim=1)
+        
+        return predictions.cpu().numpy()
+    
+    def predict_proba(self, urls):
+        if isinstance(urls, str):
+            urls = [urls]
+        
+        encodings = self.tokenizer(
+            urls,
+            padding=True,
+            truncation=True,
+            max_length=512,
+            return_tensors='pt'
+        )
+        
+        encodings = {key: val.to(self.device) for key, val in encodings.items()}
+        
+        with torch.no_grad():
+            outputs = self.model(**encodings)
+            probas = torch.softmax(outputs.logits, dim=1)
+        
+        return probas.cpu().numpy()
+
+class PhishingDataset(Dataset):
+    def __init__(self, X, y):
+        self.X = torch.tensor(X, dtype=torch.float32)
+        self.y = torch.tensor(y, dtype=torch.float32)
+
+    def __len__(self):
+        return len(self.y)
+
+    def __getitem__(self, idx):
+        return self.X[idx], self.y[idx].view(-1)
+
+def get_dl_models(input_dim):
+    models = {
+        'attention_blstm': Attention_BLSTM(input_dim),
+        'rcnn': RCNN(input_dim)
+    }
+    return models

models/attention_blstm.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa49103ee969cc2194fc6369905ccb6942e0a472263544a40072d678cf4abae9
+size 2283420

models/dl_scaler.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5684232d23cab5e5579c68e653318a032867e2f22d101fdfa6756b80fe097042
+size 1191

models/logistic.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f030cc6dd0405aab6f4ad3813d58cf1d17723426e6ba6e0fe5b54466776be921
+size 848679

models/rcnn.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9195f2166c067d0a1d9f15476f94cf13b6ea7cc6946745be1c4eaccb9c968a7
+size 425800

models/svm.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fb6afd8b17462766f22b0a24ec31d9012b41430de30cb5ae816d0933e0be124
+size 1811267

models/xgboost.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d2c829005e6249c5aeca9e80ff272660c65268b69ce482b00ef436cc2c10841
+size 861278

report.py

@@ -0,0 +1,389 @@
+import os
+import pandas as pd
+import numpy as np
+import joblib
+import torch
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
+from sklearn.preprocessing import StandardScaler
+import config
+from models import get_dl_models, PhishingDataset, FinetunedBERT
+
+sns.set_style("whitegrid")
+plt.rcParams['figure.figsize'] = (12, 8)
+plt.rcParams['font.size'] = 11
+
+COLORS = {
+    'primary': '#FF6B6B',
+    'secondary': '#4ECDC4',
+    'tertiary': '#45B7D1',
+    'quaternary': '#FFA07A',
+    'quinary': '#98D8C8',
+    'bg': '#F7F7F7',
+    'text': '#2C3E50'
+}
+
+MODEL_THRESHOLDS = {
+    'attention_blstm': 0.8,
+    'rcnn': 0.8,
+    'logistic': 0.5,
+    'svm': 0.5,
+    'xgboost': 0.5,
+    'bert': 0.5
+}
+
+def load_sample_data(sample_fraction=0.05):
+    print(f"Loading {sample_fraction*100}% sample from data...")
+    
+    if os.path.exists(config.ENGINEERED_TEST_FILE):
+        df = pd.read_csv(config.ENGINEERED_TEST_FILE)
+        print(f"Loaded test data: {len(df)} samples")
+    elif os.path.exists(config.ENGINEERED_TRAIN_FILE):
+        df = pd.read_csv(config.ENGINEERED_TRAIN_FILE)
+        print(f"Loaded train data: {len(df)} samples")
+    else:
+        data_files = [
+            os.path.join(config.DATA_DIR, 'url_data_labeled.csv'),
+            os.path.join(config.DATA_DIR, 'data_bal - 20000.csv')
+        ]
+        df = None
+        for file in data_files:
+            if os.path.exists(file):
+                df = pd.read_csv(file)
+                print(f"Loaded raw data: {len(df)} samples")
+                break
+        
+        if df is None:
+            raise FileNotFoundError("No data file found!")
+    
+    sample_size = max(int(len(df) * sample_fraction), config.REPORT_SAMPLE_SIZE)
+    sample_size = min(sample_size, len(df))
+    df_sample = df.sample(n=sample_size, random_state=42)
+    
+    print(f"Sampled {len(df_sample)} URLs for report generation")
+    return df_sample
+
+def prepare_ml_data(df):
+    X = df[config.NUMERICAL_FEATURES + config.CATEGORICAL_FEATURES]
+    y = df['label'].values
+    
+    X.loc[:, config.NUMERICAL_FEATURES] = X.loc[:, config.NUMERICAL_FEATURES].fillna(-1)
+    X.loc[:, config.CATEGORICAL_FEATURES] = X.loc[:, config.CATEGORICAL_FEATURES].fillna('N/A')
+    
+    return X, y
+
+def prepare_dl_data(df):
+    X = df[config.NUMERICAL_FEATURES].fillna(-1).values
+    y = df['label'].values
+    
+    scaler_path = os.path.join(config.MODELS_DIR, "dl_scaler.pkl")
+    if os.path.exists(scaler_path):
+        scaler = joblib.load(scaler_path)
+        X_scaled = scaler.transform(X)
+    else:
+        scaler = StandardScaler()
+        X_scaled = scaler.fit_transform(X)
+    
+    return X_scaled, y
+
+def predict_ml_models(X, y):
+    predictions = {}
+    scores = {}
+    
+    ml_models = ['logistic', 'svm', 'xgboost']
+    
+    for model_name in ml_models:
+        model_path = os.path.join(config.MODELS_DIR, f"{model_name}.joblib")
+        if not os.path.exists(model_path):
+            print(f"WARNING: Model {model_name} not found, skipping...")
+            continue
+        
+        print(f"Loading {model_name} model...")
+        model = joblib.load(model_path)
+        
+        y_pred = model.predict(X)
+        y_proba = model.predict_proba(X)[:, 1]
+        
+        predictions[model_name] = y_pred
+        scores[model_name] = y_proba
+        
+        acc = accuracy_score(y, y_pred)
+        print(f"  {model_name} accuracy: {acc:.4f}")
+    
+    return predictions, scores
+
+def predict_dl_models(X, y):
+    predictions = {}
+    scores = {}
+    
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    input_dim = X.shape[1]
+    
+    dl_models_dict = get_dl_models(input_dim)
+    
+    for model_name, model in dl_models_dict.items():
+        model_path = os.path.join(config.MODELS_DIR, f"{model_name}.pt")
+        if not os.path.exists(model_path):
+            print(f"WARNING: Model {model_name} not found, skipping...")
+            continue
+        
+        print(f"Loading {model_name} model...")
+        model.load_state_dict(torch.load(model_path, map_location=device, weights_only=True))
+        model.to(device)
+        model.eval()
+        
+        X_tensor = torch.tensor(X, dtype=torch.float32).to(device)
+        
+        with torch.no_grad():
+            outputs = model(X_tensor).cpu().numpy().flatten()
+        
+        threshold = MODEL_THRESHOLDS.get(model_name, 0.5)
+        y_pred = (outputs > threshold).astype(int)
+        
+        predictions[model_name] = y_pred
+        scores[model_name] = outputs
+        
+        acc = accuracy_score(y, y_pred)
+        print(f"  {model_name} accuracy: {acc:.4f} (threshold: {threshold})")
+        
+        del model, X_tensor
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    
+    return predictions, scores
+
+def predict_bert_model(df, y):
+    bert_path = os.path.join(config.BASE_DIR, 'finetuned_bert')
+    if not os.path.exists(bert_path):
+        print(f"WARNING: BERT model not found at {bert_path}, skipping...")
+        return None, None
+    
+    if 'url' not in df.columns:
+        print("WARNING: 'url' column not found in data, skipping BERT...")
+        return None, None
+    
+    try:
+        print("Loading BERT model...")
+        bert_model = FinetunedBERT(bert_path)
+        
+        urls = df['url'].tolist()
+        
+        batch_size = 32
+        all_preds = []
+        all_probas = []
+        
+        print(f"Processing {len(urls)} URLs in batches of {batch_size}...")
+        for i in range(0, len(urls), batch_size):
+            batch_urls = urls[i:i+batch_size]
+            batch_preds = bert_model.predict(batch_urls)
+            batch_probas = bert_model.predict_proba(batch_urls)[:, 1]
+            all_preds.extend(batch_preds)
+            all_probas.extend(batch_probas)
+            
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+        
+        y_pred = 1-np.array(all_preds)
+        y_proba = 1-np.array(all_probas)
+        
+        acc = accuracy_score(y, y_pred)
+        print(f"  BERT accuracy: {acc:.4f}")
+        
+        return y_pred, y_proba
+        
+    except torch.cuda.OutOfMemoryError:
+        print("WARNING: CUDA out of memory for BERT model, skipping...")
+        print("  Try reducing batch size or use CPU by setting CUDA_VISIBLE_DEVICES=''")
+        return None, None
+    except Exception as e:
+        print(f"WARNING: Error loading BERT model: {e}")
+        return None, None
+
+def plot_confusion_matrices(y_true, all_predictions, save_dir):
+    print("\nGenerating confusion matrices...")
+    
+    n_models = len(all_predictions)
+    if n_models == 0:
+        print("No predictions to plot!")
+        return
+    
+    cols = min(3, n_models)
+    rows = (n_models + cols - 1) // cols
+    
+    fig, axes = plt.subplots(rows, cols, figsize=(6*cols, 5*rows))
+    if n_models == 1:
+        axes = [axes]
+    else:
+        axes = axes.flatten() if rows > 1 else axes
+    
+    cmap = sns.color_palette("RdYlGn_r", as_cmap=True)
+    
+    for idx, (model_name, y_pred) in enumerate(all_predictions.items()):
+        ax = axes[idx]
+        
+        cm = confusion_matrix(y_true, y_pred)
+        
+        sns.heatmap(cm, annot=True, fmt='d', cmap=cmap, ax=ax,
+                    cbar_kws={'label': 'Count'},
+                    annot_kws={'size': 14, 'weight': 'bold'})
+        
+        ax.set_title(f'{model_name.upper()} Confusion Matrix', 
+                     fontsize=14, fontweight='bold', color=COLORS['text'])
+        ax.set_xlabel('Predicted Label', fontsize=12, fontweight='bold')
+        ax.set_ylabel('True Label', fontsize=12, fontweight='bold')
+        ax.set_xticklabels(['Legitimate (0)', 'Phishing (1)'])
+        ax.set_yticklabels(['Legitimate (0)', 'Phishing (1)'])
+    
+    for idx in range(n_models, len(axes)):
+        fig.delaxes(axes[idx])
+    
+    plt.tight_layout()
+    save_path = os.path.join(save_dir, 'confusion_matrices.png')
+    plt.savefig(save_path, dpi=300, bbox_inches='tight', facecolor='white')
+    print(f"Saved confusion matrices to {save_path}")
+    plt.close()
+
+def plot_accuracy_comparison(y_true, all_predictions, save_dir):
+    print("\nGenerating accuracy comparison plot...")
+    
+    if len(all_predictions) == 0:
+        print("No predictions to plot!")
+        return
+    
+    accuracies = {}
+    for model_name, y_pred in all_predictions.items():
+        acc = accuracy_score(y_true, y_pred)
+        accuracies[model_name] = acc
+    
+    models = list(accuracies.keys())
+    accs = list(accuracies.values())
+    
+    colors_list = [COLORS['primary'], COLORS['secondary'], COLORS['tertiary'], 
+                   COLORS['quaternary'], COLORS['quinary']]
+    bar_colors = [colors_list[i % len(colors_list)] for i in range(len(models))]
+    
+    fig, ax = plt.subplots(figsize=(12, 7))
+    
+    bars = ax.bar(models, accs, color=bar_colors, edgecolor='black', linewidth=2, alpha=0.8)
+    
+    for bar, acc in zip(bars, accs):
+        height = bar.get_height()
+        ax.text(bar.get_x() + bar.get_width()/2., height + 0.01,
+                f'{acc:.4f}',
+                ha='center', va='bottom', fontsize=13, fontweight='bold')
+    
+    ax.set_xlabel('Models', fontsize=14, fontweight='bold', color=COLORS['text'])
+    ax.set_ylabel('Accuracy', fontsize=14, fontweight='bold', color=COLORS['text'])
+    ax.set_title('Model Accuracy Comparison', fontsize=18, fontweight='bold', 
+                 color=COLORS['text'], pad=20)
+    ax.set_ylim([0, 1.1])
+    ax.grid(axis='y', alpha=0.3, linestyle='--')
+    ax.set_axisbelow(True)
+    
+    plt.xticks(rotation=45, ha='right', fontsize=12)
+    plt.tight_layout()
+    
+    save_path = os.path.join(save_dir, 'accuracy_comparison.png')
+    plt.savefig(save_path, dpi=300, bbox_inches='tight', facecolor='white')
+    print(f"Saved accuracy comparison to {save_path}")
+    plt.close()
+
+def plot_score_vs_label(y_true, all_scores, save_dir):
+    print("\nGenerating score vs label scatter plots...")
+    
+    if len(all_scores) == 0:
+        print("No scores to plot!")
+        return
+    
+    n_models = len(all_scores)
+    cols = min(3, n_models)
+    rows = (n_models + cols - 1) // cols
+    
+    fig, axes = plt.subplots(rows, cols, figsize=(6*cols, 5*rows))
+    if n_models == 1:
+        axes = [axes]
+    else:
+        axes = axes.flatten() if rows > 1 else axes
+    
+    colors_map = {0: COLORS['secondary'], 1: COLORS['primary']}
+    
+    for idx, (model_name, scores) in enumerate(all_scores.items()):
+        ax = axes[idx]
+        
+        for label in [0, 1]:
+            mask = y_true == label
+            label_name = 'Legitimate' if label == 0 else 'Phishing'
+            ax.scatter(np.where(mask)[0], scores[mask], 
+                      c=colors_map[label], label=label_name, 
+                      alpha=0.6, s=50, edgecolors='black', linewidth=0.5)
+        
+        threshold = MODEL_THRESHOLDS.get(model_name, 0.5)
+        ax.axhline(y=threshold, color='red', linestyle='--', linewidth=2, 
+                   label=f'Threshold ({threshold})', alpha=0.7)
+        
+        ax.set_title(f'{model_name.upper()} Prediction Scores', 
+                     fontsize=14, fontweight='bold', color=COLORS['text'])
+        ax.set_xlabel('Sample Index', fontsize=11, fontweight='bold')
+        ax.set_ylabel('Prediction Score', fontsize=11, fontweight='bold')
+        ax.set_ylim([-0.1, 1.1])
+        ax.legend(loc='best', framealpha=0.9)
+        ax.grid(True, alpha=0.3, linestyle='--')
+    
+    for idx in range(n_models, len(axes)):
+        fig.delaxes(axes[idx])
+    
+    plt.tight_layout()
+    save_path = os.path.join(save_dir, 'score_vs_label.png')
+    plt.savefig(save_path, dpi=300, bbox_inches='tight', facecolor='white')
+    print(f"Saved score vs label plots to {save_path}")
+    plt.close()
+
+def main():
+    print("="*60)
+    print("PHISHING DETECTION MODEL EVALUATION REPORT")
+    print("="*60)
+    print("\nCustom Thresholds Configuration:")
+    for model, threshold in MODEL_THRESHOLDS.items():
+        print(f"   • {model}: {threshold}")
+    print()
+    
+    os.makedirs(config.REPORTS_DIR, exist_ok=True)
+    os.makedirs(config.MODELS_DIR, exist_ok=True)
+    
+    df = load_sample_data(sample_fraction=0.05)
+    
+    all_predictions = {}
+    all_scores = {}
+    
+    X_ml, y = prepare_ml_data(df)
+    ml_preds, ml_scores = predict_ml_models(X_ml, y)
+    all_predictions.update(ml_preds)
+    all_scores.update(ml_scores)
+    
+    X_dl, y_dl = prepare_dl_data(df)
+    dl_preds, dl_scores = predict_dl_models(X_dl, y_dl)
+    all_predictions.update(dl_preds)
+    all_scores.update(dl_scores)
+    
+    bert_pred, bert_score = predict_bert_model(df, y)
+    if bert_pred is not None:
+        all_predictions['bert'] = bert_pred
+        all_scores['bert'] = bert_score
+    
+    if len(all_predictions) == 0:
+        print("\nWARNING: No models found! Please train models first.")
+        print("Run: python train_ml.py && python train_dl.py")
+        return
+    
+    plot_confusion_matrices(y, all_predictions, config.REPORTS_DIR)
+    plot_accuracy_comparison(y, all_predictions, config.REPORTS_DIR)
+    plot_score_vs_label(y, all_scores, config.REPORTS_DIR)
+    
+    print("\n" + "="*60)
+    print("REPORT GENERATION COMPLETE!")
+    print(f"All visualizations saved to: {config.REPORTS_DIR}")
+    print("="*60)
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,23 @@
+pandas>=1.5.0
+numpy>=1.23.0,<2.0.0
+scikit-learn>=1.2.0
+matplotlib>=3.6.0
+seaborn>=0.12.0
+torch>=2.0.0
+xgboost>=1.7.0
+transformers>=4.30.0
+python-whois>=0.8.0
+tldextract>=3.4.0
+pyOpenSSL>=23.0.0
+joblib>=1.2.0
+tqdm>=4.65.0
+fastapi>=0.104.0
+uvicorn>=0.24.0
+google-generativeai>=0.3.0
+scipy>=1.10.0
+pydantic>=2.0.0
+python-multipart>=0.0.6
+httpx>=0.24.0
+groq
+python-dotenv>=1.0.0
+beautifulsoup4>=4.12.0

test_api.py

@@ -0,0 +1,157 @@
+from asyncio import sleep
+import requests
+import json
+import textwrap
+
+ 
+API_URL = "https://sharyl-liberalistic-procrastinatingly.ngrok-free.dev/predict" 
+
+ 
+HEADERS = {
+    'Content-Type': 'application/json',
+    'ngrok-skip-browser-warning': 'true'
+}
+
+ 
+messages_to_test = [
+    """Dear Akshat Nitinkumar Bhatt of DAIICT
+Become a Campus Manager at Unlox: Lead, Earn, and Grow with AI EdTech!
+Join Unlox as a Campus Manager and be the face of India's top AI-powered EdTech revolution right on your campus! Unlox is a next-gen platform dedicated to revolutionising student learning and career preparation. We're looking for driven, tech-savvy student leaders to champion this movement at their colleges.
+What You'll Gain:
+Earn up to ₹21,000 in Stipends* (get ₹1,000 instantly for every 4 enrollments!).
+Free Edulet after 15 enrollments (Tab with smartlab,blu, job bridge program)
+Official Certificate of Completion & Appraisal Letter for your resume.
+Invaluable Real Leadership & Management Experience.
+FREE Access to PrepFree, our exclusive career platform, featuring:
+AI Resume Builder
+Mock Interviews
+Smart Job Matching
+Dedicated Career Assistance and Job Portals
+Masterclasses and bootcamps
+ Your Mission:
+Lead a simple WhatsApp campaign to engage 50+ students from your college. Your goal is to grow the Unlox Student Community—all while gaining excellent real-world experience and significantly boosting your resume.
+
+Ready to lead and make a tangible impact?
+Application form - Apply by clicking this link
+
+Regards
+Aman K
+Team Leader Unlox.
+https://unlox.com/
+https://in.linkedin.com/company/unloxacademy""",
+
+    """You're invited to join the Internshala Student Partner program to develop professional skills while gaining valuable experience.
+Ready to represent us
+in DAIICT Gandhinagar?
+Join the Internshala Student Partner Program
+Program benefits include:
+ Learning opportunity - Masterclasses by industry professionals
+ Career development - Certificates and recommendation letters
+ Skill building - Develop leadership and communication skills
+Who can apply? B.Tech students at DAIICT Gandhinagar can learn more about the program by clicking below.
+LEARN MOREInternshala (Scholiverse Educare Pvt. Ltd.)
+901A and 901B, Iris Tech Park, Sector - 48, Sohna Road, Gurugram
+Not interested? Unsubscribe""",
+
+    """Hey Akshat,
+While you may still be (or not be!) on the hunt for a full-time opportunity, I wanted to share that we are also starting to partner with organizations that have part-time roles and tasks that you can complete to earn some additional income while still looking for full-time work.
+
+We’re currently collaborating with an AI research lab on a project to help train AI models to better understand how real people use computers in day-to-day tasks. As a part of this project, you will have a front-row seat into helping shape the way future AI systems interact with common software, making technology more accessible and useful for everyone.
+
+What's involved?
+You'll be asked to perform simple, everyday computer tasks—like creating a Word document, making a presentation slide, or organizing videos online.
+Each task is quick (around 2–3 minutes) and you can do as many or as few as you’d like within a 24-hour period. (Note: Project expires on November 6th, 2025)
+
+Requirements:
+For verification, you'll need to install two small programs and allow temporary recording of your screen and keyboard activities, strictly for checking that tasks are completed as described. Your data is protected and only used for research purposes.
+
+Compensation:
+You'll earn $0.30 (or Rs. 30 if in India) for each correctly completed and verified task (minimum quality standard: 75% accuracy).
+There are at least 1,000 tasks available—meaning you could earn a upto ~$350 in one day if you choose to finish all. The most active participants can take on up to 20,000 tasks.
+
+Next Steps and How to Start:
+Just fill out the form here to get started: https://tinyurl.com/lightning-puneet
+You will receive onboarding instructions over email in 5-10 minutes after filling out the form.
+
+You can also join our WhatsApp group for this project to discuss with other contributors - cheers (Link will be provided after registration)!
+
+We appreciate your help building smarter, fairer AI systems. Happy to answer any questions!
+
+Regards,
+Puneet Kohli
+careerflow.ai""",
+    """Dear Akshat!
+😎 Turn your Curiosity into Epic rewards with the TATA group*!
+
+Yes, its true! Join the Tata Crucible Campus Quiz 2025  🎉 open to all in-college students from every stream and background.
+
+✨  Here's what you can win:
+A brand-new iPhone 17
+Win cash prizes worth up to ₹2.5 Lakh*
+Internships* with the TATA group
+A Luxury holiday worth ₹50,000
+Certificates & loads of other rewards (every quiz taker gets a reward)
+👉  How to Participate? (follow 3 simple steps):
+
+1️⃣ Click the button below to Register
+2️⃣ Once logged in, click on continue and hit "Complete Details.”
+3️⃣ Fill in your details, take the quiz, and DM your quiz completion screenshot to Tata Crucible’s official Instagram by following them.
+⚠️ Important: Registration can only be completed once the basic details are filled in.
+
+🔥 Thousands of students are already in. Don’t miss your chance to shine at India’s biggest campus quiz!
+Register Now
+ 
+ 
+internshala (scholiverse educare pvt. ltd.)
+iris tech park, sohna road, gurugram 
+
+view it in your browser.
+unsubscribe me from this list
+    """
+]
+
+def analyze_message(text):
+     
+    payload = {"text": text}  
+    
+    try:
+       
+        response = requests.post(API_URL, headers=HEADERS, json=payload, timeout=10)
+        
+       
+        if response.status_code == 200:
+            prediction = response.json()  
+             
+            print(json.dumps(prediction, indent=4))
+            
+           
+            print("-" * 20)
+            print(f"Decision:     {prediction.get('final_decision')}")
+            print(f"Confidence:   {prediction.get('confidence')}%")
+            print(f"Reasoning:    {prediction.get('reasoning')}")
+            print(f"Suggestion:   {prediction.get('suggestion')}")
+            print("-" * 20)
+
+        else:
+            
+            print(f"Error: Received status code {response.status_code}")
+            print(f"Response text: {response.text}")  
+
+    except requests.exceptions.ConnectionError as e:
+        print(f"Connection Error: Could not connect to {API_URL}.")
+        print("Please ensure the ngrok tunnel is running and the URL is correct.")  
+    except requests.exceptions.RequestException as e:
+       
+        print(f"An error occurred during the request: {e}")
+
+ 
+if __name__ == "__main__":
+    for i, message in enumerate(messages_to_test):
+        print(f"================== TESTING MESSAGE {i+1} ==================")
+        
+        print(f"Message Snippet: {textwrap.shorten(message, width=70, placeholder='...')}\n")
+        
+        analyze_message(message)
+        
+        print(f"================END OF TEST FOR MESSAGE {i+1} ================\n\n")
+        sleep(6.0)

train_dl.py

@@ -0,0 +1,129 @@
+import os
+import pandas as pd
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import DataLoader
+import config
+from models import get_dl_models, PhishingDataset
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+import warnings
+
+warnings.filterwarnings('ignore', category=UserWarning)
+warnings.filterwarnings('ignore', category=FutureWarning)
+
+def prepare_data(df, numerical_cols):
+    print("Preparing data for DL training...")
+    X = df[numerical_cols].fillna(-1).values
+    y = df['label'].values
+    
+    scaler = StandardScaler()
+    X_scaled = scaler.fit_transform(X)
+    
+    return X_scaled, y, scaler
+
+def train_dl_model(model, train_loader, val_loader, device, epochs=50, lr=0.001):
+    criterion = nn.BCELoss()
+    optimizer = optim.Adam(model.parameters(), lr=lr)
+    
+    model.to(device)
+    best_val_loss = float('inf')
+    
+    for epoch in range(epochs):
+        model.train()
+        train_loss = 0.0
+        for X_batch, y_batch in train_loader:
+            X_batch, y_batch = X_batch.to(device), y_batch.to(device)
+            
+            optimizer.zero_grad()
+            outputs = model(X_batch)
+            loss = criterion(outputs, y_batch)
+            loss.backward()
+            optimizer.step()
+            
+            train_loss += loss.item()
+        
+        model.eval()
+        val_loss = 0.0
+        correct = 0
+        total = 0
+        
+        with torch.no_grad():
+            for X_batch, y_batch in val_loader:
+                X_batch, y_batch = X_batch.to(device), y_batch.to(device)
+                outputs = model(X_batch)
+                loss = criterion(outputs, y_batch)
+                val_loss += loss.item()
+                
+                predicted = (outputs > 0.5).float()
+                total += y_batch.size(0)
+                correct += (predicted == y_batch).sum().item()
+        
+        val_accuracy = correct / total
+        
+        if (epoch + 1) % 10 == 0:
+            print(f"Epoch [{epoch+1}/{epochs}], Train Loss: {train_loss/len(train_loader):.4f}, Val Loss: {val_loss/len(val_loader):.4f}, Val Acc: {val_accuracy:.4f}")
+        
+        if val_loss < best_val_loss:
+            best_val_loss = val_loss
+    
+    return model
+
+def main():
+    print("--- Starting DL Model Training ---")
+    os.makedirs(config.MODELS_DIR, exist_ok=True)
+    
+    try:
+        df = pd.read_csv(config.ENGINEERED_TRAIN_FILE)
+    except FileNotFoundError:
+        print(f"Error: '{config.ENGINEERED_TRAIN_FILE}' not found.")
+        print("Please run `python data_pipeline.py` first.")
+        return
+    
+    X_scaled, y, scaler = prepare_data(df, config.NUMERICAL_FEATURES)
+    
+    X_train, X_val, y_train, y_val = train_test_split(
+        X_scaled, y,
+        test_size=config.ML_TEST_SIZE,
+        random_state=config.ML_MODEL_RANDOM_STATE,
+        stratify=y
+    )
+    
+    print(f"Training on {len(X_train)} samples, validating on {len(X_val)} samples.")
+    
+    train_dataset = PhishingDataset(X_train, y_train)
+    val_dataset = PhishingDataset(X_val, y_val)
+    
+    train_loader = DataLoader(train_dataset, batch_size=config.DL_BATCH_SIZE, shuffle=True)
+    val_loader = DataLoader(val_dataset, batch_size=config.DL_BATCH_SIZE, shuffle=False)
+    
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    print(f"Using device: {device}")
+    
+    input_dim = X_train.shape[1]
+    dl_models = get_dl_models(input_dim)
+    
+    for name, model in dl_models.items():
+        print(f"\n--- Training {name} ---")
+        
+        trained_model = train_dl_model(
+            model, train_loader, val_loader, device,
+            epochs=config.DL_EPOCHS,
+            lr=config.DL_LEARNING_RATE
+        )
+        
+        save_path = os.path.join(config.MODELS_DIR, f"{name}.pt")
+        torch.save(trained_model.state_dict(), save_path)
+        print(f"Model saved to {save_path}")
+    
+    scaler_path = os.path.join(config.MODELS_DIR, "dl_scaler.pkl")
+    import joblib
+    joblib.dump(scaler, scaler_path)
+    print(f"Scaler saved to {scaler_path}")
+    
+    print("\n--- DL Model Training Complete ---")
+
+if __name__ == "__main__":
+    main()

train_ml.py

@@ -0,0 +1,85 @@
+import os
+import pandas as pd
+import joblib
+import config
+from models import get_ml_models
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from sklearn.pipeline import Pipeline
+from sklearn.metrics import accuracy_score
+import warnings
+
+warnings.filterwarnings('ignore', category=UserWarning)
+warnings.filterwarnings('ignore', category=FutureWarning)
+
+def prepare_data(df, numerical_cols, categorical_cols):
+    print("Preparing data for ML training...")
+    X = df[numerical_cols + categorical_cols]
+    y = df['label']
+    numerical_transformer = Pipeline(steps=[
+        ('imputer', 'passthrough'),
+        ('scaler', StandardScaler())
+    ])
+    categorical_transformer = Pipeline(steps=[
+        ('imputer', 'passthrough'),
+        ('onehot', OneHotEncoder(handle_unknown='ignore'))
+    ])
+    X.loc[:, numerical_cols] = X.loc[:, numerical_cols].fillna(-1)
+    X.loc[:, categorical_cols] = X.loc[:, categorical_cols].fillna('N/A')
+
+    preprocessor = ColumnTransformer(
+        transformers=[
+            ('num', numerical_transformer, numerical_cols),
+            ('cat', categorical_transformer, categorical_cols)
+        ],
+        remainder='passthrough'
+    )
+    
+    return preprocessor, X, y
+
+def main():
+    print("--- Starting ML Model Training ---")
+    os.makedirs(config.MODELS_DIR, exist_ok=True)
+    try:
+        df = pd.read_csv(config.ENGINEERED_TRAIN_FILE)
+    except FileNotFoundError:
+        print(f"Error: '{config.ENGINEERED_TRAIN_FILE}' not found.")
+        print("Please run `python data_pipeline.py` first.")
+        return
+
+    preprocessor, X, y = prepare_data(
+        df, 
+        config.NUMERICAL_FEATURES, 
+        config.CATEGORICAL_FEATURES
+    )
+    X_train, X_val, y_train, y_val = train_test_split(
+        X, y, 
+        test_size=config.ML_TEST_SIZE, 
+        random_state=config.ML_MODEL_RANDOM_STATE,
+        stratify=y
+    )
+    
+    print(f"Training on {len(X_train)} samples, validating on {len(X_val)} samples.")
+    ml_models = get_ml_models()
+    for name, model in ml_models.items():
+        print(f"\n--- Training {name} ---")
+        
+        model_pipeline = Pipeline(steps=[
+            ('preprocessor', preprocessor),
+            ('classifier', model)
+        ])
+        
+        model_pipeline.fit(X_train, y_train)
+        y_pred = model_pipeline.predict(X_val)
+        val_accuracy = accuracy_score(y_val, y_pred)
+        print(f"Validation Accuracy for {name}: {val_accuracy:.4f}")
+        
+        save_path = os.path.join(config.MODELS_DIR, f"{name}.joblib")
+        joblib.dump(model_pipeline, save_path)
+        print(f"Model saved to {save_path}")
+
+    print("\n--- ML Model Training Complete ---")
+
+if __name__ == "__main__":
+    main()