┌─────────────────────────────────────────────────────────────────────────────┐
│                          DEPLOYMENT ARCHITECTURE                           │
├─────────────────────────────────────────────────────────────────────────────┤
│                                                                             │
│  ┌─────────────┐     ┌─────────────┐     ┌─────────────┐                   │
│  │  Loan       │     │  API        │     │  Model      │                   │
│  │  Application│ ──→ │  Gateway    │ ──→ │  Serving    │ ──→ Prediction   │
│  │  System     │     │  (Flask)    │     │  (Container)│                   │
│  └─────────────┘     └─────────────┘     └─────────────┘                   │
│                            │                     │                         │
│                            v                     v                         │
│  ┌─────────────┐     ┌─────────────┐     ┌─────────────┐                   │
│  │  Monitoring │     │  Logging    │     │  Model      │                   │
│  │  Dashboard  │ ←── │  (Prometheus│ ←── │  Registry   │                   │
│  │  (Grafana)  │     │   + Loki)   │     │  (MLflow)   │                   │
│  └─────────────┘     └─────────────┘     └─────────────┘                   │
│                            │                     │                         │
│                            v                     v                         │
│  ┌─────────────┐     ┌─────────────┐     ┌─────────────┐                   │
│  │  Alerting   │     │  Retraining │     │  CI/CD      │                   │
│  │  (PagerDuty)│     │  Pipeline   │     │  Pipeline   │                   │
│  └─────────────┘     └─────────────┘     └─────────────┘                   │
│                                                                             │
└─────────────────────────────────────────────────────────────────────────────┘

# ===================================================================
# MODULE 9, LESSON 5: MODEL DEPLOYMENT AND MONITORING
# ===================================================================

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import joblib
import json
from datetime import datetime, timedelta
import warnings
warnings.filterwarnings('ignore')

# Set style
sns.set_style("whitegrid")
np.random.seed(42)

print("="*70)
print("CAPSTONE PROJECT – MODEL DEPLOYMENT AND MONITORING")
print("="*70)

# ----------------------------------------------------------------
# PART A: MODEL SERIALISATION AND PREPARATION
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART A: Model Serialisation and Preparation")
print("-"*60)

# Load models and scaler
lr_model = joblib.load('logistic_regression_model.pkl')
best_xgb = joblib.load('xgboost_model.pkl')
scaler = joblib.load('scaler.pkl')

# Feature names
feature_names = ['applicant_age', 'income', 'credit_score', 'dti', 'loan_amount', 
                 'loan_term', 'employment_years', 'home_owner', 'marital_status', 
                 'education', 'loan_to_income', 'dti_credit_interaction', 
                 'dti_squared', 'loan_amount_log']

print(f"Loaded {len(feature_names)} features.")
print("Models and scaler loaded successfully.")

# ----------------------------------------------------------------
# PART B: SIMULATED API SERVICE (FLASK-LIKE)
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART B: Simulated API Service")
print("-"*60)

class ModelAPI:
    """Simulated model serving API."""
    
    def __init__(self, model, scaler, feature_names, model_name='XGBoost', threshold=0.5):
        self.model = model
        self.scaler = scaler
        self.feature_names = feature_names
        self.model_name = model_name
        self.threshold = threshold
        self.request_log = []
        self.prediction_log = []
        self.start_time = datetime.now()
        self.request_count = 0
        
    def predict(self, data):
        """
        Make a prediction from input data.
        Expects a dictionary with feature values.
        """
        self.request_count += 1
        request_time = datetime.now()
        
        # Validate input
        missing = [f for f in self.feature_names if f not in data]
        if missing:
            return {
                'error': f'Missing features: {missing}',
                'status': 'error'
            }
        
        # Create feature vector
        X = np.array([[data.get(f, 0) for f in self.feature_names]])
        
        # Standardise
        X_scaled = self.scaler.transform(X)
        
        # Predict
        prob = self.model.predict_proba(X_scaled)[0, 1]
        pred = 1 if prob >= self.threshold else 0
        
        # Log
        self.prediction_log.append({
            'request_id': self.request_count,
            'timestamp': request_time.isoformat(),
            'input': data,
            'probability': float(prob),
            'prediction': int(pred),
            'model': self.model_name
        })
        
        return {
            'request_id': self.request_count,
            'probability': prob,
            'prediction': pred,
            'decision': 'Approved' if pred == 0 else 'Declined',
            'model': self.model_name,
            'timestamp': request_time.isoformat()
        }
    
    def get_stats(self):
        """Get API statistics."""
        total_requests = len(self.prediction_log)
        if total_requests == 0:
            return {'total_requests': 0}
        
        approved = sum(1 for p in self.prediction_log if p['prediction'] == 0)
        declined = sum(1 for p in self.prediction_log if p['prediction'] == 1)
        
        return {
            'total_requests': total_requests,
            'approved': approved,
            'declined': declined,
            'approval_rate': approved / total_requests,
            'uptime': (datetime.now() - self.start_time).total_seconds() / 3600
        }

# Create API instance
api = ModelAPI(best_xgb, scaler, feature_names, model_name='XGBoost (Tuned)', threshold=0.5)

# Test with a sample application
sample_application = {
    'applicant_age': 42,
    'income': 65.4,
    'credit_score': 710,
    'dti': 28.5,
    'loan_amount': 145,
    'loan_term': 36,
    'employment_years': 8,
    'home_owner': 1,
    'marital_status': 1,
    'education': 2,
    'loan_to_income': 145 / 65.4,
    'dti_credit_interaction': 28.5 * 710 / 1000,
    'dti_squared': 28.5 ** 2,
    'loan_amount_log': np.log(145 + 1)
}

result = api.predict(sample_application)
print("Sample Prediction:")
print(json.dumps(result, indent=2))

# ----------------------------------------------------------------
# PART C: SIMULATED PRODUCTION TRAFFIC
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART C: Simulating Production Traffic")
print("-"*60)

def generate_sample_application():
    """Generate a random loan application."""
    age = np.random.normal(42, 14).clip(18, 80)
    income = np.random.gamma(5, 15) + 20
    credit_score = np.random.normal(700, 50).clip(550, 850)
    dti = np.random.beta(2, 5) * 60
    loan_amount = np.random.gamma(4, 50) + 30
    loan_term = np.random.choice([12, 24, 36, 48, 60, 72])
    employment_years = np.random.gamma(3, 5).clip(0, 40)
    home_owner = np.random.binomial(1, 0.65)
    marital_status = np.random.choice([0, 1, 2], p=[0.35, 0.45, 0.20])
    education = np.random.choice([0, 1, 2, 3], p=[0.15, 0.25, 0.35, 0.25])
    
    return {
        'applicant_age': int(age),
        'income': float(income),
        'credit_score': int(credit_score),
        'dti': float(dti),
        'loan_amount': float(loan_amount),
        'loan_term': int(loan_term),
        'employment_years': int(employment_years),
        'home_owner': int(home_owner),
        'marital_status': int(marital_status),
        'education': int(education),
        'loan_to_income': loan_amount / income if income > 0 else 0,
        'dti_credit_interaction': dti * credit_score / 1000,
        'dti_squared': dti ** 2,
        'loan_amount_log': np.log(loan_amount + 1)
    }

# Simulate 1000 requests
print("Simulating 1000 API requests...")
for i in range(1000):
    app = generate_sample_application()
    api.predict(app)

stats = api.get_stats()
print(f"\nAPI Statistics:")
print(f"  Total Requests: {stats['total_requests']}")
print(f"  Approved: {stats['approved']} ({stats['approval_rate']*100:.1f}%)")
print(f"  Declined: {stats['declined']} ({(1-stats['approval_rate'])*100:.1f}%)")
print(f"  Uptime: {stats['uptime']:.1f} hours")

# ----------------------------------------------------------------
# PART D: MODEL MONITORING – PERFORMANCE OVER TIME
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART D: Model Monitoring – Performance Over Time")
print("-"*60)

def generate_monitoring_data(n_days=30):
    """Generate simulated monitoring data over time."""
    dates = [datetime.now() - timedelta(days=i) for i in range(n_days)]
    dates = sorted(dates)
    
    data = []
    for i, date in enumerate(dates):
        # Simulate performance drift
        drift = np.sin(i / 10) * 0.03
        auc = 0.85 + drift + np.random.normal(0, 0.01)
        ks = 0.40 + drift * 0.5 + np.random.normal(0, 0.01)
        psi = np.random.gamma(0.1, 0.05) + max(0, drift * 2)
        requests = np.random.poisson(500 + i * 2)
        latency = np.random.normal(50 + drift * 10, 10).clip(20, 150)
        
        data.append({
            'date': date,
            'auc': auc.clip(0, 1),
            'ks': ks.clip(0, 1),
            'psi': psi,
            'requests': requests,
            'latency': latency,
            'approval_rate': np.random.normal(0.75, 0.02).clip(0.6, 0.9)
        })
    
    return pd.DataFrame(data)

monitoring_df = generate_monitoring_data(30)
print("Monitoring data generated.")

# ----------------------------------------------------------------
# PART E: MONITORING DASHBOARD VISUALISATION
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART E: Monitoring Dashboard Visualisation")
print("-"*60)

fig, axes = plt.subplots(2, 3, figsize=(15, 10))

# 1. AUC over time
ax = axes[0, 0]
ax.plot(monitoring_df['date'], monitoring_df['auc'], 'b-', linewidth=2)
ax.axhline(y=0.80, color='green', linestyle='--', label='Acceptable (0.80)')
ax.axhline(y=0.75, color='orange', linestyle='--', label='Warning (0.75)')
ax.axhline(y=0.70, color='red', linestyle='--', label='Critical (0.70)')
ax.set_xlabel('Date')
ax.set_ylabel('AUC')
ax.set_title('Model Performance (AUC)')
ax.legend(fontsize=8)
ax.grid(True, alpha=0.3)

# 2. KS over time
ax = axes[0, 1]
ax.plot(monitoring_df['date'], monitoring_df['ks'], 'g-', linewidth=2)
ax.axhline(y=0.35, color='green', linestyle='--', label='Acceptable (0.35)')
ax.axhline(y=0.30, color='orange', linestyle='--', label='Warning (0.30)')
ax.axhline(y=0.25, color='red', linestyle='--', label='Critical (0.25)')
ax.set_xlabel('Date')
ax.set_ylabel('KS')
ax.set_title('Model Performance (KS)')
ax.legend(fontsize=8)
ax.grid(True, alpha=0.3)

# 3. PSI (Data Drift)
ax = axes[0, 2]
ax.plot(monitoring_df['date'], monitoring_df['psi'], 'r-', linewidth=2)
ax.axhline(y=0.10, color='green', linestyle='--', label='Stable (<0.10)')
ax.axhline(y=0.25, color='orange', linestyle='--', label='Warning (0.10-0.25)')
ax.axhline(y=0.25, color='red', linestyle='--', label='Critical (>0.25)')
ax.set_xlabel('Date')
ax.set_ylabel('PSI')
ax.set_title('Data Drift (PSI)')
ax.legend(fontsize=8)
ax.grid(True, alpha=0.3)

# 4. Request volume
ax = axes[1, 0]
ax.bar(monitoring_df['date'], monitoring_df['requests'], color='blue', alpha=0.7)
ax.set_xlabel('Date')
ax.set_ylabel('Requests')
ax.set_title('API Request Volume')
ax.grid(True, alpha=0.3)
plt.xticks(rotation=45)

# 5. Latency
ax = axes[1, 1]
ax.plot(monitoring_df['date'], monitoring_df['latency'], 'purple', linewidth=2)
ax.axhline(y=100, color='red', linestyle='--', label='SLA (100ms)')
ax.set_xlabel('Date')
ax.set_ylabel('Latency (ms)')
ax.set_title('API Latency')
ax.legend(fontsize=8)
ax.grid(True, alpha=0.3)

# 6. Approval Rate
ax = axes[1, 2]
ax.plot(monitoring_df['date'], monitoring_df['approval_rate'], 'orange', linewidth=2)
ax.set_xlabel('Date')
ax.set_ylabel('Approval Rate')
ax.set_title('Loan Approval Rate')
ax.axhline(y=0.70, color='red', linestyle='--', label='Target (70%)')
ax.legend(fontsize=8)
ax.grid(True, alpha=0.3)

plt.tight_layout()
plt.savefig('monitoring_dashboard.png', dpi=300, bbox_inches='tight')
plt.show()
print("Monitoring dashboard saved as 'monitoring_dashboard.png'")

# ----------------------------------------------------------------
# PART F: DRIFT DETECTION AND ALERTING
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART F: Drift Detection and Alerting")
print("-"*60)

def detect_drift(monitoring_df, thresholds):
    """
    Detect drift and generate alerts based on thresholds.
    """
    alerts = []
    latest = monitoring_df.iloc[-1]
    
    # Check performance metrics
    if latest['auc'] < thresholds.get('auc_warning', 0.75):
        alerts.append({
            'severity': 'WARNING' if latest['auc'] >= 0.70 else 'CRITICAL',
            'metric': 'AUC',
            'value': latest['auc'],
            'threshold': thresholds.get('auc_warning', 0.75),
            'message': f"AUC dropped to {latest['auc']:.3f}"
        })
    
    if latest['ks'] < thresholds.get('ks_warning', 0.30):
        alerts.append({
            'severity': 'WARNING' if latest['ks'] >= 0.25 else 'CRITICAL',
            'metric': 'KS',
            'value': latest['ks'],
            'threshold': thresholds.get('ks_warning', 0.30),
            'message': f"KS dropped to {latest['ks']:.3f}"
        })
    
    if latest['psi'] > thresholds.get('psi_warning', 0.10):
        alerts.append({
            'severity': 'WARNING' if latest['psi'] < 0.25 else 'CRITICAL',
            'metric': 'PSI',
            'value': latest['psi'],
            'threshold': thresholds.get('psi_warning', 0.10),
            'message': f"PSI increased to {latest['psi']:.3f}"
        })
    
    if latest['latency'] > thresholds.get('latency_warning', 100):
        alerts.append({
            'severity': 'WARNING',
            'metric': 'Latency',
            'value': latest['latency'],
            'threshold': thresholds.get('latency_warning', 100),
            'message': f"Latency exceeded {thresholds.get('latency_warning', 100)}ms"
        })
    
    return alerts

# Define thresholds
thresholds = {
    'auc_warning': 0.75,
    'auc_critical': 0.70,
    'ks_warning': 0.30,
    'ks_critical': 0.25,
    'psi_warning': 0.10,
    'psi_critical': 0.25,
    'latency_warning': 100,
    'latency_critical': 150
}

alerts = detect_drift(monitoring_df, thresholds)

print("Current Alerts:")
if alerts:
    for alert in alerts:
        print(f"  [{alert['severity']}] {alert['metric']}: {alert['message']}")
else:
    print("  ✅ All systems nominal")

# ----------------------------------------------------------------
# PART G: AUTOMATED RETRAINING STRATEGY
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART G: Automated Retraining Strategy")
print("-"*60)

def retraining_check(monitoring_df, thresholds):
    """
    Determine if retraining is needed based on monitoring data.
    """
    latest = monitoring_df.iloc[-1]
    retrain = False
    reasons = []
    
    # Check performance degradation
    if latest['auc'] < thresholds.get('auc_critical', 0.70):
        retrain = True
        reasons.append("AUC below critical threshold")
    
    if latest['ks'] < thresholds.get('ks_critical', 0.25):
        retrain = True
        reasons.append("KS below critical threshold")
    
    if latest['psi'] > thresholds.get('psi_critical', 0.25):
        retrain = True
        reasons.append("PSI above critical threshold (data drift)")
    
    # Check trend: if performance has been declining for 7 days
    last_7 = monitoring_df.tail(7)
    auc_trend = np.polyfit(range(len(last_7)), last_7['auc'], 1)[0]
    if auc_trend < -0.01:  # Decreasing more than 1% per day
        retrain = True
        reasons.append("Performance decline trend detected")
    
    return retrain, reasons

retrain_needed, reasons = retraining_check(monitoring_df, thresholds)

print("Retraining Analysis:")
print(f"  Retraining Needed: {'YES' if retrain_needed else 'NO'}")
if retrain_needed:
    print("  Reasons:")
    for reason in reasons:
        print(f"    • {reason}")
else:
    print("  ✅ Model performance stable; retraining not required.")

# ----------------------------------------------------------------
# PART H: MODEL CARD FOR DEPLOYMENT
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART H: Model Card for Deployment")
print("-"*60)

model_card = """
--- MODEL CARD: XGBoost Default Prediction Model ---

1. MODEL DETAILS
   - Model Name: XGBoost Default Predictor v1.0
   - Model Type: XGBoost Classifier
   - Development Date: 2024-01-15
   - Deployment Date: 2024-06-01
   - Owner: Atlantic Bank Data Science Team
   - Contact: datascience@atlanticbank.com

2. INTENDED USE
   - Primary Use: Credit risk assessment for personal loans.
   - Secondary Use: Portfolio risk monitoring and stress testing.
   - Out-of-Scope: Automated lending without human review.

3. PERFORMANCE (At Deployment)
   - AUC: 0.85
   - KS Statistic: 0.42
   - Gini Coefficient: 0.70
   - Calibration: HL p-value = 0.18
   - Accuracy: 0.81

4. FEATURES
   - Number of Features: 14 (original + engineered)
   - Key Features: credit_score, dti, loan_to_income, dti_credit_interaction
   - Feature Engineering: loan_to_income, dti_credit_interaction, dti_squared, loan_amount_log

5. DATA
   - Training Data: 2019-2023 loan applications (70,000 rows)
   - Validation Data: 2023-2024 loan applications (30,000 rows)
   - Default Rate: 4.2%

6. MONITORING
   - Performance: AUC, KS, Calibration (daily)
   - Data Drift: PSI (weekly)
   - Alerts: AUC < 0.75, KS < 0.30, PSI > 0.25
   - Retraining: Triggered by alerts or monthly schedule

7. GOVERNANCE
   - Model Validator: Model Validation Team
   - Approval Date: 2024-05-15
   - Review Frequency: Annual (or trigger-based)
   - Compliance: SR 11-7, Fair Lending

8. DEPLOYMENT
   - Environment: Production (AWS)
   - Serving: Containerised (Docker) + API (Flask)
   - Monitoring: Prometheus + Grafana
   - CI/CD: GitHub Actions
"""

print(model_card)

# ----------------------------------------------------------------
# PART I: DEPLOYMENT CHECKLIST
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART I: Deployment Checklist")
print("-"*60)

deployment_checklist = [
    "✅ Model validated and approved by independent validator",
    "✅ Model performance meets thresholds (AUC > 0.80, KS > 0.35)",
    "✅ Model documentation (Model Card) completed",
    "✅ API developed and tested",
    "✅ Performance monitoring dashboard set up",
    "✅ Drift detection and alerting configured",
    "✅ Retraining strategy documented",
    "✅ Rollback plan in place",
    "✅ User training materials prepared",
    "✅ Regulatory compliance review completed",
    "✅ Security review completed (access controls, encryption)",
    "✅ Load testing completed (1000+ requests/second)",
    "✅ Business continuity plan in place",
    "✅ Go-live approval obtained from steering committee"
]

print("Deployment Checklist:")
for item in deployment_checklist:
    print(item)

# ----------------------------------------------------------------
# PART J: SUMMARY
# ----------------------------------------------------------------

print("\n" + "="*70)
print("PART J: Summary")
print("="*70)

print("""
Model Deployment and Monitoring – Key Takeaways:

1. Model Serialisation:
   - Models are saved using joblib/pickle for production.
   - Feature engineering logic must be packaged with the model.

2. API Service:
   - Flask/FastAPI provides a RESTful interface for predictions.
   - Input validation and standardisation are critical.

3. Monitoring:
   - Track performance metrics (AUC, KS) over time.
   - Monitor data drift using PSI (Population Stability Index).
   - Monitor concept drift via performance decline.

4. Alerting:
   - Alerts for performance drops, drift, and latency issues.
   - Severity levels: WARNING and CRITICAL.

5. Retraining:
   - Triggered by alerts (critical thresholds) or schedule.
   - Automated retraining pipeline (Airflow/Dagster).

6. Documentation:
   - Model Card summarises model details, performance, and governance.
   - Deployment checklist ensures all requirements are met.

7. Regulatory Compliance:
   - SR 11-7 requires validation and monitoring.
   - Fair Lending requires ongoing bias testing.
""")

print("="*70)
print("END OF LESSON 5 – MODULE 9")
print("="*70)