┌─────────────────────────────────────────────────────────────────┐
│                    MODEL LIFECYCLE (SR 11-7)                   │
├─────────────────────────────────────────────────────────────────┤
│                                                                 │
│  1. DEVELOPMENT                                                 │
│     ├── Data collection and preparation                        │
│     ├── Feature engineering                                    │
│     ├── Model training and tuning                              │
│     ├── Model selection and documentation                     │
│     └── Initial validation                                     │
│                                                                 │
│  2. VALIDATION (Independent Review)                            │
│     ├── Conceptual soundness                                   │
│     ├── Data quality assessment                               │
│     ├── Performance assessment (AUC, KS, etc.)                │
│     ├── Robustness testing (stress testing)                   │
│     └── Documentation review                                   │
│                                                                 │
│  3. DEPLOYMENT                                                 │
│     ├── Model packaging and serialisation                     │
│     ├── Integration with production systems                    │
│     ├── API development                                        │
│     └── User training                                          │
│                                                                 │
│  4. MONITORING (Ongoing)                                       │
│     ├── Performance tracking (AUC, KS over time)              │
│     ├── Data drift detection (PSI, CSI)                       │
│     ├── Concept drift detection                               │
│     ├── Alerting and reporting                                │
│     └── Trigger for revalidation                               │
│                                                                 │
│  5. REVALIDATION / RETRAINING                                  │
│     ├── Periodic revalidation (annually or as triggered)      │
│     ├── Model retraining (with new data)                      │
│     ├── Re-documentation                                      │
│     └── Re-deployment                                         │
│                                                                 │
└─────────────────────────────────────────────────────────────────┘

# ===================================================================
# MODULE 4, LESSON 6: MODEL MONITORING – DRIFT DETECTION
# ===================================================================

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import roc_auc_score, confusion_matrix
from sklearn.preprocessing import KBinsDiscretizer
import warnings
warnings.filterwarnings('ignore')

print("="*70)
print("MODEL MONITORING – DRIFT DETECTION AND PERFORMANCE TRACKING")
print("="*70)

# ----------------------------------------------------------------
# PART A: GENERATE SYNTHETIC MONITORING DATA
# ----------------------------------------------------------------

# Simulate model scores and actual outcomes over 12 quarters (3 years)
np.random.seed(42)
n_periods = 12
n_samples = 500

# Simulate 4 features (like income, DTI, credit_score, loan_amount)
features = ['income', 'dti', 'credit_score', 'loan_amount']

# Create baseline data (Period 0)
baseline_data = pd.DataFrame({
    'income': np.random.gamma(5, 15, n_samples) + 20,
    'dti': np.random.beta(2, 5, n_samples) * 60,
    'credit_score': np.random.normal(700, 50, n_samples).clip(550, 850),
    'loan_amount': np.random.gamma(4, 50, n_samples) + 50
})

# Simulate a model score (0-1) based on the features
log_odds = -4.5 + 0.04 * baseline_data['dti'] - 0.005 * baseline_data['credit_score']
baseline_score = 1 / (1 + np.exp(-log_odds))
baseline_score = np.clip(baseline_score, 0.01, 0.99)
baseline_default = np.random.binomial(1, baseline_score)

# Store baseline
baseline_df = baseline_data.copy()
baseline_df['score'] = baseline_score
baseline_df['default'] = baseline_default

# Simulate drift in features and performance over time
periodic_dfs = []

for period in range(1, n_periods + 1):
    # Simulate drift: income shifts up, credit_score shifts down slightly, DTI increases over time
    drift_factor = period * 0.02  # gradually increasing drift
    
    # Add drift to features
    period_data = baseline_data.copy()
    period_data['income'] = period_data['income'] * (1 + 0.005 * period)
    period_data['dti'] = period_data['dti'] * (1 + 0.01 * period)
    period_data['credit_score'] = period_data['credit_score'] - 2 * period
    period_data['loan_amount'] = period_data['loan_amount'] * (1 + 0.005 * period)
    
    # Score (model still uses original coefficients, leading to performance decay)
    log_odds_drift = -4.5 + 0.04 * period_data['dti'] - 0.005 * period_data['credit_score']
    score = 1 / (1 + np.exp(-log_odds_drift))
    score = np.clip(score, 0.01, 0.99)
    
    # Actual outcome may have concept drift (relationship changes)
    # In later periods, defaults become more likely for a given score
    default_prob = score * (1 + 0.05 * period / 4)  # gradual increase in default rate
    default_prob = np.clip(default_prob, 0.01, 0.95)
    default = np.random.binomial(1, default_prob)
    
    period_df = period_data.copy()
    period_df['score'] = score
    period_df['default'] = default
    period_df['period'] = period
    periodic_dfs.append(period_df)

all_data = pd.concat([baseline_df.assign(period=0), *periodic_dfs], ignore_index=True)

# Compute period-level metrics
period_metrics = []
for period in sorted(all_data['period'].unique()):
    data = all_data[all_data['period'] == period]
    auc = roc_auc_score(data['default'], data['score'])
    default_rate = data['default'].mean()
    period_metrics.append({
        'period': period,
        'auc': auc,
        'default_rate': default_rate,
        'n_samples': len(data)
    })

metrics_df = pd.DataFrame(period_metrics)
print("\nModel Performance Over Time:")
print(metrics_df.to_string(index=False))

# ----------------------------------------------------------------
# PART B: POPULATION STABILITY INDEX (PSI) – FEATURE DRIFT
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART B: POPULATION STABILITY INDEX (PSI) – Feature Drift")
print("-"*60)

def calculate_psi(expected, actual, bins=10):
    """
    Calculate Population Stability Index between expected and actual distributions.
    PSI = sum((actual_i - expected_i) * ln(actual_i / expected_i))
    """
    # Bin the data
    expected_bins = np.percentile(expected, np.linspace(0, 100, bins+1))
    expected_counts = np.histogram(expected, bins=expected_bins)[0]
    actual_counts = np.histogram(actual, bins=expected_bins)[0]
    
    # Convert to proportions
    expected_prop = expected_counts / len(expected)
    actual_prop = actual_counts / len(actual)
    
    # Add small epsilon to avoid division by zero
    expected_prop = np.clip(expected_prop, 1e-10, 1)
    actual_prop = np.clip(actual_prop, 1e-10, 1)
    
    # Calculate PSI
    psi = np.sum((actual_prop - expected_prop) * np.log(actual_prop / expected_prop))
    return psi

# Baseline (period 0) as reference
baseline_scores = all_data[all_data['period'] == 0]['score']

# Calculate PSI for each period
psi_results = []
for period in sorted(all_data['period'].unique())[1:]:
    actual_scores = all_data[all_data['period'] == period]['score']
    psi = calculate_psi(baseline_scores, actual_scores)
    psi_results.append({'period': period, 'psi': psi})
psi_df = pd.DataFrame(psi_results)

print("\nPSI by Period (Score Distribution):")
print(psi_df.to_string(index=False))

# Interpret PSI
for _, row in psi_df.iterrows():
    psi_val = row['psi']
    if psi_val < 0.1:
        status = "STABLE"
        action = "No action required."
    elif psi_val < 0.25:
        status = "MODERATE"
        action = "Investigate feature distributions."
    else:
        status = "SIGNIFICANT"
        action = "Revalidate model or retrain."
    print(f"Period {row['period']}: PSI={psi_val:.4f} → {status} → {action}")

# ----------------------------------------------------------------
# PART C: FEATURE-LEVEL DRIFT DETECTION
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART C: Feature-Level Drift (PSI per feature)")
print("-"*60)

# Baseline data (period 0)
baseline_full = all_data[all_data['period'] == 0][['income', 'dti', 'credit_score', 'loan_amount']]

# Recent data (period 11)
recent_full = all_data[all_data['period'] == 11][['income', 'dti', 'credit_score', 'loan_amount']]

feature_psi = {}
for feature in baseline_full.columns:
    psi = calculate_psi(baseline_full[feature], recent_full[feature])
    feature_psi[feature] = psi

feature_psi_df = pd.DataFrame(list(feature_psi.items()), columns=['Feature', 'PSI'])
print(feature_psi_df.to_string(index=False))

# Visualise feature drift
fig, axes = plt.subplots(2, 2, figsize=(14, 10))

for i, feature in enumerate(['income', 'dti', 'credit_score', 'loan_amount']):
    ax = axes[i // 2, i % 2]
    ax.hist(baseline_full[feature], bins=30, alpha=0.5, label='Baseline', edgecolor='black')
    ax.hist(recent_full[feature], bins=30, alpha=0.5, label='Recent', edgecolor='black')
    ax.set_title(f'{feature} (PSI={feature_psi[feature]:.4f})')
    ax.set_xlabel(feature)
    ax.set_ylabel('Frequency')
    ax.legend()
    ax.grid(True, alpha=0.3)

plt.tight_layout()
plt.savefig('feature_drift_psi.png', dpi=300)
plt.show()

# ----------------------------------------------------------------
# PART D: PERFORMANCE MONITORING DASHBOARD
# -----------------------------------------------------------------

print("\n" + "-"*60)
print("PART D: Performance Monitoring Dashboard")
print("-"*60)

# Create dashboard visualisation
fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# 1. AUC over time
ax = axes[0, 0]
ax.plot(metrics_df['period'], metrics_df['auc'], 'bo-', linewidth=2, markersize=8)
ax.axhline(metrics_df['auc'].iloc[0], color='green', linestyle='--', 
           label=f'Baseline AUC: {metrics_df["auc"].iloc[0]:.3f}')
ax.axhline(metrics_df['auc'].iloc[0] - 0.05, color='red', linestyle='--', 
           label='Warning threshold (AUC - 5%)')
ax.set_xlabel('Time Period (Quarter)')
ax.set_ylabel('AUC')
ax.set_title('Model Performance (AUC) Over Time')
ax.legend()
ax.grid(True, alpha=0.3)

# 2. Default Rate over time
ax = axes[0, 1]
ax.bar(metrics_df['period'], metrics_df['default_rate'], color='orange', alpha=0.7)
ax.set_xlabel('Time Period (Quarter)')
ax.set_ylabel('Default Rate')
ax.set_title('Actual Default Rate Over Time')
ax.grid(True, alpha=0.3)

# 3. PSI over time
ax = axes[1, 0]
ax.plot(psi_df['period'], psi_df['psi'], 'ro-', linewidth=2, markersize=8)
ax.axhline(0.1, color='green', linestyle='--', label='Stable (PSI < 0.1)')
ax.axhline(0.25, color='orange', linestyle='--', label='Moderate (PSI 0.1-0.25)')
ax.axhline(0.25, color='red', linestyle='--', label='Significant (PSI > 0.25)')
ax.fill_between(psi_df['period'], 0, psi_df['psi'], alpha=0.3, color='red', 
                where=(psi_df['psi'] > 0.25))
ax.fill_between(psi_df['period'], 0, psi_df['psi'], alpha=0.3, color='orange', 
                where=(psi_df['psi'] > 0.1) & (psi_df['psi'] <= 0.25))
ax.set_xlabel('Time Period (Quarter)')
ax.set_ylabel('PSI')
ax.set_title('Population Stability Index (Score Drift)')
ax.legend()
ax.grid(True, alpha=0.3)

# 4. Confusion Matrix for current period
ax = axes[1, 1]
current_data = all_data[all_data['period'] == 11]
threshold = 0.5
y_pred = (current_data['score'] >= threshold).astype(int)
cm = confusion_matrix(current_data['default'], y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
            xticklabels=['Pred No Default', 'Pred Default'],
            yticklabels=['Actual No Default', 'Actual Default'],
            ax=ax)
ax.set_title(f'Confusion Matrix (Period 11, Threshold={threshold})')

plt.tight_layout()
plt.savefig('monitoring_dashboard.png', dpi=300)
plt.show()

# ----------------------------------------------------------------
# PART E: ALERTING AND ACTION TRIGGERS
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART E: Alerting System – Trigger Conditions")
print("-"*60)

# Define alert thresholds
ALERT_CONFIG = {
    'auc_drop': 0.05,        # 5% drop in AUC
    'psi_threshold': 0.25,   # Significant drift
    'psi_warning': 0.10,     # Moderate drift
    'default_rate_change': 0.10  # 10% change in default rate
}

# Check each period for alerts
alerts = []
for period in sorted(all_data['period'].unique())[1:]:
    period_metrics = metrics_df[metrics_df['period'] == period].iloc[0]
    baseline_metrics = metrics_df[metrics_df['period'] == 0].iloc[0]
    
    # AUC drop
    auc_drop = baseline_metrics['auc'] - period_metrics['auc']
    if auc_drop > ALERT_CONFIG['auc_drop']:
        alerts.append({
            'period': period,
            'type': 'PERFORMANCE_DECAY',
            'detail': f'AUC dropped from {baseline_metrics["auc"]:.3f} to {period_metrics["auc"]:.3f} (drop: {auc_drop:.3f})'
        })
    
    # PSI
    psi_val = psi_df[psi_df['period'] == period]['psi'].iloc[0]
    if psi_val > ALERT_CONFIG['psi_threshold']:
        alerts.append({
            'period': period,
            'type': 'SIGNIFICANT_DRIFT',
            'detail': f'PSI = {psi_val:.3f} (threshold: {ALERT_CONFIG["psi_threshold"]})'
        })
    elif psi_val > ALERT_CONFIG['psi_warning']:
        alerts.append({
            'period': period,
            'type': 'MODERATE_DRIFT',
            'detail': f'PSI = {psi_val:.3f} (warning threshold: {ALERT_CONFIG["psi_warning"]})'
        })
    
    # Default rate change
    default_change = abs(period_metrics['default_rate'] - baseline_metrics['default_rate']) / baseline_metrics['default_rate']
    if default_change > ALERT_CONFIG['default_rate_change']:
        alerts.append({
            'period': period,
            'type': 'DEFAULT_RATE_SHIFT',
            'detail': f'Default rate changed from {baseline_metrics["default_rate"]:.3f} to {period_metrics["default_rate"]:.3f} ({default_change:.1%})'
        })

# Display alerts
if alerts:
    alert_df = pd.DataFrame(alerts)
    print("\nALERTS TRIGGERED:")
    print(alert_df.to_string(index=False))
else:
    print("\nNo alerts triggered. Model is stable.")

print("\nRecommended Actions:")
print("  • PERFORMANCE_DECAY → Revalidate model; consider retraining.")
print("  • SIGNIFICANT_DRIFT → Investigate data sources; update preprocessing.")
print("  • MODERATE_DRIFT → Monitor closely; plan for upcoming revalidation.")
print("  • DEFAULT_RATE_SHIFT → Review business conditions; adjust risk appetite.")

# ----------------------------------------------------------------
# PART F: MODEL LIFECYCLE DOCUMENTATION
# ----------------------------------------------------------------

print("\n" + "="*70)
print("PART F: MODEL LIFECYCLE DOCUMENTATION TEMPLATE")
print("="*70)

documentation_template = """
--- MODEL DOCUMENTATION (SR 11-7 COMPLIANCE) ---

1. MODEL IDENTIFICATION
   - Model Name: Credit Default Predictor v3.2
   - Model Type: XGBoost Classifier
   - Development Date: 2025-12-01
   - Deployment Date: 2026-02-15
   - Owner: Credit Risk Analytics Team

2. DATA & FEATURES
   - Training Data: Loans originated 2022-2024 (48 months)
   - Features Used: income, age, dti, credit_score, loan_amount
   - Data Source: Internal Core Banking System (DWH)

3. MODEL PERFORMANCE (At Deployment)
   - AUC: 0.84
   - KS: 0.42
   - Gini: 0.68
   - Calibration: Hosmer-Lemeshow p-value = 0.12

4. PERFORMANCE MONITORING
   - Current AUC: {current_auc:.3f}
   - AUC Drop: {auc_drop:.3f}
   - Current PSI: {current_psi:.3f}
   - Status: {status}

5. NEXT STEPS
   - Next Validation Date: {next_validation}
   - Retraining Trigger: AUC < 0.75 OR PSI > 0.25
   - Action Plan: {action_plan}
"""

# Fill template with latest data
latest_period = 11
current_auc = metrics_df[metrics_df['period'] == latest_period]['auc'].iloc[0]
auc_drop = metrics_df[metrics_df['period'] == 0]['auc'].iloc[0] - current_auc
current_psi = psi_df[psi_df['period'] == latest_period]['psi'].iloc[0] if latest_period in psi_df['period'].values else 0

if current_psi < 0.1 and auc_drop < 0.05:
    status = "STABLE – No action required"
    action_plan = "Continue monitoring; quarterly review"
elif current_psi < 0.25 or auc_drop < 0.05:
    status = "MODERATE – Investigate"
    action_plan = "Perform detailed feature analysis; plan validation in 3 months"
else:
    status = "SIGNIFICANT – Action required"
    action_plan = "Immediate model validation; consider retraining"

print(documentation_template.format(
    current_auc=current_auc,
    auc_drop=auc_drop,
    current_psi=current_psi,
    status=status,
    next_validation="2026-09-15" if status != "SIGNIFICANT" else "2026-06-15",
    action_plan=action_plan
))