┌─────────────────────────────────────────────────────────────────────────────┐
│                    DATA GOVERNANCE FRAMEWORK                              │
├─────────────────────────────────────────────────────────────────────────────┤
│                                                                             │
│  ┌──────────────────────────────────────────────────────────────────────┐   │
│  │                    STRATEGY & POLICIES                              │   │
│  │  Data strategy, policies, standards, principles                    │   │
│  └──────────────────────────────────────────────────────────────────────┘   │
│                                    │                                        │
│                                    v                                        │
│  ┌──────────────────────────────────────────────────────────────────────┐   │
│  │                    ORGANISATION & ROLES                             │   │
│  │  Data owners, stewards, custodians, committee                      │   │
│  └──────────────────────────────────────────────────────────────────────┘   │
│                                    │                                        │
│                                    v                                        │
│  ┌──────────────────────────────────────────────────────────────────────┐   │
│  │                    PROCESSES                                        │   │
│  │  Data quality, lineage, metadata, lifecycle management              │   │
│  └──────────────────────────────────────────────────────────────────────┘   │
│                                    │                                        │
│                                    v                                        │
│  ┌──────────────────────────────────────────────────────────────────────┐   │
│  │                    TECHNOLOGY                                       │   │
│  │  Data catalog, data quality tools, MDM, security                   │   │
│  └──────────────────────────────────────────────────────────────────────┘   │
│                                    │                                        │
│                                    v                                        │
│  ┌──────────────────────────────────────────────────────────────────────┐   │
│  │                    MONITORING & METRICS                             │   │
│  │  Data quality metrics, compliance reporting, dashboards             │   │
│  └──────────────────────────────────────────────────────────────────────┘   │
│                                                                             │
└─────────────────────────────────────────────────────────────────────────────┘

# ===================================================================
# MODULE 4, LESSON 8: DATA GOVERNANCE AND ETHICS IN BANKING
# ===================================================================

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
import warnings
warnings.filterwarnings('ignore')

print("="*70)
print("DATA GOVERNANCE AND ETHICS IN DIGITAL BANKING")
print("="*70)

# ----------------------------------------------------------------
# PART A: DATA GOVERNANCE FRAMEWORK
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART A: Data Governance Framework")
print("-"*60)

governance_framework = {
    "1. Strategy & Policies": {
        "Components": [
            "Data Strategy",
            "Data Quality Policy",
            "Data Privacy Policy",
            "Data Retention Policy",
            "Data Access Policy"
        ],
        "Owner": "Chief Data Officer"
    },
    "2. Organisation & Roles": {
        "Components": [
            "Data Governance Committee",
            "Data Owners",
            "Data Stewards",
            "Data Custodians",
            "Data Scientists"
        ],
        "Owner": "Head of Data Governance"
    },
    "3. Processes": {
        "Components": [
            "Data Quality Management",
            "Data Lineage Tracking",
            "Metadata Management",
            "Data Lifecycle Management",
            "Data Issue Resolution"
        ],
        "Owner": "Data Governance Manager"
    },
    "4. Technology": {
        "Components": [
            "Data Catalog",
            "Data Quality Tools",
            "Metadata Repository",
            "Master Data Management",
            "Data Security Tools"
        ],
        "Owner": "Head of Data Engineering"
    },
    "5. Monitoring & Metrics": {
        "Components": [
            "Data Quality Dashboards",
            "Compliance Reports",
            "Data Usage Metrics",
            "Issue Tracking",
            "Audit Trails"
        ],
        "Owner": "Data Governance Manager"
    }
}

print("Data Governance Framework:")
for area, details in governance_framework.items():
    print(f"\n{area}:")
    print(f"  Owner: {details['Owner']}")
    print("  Components:")
    for component in details['Components']:
        print(f"    • {component}")

# ----------------------------------------------------------------
# PART B: DATA QUALITY DASHBOARD
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART B: Data Quality Dashboard")
print("-"*60)

# Simulate data quality metrics
datasets = ['Customer', 'Transactions', 'Accounts', 'Loans', 'Products', 'Employees']
quality_metrics = {
    'Completeness (%)': [92, 88, 85, 78, 82, 90],
    'Accuracy (%)': [95, 92, 90, 85, 88, 94],
    'Consistency (%)': [88, 85, 82, 80, 85, 90],
    'Timeliness (%)': [90, 95, 88, 82, 80, 92]
}

quality_df = pd.DataFrame(quality_metrics, index=datasets)
quality_df['Overall Score (%)'] = quality_df.mean(axis=1)

print("Data Quality Dashboard:")
print(quality_df.round(2).to_string())

# Visualise
fig, ax = plt.subplots(figsize=(12, 6))
quality_df[['Completeness (%)', 'Accuracy (%)', 'Consistency (%)', 'Timeliness (%)']].plot(kind='bar', ax=ax)
ax.set_ylabel('Score (%)')
ax.set_title('Data Quality Metrics by Dataset')
ax.axhline(y=90, color='green', linestyle='--', label='Target (90%)')
ax.legend(loc='best')
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('data_quality_dashboard.png', dpi=300, bbox_inches='tight')
plt.show()
print("Data quality dashboard saved as 'data_quality_dashboard.png'")

# ----------------------------------------------------------------
# PART C: DATA LINEAGE SIMULATION
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART C: Data Lineage Simulation")
print("-"*60)

# Simulate data lineage
lineage = {
    "Customer Data": {
        "Sources": ["Core Banking", "CRM", "Digital Channels"],
        "Transformations": ["Data Cleaning", "Data Enrichment", "Identity Resolution"],
        "Consumers": ["Credit Risk Model", "Marketing Campaign", "Customer Analytics"]
    },
    "Transaction Data": {
        "Sources": ["Core Banking", "Payment Systems"],
        "Transformations": ["Data Standardisation", "Transaction Categorisation"],
        "Consumers": ["Fraud Detection", "Customer Analytics", "AML Monitoring"]
    },
    "Loan Data": {
        "Sources": ["Loan Origination System", "Core Banking"],
        "Transformations": ["Data Validation", "Credit Scoring"],
        "Consumers": ["Credit Risk Model", "Regulatory Reporting", "Collections"]
    }
}

print("Data Lineage Examples:")
for dataset, details in lineage.items():
    print(f"\n{dataset}:")
    print(f"  Sources: {', '.join(details['Sources'])}")
    print(f"  Transformations: {', '.join(details['Transformations'])}")
    print(f"  Consumers: {', '.join(details['Consumers'])}")

# ----------------------------------------------------------------
# PART D: FAIRNESS TESTING
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART D: Fairness Testing")
print("-"*60)

# Generate data with protected attribute
np.random.seed(42)
n = 2000

# Group A (favoured) and Group B (protected)
group_a = np.random.choice([0, 1], n, p=[0.7, 0.3])

# Simulate credit data
df_fairness = pd.DataFrame({
    'income': np.random.gamma(5, 20, n) + 20,
    'credit_score': np.random.normal(700, 50, n).clip(550, 850).astype(int),
    'dti': np.random.beta(2, 5, n) * 60,
    'group': group_a
})

# Generate default with slight bias
log_odds = (-4.5 + 0.04 * df_fairness['dti'] - 0.005 * df_fairness['credit_score'])
# Add bias: Group B has higher default probability
log_odds += 0.3 * df_fairness['group']
prob = 1 / (1 + np.exp(-log_odds))
df_fairness['default'] = np.random.binomial(1, prob)

# Train model
features = ['income', 'credit_score', 'dti']
X = df_fairness[features]
y = df_fairness['default']
sensitive = df_fairness['group']

X_train, X_test, y_train, y_test, s_train, s_test = train_test_split(
    X, y, sensitive, test_size=0.3, random_state=42
)

model = RandomForestClassifier(n_estimators=100, max_depth=8, random_state=42)
model.fit(X_train, y_train)
y_pred_proba = model.predict_proba(X_test)[:, 1]
y_pred = (y_pred_proba >= 0.5).astype(int)

# Fairness metrics
def fairness_metrics(y_true, y_pred, sensitive):
    groups = np.unique(sensitive)
    metrics = {}
    
    for g in groups:
        mask = sensitive == g
        metrics[g] = {
            'approval_rate': y_pred[mask].mean(),
            'tpr': np.mean(y_pred[mask] & y_true[mask]) / np.mean(y_true[mask]) if np.mean(y_true[mask]) > 0 else 0,
            'count': mask.sum()
        }
    
    rates = [metrics[g]['approval_rate'] for g in groups]
    tprs = [metrics[g]['tpr'] for g in groups]
    
    return {
        'demographic_parity_difference': max(rates) - min(rates),
        'demographic_parity_ratio': min(rates) / max(rates) if max(rates) > 0 else 0,
        'equal_opportunity_difference': max(tprs) - min(tprs)
    }

fairness = fairness_metrics(y_test, y_pred, s_test)

print("Fairness Metrics:")
print(f"  Demographic Parity Difference: {fairness['demographic_parity_difference']:.4f}")
print(f"  Demographic Parity Ratio: {fairness['demographic_parity_ratio']:.4f}")
print(f"  Equal Opportunity Difference: {fairness['equal_opportunity_difference']:.4f}")

if fairness['demographic_parity_ratio'] >= 0.8:
    print("  ✅ Passes 4/5 Rule")
else:
    print("  ⚠️ Fails 4/5 Rule - Bias Detected")

# ----------------------------------------------------------------
# PART E: ETHICAL AI CHECKLIST
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART E: Ethical AI Checklist")
print("-"*60)

ethical_ai_checklist = [
    "✅ Conducted fairness testing for protected attributes.",
    "✅ Implemented model explainability (SHAP/LIME).",
    "✅ Tested for disparate impact (4/5 rule).",
    "✅ Documented model limitations and assumptions.",
    "✅ Implemented human oversight for critical decisions.",
    "✅ Established model governance and monitoring.",
    "✅ Ensured data privacy (encryption, anonymisation).",
    "✅ Obtained customer consent for data usage.",
    "✅ Provided transparency on AI-driven decisions.",
    "✅ Established appeals process for automated decisions.",
    "✅ Regular ethics reviews and audits.",
    "✅ Engaged diverse stakeholders in model development."
]

print("Ethical AI Checklist:")
for item in ethical_ai_checklist:
    print(f"  {item}")

# ----------------------------------------------------------------
# PART F: DATA GOVERNANCE MATURITY ASSESSMENT
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART F: Data Governance Maturity Assessment")
print("-"*60)

maturity = pd.DataFrame({
    'Dimension': [
        'Data Strategy',
        'Data Quality',
        'Data Lineage',
        'Data Privacy',
        'Data Security',
        'Data Culture',
        'Data Ethics',
        'Regulatory Compliance'
    ],
    'Current Score (1-5)': [3, 3, 2, 3, 3, 2, 2, 3],
    'Target Score (1-5)': [5, 5, 4, 5, 5, 4, 5, 5],
    'Gap': [2, 2, 2, 2, 2, 2, 3, 2]
})

print("Data Governance Maturity Assessment:")
print(maturity.to_string(index=False))

# Visualise
fig, ax = plt.subplots(figsize=(10, 6))
dimensions = maturity['Dimension'].tolist()
current = maturity['Current Score (1-5)'].tolist()
target = maturity['Target Score (1-5)'].tolist()

x = np.arange(len(dimensions))
width = 0.35

ax.barh(x - width/2, current, width, label='Current', color='blue', alpha=0.7)
ax.barh(x + width/2, target, width, label='Target', color='green', alpha=0.7)

ax.set_yticks(x)
ax.set_yticklabels(dimensions)
ax.set_xlabel('Maturity Score (1-5)')
ax.set_title('Data Governance Maturity Assessment')
ax.legend()
ax.grid(True, alpha=0.3, axis='x')

plt.tight_layout()
plt.savefig('governance_maturity.png', dpi=300, bbox_inches='tight')
plt.show()
print("Governance maturity visualisation saved as 'governance_maturity.png'")

# ----------------------------------------------------------------
# PART G: SUMMARY AND RECOMMENDATIONS
# ----------------------------------------------------------------

print("\n" + "="*70)
print("PART G: Summary and Recommendations")
print("="*70)

print("""
Data Governance and Ethics – Key Takeaways:

1. Data governance ensures data quality, security, and compliance.
2. Key principles: fairness, transparency, accountability, privacy, robustness.
3. Data governance framework: strategy, roles, processes, technology, monitoring.
4. Data quality dimensions: accuracy, completeness, consistency, timeliness.
5. Data lineage tracks data from source to consumption.
6. Regulatory landscape: GDPR, CCPA, EU AI Act, SR 11-7, BCBS 239.
7. Ethical AI requires fairness testing, explainability, and human oversight.

Recommendations:
  - Establish a data governance framework.
  - Implement data quality monitoring.
  - Conduct fairness testing for all AI models.
  - Ensure model explainability (SHAP/LIME).
  - Maintain clear documentation for regulatory submissions.
  - Foster a culture of data ethics and responsibility.
""")

print("="*70)
print("END OF LESSON 8 – MODULE 4")
print("="*70)