EDA WORKFLOW FOR BANKING DATA:

Step 1: Define Business Questions
    ↓
Step 2: Collect and Load Data
    ↓
Step 3: Data Profiling (Understand Structure)
    ↓
Step 4: Handle Missing Values
    ↓
Step 5: Univariate Analysis (Single Variables)
    ↓
Step 6: Bivariate Analysis (Two Variables)
    ↓
Step 7: Multivariate Analysis (Multiple Variables)
    ↓
Step 8: Generate Insights
    ↓
Step 9: Document Findings

# ============= IMPORT REQUIRED LIBRARIES =============

# Core data manipulation
import pandas as pd
import numpy as np

# Visualization
import matplotlib.pyplot as plt
import seaborn as sns

# Statistical analysis
from scipy import stats
from scipy.stats import skew, kurtosis

# Handling warnings
import warnings
warnings.filterwarnings('ignore')

# Set visualization style for banking reports
plt.style.use('seaborn-v0_8-darkgrid')
sns.set_palette("Set2")

# Configure pandas display
pd.set_option('display.max_columns', 20)
pd.set_option('display.max_rows', 10)
pd.set_option('display.float_format', '{:.2f}'.format)

print("✅ Environment ready for financial EDA!")

# ============= GENERATE SAMPLE BANKING DATA =============

def generate_banking_dataset():
    """Generate realistic banking data for EDA practice."""
    
    np.random.seed(42)  # For reproducibility
    
    # ---------- Customer Data ----------
    n_customers = 1000
    
    customers = pd.DataFrame({
        'customer_id': range(1, n_customers + 1),
        'age': np.random.normal(45, 15, n_customers).astype(int).clip(18, 85),
        'annual_income': np.random.lognormal(10.5, 0.5, n_customers).astype(int).clip(15000, 300000),
        'credit_score': np.random.normal(680, 60, n_customers).astype(int).clip(500, 850),
        'state': np.random.choice(['CA', 'NY', 'TX', 'FL', 'IL', 'PA', 'OH', 'GA', 'NC', 'MI'], n_customers),
        'segment': np.random.choice(['Premium', 'Standard', 'Basic'], n_customers, p=[0.15, 0.55, 0.30]),
        'tenure_months': np.random.exponential(36, n_customers).astype(int).clip(1, 240),
        'has_mortgage': np.random.choice([0, 1], n_customers, p=[0.4, 0.6]),
        'has_credit_card': np.random.choice([0, 1], n_customers, p=[0.3, 0.7]),
    })
    
    # Introduce some missing values (realistic)
    customers.loc[np.random.choice(customers.index, 50, replace=False), 'credit_score'] = np.nan
    customers.loc[np.random.choice(customers.index, 30, replace=False), 'annual_income'] = np.nan
    
    # ---------- Transaction Data ----------
    n_transactions = 5000
    
    # Generate transaction dates over last 180 days
    base_date = pd.Timestamp('2024-06-01')
    transaction_dates = [base_date - pd.Timedelta(days=np.random.randint(0, 180)) 
                        for _ in range(n_transactions)]
    
    transactions = pd.DataFrame({
        'transaction_id': range(1, n_transactions + 1),
        'customer_id': np.random.randint(1, n_customers + 1, n_transactions),
        'date': transaction_dates,
        'amount': np.random.lognormal(4, 1.0, n_transactions).round(2).clip(1, 10000),
        'transaction_type': np.random.choice(['Deposit', 'Withdrawal', 'Purchase', 'Transfer'], 
                                            n_transactions, p=[0.25, 0.25, 0.35, 0.15]),
        'status': np.random.choice(['Completed', 'Pending', 'Failed'], 
                                  n_transactions, p=[0.85, 0.08, 0.07]),
        'merchant': np.random.choice(['Amazon', 'Walmart', 'Target', 'Starbucks', 'Whole Foods',
                                     'Shell Gas', 'CVS', 'Home Depot', 'Apple Store', 'Netflix',
                                     'McDonalds', 'Uber', 'Spotify', 'Costco', 'Best Buy'],
                                    n_transactions),
        'category': np.random.choice(['Shopping', 'Groceries', 'Dining', 'Transportation',
                                     'Entertainment', 'Health', 'Home Improvement', 'Utilities',
                                     'Travel', 'Education'],
                                    n_transactions)
    })
    
    # ---------- Loan Data ----------
    n_loans = 300
    
    loan_balances = np.random.lognormal(10, 0.8, n_loans).astype(int).clip(10000, 500000)
    
    loans = pd.DataFrame({
        'loan_id': range(1, n_loans + 1),
        'customer_id': np.random.randint(1, n_customers + 1, n_loans),
        'loan_type': np.random.choice(['Mortgage', 'Auto', 'Personal', 'Student'], n_loans, 
                                     p=[0.4, 0.25, 0.2, 0.15]),
        'origination_date': [pd.Timestamp('2024-01-01') - pd.Timedelta(days=np.random.randint(0, 1825)) 
                            for _ in range(n_loans)],
        'original_balance': loan_balances,
        'current_balance': (loan_balances * np.random.uniform(0.3, 0.9, n_loans)).astype(int),
        'interest_rate': np.random.uniform(3.5, 8.5, n_loans).round(2),
        'term_months': np.random.choice([36, 48, 60, 120, 180, 360], n_loans),
        'status': np.random.choice(['Current', 'Delinquent', 'Default', 'Paid Off'], 
                                  n_loans, p=[0.75, 0.10, 0.05, 0.10])
    })
    
    return customers, transactions, loans

# Generate the data
customers, transactions, loans = generate_banking_dataset()

print("✅ Banking data generated successfully!")
print(f"\n📊 Dataset Sizes:")
print(f"  Customers: {len(customers)} records")
print(f"  Transactions: {len(transactions)} records")
print(f"  Loans: {len(loans)} records")

# ============= DATA PROFILING =============

def profile_dataset(df, name):
    """Generate a comprehensive profile of a dataset."""
    
    print(f"\n{'='*60}")
    print(f"DATA PROFILE: {name}")
    print(f"{'='*60}")
    
    # Basic information
    print(f"\n📊 Shape: {df.shape[0]} rows, {df.shape[1]} columns")
    print(f"💾 Memory: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB")
    
    # Column types
    print(f"\n📋 Column Types:")
    for dtype, count in df.dtypes.value_counts().items():
        print(f"  {dtype}: {count} columns")
    
    # Missing values
    missing = df.isnull().sum()
    missing_pct = (missing / len(df) * 100).round(2)
    missing_data = missing[missing > 0]
    
    if len(missing_data) > 0:
        print(f"\n⚠️ Missing Values:")
        for col, count in missing_data.items():
            print(f"  {col}: {count} missing ({missing_pct[col]}%)")
    else:
        print("\n✅ No missing values found")
    
    # Column preview
    print(f"\n🔍 Sample Data:")
    print(df.head())
    
    return missing_data

# Profile each dataset
missing_customers = profile_dataset(customers, "Customers")
missing_transactions = profile_dataset(transactions, "Transactions")
missing_loans = profile_dataset(loans, "Loans")

# ============= DATA TYPES ANALYSIS =============

print("\n" + "="*60)
print("DATA TYPES DETAILED ANALYSIS")
print("="*60)

def analyze_data_types(df, name):
    """Analyze data types and their distributions."""
    
    print(f"\n📊 {name}:")
    
    for col in df.columns:
        dtype = df[col].dtype
        
        if pd.api.types.is_numeric_dtype(dtype):
            # Numeric column
            print(f"  🔢 {col}: {dtype}")
            print(f"     Unique values: {df[col].nunique()}")
            print(f"     Min: {df[col].min():.2f}")
            print(f"     Max: {df[col].max():.2f}")
            print(f"     Mean: {df[col].mean():.2f}")
            print(f"     Median: {df[col].median():.2f}")
            
        elif pd.api.types.is_datetime64_any_dtype(dtype):
            # Date column
            print(f"  📅 {col}: {dtype}")
            print(f"     Range: {df[col].min()} to {df[col].max()}")
            print(f"     Unique values: {df[col].nunique()}")
            
        elif pd.api.types.is_string_dtype(dtype) or pd.api.types.is_object_dtype(dtype):
            # Categorical/string column
            print(f"  📋 {col}: {dtype}")
            top_values = df[col].value_counts().head(3)
            print(f"     Unique values: {df[col].nunique()}")
            print(f"     Top values:")
            for val, count in top_values.items():
                print(f"       {val}: {count} ({count/len(df)*100:.1f}%)")

# Analyze each dataset
analyze_data_types(customers, "Customers")
analyze_data_types(transactions, "Transactions")
analyze_data_types(loans, "Loans")

# ============= SUMMARY STATISTICS =============

print("\n" + "="*60)
print("SUMMARY STATISTICS")
print("="*60)

# Customer numeric columns
numeric_cols = ['age', 'annual_income', 'credit_score', 'tenure_months']

print("\n📊 Customer Numeric Variables:")
print(customers[numeric_cols].describe())

# Calculate additional statistics
print("\n📊 Additional Distribution Statistics:")

for col in numeric_cols:
    data = customers[col].dropna()
    print(f"\n{col}:")
    print(f"  Skewness: {skew(data):.3f}")
    print(f"  Kurtosis: {kurtosis(data):.3f}")
    print(f"  IQR: {data.quantile(0.75) - data.quantile(0.25):.2f}")

# Transaction amount statistics
print("\n💳 Transaction Amounts:")
print(transactions['amount'].describe())
print(f"  Skewness: {skew(transactions['amount']):.3f}")
print(f"  IQR: {transactions['amount'].quantile(0.75) - transactions['amount'].quantile(0.25):.2f}")

# Loan balance statistics
print("\n🏦 Loan Balances:")
print(loans['current_balance'].describe())

# ============= CATEGORICAL ANALYSIS =============

print("\n" + "="*60)
print("CATEGORICAL VARIABLE ANALYSIS")
print("="*60)

def analyze_categorical(df, col_name):
    """Analyze a categorical column."""
    
    print(f"\n📊 {col_name}:")
    
    value_counts = df[col_name].value_counts()
    print(f"  Unique values: {len(value_counts)}")
    
    for value, count in value_counts.head(5).items():
        pct = count / len(df) * 100
        print(f"    {value}: {count} ({pct:.1f}%)")

# Analyze categorical columns
categorical_cols = ['state', 'segment', 'has_mortgage', 'has_credit_card']

for col in categorical_cols:
    if col in customers.columns:
        analyze_categorical(customers, col)

# Transaction categorical analysis
print("\n📊 transaction_type:")
txn_counts = transactions['transaction_type'].value_counts()
for txn_type, count in txn_counts.items():
    pct = count / len(transactions) * 100
    print(f"    {txn_type}: {count} ({pct:.1f}%)")

print("\n📊 status:")
status_counts = transactions['status'].value_counts()
for status, count in status_counts.items():
    pct = count / len(transactions) * 100
    print(f"    {status}: {count} ({pct:.1f}%)")

# ============= DISTRIBUTION VISUALIZATION =============

fig, axes = plt.subplots(2, 3, figsize=(15, 10))

# 1. Customer Age Distribution
customers['age'].hist(bins=30, ax=axes[0, 0], edgecolor='black', alpha=0.7)
axes[0, 0].axvline(customers['age'].mean(), color='red', linestyle='--', 
                   label=f"Mean: {customers['age'].mean():.1f}")
axes[0, 0].axvline(customers['age'].median(), color='blue', linestyle='--', 
                   label=f"Median: {customers['age'].median():.1f}")
axes[0, 0].set_title('Customer Age Distribution', fontsize=12)
axes[0, 0].set_xlabel('Age')
axes[0, 0].set_ylabel('Count')
axes[0, 0].legend()

# 2. Annual Income Distribution
customers['annual_income'].hist(bins=50, ax=axes[0, 1], edgecolor='black', alpha=0.7)
axes[0, 1].axvline(customers['annual_income'].mean(), color='red', linestyle='--', 
                   label=f"Mean: ${customers['annual_income'].mean():,.0f}")
axes[0, 1].axvline(customers['annual_income'].median(), color='blue', linestyle='--', 
                   label=f"Median: ${customers['annual_income'].median():,.0f}")
axes[0, 1].set_title('Annual Income Distribution', fontsize=12)
axes[0, 1].set_xlabel('Annual Income ($)')
axes[0, 1].set_ylabel('Count')
axes[0, 1].legend()

# 3. Credit Score Distribution
customers['credit_score'].dropna().hist(bins=30, ax=axes[0, 2], edgecolor='black', alpha=0.7)
axes[0, 2].axvline(customers['credit_score'].mean(), color='red', linestyle='--', 
                   label=f"Mean: {customers['credit_score'].mean():.1f}")
axes[0, 2].axvline(customers['credit_score'].median(), color='blue', linestyle='--', 
                   label=f"Median: {customers['credit_score'].median():.1f}")
axes[0, 2].set_title('Credit Score Distribution', fontsize=12)
axes[0, 2].set_xlabel('Credit Score')
axes[0, 2].set_ylabel('Count')
axes[0, 2].legend()

# 4. Transaction Amount Distribution
transactions['amount'].hist(bins=50, ax=axes[1, 0], edgecolor='black', alpha=0.7)
axes[1, 0].axvline(transactions['amount'].mean(), color='red', linestyle='--', 
                   label=f"Mean: ${transactions['amount'].mean():.2f}")
axes[1, 0].axvline(transactions['amount'].median(), color='blue', linestyle='--', 
                   label=f"Median: ${transactions['amount'].median():.2f}")
axes[1, 0].set_title('Transaction Amount Distribution', fontsize=12)
axes[1, 0].set_xlabel('Amount ($)')
axes[1, 0].set_ylabel('Count')
axes[1, 0].legend()

# 5. Tenure Distribution
customers['tenure_months'].hist(bins=30, ax=axes[1, 1], edgecolor='black', alpha=0.7)
axes[1, 1].axvline(customers['tenure_months'].mean(), color='red', linestyle='--', 
                   label=f"Mean: {customers['tenure_months'].mean():.1f} months")
axes[1, 1].axvline(customers['tenure_months'].median(), color='blue', linestyle='--', 
                   label=f"Median: {customers['tenure_months'].median():.1f} months")
axes[1, 1].set_title('Customer Tenure Distribution', fontsize=12)
axes[1, 1].set_xlabel('Tenure (months)')
axes[1, 1].set_ylabel('Count')
axes[1, 1].legend()

# 6. Log of Annual Income (to handle skewness)
np.log(customers['annual_income'].dropna()).hist(bins=30, ax=axes[1, 2], edgecolor='black', alpha=0.7)
axes[1, 2].set_title('Log Annual Income Distribution', fontsize=12)
axes[1, 2].set_xlabel('Log(Annual Income)')
axes[1, 2].set_ylabel('Count')

plt.tight_layout()
plt.savefig('distribution_plots.png', dpi=300)
plt.show()

# ============= BOX PLOTS =============

fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# 1. Income by Segment
customers.boxplot(column='annual_income', by='segment', ax=axes[0, 0])
axes[0, 0].set_title('Annual Income by Customer Segment', fontsize=12)
axes[0, 0].set_xlabel('Segment')
axes[0, 0].set_ylabel('Annual Income ($)')

# 2. Credit Score by Segment
customers.boxplot(column='credit_score', by='segment', ax=axes[0, 1])
axes[0, 1].set_title('Credit Score by Customer Segment', fontsize=12)
axes[0, 1].set_xlabel('Segment')
axes[0, 1].set_ylabel('Credit Score')

# 3. Transaction Amount by Type
transactions.boxplot(column='amount', by='transaction_type', ax=axes[1, 0])
axes[1, 0].set_title('Transaction Amount by Type', fontsize=12)
axes[1, 0].set_xlabel('Transaction Type')
axes[1, 0].set_ylabel('Amount ($)')

# 4. Transaction Amount by Status
transactions.boxplot(column='amount', by='status', ax=axes[1, 1])
axes[1, 1].set_title('Transaction Amount by Status', fontsize=12)
axes[1, 1].set_xlabel('Status')
axes[1, 1].set_ylabel('Amount ($)')

plt.tight_layout()
plt.savefig('box_plots.png', dpi=300)
plt.show()

# ============= CATEGORICAL BAR CHARTS =============

fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# 1. Customer Segment Distribution
segment_counts = customers['segment'].value_counts()
segment_counts.plot(kind='bar', ax=axes[0, 0], color=['#2ecc71', '#f1c40f', '#e74c3c'])
axes[0, 0].set_title('Customer Segment Distribution', fontsize=12)
axes[0, 0].set_xlabel('Segment')
axes[0, 0].set_ylabel('Count')
for i, v in enumerate(segment_counts.values):
    axes[0, 0].text(i, v + 10, str(v), ha='center', va='bottom')

# 2. State Distribution
state_counts = customers['state'].value_counts().head(10)
state_counts.plot(kind='bar', ax=axes[0, 1], color='steelblue')
axes[0, 1].set_title('Top 10 States', fontsize=12)
axes[0, 1].set_xlabel('State')
axes[0, 1].set_ylabel('Count')

# 3. Transaction Type Distribution
txn_counts = transactions['transaction_type'].value_counts()
txn_counts.plot(kind='bar', ax=axes[1, 0], color=['#3498db', '#e67e22', '#2ecc71', '#9b59b6'])
axes[1, 0].set_title('Transaction Type Distribution', fontsize=12)
axes[1, 0].set_xlabel('Transaction Type')
axes[1, 0].set_ylabel('Count')

# 4. Top Merchants
top_merchants = transactions['merchant'].value_counts().head(10)
top_merchants.plot(kind='bar', ax=axes[1, 1], color='#e67e22')
axes[1, 1].set_title('Top 10 Merchants', fontsize=12)
axes[1, 1].set_xlabel('Merchant')
axes[1, 1].set_ylabel('Count')
axes[1, 1].tick_params(axis='x', rotation=45)

plt.tight_layout()
plt.savefig('categorical_bar_charts.png', dpi=300)
plt.show()

# ============= MISSING DATA ANALYSIS =============

def analyze_missing_data(df, name):
    """Comprehensive missing data analysis."""
    
    print(f"\n{'='*60}")
    print(f"MISSING DATA ANALYSIS: {name}")
    print(f"{'='*60}")
    
    missing_count = df.isnull().sum()
    missing_pct = (missing_count / len(df) * 100).round(2)
    missing_df = pd.DataFrame({
        'Column': missing_count.index,
        'Missing Count': missing_count.values,
        'Missing %': missing_pct.values
    })
    missing_df = missing_df[missing_df['Missing Count'] > 0]
    
    if missing_df.empty:
        print("✅ No missing values found!")
    else:
        print(f"Total missing values: {missing_df['Missing Count'].sum()}")
        print(f"Overall completeness: {(1 - missing_df['Missing Count'].sum() / (len(df) * len(df.columns))) * 100:.1f}%")
        print("\nMissing by Column:")
        print(missing_df.to_string(index=False))
    
    return missing_df

# Analyze missing data
missing_customers = analyze_missing_data(customers, "Customers")
missing_transactions = analyze_missing_data(transactions, "Transactions")
missing_loans = analyze_missing_data(loans, "Loans")

# Visualize missing data
fig, axes = plt.subplots(1, 3, figsize=(15, 5))

datasets = [('Customers', customers), ('Transactions', transactions), ('Loans', loans)]

for ax, (name, data) in zip(axes, datasets):
    missing = data.isnull().sum()
    missing = missing[missing > 0]
    
    if len(missing) > 0:
        missing.sort_values().plot(kind='barh', ax=ax, color='#e74c3c')
        ax.set_title(f'{name} - Missing Data', fontsize=12)
        ax.set_xlabel('Missing Count')
        ax.set_ylabel('Column')
    else:
        ax.text(0.5, 0.5, '✅ No Missing Data', ha='center', va='center', fontsize=14)
        ax.set_title(f'{name} - Missing Data', fontsize=12)
        ax.axis('off')

plt.tight_layout()
plt.savefig('missing_data_analysis.png', dpi=300)
plt.show()

# ============= OUTLIER DETECTION =============

def detect_outliers_iqr(data, column):
    """Detect outliers using IQR method."""
    q1 = data[column].quantile(0.25)
    q3 = data[column].quantile(0.75)
    iqr = q3 - q1
    
    lower_bound = q1 - 1.5 * iqr
    upper_bound = q3 + 1.5 * iqr
    
    outliers = data[(data[column] < lower_bound) | (data[column] > upper_bound)]
    
    return outliers, lower_bound, upper_bound

print("\n" + "="*60)
print("OUTLIER DETECTION (IQR Method)")
print("="*60)

# Analyze outliers in customer data
columns_to_check = ['age', 'annual_income', 'credit_score', 'tenure_months']

for col in columns_to_check:
    if col in customers.columns and customers[col].notna().any():
        outliers, lower, upper = detect_outliers_iqr(customers, col)
        pct_outliers = len(outliers) / len(customers) * 100
        print(f"\n📊 {col}:")
        print(f"  Normal range: [{lower:.2f}, {upper:.2f}]")
        print(f"  Outliers: {len(outliers)} ({pct_outliers:.1f}%)")
        if len(outliers) > 0:
            print(f"  Outlier range: {outliers[col].min():.2f} to {outliers[col].max():.2f}")

# Transaction amount outliers
outliers, lower, upper = detect_outliers_iqr(transactions, 'amount')
print(f"\n💳 Transaction Amount:")
print(f"  Normal range: [${lower:.2f}, ${upper:.2f}]")
print(f"  Outliers: {len(outliers)} ({len(outliers)/len(transactions)*100:.1f}%)")

# ============= REGULATORY CONSIDERATIONS =============

print("\n" + "="*60)
print("REGULATORY CONSIDERATIONS FOR EDA")
print("="*60)

regulations = [
    {
        'regulation': 'SR 11-7',
        'description': 'Model Risk Management',
        'eda_requirement': 'Must demonstrate data understanding and validation'
    },
    {
        'regulation': 'BASEL III',
        'description': 'Capital Requirements',
        'eda_requirement': 'Data quality and completeness validation'
    },
    {
        'regulation': 'GDPR/CCPA',
        'description': 'Data Privacy',
        'eda_requirement': 'Identify and document personal data'
    },
    {
        'regulation': 'Fair Lending',
        'description': 'Anti-Discrimination',
        'eda_requirement': 'Analyze fairness of models and data'
    }
]

for reg in regulations:
    print(f"\n📋 {reg['regulation']} - {reg['description']}")
    print(f"   EDA Requirement: {reg['eda_requirement']}")