# ============= UNDERSTANDING MISSING DATA TYPES =============

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats

print("="*60)
print("TYPES OF MISSING DATA IN FINANCE")
print("="*60)

# Create examples of each type
np.random.seed(42)
n = 200

# 1. MCAR - Missing Completely at Random
# Example: Random system glitch affects random records
mcar_data = pd.DataFrame({
    'customer_id': range(1, n+1),
    'income': np.random.normal(70000, 20000, n),
    'credit_score': np.random.normal(700, 50, n)
})
# Randomly remove some credit scores
mcar_idx = np.random.choice(mcar_data.index, 30, replace=False)
mcar_data.loc[mcar_idx, 'credit_score'] = np.nan

# 2. MAR - Missing at Random
# Example: Higher income customers less likely to report income
mar_data = pd.DataFrame({
    'customer_id': range(1, n+1),
    'income': np.random.normal(70000, 20000, n),
    'credit_score': np.random.normal(700, 50, n)
})
# Missing income depends on credit_score
mar_idx = mar_data[mar_data['credit_score'] > 750].sample(20).index
mar_data.loc[mar_idx, 'income'] = np.nan

# 3. MNAR - Missing Not at Random
# Example: Customers with bad credit hide their scores
mnar_data = pd.DataFrame({
    'customer_id': range(1, n+1),
    'income': np.random.normal(70000, 20000, n),
    'credit_score': np.random.normal(700, 50, n)
})
# Missing credit_score depends on credit_score itself
mnar_idx = mnar_data[mnar_data['credit_score'] < 600].sample(25).index
mnar_data.loc[mnar_idx, 'credit_score'] = np.nan

print("\n📊 Missing Data Examples:")
print(f"  MCAR: {mcar_data['credit_score'].isnull().sum()} missing values (random)")
print(f"  MAR: {mar_data['income'].isnull().sum()} missing values (depends on credit_score)")
print(f"  MNAR: {mnar_data['credit_score'].isnull().sum()} missing values (depends on itself)")

# Visualize the patterns
fig, axes = plt.subplots(1, 3, figsize=(15, 4))

# MCAR: Missing doesn't depend on other variables
sns.boxplot(data=mcar_data, x=mcar_data['credit_score'].isnull(), y='income', ax=axes[0])
axes[0].set_title('MCAR: Missing Credit Score vs Income', fontsize=12)
axes[0].set_xlabel('Credit Score Missing')
axes[0].set_ylabel('Income')

# MAR: Missing depends on other variable
sns.boxplot(data=mar_data, x=mar_data['income'].isnull(), y='credit_score', ax=axes[1])
axes[1].set_title('MAR: Missing Income vs Credit Score', fontsize=12)
axes[1].set_xlabel('Income Missing')
axes[1].set_ylabel('Credit Score')

# MNAR: Missing depends on itself
sns.boxplot(data=mnar_data, x=mnar_data['credit_score'].isnull(), y='credit_score', ax=axes[2])
axes[2].set_title('MNAR: Missing Credit Score vs Credit Score', fontsize=12)
axes[2].set_xlabel('Credit Score Missing')
axes[2].set_ylabel('Credit Score')

plt.tight_layout()
plt.savefig('missing_data_types.png', dpi=300)
plt.show()

# Statistical test for missing data type
print("\n📊 Missing Data Type Identification:")

# MCAR test: Compare distributions of non-missing vs missing
def test_mcar(data, missing_col, test_col):
    """Test if missing is MCAR by comparing groups."""
    missing_group = data[data[missing_col].isnull()][test_col]
    non_missing_group = data[data[missing_col].notnull()][test_col]
    
    if len(missing_group) > 0 and len(non_missing_group) > 0:
        stat, p_value = stats.ttest_ind(missing_group, non_missing_group)
        return p_value
    return None

# Test if credit_score missing is MCAR (should be random)
p_value = test_mcar(mcar_data, 'credit_score', 'income')
print(f"\n  MCAR Test (Credit Score vs Income): p-value = {p_value:.4f}")
if p_value > 0.05:
    print("    → Likely MCAR (no significant difference)")
else:
    print("    → Not MCAR (significant difference detected)")

# Test if income missing is MAR
p_value = test_mcar(mar_data, 'income', 'credit_score')
print(f"\n  MAR Test (Income vs Credit Score): p-value = {p_value:.4f}")
if p_value < 0.05:
    print("    → Likely MAR (missing depends on credit_score)")
else:
    print("    → Not clearly MAR")

# Test if credit_score missing is MNAR
p_value = test_mcar(mnar_data, 'credit_score', 'credit_score')
print(f"\n  MNAR Test (Credit Score vs Credit Score): p-value = {p_value:.4f}")
if p_value < 0.05:
    print("    → Likely MNAR (missing depends on the value itself)")
else:
    print("    → Not clearly MNAR")

# ============= COMPREHENSIVE MISSING DATA ANALYSIS =============

# Load sample banking data with missing values
np.random.seed(42)
n = 500

banking_data = pd.DataFrame({
    'customer_id': range(1, n+1),
    'age': np.random.normal(45, 15, n).astype(int).clip(18, 85),
    'income': np.random.lognormal(10.5, 0.5, n).astype(int).clip(15000, 300000),
    'credit_score': np.random.normal(680, 60, n).astype(int).clip(500, 850),
    'dti_ratio': np.random.uniform(0.1, 0.5, n),
    'savings': np.random.lognormal(9, 1.2, n).astype(int).clip(0, 200000),
    'tenure_years': np.random.exponential(5, n).round(1).clip(0.5, 20),
    'num_late_payments': np.random.poisson(0.5, n).clip(0, 5)
})

# Introduce missing values with realistic patterns
# Income missing (MAR - depends on credit score)
income_missing = banking_data[banking_data['credit_score'] > 750].sample(30).index
banking_data.loc[income_missing, 'income'] = np.nan

# Credit score missing (MNAR - depends on itself)
credit_missing = banking_data[banking_data['credit_score'] < 600].sample(20).index
banking_data.loc[credit_missing, 'credit_score'] = np.nan

# DTI ratio missing (MCAR - random)
dti_missing = np.random.choice(banking_data.index, 25, replace=False)
banking_data.loc[dti_missing, 'dti_ratio'] = np.nan

# Savings missing (MCAR - random)
savings_missing = np.random.choice(banking_data.index, 15, replace=False)
banking_data.loc[savings_missing, 'savings'] = np.nan

# Function for comprehensive missing data analysis
class MissingDataAnalyzer:
    """Comprehensive missing data analysis class."""
    
    def __init__(self, data):
        self.data = data
        self.missing_info = None
        self.visual_patterns = {}
    
    def analyze_missing(self):
        """Analyze missing data patterns."""
        missing_count = self.data.isnull().sum()
        missing_pct = (missing_count / len(self.data) * 100).round(2)
        
        self.missing_info = pd.DataFrame({
            'Column': missing_count.index,
            'Missing Count': missing_count.values,
            'Missing %': missing_pct.values,
            'Data Type': self.data.dtypes.values
        })
        self.missing_info = self.missing_info[self.missing_info['Missing Count'] > 0]
        self.missing_info = self.missing_info.sort_values('Missing %', ascending=False)
        
        return self.missing_info
    
    def missing_by_column_type(self):
        """Analyze missing patterns by column type."""
        result = {}
        for col in self.missing_info['Column']:
            dtype = self.data[col].dtype
            if pd.api.types.is_numeric_dtype(dtype):
                result[col] = 'Numeric'
            else:
                result[col] = 'Categorical'
        return result
    
    def missing_vs_complete_comparison(self, numeric_cols):
        """Compare distributions of missing vs complete records."""
        comparisons = {}
        
        for col in numeric_cols:
            if col in self.data.columns:
                missing_group = self.data[self.data[col].isnull()]
                complete_group = self.data[self.data[col].notnull()]
                
                if len(missing_group) > 0 and len(complete_group) > 0:
                    comparisons[col] = {
                        'missing_mean': missing_group.mean(numeric_only=True).to_dict(),
                        'complete_mean': complete_group.mean(numeric_only=True).to_dict(),
                        'difference': (complete_group.mean(numeric_only=True) - missing_group.mean(numeric_only=True)).to_dict()
                    }
        
        return comparisons
    
    def create_visualization(self, figsize=(14, 8)):
        """Create missing data visualization."""
        fig, axes = plt.subplots(2, 2, figsize=figsize)
        
        # 1. Missing data heatmap
        missing_matrix = self.data.isnull().astype(int)
        sns.heatmap(missing_matrix.T, ax=axes[0, 0], cmap='RdYlBu_r', 
                   cbar_kws={'label': 'Missing'})
        axes[0, 0].set_title('Missing Data Heatmap', fontsize=12)
        axes[0, 0].set_xlabel('Row Index')
        axes[0, 0].set_ylabel('Columns')
        
        # 2. Missing percentage bar chart
        if len(self.missing_info) > 0:
            bars = axes[0, 1].bar(self.missing_info['Column'], self.missing_info['Missing %'], 
                                color='#e74c3c')
            axes[0, 1].set_title('Missing Percentage by Column', fontsize=12)
            axes[0, 1].set_xlabel('Column')
            axes[0, 1].set_ylabel('Missing %')
            axes[0, 1].axhline(5, color='orange', linestyle='--', label='5% threshold')
            axes[0, 1].axhline(20, color='red', linestyle='--', label='20% threshold')
            axes[0, 1].legend()
            axes[0, 1].tick_params(axis='x', rotation=45)
        else:
            axes[0, 1].text(0.5, 0.5, '✅ No Missing Values', 
                           ha='center', va='center', fontsize=14)
            axes[0, 1].set_title('Missing Values', fontsize=12)
            axes[0, 1].axis('off')
        
        # 3. Distribution comparison (missing vs complete)
        numeric_cols = self.data.select_dtypes(include=[np.number]).columns
        cols_with_missing = [col for col in numeric_cols if self.data[col].isnull().any()]
        
        if cols_with_missing:
            col_to_plot = cols_with_missing[0] if cols_with_missing else None
            if col_to_plot:
                missing = self.data[self.data[col_to_plot].isnull()][col_to_plot]
                complete = self.data[self.data[col_to_plot].notnull()][col_to_plot]
                
                axes[1, 0].hist(complete, bins=30, alpha=0.6, label='Complete', color='green')
                axes[1, 0].hist(missing, bins=20, alpha=0.6, label='Missing', color='red')
                axes[1, 0].set_title(f'Distribution: {col_to_plot}', fontsize=12)
                axes[1, 0].set_xlabel(col_to_plot)
                axes[1, 0].set_ylabel('Frequency')
                axes[1, 0].legend()
        else:
            axes[1, 0].text(0.5, 0.5, 'No numeric columns with missing data', 
                           ha='center', va='center', fontsize=14)
            axes[1, 0].set_title('Distribution Comparison', fontsize=12)
        
        # 4. Missing pattern matrix
        if len(self.missing_info) > 1:
            # Show which columns tend to be missing together
            missing_matrix = self.data.isnull()
            correlation = missing_matrix.corr()
            sns.heatmap(correlation, ax=axes[1, 1], cmap='RdBu_r', center=0,
                       annot=True, fmt='.2f', cbar_kws={'label': 'Correlation'})
            axes[1, 1].set_title('Missing Data Correlation', fontsize=12)
        else:
            axes[1, 1].text(0.5, 0.5, 'Too few missing columns for correlation', 
                           ha='center', va='center', fontsize=14)
            axes[1, 1].set_title('Missing Pattern', fontsize=12)
            axes[1, 1].axis('off')
        
        plt.tight_layout()
        plt.savefig('missing_data_analysis.png', dpi=300)
        plt.show()
    
    def generate_report(self):
        """Generate comprehensive missing data report."""
        self.analyze_missing()
        
        print("\n" + "="*80)
        print("MISSING DATA ANALYSIS REPORT")
        print("="*80)
        
        if len(self.missing_info) == 0:
            print("\n✅ No missing data found in the dataset!")
            return
        
        total_missing = self.missing_info['Missing Count'].sum()
        total_cells = len(self.data) * len(self.data.columns)
        overall_completeness = (1 - total_missing / total_cells) * 100
        
        print(f"\n📊 Overall Statistics:")
        print(f"  Total Missing Values: {total_missing}")
        print(f"  Overall Completeness: {overall_completeness:.1f}%")
        print(f"  Columns with Missing Data: {len(self.missing_info)}")
        
        print(f"\n📋 Missing Data by Column:")
        print(self.missing_info.to_string(index=False))
        
        # Recommendations
        print("\n💡 Recommendations:")
        for _, row in self.missing_info.iterrows():
            pct = row['Missing %']
            col = row['Column']
            if pct > 20:
                print(f"  • {col}: {pct}% missing - Consider dropping or advanced imputation")
            elif pct > 5:
                print(f"  • {col}: {pct}% missing - Consider imputation with caution")
            else:
                print(f"  • {col}: {pct}% missing - Safe to impute with simple methods")

# Analyze missing data
analyzer = MissingDataAnalyzer(banking_data)
analyzer.analyze_missing()
analyzer.create_visualization()
analyzer.generate_report()

# ============= DELETION METHODS =============

class DeletionMethods:
    """Methods for handling missing data through deletion."""
    
    def __init__(self, data):
        self.data = data
        self.results = {}
    
    def listwise_deletion(self):
        """Delete all rows with any missing values."""
        original_shape = self.data.shape
        cleaned = self.data.dropna()
        removed = original_shape[0] - cleaned.shape[0]
        
        self.results['listwise'] = {
            'original_rows': original_shape[0],
            'remaining_rows': cleaned.shape[0],
            'removed_rows': removed,
            'removed_pct': removed / original_shape[0] * 100,
            'data': cleaned
        }
        
        return cleaned
    
    def pairwise_deletion(self):
        """
        Delete only the missing values for each analysis.
        (This is a concept - in practice, implemented per analysis)
        """
        # Just for reporting
        missing_counts = self.data.isnull().sum()
        self.results['pairwise'] = {
            'description': 'Deletes missing values only for the specific analysis being performed',
            'note': 'Useful when different analyses use different variables'
        }
        
        return self.data
    
    def column_deletion(self, threshold=0.5):
        """Delete columns with more than threshold missing values."""
        missing_pct = self.data.isnull().mean()
        cols_to_drop = missing_pct[missing_pct > threshold].index.tolist()
        
        cleaned = self.data.drop(columns=cols_to_drop)
        
        self.results['column_deletion'] = {
            'dropped_columns': cols_to_drop,
            'remaining_columns': cleaned.shape[1],
            'data': cleaned
        }
        
        return cleaned
    
    def row_deletion_threshold(self, threshold=0.5):
        """Delete rows with more than threshold missing values."""
        missing_pct_per_row = self.data.isnull().mean(axis=1)
        rows_to_drop = self.data[missing_pct_per_row > threshold].index
        
        cleaned = self.data.drop(index=rows_to_drop)
        
        self.results['row_deletion_threshold'] = {
            'dropped_rows': len(rows_to_drop),
            'remaining_rows': cleaned.shape[0],
            'threshold': threshold,
            'data': cleaned
        }
        
        return cleaned
    
    def compare_deletion_methods(self):
        """Compare different deletion methods."""
        print("\n" + "="*60)
        print("DELETION METHODS COMPARISON")
        print("="*60)
        
        # Run listwise deletion
        listwise_result = self.listwise_deletion()
        
        # Run column deletion (50% threshold)
        column_result = self.column_deletion(threshold=0.5)
        
        # Run row deletion (50% threshold)
        row_result = self.row_deletion_threshold(threshold=0.5)
        
        print(f"\n📊 Original Data: {self.data.shape[0]} rows, {self.data.shape[1]} columns")
        
        print(f"\n📋 Listwise Deletion:")
        print(f"  Remaining: {listwise_result.shape[0]} rows ({listwise_result.shape[0]/self.data.shape[0]*100:.1f}%)")
        print(f"  Removed: {self.data.shape[0] - listwise_result.shape[0]} rows")
        
        print(f"\n📋 Column Deletion (>50% missing):")
        dropped_cols = [col for col in self.data.columns if self.data[col].isnull().mean() > 0.5]
        print(f"  Dropped columns: {dropped_cols if dropped_cols else 'None'}")
        print(f"  Remaining columns: {column_result.shape[1]}")
        
        print(f"\n📋 Row Deletion (>50% missing):")
        print(f"  Remaining: {row_result.shape[0]} rows ({row_result.shape[0]/self.data.shape[0]*100:.1f}%)")
        print(f"  Dropped: {self.data.shape[0] - row_result.shape[0]} rows")
        
        print("\n💡 Recommendation:")
        if self.data.isnull().sum().sum() < self.data.shape[0] * 0.01:
            print("  • Missing data is minimal (<1%) - listwise deletion is safe")
        elif any(self.data.isnull().mean() > 0.5):
            print("  • Some columns have >50% missing - consider column deletion")
        else:
            print("  • Moderate missing data - consider imputation rather than deletion")

# Compare deletion methods
deletion = DeletionMethods(banking_data)
deletion.compare_deletion_methods()

# ============= SIMPLE IMPUTATION METHODS =============

from sklearn.impute import SimpleImputer

class SimpleImputation:
    """Simple imputation methods for financial data."""
    
    def __init__(self, data):
        self.data = data.copy()
        self.imputed_data = None
        self.imputation_stats = {}
    
    def impute_mean(self):
        """Impute missing values with column mean."""
        numeric_cols = self.data.select_dtypes(include=[np.number]).columns
        self.imputed_data = self.data.copy()
        
        for col in numeric_cols:
            if self.imputed_data[col].isnull().any():
                mean_val = self.imputed_data[col].mean()
                self.imputed_data[col] = self.imputed_data[col].fillna(mean_val)
                self.imputation_stats[col] = {'method': 'mean', 'value': mean_val}
        
        return self.imputed_data
    
    def impute_median(self):
        """Impute missing values with column median."""
        numeric_cols = self.data.select_dtypes(include=[np.number]).columns
        self.imputed_data = self.data.copy()
        
        for col in numeric_cols:
            if self.imputed_data[col].isnull().any():
                median_val = self.imputed_data[col].median()
                self.imputed_data[col] = self.imputed_data[col].fillna(median_val)
                self.imputation_stats[col] = {'method': 'median', 'value': median_val}
        
        return self.imputed_data
    
    def impute_mode(self):
        """Impute missing values with column mode."""
        self.imputed_data = self.data.copy()
        
        for col in self.data.columns:
            if self.imputed_data[col].isnull().any():
                mode_val = self.imputed_data[col].mode()[0] if not self.imputed_data[col].mode().empty else None
                if mode_val is not None:
                    self.imputed_data[col] = self.imputed_data[col].fillna(mode_val)
                    self.imputation_stats[col] = {'method': 'mode', 'value': mode_val}
        
        return self.imputed_data
    
    def impute_constant(self, constant_value=0):
        """Impute missing values with a constant."""
        self.imputed_data = self.data.copy()
        
        for col in self.data.columns:
            if self.imputed_data[col].isnull().any():
                self.imputed_data[col] = self.imputed_data[col].fillna(constant_value)
                self.imputation_stats[col] = {'method': 'constant', 'value': constant_value}
        
        return self.imputed_data
    
    def compare_methods(self):
        """Compare different imputation methods."""
        print("\n" + "="*60)
        print("SIMPLE IMPUTATION METHODS COMPARISON")
        print("="*60)
        
        # Store each imputed dataset
        methods = {
            'Original': self.data,
            'Mean': self.impute_mean(),
            'Median': self.impute_median(),
            'Mode': self.impute_mode(),
            'Constant (0)': self.impute_constant(0)
        }
        
        # Compare statistics for a column with missing data
        cols_with_missing = [col for col in self.data.columns if self.data[col].isnull().any()]
        
        for col in cols_with_missing[:2]:  # Show first 2 columns
            print(f"\n📊 Column: {col}")
            print(f"  Original missing count: {self.data[col].isnull().sum()}")
            
            for method_name, method_data in methods.items():
                if method_name != 'Original':
                    print(f"  {method_name}: imputed {method_data[col].isnull().sum()} missing")
                    print(f"    Mean: {method_data[col].mean():.2f}")
                    print(f"    Std: {method_data[col].std():.2f}")
                    print(f"    Min: {method_data[col].min():.2f}")
                    print(f"    Max: {method_data[col].max():.2f}")
        
        print("\n💡 Recommendation:")
        print("  • Mean/Median: Best for normally distributed data")
        print("  • Mode: Best for categorical data")
        print("  • Constant: Use for business rules (e.g., default values)")
        print("  • Consider the business context when choosing")

# Compare simple imputation methods
simple_imputer = SimpleImputation(banking_data)
simple_imputer.compare_methods()

# ============= ADVANCED IMPUTATION METHODS =============

from sklearn.impute import KNNImputer
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.ensemble import RandomForestRegressor

class AdvancedImputation:
    """Advanced imputation methods for financial data."""
    
    def __init__(self, data):
        self.data = data.copy()
        self.imputed_data = None
        self.imputation_stats = {}
    
    def impute_knn(self, n_neighbors=5):
        """K-Nearest Neighbors imputation."""
        numeric_cols = self.data.select_dtypes(include=[np.number]).columns
        data_numeric = self.data[numeric_cols].copy()
        
        imputer = KNNImputer(n_neighbors=n_neighbors)
        imputed_array = imputer.fit_transform(data_numeric)
        
        self.imputed_data = self.data.copy()
        self.imputed_data[numeric_cols] = imputed_array
        
        self.imputation_stats['knn'] = {
            'method': 'KNN',
            'n_neighbors': n_neighbors,
            'columns_imputed': numeric_cols.tolist()
        }
        
        return self.imputed_data
    
    def impute_mice(self, max_iter=10, random_state=42):
        """
        Multiple Imputation by Chained Equations (MICE).
        """
        numeric_cols = self.data.select_dtypes(include=[np.number]).columns
        data_numeric = self.data[numeric_cols].copy()
        
        imputer = IterativeImputer(max_iter=max_iter, random_state=random_state)
        imputed_array = imputer.fit_transform(data_numeric)
        
        self.imputed_data = self.data.copy()
        self.imputed_data[numeric_cols] = imputed_array
        
        self.imputation_stats['mice'] = {
            'method': 'MICE',
            'max_iter': max_iter,
            'columns_imputed': numeric_cols.tolist()
        }
        
        return self.imputed_data
    
    def impute_random_forest(self, columns_to_impute):
        """
        Impute using Random Forest (predict missing values).
        """
        self.imputed_data = self.data.copy()
        
        for col in columns_to_impute:
            if col not in self.data.columns:
                continue
                
            # Split into training (complete) and prediction (missing)
            train_data = self.imputed_data[self.imputed_data[col].notnull()]
            pred_data = self.imputed_data[self.imputed_data[col].isnull()]
            
            if len(pred_data) == 0:
                continue
                
            # Prepare features (other numeric columns)
            feature_cols = [c for c in self.data.columns 
                          if c != col and c in self.data.select_dtypes(include=[np.number]).columns]
            
            if len(feature_cols) == 0:
                continue
                
            # Train model
            X_train = train_data[feature_cols]
            y_train = train_data[col]
            
            # Handle any missing in features
            X_train = X_train.fillna(X_train.mean())
            
            model = RandomForestRegressor(n_estimators=50, random_state=42)
            model.fit(X_train, y_train)
            
            # Predict missing values
            X_pred = pred_data[feature_cols].fillna(X_train.mean())
            y_pred = model.predict(X_pred)
            
            # Fill missing values
            self.imputed_data.loc[self.imputed_data[col].isnull(), col] = y_pred
            
            self.imputation_stats[f'rf_{col}'] = {
                'method': 'Random Forest',
                'feature_columns': feature_cols,
                'n_predictions': len(y_pred)
            }
        
        return self.imputed_data
    
    def impute_by_group(self, group_col, columns_to_impute):
        """
        Impute missing values using group statistics.
        """
        self.imputed_data = self.data.copy()
        
        for col in columns_to_impute:
            if col not in self.data.columns:
                continue
                
            # Calculate group means
            group_means = self.data.groupby(group_col)[col].mean()
            
            # Impute missing values with group mean
            for group, mean_val in group_means.items():
                mask = (self.imputed_data[group_col] == group) & (self.imputed_data[col].isnull())
                self.imputed_data.loc[mask, col] = mean_val
        
        self.imputation_stats['by_group'] = {
            'method': 'Group Imputation',
            'group_column': group_col,
            'columns_imputed': columns_to_impute
        }
        
        return self.imputed_data
    
    def compare_advanced_methods(self):
        """Compare advanced imputation methods."""
        print("\n" + "="*60)
        print("ADVANCED IMPUTATION METHODS COMPARISON")
        print("="*60)
        
        # Test on a column with missing data
        cols_with_missing = [col for col in self.data.columns 
                           if self.data[col].isnull().any() and 
                           pd.api.types.is_numeric_dtype(self.data[col])]
        
        if not cols_with_missing:
            print("No numeric columns with missing data to compare")
            return
        
        test_col = cols_with_missing[0]
        
        print(f"\n📊 Testing on column: {test_col}")
        print(f"  Original missing: {self.data[test_col].isnull().sum()} values")
        
        # Different imputation methods
        methods = {
            'KNN (k=3)': lambda: self.impute_knn(n_neighbors=3)[test_col],
            'KNN (k=5)': lambda: self.impute_knn(n_neighbors=5)[test_col],
            'MICE': lambda: self.impute_mice(max_iter=10)[test_col],
        }
        
        for method_name, method_func in methods.items():
            # Reset data
            self.imputed_data = self.data.copy()
            imputed_col = method_func()
            print(f"\n  {method_name}:")
            print(f"    Missing after: {imputed_col.isnull().sum()}")
            print(f"    Mean: {imputed_col.mean():.2f}")
            print(f"    Std: {imputed_col.std():.2f}")
            print(f"    Min: {imputed_col.min():.2f}")
            print(f"    Max: {imputed_col.max():.2f}")

# Compare advanced imputation methods
advanced_imputer = AdvancedImputation(banking_data)
advanced_imputer.compare_advanced_methods()

# ============= IMPUTATION EVALUATION =============

class ImputationEvaluator:
    """Evaluate the quality of different imputation methods."""
    
    def __init__(self, original_data, imputed_data, method_name):
        self.original = original_data
        self.imputed = imputed_data
        self.method_name = method_name
        
    def evaluate_numeric(self, column):
        """Evaluate imputation for a numeric column."""
        # Get original complete data (for comparison)
        original_complete = self.original[self.original[column].notnull()]
        imputed_values = self.imputed[self.original[column].isnull()]
        
        if len(imputed_values) == 0:
            return {'status': 'No missing values to impute'}
        
        # Compare distributions
        original_mean = original_complete[column].mean()
        imputed_mean = imputed_values[column].mean()
        
        original_std = original_complete[column].std()
        imputed_std = imputed_values[column].std()
        
        # Calculate metrics
        mean_diff = abs(original_mean - imputed_mean) / original_mean * 100
        std_ratio = imputed_std / original_std if original_std > 0 else 1
        
        return {
            'column': column,
            'n_imputed': len(imputed_values),
            'original_mean': original_mean,
            'imputed_mean': imputed_mean,
            'mean_diff_pct': mean_diff,
            'original_std': original_std,
            'imputed_std': imputed_std,
            'std_ratio': std_ratio,
            'quality': 'Good' if mean_diff < 5 and 0.5 < std_ratio < 2 else 'Fair' if mean_diff < 15 else 'Poor'
        }
    
    def evaluate_categorical(self, column):
        """Evaluate imputation for a categorical column."""
        # Get original complete data (for comparison)
        original_complete = self.original[self.original[column].notnull()]
        imputed_values = self.imputed[self.original[column].isnull()]
        
        if len(imputed_values) == 0:
            return {'status': 'No missing values to impute'}
        
        # Compare mode distribution
        original_mode = original_complete[column].mode()[0] if not original_complete[column].mode().empty else None
        imputed_mode = imputed_values[column].mode()[0] if not imputed_values[column].mode().empty else None
        
        # Compare category distributions
        original_dist = original_complete[column].value_counts(normalize=True)
        imputed_dist = imputed_values[column].value_counts(normalize=True)
        
        # Calculate similarity (shared categories)
        common_categories = set(original_dist.index) & set(imputed_dist.index)
        similarity = len(common_categories) / len(set(original_dist.index) | set(imputed_dist.index)) if len(set(original_dist.index) | set(imputed_dist.index)) > 0 else 0
        
        return {
            'column': column,
            'n_imputed': len(imputed_values),
            'original_mode': original_mode,
            'imputed_mode': imputed_mode,
            'similarity': similarity,
            'quality': 'Good' if similarity > 0.8 else 'Fair' if similarity > 0.5 else 'Poor'
        }
    
    def generate_report(self):
        """Generate imputation quality report."""
        print(f"\n{'='*60}")
        print(f"IMPUTATION QUALITY REPORT: {self.method_name}")
        print(f"{'='*60}")
        
        results = []
        
        # Evaluate numeric columns
        numeric_cols = self.original.select_dtypes(include=[np.number]).columns
        for col in numeric_cols:
            if self.original[col].isnull().any():
                result = self.evaluate_numeric(col)
                if 'status' not in result:
                    results.append(result)
                    print(f"\n📊 {col}:")
                    print(f"  Imputed: {result['n_imputed']} values")
                    print(f"  Original Mean: {result['original_mean']:.2f}")
                    print(f"  Imputed Mean: {result['imputed_mean']:.2f}")
                    print(f"  Mean Difference: {result['mean_diff_pct']:.1f}%")
                    print(f"  Quality: {result['quality']}")
        
        # Evaluate categorical columns
        cat_cols = self.original.select_dtypes(include=['object']).columns
        for col in cat_cols:
            if self.original[col].isnull().any():
                result = self.evaluate_categorical(col)
                if 'status' not in result:
                    results.append(result)
                    print(f"\n📋 {col}:")
                    print(f"  Imputed: {result['n_imputed']} values")
                    print(f"  Original Mode: {result['original_mode']}")
                    print(f"  Imputed Mode: {result['imputed_mode']}")
                    print(f"  Similarity: {result['similarity']:.1%}")
                    print(f"  Quality: {result['quality']}")
        
        # Summary
        if results:
            good = sum(1 for r in results if r['quality'] == 'Good')
            fair = sum(1 for r in results if r['quality'] == 'Fair')
            poor = sum(1 for r in results if r['quality'] == 'Poor')
            
            print(f"\n{'='*60}")
            print(f"SUMMARY: {good} Good, {fair} Fair, {poor} Poor")
            print(f"{'='*60}")

# Evaluate imputation
# Use one of the imputation methods
imputer = AdvancedImputation(banking_data)
imputed_data = imputer.impute_knn(n_neighbors=5)

evaluator = ImputationEvaluator(banking_data, imputed_data, "KNN Imputation")
evaluator.generate_report()

# ============= BUSINESS IMPACT ANALYSIS =============

def analyze_business_impact():
    """Analyze the business impact of improper missing data handling."""
    
    print("\n" + "="*80)
    print("BUSINESS IMPACT OF IMPROPER MISSING DATA HANDLING")
    print("="*80)
    
    impacts = {
        'Credit Risk Models': {
            'risk': 'High',
            'impact': 'Incorrect credit decisions, increased defaults',
            'estimated_cost': '$100M - $500M',
            'scenario': 'Missing income data leads to underestimating debt-to-income ratio'
        },
        'Fraud Detection': {
            'risk': 'High',
            'impact': 'Undetected fraud, financial losses',
            'estimated_cost': '$50M - $200M',
            'scenario': 'Missing transaction data hides fraud patterns'
        },
        'Regulatory Reporting': {
            'risk': 'Critical',
            'impact': 'Regulatory fines, reputational damage',
            'estimated_cost': '$100M - $1B',
            'scenario': 'Incomplete BASEL III reporting due to missing data'
        },
        'Customer Analytics': {
            'risk': 'Medium',
            'impact': 'Poor customer targeting, lost revenue',
            'estimated_cost': '$10M - $50M',
            'scenario': 'Missing customer attributes leads to incorrect segmentation'
        }
    }
    
    for area, details in impacts.items():
        print(f"\n🔴 {area}")
        print(f"   Risk Level: {details['risk']}")
        print(f"   Impact: {details['impact']}")
        print(f"   Estimated Cost: {details['estimated_cost']}")
        print(f"   Scenario: {details['scenario']}")
    
    print("\n" + "-"*40)
    print("💡 Best Practices for Missing Data Handling:")
    print("  1. Always document missing data patterns")
    print("  2. Use appropriate imputation methods for each variable")
    print("  3. Validate imputation quality")
    print("  4. Consider the business context")
    print("  5. Maintain data lineage for regulatory compliance")

analyze_business_impact()

# ============= TYPES OF OUTLIERS =============

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats

print("="*60)
print("TYPES OF OUTLIERS IN FINANCIAL DATA")
print("="*60)

# Create example data
np.random.seed(42)
n = 200

# Point outlier (single extreme value)
point_data = np.random.normal(100, 20, n)
point_data[175] = 350  # Point outlier

# Contextual outlier (value is extreme in context)
contextual_data = np.random.normal(100, 20, n)
contextual_data[50:55] = contextual_data[50:55] * 2  # Contextual outliers

# Collective outlier (group of extreme values)
collective_data = np.random.normal(100, 20, n)
collective_data[180:190] = np.random.normal(250, 10, 10)  # Collective outliers

# Create DataFrame
outlier_data = pd.DataFrame({
    'point_outliers': point_data,
    'contextual_outliers': contextual_data,
    'collective_outliers': collective_data
})

# Visualize different types
fig, axes = plt.subplots(1, 3, figsize=(15, 4))

for i, (col, ax) in enumerate(zip(outlier_data.columns, axes)):
    sns.boxplot(y=outlier_data[col], ax=ax)
    ax.set_title(f'{col.replace("_", " ").title()}', fontsize=12)
    ax.set_ylabel('Value')

plt.tight_layout()
plt.savefig('outlier_types.png', dpi=300)
plt.show()

print("\n📊 Outlier Types:")
print("  1. Point Outlier: Single extreme value")
print("  2. Contextual Outlier: Value extreme in specific context")
print("  3. Collective Outlier: Group of extreme values together")

# ============= IQR OUTLIER DETECTION =============

class IQROutlierDetector:
    """Outlier detection using IQR method."""
    
    def __init__(self, data, multiplier=1.5):
        self.data = data
        self.multiplier = multiplier
        self.outliers = {}
        self.bounds = {}
    
    def detect(self, column):
        """Detect outliers using IQR method."""
        if column not in self.data.columns:
            return None
        
        data = self.data[column].dropna()
        
        q1 = data.quantile(0.25)
        q3 = data.quantile(0.75)
        iqr = q3 - q1
        
        lower_bound = q1 - self.multiplier * iqr
        upper_bound = q3 + self.multiplier * iqr
        
        outliers = data[(data < lower_bound) | (data > upper_bound)]
        
        self.outliers[column] = outliers
        self.bounds[column] = (lower_bound, upper_bound)
        
        return outliers
    
    def detect_all(self):
        """Detect outliers for all numeric columns."""
        numeric_cols = self.data.select_dtypes(include=[np.number]).columns
        
        for col in numeric_cols:
            self.detect(col)
        
        return self.outliers
    
    def get_outlier_info(self):
        """Get information about detected outliers."""
        info = {}
        
        for col, outliers in self.outliers.items():
            if len(outliers) > 0:
                lower, upper = self.bounds[col]
                info[col] = {
                    'n_outliers': len(outliers),
                    'percentage': len(outliers) / len(self.data) * 100,
                    'lower_bound': lower,
                    'upper_bound': upper,
                    'min_outlier': outliers.min(),
                    'max_outlier': outliers.max(),
                    'outlier_values': outliers.head(10).tolist()
                }
        
        return info
    
    def report(self):
        """Generate outlier report."""
        self.detect_all()
        
        print("\n" + "="*60)
        print("IQR OUTLIER DETECTION REPORT")
        print("="*60)
        
        info = self.get_outlier_info()
        
        if not info:
            print("\n✅ No outliers detected!")
            return
        
        for col, details in info.items():
            print(f"\n📊 {col}:")
            print(f"  Outliers: {details['n_outliers']} ({details['percentage']:.1f}%)")
            print(f"  Normal Range: [{details['lower_bound']:.2f}, {details['upper_bound']:.2f}]")
            print(f"  Outlier Range: [{details['min_outlier']:.2f}, {details['max_outlier']:.2f}]")
            print(f"  Sample Outliers: {details['outlier_values'][:5]}")

# Test IQR detection
iqr_detector = IQROutlierDetector(banking_data)
iqr_detector.report()

# ============= Z-SCORE OUTLIER DETECTION =============

from scipy import stats

class ZScoreOutlierDetector:
    """Outlier detection using Z-score and Modified Z-score."""
    
    def __init__(self, data, threshold=3):
        self.data = data
        self.threshold = threshold
        self.outliers = {}
    
    def detect_zscore(self, column):
        """Detect outliers using Z-score."""
        if column not in self.data.columns:
            return None
        
        data = self.data[column].dropna()
        z_scores = np.abs(stats.zscore(data))
        outliers = data[z_scores > self.threshold]
        
        self.outliers[f'{column}_zscore'] = {
            'column': column,
            'method': 'Z-score',
            'threshold': self.threshold,
            'outliers': outliers,
            'n_outliers': len(outliers),
            'percentage': len(outliers) / len(data) * 100 if len(data) > 0 else 0
        }
        
        return outliers
    
    def detect_modified_zscore(self, column):
        """Detect outliers using Modified Z-score (based on median)."""
        if column not in self.data.columns:
            return None
        
        data = self.data[column].dropna()
        median = data.median()
        mad = stats.median_abs_deviation(data)
        
        if mad == 0:
            return None
        
        modified_z_scores = 0.6745 * (data - median) / mad
        outliers = data[np.abs(modified_z_scores) > self.threshold]
        
        self.outliers[f'{column}_modified'] = {
            'column': column,
            'method': 'Modified Z-score',
            'threshold': self.threshold,
            'outliers': outliers,
            'n_outliers': len(outliers),
            'percentage': len(outliers) / len(data) * 100 if len(data) > 0 else 0
        }
        
        return outliers
    
    def detect_all(self):
        """Detect outliers for all numeric columns."""
        numeric_cols = self.data.select_dtypes(include=[np.number]).columns
        
        for col in numeric_cols:
            self.detect_zscore(col)
            self.detect_modified_zscore(col)
        
        return self.outliers
    
    def compare_methods(self, column):
        """Compare Z-score and Modified Z-score methods."""
        self.detect_all()
        
        print(f"\n{'='*60}")
        print(f"OUTLIER DETECTION COMPARISON: {column}")
        print(f"{'='*60}")
        
        # Z-score results
        z_key = f'{column}_zscore'
        mod_key = f'{column}_modified'
        
        if z_key in self.outliers:
            z_info = self.outliers[z_key]
            print(f"\n📊 Z-score Method:")
            print(f"  Outliers: {z_info['n_outliers']} ({z_info['percentage']:.1f}%)")
            if z_info['n_outliers'] > 0:
                print(f"  Values: {z_info['outliers'].head(5).tolist()}")
        
        if mod_key in self.outliers:
            mod_info = self.outliers[mod_key]
            print(f"\n📊 Modified Z-score Method:")
            print(f"  Outliers: {mod_info['n_outliers']} ({mod_info['percentage']:.1f}%)")
            if mod_info['n_outliers'] > 0:
                print(f"  Values: {mod_info['outliers'].head(5).tolist()}")
        
        print("\n💡 Recommendation:")
        print("  • Z-score: Works well for normally distributed data")
        print("  • Modified Z-score: More robust, works for skewed data")

# Test Z-score methods
zscore_detector = ZScoreOutlierDetector(banking_data)
zscore_detector.compare_methods('income')

# ============= ISOLATION FOREST OUTLIER DETECTION =============

from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler

class IsolationForestDetector:
    """Outlier detection using Isolation Forest."""
    
    def __init__(self, data, contamination=0.05, random_state=42):
        self.data = data
        self.contamination = contamination
        self.random_state = random_state
        self.outlier_predictions = None
        self.outlier_scores = None
    
    def detect(self, columns=None):
        """Detect outliers using Isolation Forest."""
        if columns is None:
            columns = self.data.select_dtypes(include=[np.number]).columns
        
        # Prepare data
        X = self.data[columns].copy()
        
        # Handle missing values
        X = X.fillna(X.mean())
        
        # Scale data
        scaler = StandardScaler()
        X_scaled = scaler.fit_transform(X)
        
        # Fit Isolation Forest
        iso_forest = IsolationForest(
            contamination=self.contamination,
            random_state=self.random_state,
            n_estimators=100
        )
        
        self.outlier_predictions = iso_forest.fit_predict(X_scaled)
        self.outlier_scores = iso_forest.score_samples(X_scaled)
        
        # Get outlier indices
        outlier_indices = np.where(self.outlier_predictions == -1)[0]
        
        return outlier_indices
    
    def get_outlier_data(self):
        """Get data points flagged as outliers."""
        if self.outlier_predictions is None:
            self.detect()
        
        outlier_indices = np.where(self.outlier_predictions == -1)[0]
        return self.data.iloc[outlier_indices]
    
    def report(self):
        """Generate isolation forest report."""
        if self.outlier_predictions is None:
            self.detect()
        
        outlier_indices = np.where(self.outlier_predictions == -1)[0]
        n_outliers = len(outlier_indices)
        
        print("\n" + "="*60)
        print("ISOLATION FOREST OUTLIER DETECTION REPORT")
        print("="*60)
        
        print(f"\n📊 Results:")
        print(f"  Total Records: {len(self.data)}")
        print(f"  Outliers Detected: {n_outliers} ({n_outliers/len(self.data)*100:.1f}%)")
        
        if n_outliers > 0:
            outlier_data = self.data.iloc[outlier_indices]
            print(f"\n📋 Outlier Sample (first 5):")
            print(outlier_data.head())
            
            # Show score distribution
            print(f"\n📊 Outlier Scores:")
            print(f"  Mean: {self.outlier_scores.mean():.3f}")
            print(f"  Min: {self.outlier_scores.min():.3f}")
            print(f"  Max: {self.outlier_scores.max():.3f}")
            print(f"  Std: {self.outlier_scores.std():.3f}")

# Test Isolation Forest
iso_detector = IsolationForestDetector(banking_data, contamination=0.05)
iso_detector.detect()
iso_detector.report()

# ============= LOCAL OUTLIER FACTOR =============

from sklearn.neighbors import LocalOutlierFactor
from sklearn.preprocessing import StandardScaler

class LOFDetector:
    """Outlier detection using Local Outlier Factor."""
    
    def __init__(self, data, n_neighbors=20, contamination=0.05):
        self.data = data
        self.n_neighbors = n_neighbors
        self.contamination = contamination
        self.outlier_predictions = None
        self.outlier_scores = None
    
    def detect(self, columns=None):
        """Detect outliers using Local Outlier Factor."""
        if columns is None:
            columns = self.data.select_dtypes(include=[np.number]).columns
        
        # Prepare data
        X = self.data[columns].copy()
        X = X.fillna(X.mean())
        
        # Scale data
        scaler = StandardScaler()
        X_scaled = scaler.fit_transform(X)
        
        # Fit LOF
        lof = LocalOutlierFactor(
            n_neighbors=self.n_neighbors,
            contamination=self.contamination,
            novelty=False
        )
        
        self.outlier_predictions = lof.fit_predict(X_scaled)
        self.outlier_scores = lof.negative_outlier_factor_
        
        outlier_indices = np.where(self.outlier_predictions == -1)[0]
        
        return outlier_indices
    
    def report(self):
        """Generate LOF report."""
        if self.outlier_predictions is None:
            self.detect()
        
        outlier_indices = np.where(self.outlier_predictions == -1)[0]
        
        print("\n" + "="*60)
        print("LOCAL OUTLIER FACTOR (LOF) REPORT")
        print("="*60)
        
        print(f"\n📊 Results:")
        print(f"  Total Records: {len(self.data)}")
        print(f"  Outliers Detected: {len(outlier_indices)} ({len(outlier_indices)/len(self.data)*100:.1f}%)")
        print(f"  Parameters: n_neighbors={self.n_neighbors}, contamination={self.contamination}")
        
        if len(outlier_indices) > 0:
            print(f"\n📊 Outlier Scores (negative LOF):")
            print(f"  Mean: {self.outlier_scores.mean():.3f}")
            print(f"  Min: {self.outlier_scores.min():.3f}")
            print(f"  Max: {self.outlier_scores.max():.3f}")

# Test LOF
lof_detector = LOFDetector(banking_data, n_neighbors=20, contamination=0.05)
lof_detector.detect()
lof_detector.report()

# ============= WINSORIZATION =============

class Winsorization:
    """Winsorize outliers (cap extreme values)."""
    
    def __init__(self, data):
        self.data = data.copy()
        self.winsorized_data = None
        self.winsorization_stats = {}
    
    def winsorize(self, column, lower_percentile=0.01, upper_percentile=0.99):
        """Winsorize a column."""
        if column not in self.data.columns:
            return None
        
        data = self.data[column].dropna()
        
        lower_bound = data.quantile(lower_percentile)
        upper_bound = data.quantile(upper_percentile)
        
        self.winsorized_data = self.data.copy()
        self.winsorized_data[column] = self.winsorized_data[column].clip(lower=lower_bound, upper=upper_bound)
        
        self.winsorization_stats[column] = {
            'lower_percentile': lower_percentile,
            'upper_percentile': upper_percentile,
            'lower_bound': lower_bound,
            'upper_bound': upper_bound,
            'original_min': data.min(),
            'original_max': data.max(),
            'winsorized_min': self.winsorized_data[column].min(),
            'winsorized_max': self.winsorized_data[column].max()
        }
        
        return self.winsorized_data[column]
    
    def compare_distributions(self, column):
        """Compare original and winsorized distributions."""
        if column not in self.data.columns:
            return
        
        self.winsorize(column)
        
        print(f"\n{'='*60}")
        print(f"WINSORIZATION COMPARISON: {column}")
        print(f"{'='*60}")
        
        original = self.data[column].dropna()
        winsorized = self.winsorized_data[column].dropna()
        
        print(f"\n📊 Original:")
        print(f"  Mean: {original.mean():.2f}")
        print(f"  Median: {original.median():.2f}")
        print(f"  Min: {original.min():.2f}")
        print(f"  Max: {original.max():.2f}")
        print(f"  Std: {original.std():.2f}")
        
        print(f"\n📊 Winsorized ({self.winsorization_stats[column]['lower_percentile']*100:.0f}%-{self.winsorization_stats[column]['upper_percentile']*100:.0f}%):")
        print(f"  Mean: {winsorized.mean():.2f}")
        print(f"  Median: {winsorized.median():.2f}")
        print(f"  Min: {winsorized.min():.2f}")
        print(f"  Max: {winsorized.max():.2f}")
        print(f"  Std: {winsorized.std():.2f}")
        
        # Visualize
        fig, axes = plt.subplots(1, 2, figsize=(12, 4))
        
        original.hist(bins=30, ax=axes[0], edgecolor='black', alpha=0.7)
        axes[0].set_title('Original Distribution', fontsize=12)
        axes[0].set_xlabel(column)
        axes[0].set_ylabel('Frequency')
        
        winsorized.hist(bins=30, ax=axes[1], edgecolor='black', alpha=0.7)
        axes[1].set_title('Winsorized Distribution', fontsize=12)
        axes[1].set_xlabel(column)
        axes[1].set_ylabel('Frequency')
        
        plt.tight_layout()
        plt.show()

# Test winsorization
winsorizer = Winsorization(banking_data)
winsorizer.compare_distributions('income')

# ============= TRANSFORMATIONS AND REMOVAL =============

class OutlierHandler:
    """Handle outliers through transformations or removal."""
    
    def __init__(self, data):
        self.data = data.copy()
        self.handled_data = None
        self.handling_stats = {}
    
    def log_transform(self, column):
        """Apply log transformation to reduce impact of outliers."""
        if column not in self.data.columns:
            return None
        
        # Ensure all values are positive
        min_val = self.data[column].min()
        if min_val <= 0:
            shift = abs(min_val) + 1
            self.data[f'{column}_log'] = np.log(self.data[column] + shift)
            self.handling_stats[f'{column}_log'] = {
                'method': 'log_transform',
                'shift': shift
            }
        else:
            self.data[f'{column}_log'] = np.log(self.data[column])
            self.handling_stats[f'{column}_log'] = {
                'method': 'log_transform',
                'shift': 0
            }
        
        return self.data[f'{column}_log']
    
    def square_root_transform(self, column):
        """Apply square root transformation."""
        if column not in self.data.columns:
            return None
        
        min_val = self.data[column].min()
        if min_val < 0:
            shift = abs(min_val)
            self.data[f'{column}_sqrt'] = np.sqrt(self.data[column] + shift)
            self.handling_stats[f'{column}_sqrt'] = {
                'method': 'sqrt_transform',
                'shift': shift
            }
        else:
            self.data[f'{column}_sqrt'] = np.sqrt(self.data[column])
            self.handling_stats[f'{column}_sqrt'] = {
                'method': 'sqrt_transform',
                'shift': 0
            }
        
        return self.data[f'{column}_sqrt']
    
    def remove_outliers(self, column, method='iqr', threshold=1.5):
        """Remove outliers from dataset."""
        if column not in self.data.columns:
            return None
        
        self.handled_data = self.data.copy()
        
        if method == 'iqr':
            q1 = self.data[column].quantile(0.25)
            q3 = self.data[column].quantile(0.75)
            iqr = q3 - q1
            lower = q1 - threshold * iqr
            upper = q3 + threshold * iqr
            
            original_count = len(self.handled_data)
            self.handled_data = self.handled_data[
                (self.handled_data[column] >= lower) & 
                (self.handled_data[column] <= upper)
            ]
            
            removed = original_count - len(self.handled_data)
            
            self.handling_stats[f'removed_{column}'] = {
                'method': 'remove',
                'removed_count': removed,
                'remaining_count': len(self.handled_data),
                'threshold': threshold,
                'lower_bound': lower,
                'upper_bound': upper
            }
        
        return self.handled_data
    
    def compare_transformations(self, column):
        """Compare different transformation methods."""
        print(f"\n{'='*60}")
        print(f"TRANSFORMATION COMPARISON: {column}")
        print(f"{'='*60}")
        
        original = self.data[column].dropna()
        print(f"\n📊 Original:")
        print(f"  Skewness: {original.skew():.3f}")
        print(f"  Kurtosis: {original.kurtosis():.3f}")
        print(f"  Mean: {original.mean():.2f}")
        print(f"  Std: {original.std():.2f}")
        
        # Log transform
        log_data = self.log_transform(column).dropna()
        print(f"\n📊 Log Transform:")
        print(f"  Skewness: {log_data.skew():.3f}")
        print(f"  Kurtosis: {log_data.kurtosis():.3f}")
        print(f"  Mean: {log_data.mean():.2f}")
        print(f"  Std: {log_data.std():.2f}")
        
        # Square root transform
        sqrt_data = self.square_root_transform(column).dropna()
        print(f"\n📊 Square Root Transform:")
        print(f"  Skewness: {sqrt_data.skew():.3f}")
        print(f"  Kurtosis: {sqrt_data.kurtosis():.3f}")
        print(f"  Mean: {sqrt_data.mean():.2f}")
        print(f"  Std: {sqrt_data.std():.2f}")
        
        print("\n💡 Recommendation:")
        if abs(original.skew()) > 1:
            print("  • Data is highly skewed - consider log or sqrt transform")
            if abs(log_data.skew()) < abs(original.skew()):
                print("  • Log transform reduced skewness - recommended")
            elif abs(sqrt_data.skew()) < abs(original.skew()):
                print("  • Square root transform reduced skewness - recommended")
        else:
            print("  • Data is not highly skewed - no transform needed")

# Test transformations
handler = OutlierHandler(banking_data)
handler.compare_transformations('income')

# ============= BUSINESS IMPACT EVALUATION =============

class OutlierBusinessImpact:
    """Evaluate business impact of outlier handling."""
    
    def __init__(self, original_data):
        self.original = original_data.copy()
        self.results = {}
    
    def evaluate_handling_method(self, method_name, handled_data):
        """Evaluate the impact of a handling method."""
        
        metrics = {}
        
        # Compare key statistics
        for col in self.original.select_dtypes(include=[np.number]).columns:
            if col in handled_data.columns:
                orig = self.original[col].dropna()
                handled = handled_data[col].dropna()
                
                metrics[col] = {
                    'original_mean': orig.mean(),
                    'handled_mean': handled.mean(),
                    'mean_change': (handled.mean() - orig.mean()) / orig.mean() * 100 if orig.mean() != 0 else 0,
                    'original_std': orig.std(),
                    'handled_std': handled.std(),
                    'std_change': (handled.std() - orig.std()) / orig.std() * 100 if orig.std() != 0 else 0,
                    'original_min': orig.min(),
                    'handled_min': handled.min(),
                    'original_max': orig.max(),
                    'handled_max': handled.max()
                }
        
        self.results[method_name] = metrics
        
        return metrics
    
    def report(self):
        """Generate impact report."""
        print("\n" + "="*60)
        print("OUTLIER HANDLING BUSINESS IMPACT")
        print("="*60)
        
        for method, metrics in self.results.items():
            print(f"\n📊 Method: {method}")
            
            for col, stats in metrics.items():
                print(f"\n  {col}:")
                print(f"    Mean: {stats['original_mean']:.2f} → {stats['handled_mean']:.2f} ({stats['mean_change']:.1f}% change)")
                print(f"    Std: {stats['original_std']:.2f} → {stats['handled_std']:.2f} ({stats['std_change']:.1f}% change)")
                print(f"    Range: [{stats['original_min']:.2f}, {stats['original_max']:.2f}] → [{stats['handled_min']:.2f}, {stats['handled_max']:.2f}]")

# Test business impact
impact = OutlierBusinessImpact(banking_data)

# Apply different methods and evaluate
winsorizer = Winsorization(banking_data)
winsorized = winsorizer.winsorize('income', lower_percentile=0.01, upper_percentile=0.99)
impact.evaluate_handling_method('Winsorization', winsorizer.winsorized_data)

handler = OutlierHandler(banking_data)
removed = handler.remove_outliers('income', method='iqr')
impact.evaluate_handling_method('Removal', removed)

impact.report()