# ===================================================================
# MODULE 4, LESSON 8: UNSUPERVISED LEARNING FOR FINANCE
# ===================================================================

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering
from sklearn.decomposition import PCA
from sklearn.ensemble import IsolationForest
from sklearn.neighbors import NearestNeighbors
from scipy.cluster.hierarchy import dendrogram, linkage
from scipy.spatial.distance import cdist
import warnings
warnings.filterwarnings('ignore')

# Set style and reproducibility
np.random.seed(42)
sns.set_style("whitegrid")

print("="*70)
print("UNSUPERVISED LEARNING FOR FINANCIAL DATA ANALYTICS")
print("="*70)

# ----------------------------------------------------------------
# PART A: GENERATE SYNTHETIC BANKING DATA
# ----------------------------------------------------------------

# Customer transaction and behavioural data
n_customers = 1000

# Features: monthly_transactions, avg_amount, recency (days since last transaction), account_age, credit_usage
monthly_trans = np.random.gamma(2, 15, n_customers) + 5
avg_amount = np.random.gamma(3, 20, n_customers) + 10
recency = np.random.exponential(30, n_customers).clip(0, 180)
account_age = np.random.gamma(3, 60, n_customers).clip(6, 360)
credit_usage = np.random.beta(2, 3, n_customers) * 100

# Create some distinct segments
# Segment 0: High-value, frequent
high_val = np.random.choice([True, False], n_customers, p=[0.15, 0.85])
monthly_trans[high_val] = monthly_trans[high_val] * 2 + 20
avg_amount[high_val] = avg_amount[high_val] * 3 + 50

# Segment 1: Dormant, low usage
dormant = np.random.choice([True, False], n_customers, p=[0.10, 0.90])
monthly_trans[dormant] = monthly_trans[dormant] * 0.2
avg_amount[dormant] = avg_amount[dormant] * 0.3
recency[dormant] = recency[dormant] + 60

# Add some anomalies (fraudulent-like)
fraud_idx = np.random.choice(n_customers, 20, replace=False)
monthly_trans[fraud_idx] = np.random.exponential(200, 20) + 100
avg_amount[fraud_idx] = np.random.exponential(500, 20) + 200

# Create DataFrame
df = pd.DataFrame({
    'monthly_trans': monthly_trans,
    'avg_amount': avg_amount,
    'recency': recency,
    'account_age': account_age,
    'credit_usage': credit_usage
})

print(f"Dataset shape: {df.shape}")
print("\nFeature Statistics:")
print(df.describe().round(2))

# ----------------------------------------------------------------
# PART B: K-MEANS CLUSTERING
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART B: K-MEANS CLUSTERING")
print("-"*60)

# Standardise features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(df)

# Elbow method to find optimal K
wcss = []
K_range = range(1, 11)
for k in K_range:
    kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)
    kmeans.fit(X_scaled)
    wcss.append(kmeans.inertia_)

fig, ax = plt.subplots(figsize=(8, 5))
ax.plot(K_range, wcss, 'bo-', linewidth=2, markersize=8)
ax.set_xlabel('Number of Clusters (K)')
ax.set_ylabel('Within-Cluster Sum of Squares (WCSS)')
ax.set_title('Elbow Method for Optimal K')
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('elbow_method.png', dpi=300)
plt.show()

# Choose K=4 based on elbow
k = 4
kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)
df['cluster_kmeans'] = kmeans.fit_predict(X_scaled)

# Visualise clusters using PCA
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X_scaled)
df_pca = pd.DataFrame(X_pca, columns=['PC1', 'PC2'])
df_pca['cluster'] = df['cluster_kmeans']

fig, ax = plt.subplots(figsize=(10, 7))
for cluster in range(k):
    subset = df_pca[df_pca['cluster'] == cluster]
    ax.scatter(subset['PC1'], subset['PC2'], label=f'Cluster {cluster}', alpha=0.7)
ax.set_xlabel(f'Principal Component 1 ({pca.explained_variance_ratio_[0]:.2%} variance)')
ax.set_ylabel(f'Principal Component 2 ({pca.explained_variance_ratio_[1]:.2%} variance)')
ax.set_title('K-Means Clusters (K=4) Visualised with PCA')
ax.legend()
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('kmeans_clusters.png', dpi=300)
plt.show()

# Interpret clusters
cluster_summary = df.groupby('cluster_kmeans').mean()
print("\nCluster Characteristics (Mean Values):")
print(cluster_summary.round(2))

# Label clusters
cluster_labels = {
    0: 'High-Value Customers',
    1: 'Regular/Active',
    2: 'Dormant/Low Activity',
    3: 'Anomalous/High-Risk'
}
df['segment'] = df['cluster_kmeans'].map(cluster_labels)

print("\nSegment Distribution:")
print(df['segment'].value_counts())

# ----------------------------------------------------------------
# PART C: HIERARCHICAL CLUSTERING
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART C: HIERARCHICAL CLUSTERING")
print("-"*60)

# Perform hierarchical clustering (Ward's method)
linkage_matrix = linkage(X_scaled[:200], method='ward')  # Use subset for readability

fig, ax = plt.subplots(figsize=(14, 7))
dendrogram(linkage_matrix, ax=ax, leaf_rotation=90, leaf_font_size=8, 
           color_threshold=10, above_threshold_color='gray')
ax.set_title('Dendrogram – Hierarchical Customer Segmentation', fontsize=14)
ax.set_xlabel('Customer Index')
ax.set_ylabel('Distance')
plt.tight_layout()
plt.savefig('dendrogram.png', dpi=300)
plt.show()

# Fit hierarchical clustering with K=4
hierarchical = AgglomerativeClustering(n_clusters=4, linkage='ward')
df['cluster_hierarchical'] = hierarchical.fit_predict(X_scaled)

print("\nHierarchical Clustering vs K-Means (Cross-tabulation):")
print(pd.crosstab(df['cluster_kmeans'], df['cluster_hierarchical']))

# ----------------------------------------------------------------
# PART D: DBSCAN – DENSITY-BASED CLUSTERING
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART D: DBSCAN (Density-Based Clustering)")
print("-"*60)

# Find optimal eps using k-distance plot
neighbors = NearestNeighbors(n_neighbors=20)
neighbors_fit = neighbors.fit(X_scaled)
distances, indices = neighbors_fit.kneighbors(X_scaled)
distances = np.sort(distances[:, -1])

fig, ax = plt.subplots(figsize=(8, 5))
ax.plot(distances, 'b-', linewidth=2)
ax.set_xlabel('Points Sorted by Distance')
ax.set_ylabel('Distance to 20th Nearest Neighbor')
ax.set_title('K-Distance Plot (to find optimal eps)')
ax.axhline(y=1.5, color='red', linestyle='--', label='eps ≈ 1.5')
ax.legend()
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('dbscan_k_distance.png', dpi=300)
plt.show()

# Fit DBSCAN
dbscan = DBSCAN(eps=1.5, min_samples=15)
df['cluster_dbscan'] = dbscan.fit_predict(X_scaled)

n_clusters_db = len(set(df['cluster_dbscan'])) - (1 if -1 in df['cluster_dbscan'].values else 0)
n_noise = (df['cluster_dbscan'] == -1).sum()

print(f"Number of clusters: {n_clusters_db}")
print(f"Number of noise points: {n_noise} ({n_noise/len(df)*100:.2f}%)")

print("\nCluster sizes (DBSCAN):")
print(df['cluster_dbscan'].value_counts().sort_index())

# Visualise DBSCAN results
df_pca['cluster_dbscan'] = df['cluster_dbscan']

fig, ax = plt.subplots(figsize=(10, 7))
clusters = sorted(df_pca['cluster_dbscan'].unique())
for cluster in clusters:
    subset = df_pca[df_pca['cluster_dbscan'] == cluster]
    label = 'Noise' if cluster == -1 else f'Cluster {cluster}'
    color = 'gray' if cluster == -1 else None
    ax.scatter(subset['PC1'], subset['PC2'], label=label, alpha=0.7, color=color)
ax.set_xlabel(f'Principal Component 1 ({pca.explained_variance_ratio_[0]:.2%} variance)')
ax.set_ylabel(f'Principal Component 2 ({pca.explained_variance_ratio_[1]:.2%} variance)')
ax.set_title('DBSCAN Clustering Results')
ax.legend()
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('dbscan_clusters.png', dpi=300)
plt.show()

# ----------------------------------------------------------------
# PART E: ISOLATION FOREST – ANOMALY DETECTION
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART E: ISOLATION FOREST – FRAUD DETECTION")
print("-"*60)

# Fit Isolation Forest
iso_forest = IsolationForest(contamination=0.05, random_state=42)
df['anomaly_score'] = iso_forest.fit_predict(X_scaled)
df['anomaly'] = df['anomaly_score'] == -1

# Visualise anomalies
df_pca['anomaly'] = df['anomaly']

fig, ax = plt.subplots(figsize=(10, 7))
normal = df_pca[~df_pca['anomaly']]
anomalies = df_pca[df_pca['anomaly']]
ax.scatter(normal['PC1'], normal['PC2'], label='Normal', alpha=0.5, color='blue')
ax.scatter(anomalies['PC1'], anomalies['PC2'], label='Anomaly', color='red', s=80, marker='x')
ax.set_xlabel(f'Principal Component 1 ({pca.explained_variance_ratio_[0]:.2%} variance)')
ax.set_ylabel(f'Principal Component 2 ({pca.explained_variance_ratio_[1]:.2%} variance)')
ax.set_title('Isolation Forest – Anomaly Detection')
ax.legend()
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('isolation_forest_anomalies.png', dpi=300)
plt.show()

# Examine anomaly characteristics
anomaly_summary = df.groupby('anomaly').mean()
print("\nAnomaly vs Normal – Feature Comparison:")
print(anomaly_summary.round(2))

print(f"\nDetected {df['anomaly'].sum()} anomalies ({df['anomaly'].mean()*100:.2f}% of data).")

# ----------------------------------------------------------------
# PART F: DIMENSIONALITY REDUCTION WITH PCA
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART F: PRINCIPAL COMPONENT ANALYSIS (PCA)")
print("-"*60)

# Fit PCA (full components)
pca_full = PCA()
X_pca_full = pca_full.fit_transform(X_scaled)

# Explained variance
explained_variance = pca_full.explained_variance_ratio_
cumulative_variance = np.cumsum(explained_variance)

fig, axes = plt.subplots(1, 2, figsize=(14, 5))

ax = axes[0]
ax.bar(range(1, len(explained_variance)+1), explained_variance, alpha=0.7)
ax.set_xlabel('Principal Component')
ax.set_ylabel('Explained Variance Ratio')
ax.set_title('Variance Explained by Each Component')
ax.grid(True, alpha=0.3)

ax = axes[1]
ax.plot(range(1, len(cumulative_variance)+1), cumulative_variance, 'bo-', linewidth=2)
ax.axhline(y=0.95, color='red', linestyle='--', label='95% variance threshold')
ax.set_xlabel('Number of Components')
ax.set_ylabel('Cumulative Explained Variance')
ax.set_title('Cumulative Variance Explained')
ax.legend()
ax.grid(True, alpha=0.3)

plt.tight_layout()
plt.savefig('pca_variance.png', dpi=300)
plt.show()

print(f"Number of components to explain 95% variance: {np.argmax(cumulative_variance >= 0.95) + 1}")

# Component loadings (feature contributions)
loadings = pd.DataFrame(
    pca_full.components_.T,
    columns=[f'PC{i+1}' for i in range(pca_full.n_components_)],
    index=['monthly_trans', 'avg_amount', 'recency', 'account_age', 'credit_usage']
)
print("\nPCA Loadings (Feature Contributions):")
print(loadings.iloc[:, :3].round(3))

# ----------------------------------------------------------------
# PART G: BUSINESS APPLICATIONS AND RECOMMENDATIONS
# ----------------------------------------------------------------

print("\n" + "="*70)
print("PART G: BUSINESS APPLICATIONS AND RECOMMENDATIONS")
print("="*70)

print("""
1. Customer Segmentation (K-Means):
   → Use segments for targeted marketing, product recommendations, and pricing.
   → Example: High-Value Customers → Premium products; Dormant → Re-engagement campaigns.

2. Hierarchical Clustering:
   → Build customer personas and sub-segments.
   → Useful for creating a taxonomy of customer behaviour.

3. DBSCAN:
   → Identify clusters of similar transactions or unusual patterns.
   → Detect outliers that may indicate fraud or operational errors.

4. Anomaly Detection (Isolation Forest):
   → Flag unusual transactions for fraud investigation.
   → Monitor for operational risks (e.g., sudden changes in behaviour).

5. PCA:
   → Reduce dimensions for visualisation.
   → Preprocess data for modelling (reduce noise, improve performance).
""")

# ----------------------------------------------------------------
# PART H: CUSTOMER SEGMENT PROFILING (ACTIONABLE INSIGHTS)
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART H: ACTIONABLE BUSINESS INSIGHTS")
print("-"*60)

segment_profiles = df.groupby('segment').agg({
    'monthly_trans': 'mean',
    'avg_amount': 'mean',
    'recency': 'mean',
    'account_age': 'mean',
    'credit_usage': 'mean'
}).round(2)

print("\nSegment Profiles:")
print(segment_profiles)

print("\nBusiness Actions:")
print("• High-Value Customers → Loyalty programs, premium offers, relationship managers.")
print("• Regular/Active → Cross-sell additional products (savings, insurance).")
print("• Dormant/Low Activity → Re-engagement campaigns, incentives to transact.")
print("• Anomalous/High-Risk → Flag for manual review, enhanced monitoring.")

# Save results to CSV for reporting
df.to_csv('customer_segmentation_results.csv', index=False)
print("\nResults saved to 'customer_segmentation_results.csv'")