┌─────────────────────────────────────────────────────────────────────────────┐
│                    PAYMENTS PROCESSING LIFECYCLE                          │
├─────────────────────────────────────────────────────────────────────────────┤
│                                                                             │
│  ┌─────────────┐    ┌─────────────┐    ┌─────────────┐    ┌─────────────┐  │
│  │ Payment    │    │ Authorisation│    │ Clearing   │    │ Settlement  │  │
│  │ Initiation │ ──→ │ (Validation, │ ──→ │ (Batching, │ ──→ │ (Funds      │  │
│  │ (Customer  │    │  Fraud      │    │  Netting)  │    │  Transfer)  │  │
│  │  starts)   │    │  Check)     │    │            │    │             │  │
│  └─────────────┘    └─────────────┘    └─────────────┘    └─────────────┘  │
│                                                                             │
│  ┌─────────────┐                                                          │
│  │ Confirmation│                                                          │
│  │ (Receipt to │                                                          │
│  │  Payer &    │                                                          │
│  │  Payee)     │                                                          │
│  └─────────────┘                                                          │
│                                                                             │
└─────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────────────────┐
│                    RECONCILIATION PROCESS                                  │
├─────────────────────────────────────────────────────────────────────────────┤
│                                                                             │
│  ┌──────────────────────────────────────────────────────────────────────┐   │
│  │                    DATA COLLECTION                                  │   │
│  │  Internal records, external statements, payment files               │   │
│  └──────────────────────────────────────────────────────────────────────┘   │
│                                    │                                        │
│                                    v                                        │
│  ┌──────────────────────────────────────────────────────────────────────┐   │
│  │                    DATA MATCHING                                    │   │
│  │  Match transactions by reference number, amount, date              │   │
│  └──────────────────────────────────────────────────────────────────────┘   │
│                                    │                                        │
│                                    v                                        │
│  ┌──────────────────────────────────────────────────────────────────────┐   │
│  │                    EXCEPTION HANDLING                               │   │
│  │  Identify and resolve unmatched transactions                        │   │
│  └──────────────────────────────────────────────────────────────────────┘   │
│                                    │                                        │
│                                    v                                        │
│  ┌──────────────────────────────────────────────────────────────────────┐   │
│  │                    REPORTING                                        │   │
│  │  Generate reconciliation reports, audit trail                      │   │
│  └──────────────────────────────────────────────────────────────────────┘   │
│                                                                             │
└─────────────────────────────────────────────────────────────────────────────┘

# ===================================================================
# MODULE 3, LESSON 4: DIGITAL PAYMENTS PROCESSING
# ===================================================================

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime, timedelta
import warnings
warnings.filterwarnings('ignore')

print("="*70)
print("DIGITAL PAYMENTS PROCESSING – CLEARING, SETTLEMENT, AND RECONCILIATION")
print("="*70)

# ----------------------------------------------------------------
# PART A: PAYMENT SYSTEMS COMPARISON
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART A: Payment Systems Comparison")
print("-"*60)

payment_systems = pd.DataFrame({
    'System': ['SWIFT', 'ACH (US)', 'Faster Payments (UK)', 'SEPA (EU)', 'FedNow (US)', 'UPI (India)'],
    'Speed': ['1-4 days', '1-2 days', 'Seconds', '1 day', 'Seconds', 'Seconds'],
    'Cost Level': ['High', 'Low', 'Low', 'Low', 'Low', 'Very Low'],
    'Transaction Limit': ['Unlimited', '$25,000 (same-day)', '£1,000,000', 'Unlimited', '$500,000', '₹1,000,000'],
    '24/7 Operation': ['No', 'No', 'Yes', 'No', 'Yes', 'Yes'],
    'ISO 20022': ['Yes', 'Yes', 'Yes', 'Yes', 'Yes', 'Yes']
})

print("Payment Systems Comparison:")
print(payment_systems.to_string(index=False))

# ----------------------------------------------------------------
# PART B: PAYMENT PROCESSING FLOW
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART B: Payment Processing Flow Simulation")
print("-"*60)

# Simulate payment processing
def simulate_payment_processing(n_payments=1000):
    """Simulate the end-to-end payment processing lifecycle."""
    
    # Generate payment data
    np.random.seed(42)
    
    # Transaction data
    payment_data = pd.DataFrame({
        'payment_id': range(1, n_payments + 1),
        'amount': np.random.lognormal(3.5, 1.0, n_payments).clip(1, 10000).round(2),
        'payer': [f'Customer_{np.random.randint(1, 100)}' for _ in range(n_payments)],
        'payee': [f'Merchant_{np.random.randint(1, 100)}' for _ in range(n_payments)],
        'timestamp': [datetime.now() - timedelta(minutes=np.random.randint(0, 1440)) for _ in range(n_payments)],
        'payment_scheme': np.random.choice(['SWIFT', 'ACH', 'Faster Payments', 'FedNow', 'UPI'], n_payments,
                                          p=[0.10, 0.30, 0.20, 0.15, 0.25]),
        'status': 'Initiated'
    })
    
    # Processing steps
    steps = ['Authorisation', 'Clearing', 'Settlement', 'Confirmation']
    
    # Simulate processing times by scheme
    processing_times = {
        'SWIFT': np.random.gamma(2, 0.5, n_payments).clip(0.5, 4),
        'ACH': np.random.gamma(1.5, 0.3, n_payments).clip(0.2, 2),
        'Faster Payments': np.random.gamma(0.5, 0.1, n_payments).clip(0.05, 0.5),
        'FedNow': np.random.gamma(0.3, 0.05, n_payments).clip(0.02, 0.3),
        'UPI': np.random.gamma(0.2, 0.05, n_payments).clip(0.01, 0.2)
    }
    
    # Assign processing times
    payment_data['processing_time'] = [
        processing_times[row['payment_scheme']][i] 
        for i, row in payment_data.iterrows()
    ]
    
    # Simulate status progression
    payment_data['status'] = 'Completed'
    payment_data['authorisation_time'] = payment_data['processing_time'] * 0.3
    payment_data['clearing_time'] = payment_data['processing_time'] * 0.4
    payment_data['settlement_time'] = payment_data['processing_time'] * 0.3
    
    return payment_data

# Simulate payments
payment_data = simulate_payment_processing(1000)

print("Payment Processing Summary:")
print(f"Total Payments: {len(payment_data)}")
print(f"Total Amount: ${payment_data['amount'].sum():,.2f}")
print(f"Average Amount: ${payment_data['amount'].mean():.2f}")
print(f"Average Processing Time: {payment_data['processing_time'].mean():.2f} minutes")

# Summary by payment scheme
scheme_summary = payment_data.groupby('payment_scheme').agg({
    'amount': ['count', 'sum', 'mean'],
    'processing_time': 'mean'
}).round(2)
scheme_summary.columns = ['Count', 'Total Amount', 'Avg Amount', 'Avg Time (min)']
print("\nPayment Scheme Summary:")
print(scheme_summary)

# Visualise
fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# Payment Scheme Distribution
ax = axes[0, 0]
scheme_counts = payment_data['payment_scheme'].value_counts()
ax.pie(scheme_counts.values, labels=scheme_counts.index, autopct='%1.1f%%')
ax.set_title('Payment Scheme Distribution')

# Processing Time by Scheme
ax = axes[0, 1]
payment_data.boxplot(column='processing_time', by='payment_scheme', ax=ax)
ax.set_title('Processing Time by Scheme')
ax.set_ylabel('Processing Time (minutes)')
ax.set_xlabel('')

# Amount Distribution
ax = axes[1, 0]
ax.hist(payment_data['amount'], bins=50, edgecolor='black', alpha=0.7)
ax.set_xlabel('Amount ($)')
ax.set_ylabel('Frequency')
ax.set_title('Transaction Amount Distribution')

# Total Amount by Scheme
ax = axes[1, 1]
scheme_amounts = payment_data.groupby('payment_scheme')['amount'].sum()
ax.bar(scheme_amounts.index, scheme_amounts.values, color='teal', alpha=0.7)
ax.set_xlabel('Payment Scheme')
ax.set_ylabel('Total Amount ($)')
ax.set_title('Total Amount by Scheme')
ax.tick_params(axis='x', rotation=45)

plt.tight_layout()
plt.savefig('payment_processing.png', dpi=300, bbox_inches='tight')
plt.show()
print("Payment processing visualisation saved as 'payment_processing.png'")

# ----------------------------------------------------------------
# PART C: RECONCILIATION AUTOMATION
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART C: Reconciliation Automation")
print("-"*60)

# Simulate reconciliation process
def simulate_reconciliation(n_transactions=500):
    """Simulate automated reconciliation."""
    
    # Generate internal records
    internal_records = pd.DataFrame({
        'txn_id': range(1, n_transactions + 1),
        'amount': np.random.lognormal(3, 0.8, n_transactions).clip(1, 5000).round(2),
        'date': [datetime.now() - timedelta(days=np.random.randint(0, 30)) for _ in range(n_transactions)],
        'reference': [f'REF-{np.random.randint(1000, 9999)}' for _ in range(n_transactions)],
        'status': 'Pending'
    })
    
    # Generate external records (bank statement)
    n_external = int(n_transactions * np.random.uniform(0.95, 1.05))
    external_records = pd.DataFrame({
        'ext_txn_id': range(1, n_external + 1),
        'amount': np.random.lognormal(3, 0.8, n_external).clip(1, 5000).round(2),
        'date': [datetime.now() - timedelta(days=np.random.randint(0, 30)) for _ in range(n_external)],
        'reference': [f'REF-{np.random.randint(1000, 9999)}' for _ in range(n_external)],
        'status': 'Recorded'
    })
    
    # Reconciliation matching (simplified)
    matched = []
    unmatched_internal = []
    unmatched_external = []
    
    # Match by amount and date (approximate)
    for idx, internal in internal_records.iterrows():
        # Find matching external record
        matches = external_records[
            (external_records['amount'] == internal['amount']) &
            (external_records['date'].dt.date == internal['date'].dt.date)
        ]
        if len(matches) > 0:
            # Take the first match
            ext_idx = matches.index[0]
            matched.append({
                'internal_txn': internal['txn_id'],
                'external_txn': external_records.loc[ext_idx, 'ext_txn_id'],
                'amount': internal['amount'],
                'date': internal['date'],
                'status': 'Matched'
            })
            # Remove from external records
            external_records = external_records.drop(ext_idx)
        else:
            unmatched_internal.append(internal['txn_id'])
    
    # Remaining external records are unmatched
    unmatched_external = external_records['ext_txn_id'].tolist()
    
    reconciliation_result = {
        'total_internal': len(internal_records),
        'total_external': n_external,
        'matched_count': len(matched),
        'unmatched_internal': len(unmatched_internal),
        'unmatched_external': len(unmatched_external),
        'match_rate': len(matched) / len(internal_records) * 100,
        'matched_details': matched,
        'unmatched_internal_ids': unmatched_internal,
        'unmatched_external_ids': unmatched_external
    }
    
    return reconciliation_result

# Run reconciliation simulation
reconciliation = simulate_reconciliation(500)

print("Reconciliation Results:")
print(f"Total Internal Records: {reconciliation['total_internal']}")
print(f"Total External Records: {reconciliation['total_external']}")
print(f"Matched Records: {reconciliation['matched_count']}")
print(f"Unmatched Internal: {reconciliation['unmatched_internal']}")
print(f"Unmatched External: {reconciliation['unmatched_external']}")
print(f"Match Rate: {reconciliation['match_rate']:.2f}%")

# Visualise reconciliation
fig, ax = plt.subplots(figsize=(10, 6))
categories = ['Matched', 'Unmatched Internal', 'Unmatched External']
counts = [reconciliation['matched_count'], reconciliation['unmatched_internal'], reconciliation['unmatched_external']]
colors = ['green', 'red', 'orange']

bars = ax.bar(categories, counts, color=colors, alpha=0.7)
ax.set_ylabel('Count')
ax.set_title('Reconciliation Summary')
for bar, count in zip(bars, counts):
    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 5, 
            str(count), ha='center', va='bottom', fontweight='bold')
ax.grid(True, alpha=0.3)

plt.tight_layout()
plt.savefig('reconciliation.png', dpi=300, bbox_inches='tight')
plt.show()
print("Reconciliation visualisation saved as 'reconciliation.png'")

# ----------------------------------------------------------------
# PART D: PAYMENTS OPERATIONAL RISK
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART D: Payments Operational Risk")
print("-"*60)

operational_risks = pd.DataFrame({
    'Risk': [
        'System Failure',
        'Data Breach',
        'Fraud',
        'Clearing Failure',
        'Settlement Failure',
        'Reconciliation Error',
        'Compliance Breach',
        'Third-Party Failure'
    ],
    'Likelihood': ['Low', 'Low', 'Medium', 'Low', 'Low', 'Medium', 'Medium', 'Low'],
    'Impact': ['Critical', 'Critical', 'High', 'Critical', 'Critical', 'Medium', 'High', 'High'],
    'Mitigation': [
        'Redundancy, DR/BCP',
        'Encryption, Access Controls',
        'Real-time Fraud Monitoring, AI',
        'Multiple Clearing Providers',
        'Reserve Funds, Oversight',
        'Automated Reconciliation',
        'Compliance Monitoring, Training',
        'Vendor Due Diligence, SLAs'
    ]
})

print("Payments Operational Risk:")
print(operational_risks.to_string(index=False))

# ----------------------------------------------------------------
# PART E: PAYMENTS PROCESSING METRICS
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART E: Payments Processing Metrics")
print("-"*60)

payment_metrics = pd.DataFrame({
    'Metric': [
        'Payment Volume',
        'Payment Value',
        'STP Rate',
        'Average Processing Time',
        'Settlement Rate',
        'Reconciliation Rate',
        'Chargeback Rate',
        'Fraud Rate'
    ],
    'Current Value': [
        '2.5M/day',
        '$450M/day',
        '78%',
        '45 min',
        '99.2%',
        '85%',
        '0.8%',
        '0.3%'
    ],
    'Target Value': [
        '3.5M/day',
        '$650M/day',
        '> 95%',
        '< 5 min',
        '> 99.5%',
        '> 98%',
        '< 0.2%',
        '< 0.05%'
    ],
    'Status': ['🟡', '🟡', '🔴', '🔴', '🟡', '🔴', '🔴', '🔴']
})

print("Payments Processing Metrics:")
print(payment_metrics.to_string(index=False))

# ----------------------------------------------------------------
# PART F: PAYMENTS PROCESSING STRATEGY
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART F: Payments Processing Strategy")
print("-"*60)

strategy = {
    "1. Real-Time Payments": {
        "Actions": [
            "Integrate with FedNow and other RTP systems.",
            "Implement ISO 20022 messaging.",
            "Enable 24/7 payment processing.",
            "Develop real-time payment analytics."
        ],
        "Priority": "High",
        "Timeline": "0-12 months"
    },
    "2. Automation": {
        "Actions": [
            "Automate reconciliation with AI.",
            "Implement STP for payments.",
            "Use RPA for exception handling.",
            "Automate compliance checks."
        ],
        "Priority": "High",
        "Timeline": "0-12 months"
    },
    "3. Security": {
        "Actions": [
            "Implement AI-powered fraud detection.",
            "Enhance authentication (SCA).",
            "Encrypt payment data.",
            "Regular security audits."
        ],
        "Priority": "Critical",
        "Timeline": "0-6 months"
    },
    "4. Analytics": {
        "Actions": [
            "Build real-time payment analytics.",
            "Implement payment flow monitoring.",
            "Develop predictive analytics.",
            "Optimise payment routing."
        ],
        "Priority": "Medium",
        "Timeline": "12-24 months"
    },
    "5. Innovation": {
        "Actions": [
            "Explore blockchain for payments.",
            "Experiment with CBDC integration.",
            "Develop embedded payments.",
            "Implement programmable payments."
        ],
        "Priority": "Medium",
        "Timeline": "24+ months"
    }
}

for item, details in strategy.items():
    print(f"\n{item}:")
    for action in details['Actions']:
        print(f"  • {action}")
    print(f"  Priority: {details['Priority']}")
    print(f"  Timeline: {details['Timeline']}")

# ----------------------------------------------------------------
# PART G: SUMMARY AND RECOMMENDATIONS
# ----------------------------------------------------------------

print("\n" + "="*70)
print("PART G: Summary and Recommendations")
print("="*70)

print("""
Digital Payments Processing – Key Takeaways:

1. Payments processing lifecycle: initiation → authorisation → clearing → settlement → confirmation.
2. Clearing exchanges payment information; settlement transfers funds.
3. Payment systems include SWIFT, ACH, Faster Payments, SEPA, FedNow, UPI.
4. ISO 20022 provides richer, structured data for payments.
5. Reconciliation automation matches internal and external records.
6. Key metrics: volume, value, STP rate, processing time, reconciliation rate.
7. Operational risks: system failure, fraud, clearing/settlement failure, compliance.

Recommendations:
  - Integrate with real-time payment systems (FedNow, etc.).
  - Implement ISO 20022 messaging.
  - Automate reconciliation and exception handling.
  - Implement AI-powered fraud detection.
  - Build real-time payment analytics.
  - Ensure robust security and compliance.
""")

print("="*70)
print("END OF LESSON 4 – MODULE 3")
print("="*70)