┌─────────────────────────────────────────────────────────────────────────────┐
│                    DOCUMENT MANAGEMENT LIFECYCLE                          │
├─────────────────────────────────────────────────────────────────────────────┤
│                                                                             │
│  ┌─────────────┐    ┌─────────────┐    ┌─────────────┐    ┌─────────────┐  │
│  │  Creation   │    │  Capture    │    │  Indexing   │    │  Storage    │  │
│  │  (Generate  │ ──→ │  (Scan,     │ ──→ │  (Metadata, │ ──→ │  (Secure    │  │
│  │   document) │    │   upload)   │    │   tagging)  │    │   archive)  │  │
│  └─────────────┘    └─────────────┘    └─────────────┘    └─────────────┘  │
│                                                                             │
│  ┌─────────────┐    ┌─────────────┐    ┌─────────────┐                    │
│  │  Retrieval  │ ← │  Search     │    │  Version    │                    │
│  │  (Find and  │    │  (Find      │    │  Control    │                    │
│  │   access)   │    │   documents)│    │  (Track     │                    │
│  └─────────────┘    └─────────────┘    │   changes)  │                    │
│                                         └─────────────┘                    │
│                                                                             │
│  ┌─────────────┐                                                          │
│  │  Disposal   │                                                          │
│  │  (Archive   │                                                          │
│  │   or delete)│                                                          │
│  └─────────────┘                                                          │
│                                                                             │
└─────────────────────────────────────────────────────────────────────────────┘

# ===================================================================
# MODULE 3, LESSON 5: BACK-OFFICE AUTOMATION AND DOCUMENT MANAGEMENT
# ===================================================================

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime, timedelta
import warnings
warnings.filterwarnings('ignore')

print("="*70)
print("BACK-OFFICE AUTOMATION AND DOCUMENT MANAGEMENT")
print("="*70)

# ----------------------------------------------------------------
# PART A: BACK-OFFICE PROCESS INVENTORY
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART A: Back-Office Process Inventory")
print("-"*60)

# Define back-office processes and automation status
processes = [
    "Account Opening",
    "KYC Verification",
    "Document Processing",
    "Data Entry",
    "Reconciliation",
    "Payment Processing",
    "Loan Origination",
    "Reporting",
    "Compliance Monitoring",
    "Customer Onboarding"
]

automation_status = {
    'Process': processes,
    'Current Automation (%)': [65, 55, 40, 30, 45, 70, 50, 35, 40, 60],
    'Target Automation (%)': [90, 85, 80, 80, 85, 90, 85, 75, 80, 85],
    'Priority (1-5)': [5, 5, 4, 4, 4, 5, 5, 3, 4, 4],
    'Process Owner': [
        'Operations', 'Compliance', 'Document Services', 'Operations', 
        'Finance', 'Payments', 'Lending', 'Finance', 'Compliance', 'Operations'
    ]
}

process_df = pd.DataFrame(automation_status)
process_df['Gap'] = process_df['Target Automation (%)'] - process_df['Current Automation (%)']
process_df = process_df.sort_values('Gap', ascending=False)

print("Back-Office Process Inventory:")
print(process_df.to_string(index=False))

# Visualise
fig, ax = plt.subplots(figsize=(12, 6))
x = np.arange(len(processes))
width = 0.35

ax.barh(x - width/2, process_df['Current Automation (%)'], width, label='Current', color='blue', alpha=0.7)
ax.barh(x + width/2, process_df['Target Automation (%)'], width, label='Target', color='green', alpha=0.7)

ax.set_yticks(x)
ax.set_yticklabels(process_df['Process'])
ax.set_xlabel('Automation (%)')
ax.set_title('Back-Office Automation Status')
ax.legend()
ax.grid(True, alpha=0.3, axis='x')

plt.tight_layout()
plt.savefig('backoffice_automation.png', dpi=300, bbox_inches='tight')
plt.show()
print("Back-office automation visualisation saved as 'backoffice_automation.png'")

# ----------------------------------------------------------------
# PART B: DOCUMENT PROCESSING SIMULATION
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART B: Document Processing Simulation")
print("-"*60)

# Simulate document processing pipeline
def simulate_document_processing(n_docs=500):
    """Simulate the document processing lifecycle."""
    
    np.random.seed(42)
    
    # Document types and volumes
    doc_types = ['Invoice', 'KYC', 'Loan Application', 'Statement', 'Contract', 'Compliance Report']
    
    docs = pd.DataFrame({
        'doc_id': range(1, n_docs + 1),
        'doc_type': np.random.choice(doc_types, n_docs),
        'received': [datetime.now() - timedelta(minutes=np.random.randint(0, 1440)) for _ in range(n_docs)],
        'status': 'Pending',
        'page_count': np.random.poisson(3, n_docs).clip(1, 20)
    })
    
    # Simulate processing stages
    processing_stages = ['Receipt', 'Classification', 'OCR', 'Data Extraction', 'Validation', 'Approval', 'Archived']
    
    # Simulate processing times (minutes)
    stage_times = {
        'Receipt': np.random.gamma(0.5, 0.5, n_docs).clip(0.1, 3),
        'Classification': np.random.gamma(0.5, 0.5, n_docs).clip(0.1, 3),
        'OCR': np.random.gamma(0.5 * docs['page_count'], 0.5, n_docs).clip(0.2, 10),
        'Data Extraction': np.random.gamma(1, 0.5, n_docs).clip(0.2, 8),
        'Validation': np.random.gamma(0.5, 0.5, n_docs).clip(0.1, 4),
        'Approval': np.random.gamma(0.5, 0.3, n_docs).clip(0.1, 3),
        'Archived': np.random.gamma(0.3, 0.3, n_docs).clip(0.1, 2)
    }
    
    # Assign processing times
    docs['total_time'] = np.sum([stage_times[s] for s in processing_stages], axis=0)
    
    # Simulate errors
    docs['has_error'] = np.random.random(n_docs) < 0.05
    docs['error_type'] = np.where(docs['has_error'], 
                                 np.random.choice(['OCR Error', 'Missing Data', 'Validation Error', 'Document Quality'], n_docs),
                                 'None')
    
    # Simulate status distribution
    status_dist = {'Pending': 0.15, 'Processing': 0.20, 'Completed': 0.50, 'Error': 0.10, 'Archived': 0.05}
    docs['status'] = np.random.choice(list(status_dist.keys()), n_docs, p=list(status_dist.values()))
    
    return docs

# Simulate document processing
docs = simulate_document_processing(500)

print("Document Processing Summary:")
print(f"Total Documents: {len(docs)}")
print(f"Average Processing Time: {docs['total_time'].mean():.2f} minutes")
print(f"Error Rate: {(docs['has_error'].sum() / len(docs) * 100):.2f}%")

# Summary by document type
doc_summary = docs.groupby('doc_type').agg({
    'doc_id': 'count',
    'total_time': 'mean',
    'page_count': 'mean',
    'has_error': 'sum'
}).round(2)
doc_summary.columns = ['Count', 'Avg Time (min)', 'Avg Pages', 'Errors']
print("\nDocument Processing by Type:")
print(doc_summary)

# Visualise
fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# Document Type Distribution
ax = axes[0, 0]
doc_counts = docs['doc_type'].value_counts()
ax.pie(doc_counts.values, labels=doc_counts.index, autopct='%1.1f%%')
ax.set_title('Document Type Distribution')

# Processing Time by Type
ax = axes[0, 1]
docs.boxplot(column='total_time', by='doc_type', ax=ax)
ax.set_title('Processing Time by Document Type')
ax.set_ylabel('Time (minutes)')
ax.set_xlabel('')

# Error Rate by Type
ax = axes[1, 0]
error_by_type = docs.groupby('doc_type')['has_error'].mean() * 100
ax.bar(error_by_type.index, error_by_type.values, color='red', alpha=0.7)
ax.set_xlabel('Document Type')
ax.set_ylabel('Error Rate (%)')
ax.set_title('Error Rate by Document Type')
ax.tick_params(axis='x', rotation=45)

# Status Distribution
ax = axes[1, 1]
status_counts = docs['status'].value_counts()
ax.bar(status_counts.index, status_counts.values, color='teal', alpha=0.7)
ax.set_xlabel('Status')
ax.set_ylabel('Count')
ax.set_title('Document Status Distribution')
ax.tick_params(axis='x', rotation=45)

plt.tight_layout()
plt.savefig('document_processing.png', dpi=300, bbox_inches='tight')
plt.show()
print("Document processing visualisation saved as 'document_processing.png'")

# ----------------------------------------------------------------
# PART C: WORKFLOW AUTOMATION SIMULATION
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART C: Workflow Automation Simulation")
print("-"*60)

class WorkflowAutomator:
    """Simulate a workflow automation system."""
    
    def __init__(self, processes):
        self.processes = processes
        self.running_instances = []
        self.completed_instances = []
        self.error_instances = []
    
    def create_instance(self, process_name, data):
        """Create a new workflow instance."""
        instance = {
            'id': len(self.running_instances) + 1,
            'process': process_name,
            'data': data,
            'status': 'Started',
            'steps': [],
            'created_at': datetime.now(),
            'completed_at': None
        }
        self.running_instances.append(instance)
        return instance
    
    def execute_step(self, instance, step_name, duration=1):
        """Execute a workflow step."""
        import time
        time.sleep(0.01)  # Simulate work
        
        # Simulate step execution
        success = np.random.random() < 0.95
        
        step = {
            'step': step_name,
            'status': 'Completed' if success else 'Error',
            'timestamp': datetime.now(),
            'duration': duration
        }
        instance['steps'].append(step)
        
        if not success:
            instance['status'] = 'Error'
            self.error_instances.append(instance)
            self.running_instances.remove(instance)
            return False
        
        return True
    
    def complete_instance(self, instance):
        """Complete a workflow instance."""
        instance['status'] = 'Completed'
        instance['completed_at'] = datetime.now()
        self.completed_instances.append(instance)
        if instance in self.running_instances:
            self.running_instances.remove(instance)
    
    def get_metrics(self):
        """Get workflow metrics."""
        total = len(self.completed_instances) + len(self.error_instances) + len(self.running_instances)
        return {
            'total_instances': total,
            'completed': len(self.completed_instances),
            'running': len(self.running_instances),
            'error': len(self.error_instances),
            'completion_rate': len(self.completed_instances) / total * 100 if total > 0 else 0,
            'avg_duration': np.mean([(c['completed_at'] - c['created_at']).total_seconds() / 60 
                                    for c in self.completed_instances]) if self.completed_instances else 0
        }

# Simulate workflows
automator = WorkflowAutomator(['Account Opening', 'Loan Origination', 'Document Approval'])

# Create and process instances
for i in range(100):
    process = np.random.choice(['Account Opening', 'Loan Origination', 'Document Approval'])
    data = {'customer_id': f'CUST_{i}', 'amount': np.random.uniform(1000, 50000)}
    instance = automator.create_instance(process, data)
    
    # Execute steps
    steps = ['Step 1: Data Entry', 'Step 2: Verification', 'Step 3: Approval', 'Step 4: Completion']
    for step in steps:
        if not automator.execute_step(instance, step, np.random.gamma(1, 0.5)):
            break
    else:
        automator.complete_instance(instance)

# Get metrics
metrics = automator.get_metrics()
print("Workflow Automation Metrics:")
print(f"Total Instances: {metrics['total_instances']}")
print(f"Completed: {metrics['completed']}")
print(f"Running: {metrics['running']}")
print(f"Errors: {metrics['error']}")
print(f"Completion Rate: {metrics['completion_rate']:.2f}%")
print(f"Average Duration: {metrics['avg_duration']:.2f} minutes")

# ----------------------------------------------------------------
# PART D: BACK-OFFICE TECHNOLOGY STACK
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART D: Back-Office Technology Stack")
print("-"*60)

tech_stack = {
    "Document Management": {
        "Technologies": ["SharePoint", "OpenText", "Box", "Alfresco"],
        "Capabilities": ["Document storage", "Version control", "Search", "Collaboration"]
    },
    "OCR/ICR": {
        "Technologies": ["ABBYY", "Google Document AI", "AWS Textract", "Tesseract"],
        "Capabilities": ["Image to text", "Handwriting recognition", "Structured data extraction"]
    },
    "Workflow Automation": {
        "Technologies": ["Pega", "Appian", "Camunda", "ServiceNow"],
        "Capabilities": ["Process orchestration", "Task management", "Approval workflows"]
    },
    "RPA": {
        "Technologies": ["UiPath", "Automation Anywhere", "Blue Prism"],
        "Capabilities": ["Task automation", "Data entry", "System integration"]
    },
    "NLP/AI": {
        "Technologies": ["spaCy", "Transformers", "AWS Comprehend", "Google Cloud NLP"],
        "Capabilities": ["Document classification", "Entity extraction", "Sentiment analysis"]
    },
    "Reporting": {
        "Technologies": ["Power BI", "Tableau", "Looker", "Crystal Reports"],
        "Capabilities": ["Report generation", "Dashboards", "Data visualisation"]
    }
}

print("Back-Office Technology Stack:")
for layer, details in tech_stack.items():
    print(f"\n{layer}:")
    print(f"  Technologies: {', '.join(details['Technologies'])}")
    print(f"  Capabilities: {', '.join(details['Capabilities'])}")

# ----------------------------------------------------------------
# PART E: BACK-OFFICE METRICS DASHBOARD
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART E: Back-Office Metrics Dashboard")
print("-"*60)

bo_metrics = pd.DataFrame({
    'Metric': [
        'Document Processing Volume',
        'Average Processing Time',
        'Automation Rate',
        'Error Rate',
        'Manual Intervention Rate',
        'Cost per Document',
        'Compliance Rate',
        'Customer CSAT'
    ],
    'Current Value': [
        '2,500/day',
        '18.5 min',
        '42%',
        '4.5%',
        '28%',
        '$2.45',
        '92%',
        '78%'
    ],
    'Target Value': [
        '5,000/day',
        '< 5 min',
        '> 80%',
        '< 1%',
        '< 10%',
        '< $0.50',
        '100%',
        '> 85%'
    ],
    'Status': ['🟡', '🔴', '🔴', '🔴', '🔴', '🟡', '🟡', '🟡']
})

print("Back-Office Metrics Dashboard:")
print(bo_metrics.to_string(index=False))

# ----------------------------------------------------------------
# PART F: BACK-OFFICE AUTOMATION ROADMAP
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART F: Back-Office Automation Roadmap")
print("-"*60)

roadmap = {
    "Phase 1 (0-6 months) – Digitisation": {
        "Focus": "Convert paper processes to digital.",
        "Activities": [
            "Implement document management system (DMS).",
            "Digitise document storage and retrieval.",
            "Implement e-signatures for approvals.",
            "Enable digital forms and applications."
        ],
        "Success Metrics": ["Paper usage reduced by 70%", "Document retrieval time < 30 seconds"]
    },
    "Phase 2 (6-12 months) – Automation": {
        "Focus": "Automate repetitive back-office tasks.",
        "Activities": [
            "Implement RPA for data entry and reconciliation.",
            "Automate document classification and routing.",
            "Implement workflow automation for approvals.",
            "Automate report generation."
        ],
        "Success Metrics": ["Automation rate > 60%", "Manual effort reduced by 50%"]
    },
    "Phase 3 (12-24 months) – Intelligent Automation": {
        "Focus": "Add AI/ML capabilities to back-office.",
        "Activities": [
            "Implement AI-powered document processing (OCR + NLP).",
            "Implement predictive analytics for workload management.",
            "Enable intelligent exception handling.",
            "Implement cognitive automation."
        ],
        "Success Metrics": ["Automation rate > 80%", "Error rate < 1%"]
    },
    "Phase 4 (24+ months) – Autonomous": {
        "Focus": "Self-optimising back-office operations.",
        "Activities": [
            "Enable self-healing processes.",
            "Implement continuous process improvement.",
            "Achieve zero-touch operations.",
            "Enable predictive and prescriptive analytics."
        ],
        "Success Metrics": ["Zero-touch operations > 70%", "Continuous improvement"]
    }
}

for phase, details in roadmap.items():
    print(f"\n{phase}:")
    print(f"  Focus: {details['Focus']}")
    print("  Activities:")
    for activity in details['Activities']:
        print(f"    • {activity}")
    print("  Success Metrics:")
    for metric in details['Success Metrics']:
        print(f"    • {metric}")

# ----------------------------------------------------------------
# PART G: SUMMARY AND RECOMMENDATIONS
# ----------------------------------------------------------------

print("\n" + "="*70)
print("PART G: Summary and Recommendations")
print("="*70)

print("""
Back-Office Automation and Document Management – Key Takeaways:

1. Back-office operations support front-office activities and are ripe for automation.
2. Key processes: document processing, data entry, reconciliation, reporting, compliance.
3. Document management lifecycle: creation → capture → indexing → storage → retrieval → disposal.
4. Workflow automation orchestrates tasks across systems and people.
5. Key technologies: DMS, OCR/ICR, workflow automation, RPA, NLP/AI.
6. Key metrics: processing time, error rate, automation rate, cost per document.
7. Automation roadmap: digitisation → automation → intelligent automation → autonomous.

Recommendations:
  - Implement a document management system.
  - Automate document classification and data extraction.
  - Use workflow automation for approvals and task routing.
  - Implement RPA for repetitive tasks.
  - Add AI/ML for intelligent document processing.
  - Measure and track back-office metrics.
""")

print("="*70)
print("END OF LESSON 5 – MODULE 3")
print("="*70)