Lesson 5.2: Deep Learning Implementation – Training, Regularisation, and Advanced Optimisation for Finance

1. Learning Objectives

By the end of this lesson, you will be able to:

Implement and train neural networks in PyTorch with proper training loops.
Apply advanced regularisation techniques (dropout, batch normalisation, weight decay, data augmentation).
Implement learning rate schedules (Step, Cosine, ReduceLROnPlateau) for financial models.
Use advanced optimisers (Adam, AdamW, RMSprop, SGD with momentum) effectively.
Implement early stopping and model checkpointing for robust training.
Apply data augmentation techniques for financial time series (time warping, jittering, scaling).
Understand the importance of hyperparameter tuning for neural networks.
Avoid common pitfalls in training financial neural networks.

2. The Complete Training Pipeline

2.1 Data Preparation

import torch
from torch.utils.data import DataLoader, TensorDataset
from sklearn.preprocessing import StandardScaler

def prepare_financial_data(X_train, y_train, X_val, y_val, X_test, y_test, batch_size=128):
    """
    Prepare data for neural network training.
    """
    # Convert to tensors
    X_train_t = torch.FloatTensor(X_train.values)
    y_train_t = torch.FloatTensor(y_train.values)
    X_val_t = torch.FloatTensor(X_val.values)
    y_val_t = torch.FloatTensor(y_val.values)
    X_test_t = torch.FloatTensor(X_test.values)
    y_test_t = torch.FloatTensor(y_test.values)

    # Create datasets
    train_dataset = TensorDataset(X_train_t, y_train_t)
    val_dataset = TensorDataset(X_val_t, y_val_t)
    test_dataset = TensorDataset(X_test_t, y_test_t)

    # Create dataloaders
    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, pin_memory=True)
    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, pin_memory=True)
    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, pin_memory=True)

    return train_loader, val_loader, test_loader

2.2 Model Definition

import torch.nn as nn
import torch.nn.functional as F

class FinancialNN(nn.Module):
    def __init__(self, input_dim, hidden_dims, output_dim, dropout_rate=0.3, use_batch_norm=True):
        super(FinancialNN, self).__init__()

        layers = []
        prev_dim = input_dim

        for hidden_dim in hidden_dims:
            layers.append(nn.Linear(prev_dim, hidden_dim))
            if use_batch_norm:
                layers.append(nn.BatchNorm1d(hidden_dim))
            layers.append(nn.ReLU())
            layers.append(nn.Dropout(dropout_rate))
            prev_dim = hidden_dim

        layers.append(nn.Linear(prev_dim, output_dim))

        self.network = nn.Sequential(*layers)

        # Weight initialisation
        self._init_weights()

    def _init_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Linear):
                nn.init.kaiming_normal_(m.weight, mode='fan_in', nonlinearity='relu')
                if m.bias is not None:
                    nn.init.constant_(m.bias, 0)

    def forward(self, x):
        return self.network(x)

2.3 The Complete Training Loop

import torch.optim as optim
from torch.optim.lr_scheduler import ReduceLROnPlateau, CosineAnnealingLR, StepLR

def train_neural_network(model, train_loader, val_loader, criterion, epochs=100,
                         lr=0.001, weight_decay=1e-5, patience=20,
                         scheduler_type='plateau', checkpoint_path='best_model.pth'):
    """
    Complete training loop with early stopping and checkpointing.
    """
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model.to(device)

    # Optimiser
    optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay, betas=(0.9, 0.999))

    # Learning rate scheduler
    if scheduler_type == 'plateau':
        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10, verbose=True)
    elif scheduler_type == 'cosine':
        scheduler = CosineAnnealingLR(optimizer, T_max=epochs, eta_min=1e-6)
    elif scheduler_type == 'step':
        scheduler = StepLR(optimizer, step_size=30, gamma=0.1)
    else:
        scheduler = None

    # Training history
    history = {'train_loss': [], 'val_loss': [], 'lr': []}

    best_val_loss = float('inf')
    patience_counter = 0

    for epoch in range(epochs):
        # Training phase
        model.train()
        train_loss = 0.0
        for X_batch, y_batch in train_loader:
            X_batch = X_batch.to(device)
            y_batch = y_batch.to(device)

            # Forward pass
            y_pred = model(X_batch)
            loss = criterion(y_pred.squeeze(), y_batch)

            # Backward pass
            optimizer.zero_grad()
            loss.backward()

            # Gradient clipping
            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)

            optimizer.step()

            train_loss += loss.item() * X_batch.size(0)

        train_loss /= len(train_loader.dataset)
        history['train_loss'].append(train_loss)

        # Validation phase
        model.eval()
        val_loss = 0.0
        with torch.no_grad():
            for X_batch, y_batch in val_loader:
                X_batch = X_batch.to(device)
                y_batch = y_batch.to(device)

                y_pred = model(X_batch)
                loss = criterion(y_pred.squeeze(), y_batch)
                val_loss += loss.item() * X_batch.size(0)

        val_loss /= len(val_loader.dataset)
        history['val_loss'].append(val_loss)

        # Learning rate scheduler
        if scheduler_type == 'plateau':
            scheduler.step(val_loss)
        elif scheduler is not None:
            scheduler.step()

        # Record learning rate
        history['lr'].append(optimizer.param_groups[0]['lr'])

        # Early stopping and checkpointing
        if val_loss < best_val_loss:
            best_val_loss = val_loss
            patience_counter = 0
            torch.save({
                'epoch': epoch,
                'model_state_dict': model.state_dict(),
                'optimizer_state_dict': optimizer.state_dict(),
                'val_loss': val_loss,
                'history': history
            }, checkpoint_path)
        else:
            patience_counter += 1
            if patience_counter >= patience:
                print(f"Early stopping at epoch {epoch}")
                break

        if epoch % 10 == 0:
            print(f"Epoch {epoch}: train_loss={train_loss:.6f}, val_loss={val_loss:.6f}, lr={optimizer.param_groups[0]['lr']:.6f}")

    # Load best model
    checkpoint = torch.load(checkpoint_path)
    model.load_state_dict(checkpoint['model_state_dict'])

    return model, history

3. Regularisation Techniques for Financial Neural Networks

3.1 L1 and L2 Regularisation (Weight Decay)

# L2 regularisation (built into optimiser)
optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)

# L1 regularisation (manual)
def l1_regularisation(model, lambda_l1=1e-5):
    l1_norm = sum(p.abs().sum() for p in model.parameters())
    return lambda_l1 * l1_norm

# In training loop:
loss = criterion(y_pred, y_batch) + l1_regularisation(model)

3.2 Dropout – Preventing Co-Adaptation

class DropoutNN(nn.Module):
    def __init__(self, input_dim, hidden_dims, output_dim, dropout_rate=0.5):
        super(DropoutNN, self).__init__()
        layers = []
        prev_dim = input_dim
        for hidden_dim in hidden_dims:
            layers.append(nn.Linear(prev_dim, hidden_dim))
            layers.append(nn.ReLU())
            layers.append(nn.Dropout(dropout_rate))
            prev_dim = hidden_dim
        layers.append(nn.Linear(prev_dim, output_dim))
        self.network = nn.Sequential(*layers)

    def forward(self, x):
        return self.network(x)

3.3 Batch Normalisation – Stabilising Training

class BatchNormNN(nn.Module):
    def __init__(self, input_dim, hidden_dims, output_dim):
        super(BatchNormNN, self).__init__()
        layers = []
        prev_dim = input_dim
        for hidden_dim in hidden_dims:
            layers.append(nn.Linear(prev_dim, hidden_dim))
            layers.append(nn.BatchNorm1d(hidden_dim))
            layers.append(nn.ReLU())
            prev_dim = hidden_dim
        layers.append(nn.Linear(prev_dim, output_dim))
        self.network = nn.Sequential(*layers)

    def forward(self, x):
        return self.network(x)

3.4 Data Augmentation for Financial Time Series

Data augmentation creates synthetic training samples to improve generalisation.

def augment_financial_time_series(X, y, augmentation_methods=['jitter', 'scale', 'time_warp']):
    """
    Augment financial time series data.
    """
    X_aug = []
    y_aug = []

    for X_i, y_i in zip(X, y):
        # Original
        X_aug.append(X_i)
        y_aug.append(y_i)

        if 'jitter' in augmentation_methods:
            # Add Gaussian noise
            noise = np.random.normal(0, 0.01, X_i.shape)
            X_aug.append(X_i + noise)
            y_aug.append(y_i)

        if 'scale' in augmentation_methods:
            # Scale by random factor
            scale = np.random.uniform(0.9, 1.1)
            X_aug.append(X_i * scale)
            y_aug.append(y_i)

        if 'time_warp' in augmentation_methods:
            # Time warping (simplified)
            if len(X_i) > 10:
                warp_factor = np.random.uniform(0.9, 1.1)
                indices = np.arange(len(X_i))
                warped_indices = np.linspace(0, len(X_i)-1, int(len(X_i) * warp_factor))
                warped_indices = np.clip(warped_indices, 0, len(X_i)-1).astype(int)
                X_aug.append(X_i[warped_indices])
                y_aug.append(y_i)

    return np.array(X_aug), np.array(y_aug)

4. Learning Rate Schedulers – Optimising Convergence

4.1 StepLR – Step Decay
Reduces learning rate by gamma every step_size epochs.
lr = lr * gamma^{floor(epoch / step_size)}

scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)

4.2 ReduceLROnPlateau – Adaptive Decay
Reduces learning rate when a metric has stopped improving.

scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
    optimizer, mode='min', factor=0.5, patience=10, threshold=1e-4, min_lr=1e-6
)

4.3 Cosine Annealing – Cyclical Learning
lr_t = lr_min + (lr_max - lr_min) * (1 + cos(π * t / T_max)) / 2

scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
    optimizer, T_max=50, eta_min=1e-6
)

4.4 OneCycleLR – Fast Convergence
Increases learning rate then decreases it.

scheduler = torch.optim.lr_scheduler.OneCycleLR(
    optimizer, max_lr=0.01, steps_per_epoch=len(train_loader), epochs=epochs
)

5. Advanced Optimisers – Beyond Standard SGD

5.1 SGD with Momentum

optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-5)

5.2 RMSprop

optimizer = optim.RMSprop(model.parameters(), lr=0.001, alpha=0.99, eps=1e-8, weight_decay=1e-5)

5.3 Adam (Most Common)

optimizer = optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-8, weight_decay=1e-5)

5.4 AdamW (Adam with Decoupled Weight Decay)

optimizer = optim.AdamW(model.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-8, weight_decay=1e-5)

5.5 Optimiser Comparison for Finance

Optimiser	Pros	Cons	Financial Use
SGD + Momentum	Simple, generalises well	Requires careful tuning	Small datasets
Adam	Fast, robust, default	May not generalise as well	Default for most tasks
AdamW	Same as Adam, better regularisation	Slightly slower	Recommended default
RMSprop	Good for non-stationary	Can be unstable	Volatility forecasting

6. Hyperparameter Tuning for Neural Networks

6.1 Key Hyperparameters

Hyperparameter	Range	Financial Recommendation
Learning Rate	1e-5 to 1e-1	Start with 1e-3 (Adam) or 1e-2 (SGD)
Batch Size	16 to 512	128 (balance speed and convergence)
Hidden Layers	1 to 5	2-3 for most financial tasks
Hidden Units	32 to 1024	128-256 for moderate datasets
Dropout Rate	0.1 to 0.5	0.3 for noisy financial data
Weight Decay	0 to 1e-2	1e-5 (light regularisation)
Number of Epochs	50 to 500	Use early stopping

6.2 Hyperparameter Tuning with Optuna

import optuna

def objective(trial, X_train, y_train, X_val, y_val):
    # Define hyperparameters
    lr = trial.suggest_float('lr', 1e-5, 1e-1, log=True)
    hidden_units = trial.suggest_int('hidden_units', 32, 512)
    num_layers = trial.suggest_int('num_layers', 1, 4)
    dropout = trial.suggest_float('dropout', 0.1, 0.5)
    weight_decay = trial.suggest_float('weight_decay', 1e-6, 1e-2, log=True)
    batch_size = trial.suggest_categorical('batch_size', [32, 64, 128, 256])

    # Build model
    hidden_dims = [hidden_units] * num_layers
    model = FinancialNN(X_train.shape[1], hidden_dims, 1, dropout)

    # Train and evaluate
    train_loader, val_loader, _ = prepare_financial_data(
        X_train, y_train, X_val, y_val, None, None, batch_size
    )

    model, history = train_neural_network(
        model, train_loader, val_loader, nn.MSELoss(),
        epochs=50, lr=lr, weight_decay=weight_decay, patience=10
    )

    return min(history['val_loss'])

study = optuna.create_study(direction='minimize')
study.optimize(lambda trial: objective(trial, X_train, y_train, X_val, y_val), n_trials=100)

7. Common Pitfalls and Solutions

Pitfall	Solution
Overfitting	Increase dropout, reduce model size, add weight decay, use early stopping.
Underfitting	Increase model size, reduce regularisation, increase training time.
Vanishing Gradients	Use ReLU, batch normalisation, residual connections.
Exploding Gradients	Use gradient clipping, smaller learning rate, weight initialisation.
Non-Convergence	Adjust learning rate, use learning rate scheduler, try different optimiser.
Data Leakage	Use time series cross-validation, never use future data.
Non-Stationarity	Use rolling windows, online learning, retrain periodically.

8. Summary for the AI Practitioner

Training pipeline: Data preparation → Model definition → Training loop → Validation → Early stopping → Checkpointing.
Regularisation is mandatory for financial neural networks. Use dropout (0.3), weight decay (1e-5), and batch normalisation.
Adam/AdamW are the default optimisers. Use OneCycleLR for fast convergence.
Early stopping prevents overfitting. Save the best model based on validation loss.
Hyperparameter tuning (Optuna) significantly improves performance. Start with learning rate, hidden units, and dropout.
Gradient clipping prevents exploding gradients. Use max_norm=1.0.
Data augmentation (jittering, scaling) can improve generalisation for small datasets.
Monitor training – training loss should decrease, validation loss should not increase significantly.

Diploma in Artificial Intelligence in Finance Lesson 5.2: Deep Learning Implementation – Training, Regularisation, and Advanced Optimisation for Finance

Diploma in Artificial Intelligence in Finance