1. LEARNING OBJECTIVES

By the end of this lesson, you will be able to:

  • Understand the NumPy array structure and its advantages over Python lists.

  • Perform vectorized operations on financial data using NumPy.

  • Compute financial metrics (returns, volatility, correlations) using NumPy.

  • Use Pandas Series and DataFrames for time series financial data.

  • Perform data cleaning, filtering, and transformation with Pandas.

  • Resample and align financial time series data.

  • Compute rolling statistics (moving averages, volatility) using Pandas.

  • Merge and join multiple financial datasets.

  • Export processed financial data to CSV, Excel, and Parquet formats.

2. NUMPY – THE FOUNDATION OF FINANCIAL COMPUTING

2.1 Why NumPy?

NumPy provides:

  • Homogeneous arrays: All elements have the same data type (e.g., float64).

  • Vectorized operations: Operations are applied element-wise without Python loops.

  • Memory efficiency: Arrays are stored contiguously in memory.

  • Broadcasting: Operations between arrays of different shapes.

  • Linear algebra: Matrix operations, eigenvalues, decompositions.

2.2 Creating NumPy Arrays

python
import numpy as np

# From Python list
prices = np.array([100, 102, 98, 105, 110])

# Zeros and ones
zeros = np.zeros(10)                    # Array of 10 zeros
ones = np.ones((3, 4))                  # 3x4 array of ones

# Range and linspace
sequence = np.arange(0, 10, 0.5)        # 0, 0.5, 1.0, ..., 9.5
log_space = np.linspace(1, 100, 50)     # 50 evenly spaced values from 1 to 100

# Random numbers
normal_returns = np.random.normal(0, 1, 1000)        # Normal distribution
uniform_prices = np.random.uniform(90, 110, 100)     # Uniform distribution
log_returns = np.random.lognormal(0, 0.2, 1000)      # Log-normal distribution

2.3 Array Attributes and Operations

python
prices = np.array([100, 102, 98, 105, 110])

# Basic attributes
print(prices.shape)      # (5,)
print(prices.ndim)       # 1
print(prices.size)       # 5
print(prices.dtype)      # int64

# Vectorized arithmetic
returns = prices[1:] / prices[:-1] - 1
# returns = array([0.02, -0.03921569, 0.07142857, 0.04761905])

# Mathematical functions
log_returns = np.log(prices[1:] / prices[:-1])
mean_return = np.mean(returns)
std_return = np.std(returns)
volatility = np.std(returns) * np.sqrt(252)  # Annualized

# Cumulative product (for cumulative returns)
cumulative_return = np.cumprod(1 + returns) - 1

2.4 Indexing and Slicing

python
prices = np.array([100, 102, 98, 105, 110])

# Basic slicing
first_three = prices[:3]           # [100, 102, 98]
last_two = prices[-2:]             # [105, 110]
middle = prices[1:4]               # [102, 98, 105]

# Boolean indexing (masking)
above_100 = prices[prices > 100]   # [102, 105, 110]

# Fancy indexing
indices = [0, 2, 4]
selected = prices[indices]         # [100, 98, 110]

2.5 Broadcasting

python
prices = np.array([100, 102, 98, 105, 110])

# Scalar operations
doubled = prices * 2
shifted = prices + 5

# Vector operations
weights = np.array([0.2, 0.3, 0.1, 0.25, 0.15])
weighted_avg = np.sum(prices * weights)

# Broadcasting with 2D arrays
prices_2d = np.array([[100, 102, 98],
                      [105, 110, 115],
                      [98, 97, 99]])
normalized = prices_2d / prices_2d.mean(axis=1, keepdims=True)

2.6 Linear Algebra for Finance

python
# Portfolio variance calculation
returns_matrix = np.random.normal(0.001, 0.02, (1000, 5))  # 5 assets, 1000 days
cov_matrix = np.cov(returns_matrix.T)  # 5x5 covariance matrix

weights = np.array([0.2, 0.2, 0.2, 0.2, 0.2])
portfolio_variance = weights.T @ cov_matrix @ weights
portfolio_std = np.sqrt(portfolio_variance)

# Solving linear systems (for portfolio optimization)
# Constraint: weights sum to 1
ones = np.ones(5)
# Solve for minimum variance portfolio: w = Σ^{-1} 1 / (1^T Σ^{-1} 1)
inv_cov = np.linalg.inv(cov_matrix)
min_var_weights = inv_cov @ ones / (ones.T @ inv_cov @ ones)

# Cholesky decomposition (for simulation)
L = np.linalg.cholesky(cov_matrix)
correlated_returns = np.random.normal(0, 1, (1000, 5)) @ L.T

2.7 Performance Comparison

python
import time

# Python list vs NumPy array performance
size = 1000000

# Python list
start = time.time()
lst = list(range(size))
result = [x * 2 + 1 for x in lst]
print(f"List: {time.time() - start:.4f} seconds")

# NumPy array
start = time.time()
arr = np.arange(size)
result = arr * 2 + 1
print(f"NumPy: {time.time() - start:.4f} seconds")
# NumPy is typically 10-100x faster

3. PANDAS – TIME SERIES ANALYSIS

3.1 Series – One-Dimensional Data

python
import pandas as pd

# Creating a Series
prices = pd.Series([100, 102, 98, 105, 110],
                   index=['2024-01-01', '2024-01-02', '2024-01-03',
                          '2024-01-04', '2024-01-05'])
print(prices)
# 2024-01-01    100
# 2024-01-02    102
# 2024-01-03     98
# 2024-01-04    105
# 2024-01-05    110

# Accessing data
print(prices['2024-01-03'])        # 98
print(prices.iloc[2])               # 98 (positional)
print(prices.loc['2024-01-04'])     # 105 (label-based)

# Vectorized operations
returns = prices.pct_change()
log_returns = np.log(prices / prices.shift(1))

3.2 DataFrame – Tabular Data

python
# Creating a DataFrame
data = {
    'AAPL': [150, 152, 148, 155, 160],
    'GOOGL': [2800, 2820, 2780, 2850, 2900],
    'MSFT': [330, 332, 328, 335, 340]
}
index = pd.date_range('2024-01-01', periods=5, freq='D')
df = pd.DataFrame(data, index=index)

print(df)
#             AAPL  GOOGL  MSFT
# 2024-01-01   150   2800   330
# 2024-01-02   152   2820   332
# 2024-01-03   148   2780   328
# 2024-01-04   155   2850   335
# 2024-01-05   160   2900   340

# Accessing columns
aapl_prices = df['AAPL']
aapl_prices = df.AAPL  # Alternative

# Accessing rows
row_2 = df.iloc[2]      # Positional
row_3 = df.loc['2024-01-04']  # Label-based

# Accessing subsets
subset = df.loc['2024-01-02':'2024-01-04', ['AAPL', 'MSFT']]

3.3 Basic DataFrame Operations

python
# Descriptive statistics
summary = df.describe()
print(summary)
#              AAPL       GOOGL        MSFT
# count    5.000000    5.000000    5.000000
# mean   153.000000 2820.000000  333.000000
# std      4.636809   44.721360    4.636809
# min    148.000000 2780.000000  328.000000
# 25%    150.000000 2800.000000  330.000000
# 50%    152.000000 2820.000000  332.000000
# 75%    155.000000 2850.000000  335.000000
# max    160.000000 2900.000000  340.000000

# Adding a new column
df['AAPL_Return'] = df['AAPL'].pct_change()

# Applying functions
df['AAPL_Rounded'] = df['AAPL'].round(0)

# Column statistics
correlation = df[['AAPL', 'GOOGL']].corr()
covariance = df[['AAPL', 'GOOGL']].cov()

3.4 Handling Missing Data

python
# Creating data with missing values
import numpy as np
df_with_nan = df.copy()
df_with_nan.loc['2024-01-03', 'AAPL'] = np.nan

# Checking for missing values
print(df_with_nan.isnull())          # Boolean mask
print(df_with_nan.isnull().sum())    # Count per column

# Dropping missing values
df_dropped = df_with_nan.dropna()    # Drop any row with NaN
df_dropped_col = df_with_nan.dropna(axis=1)  # Drop any column with NaN

# Filling missing values
df_filled = df_with_nan.fillna(method='ffill')   # Forward fill
df_filled = df_with_nan.fillna(method='bfill')   # Backward fill
df_filled = df_with_nan.fillna(df.mean())        # Fill with mean

# Interpolation
df_interp = df_with_nan.interpolate(method='linear')

3.5 Date and Time Handling

python
# Creating date ranges
dates = pd.date_range('2024-01-01', periods=252, freq='B')  # Business days
monthly = pd.date_range('2024-01-01', periods=12, freq='M')  # Month end
quarterly = pd.date_range('2024-01-01', periods=4, freq='Q')  # Quarter end

# Date arithmetic
start = pd.Timestamp('2024-01-01')
end = start + pd.Timedelta(days=30)
delta = end - start  # 30 days

# Resampling
daily_data = pd.DataFrame(np.random.randn(100, 2),
                          index=pd.date_range('2024-01-01', periods=100, freq='D'))
monthly_data = daily_data.resample('M').mean()
weekly_data = daily_data.resample('W').agg(['mean', 'std'])

3.6 Resampling Financial Data

python
# Daily returns to monthly returns
def daily_to_monthly(df):
    monthly = df.resample('M').last()  # End of month prices
    monthly_returns = monthly.pct_change()
    return monthly_returns

# Volatility from daily to annual
def annualize_volatility(daily_returns):
    return daily_returns.std() * np.sqrt(252)

# Upsampling (interpolating)
hourly = df.resample('h').interpolate(method='linear')

3.7 Rolling and Expanding Windows

python
# Rolling statistics
df['AAPL_MA_50'] = df['AAPL'].rolling(window=50).mean()
df['AAPL_Std_20'] = df['AAPL'].rolling(window=20).std()
df['AAPL_Vol_20'] = df['AAPL_Std_20'] * np.sqrt(252)

# Exponential moving average
df['AAPL_EMA_20'] = df['AAPL'].ewm(span=20, adjust=False).mean()

# Expanding statistics (from inception)
df['AAPL_Cumulative_Mean'] = df['AAPL'].expanding().mean()
df['AAPL_Cumulative_Std'] = df['AAPL'].expanding().std()

# Rolling correlations
df['Corr_AAPL_GOOGL'] = df['AAPL'].rolling(60).corr(df['GOOGL'])

3.8 Merging and Joining DataFrames

python
# Creating two datasets
df1 = pd.DataFrame({'AAPL': [150, 152, 148]},
                   index=pd.date_range('2024-01-01', periods=3, freq='D'))
df2 = pd.DataFrame({'GOOGL': [2800, 2820, 2780]},
                   index=pd.date_range('2024-01-01', periods=3, freq='D'))

# Concatenation (vertical)
df_concatenated = pd.concat([df1, df2], axis=1)

# Merging (SQL-style)
df3 = pd.DataFrame({
    'Date': pd.date_range('2024-01-01', periods=5, freq='D'),
    'AAPL': [150, 152, 148, 155, 160],
    'GOOGL': [2800, 2820, 2780, 2850, 2900]
})
df4 = pd.DataFrame({
    'Date': pd.date_range('2024-01-01', periods=4, freq='D'),
    'MSFT': [330, 332, 328, 335]
})
merged = pd.merge(df3, df4, on='Date', how='inner')  # Inner join
merged_left = pd.merge(df3, df4, on='Date', how='left')  # Left join

3.9 GroupBy Operations

python
# Simulating multiple stocks with sectors
data = {
    'Symbol': ['AAPL', 'AAPL', 'GOOGL', 'GOOGL', 'MSFT', 'MSFT'],
    'Sector': ['Tech', 'Tech', 'Tech', 'Tech', 'Tech', 'Tech'],
    'Date': ['2024-01-01', '2024-01-02', '2024-01-01', '2024-01-02',
             '2024-01-01', '2024-01-02'],
    'Price': [150, 152, 2800, 2820, 330, 332]
}
df = pd.DataFrame(data)

# Group by symbol
grouped = df.groupby('Symbol')
print(grouped['Price'].mean())
# Symbol
# AAPL     151.0
# GOOGL   2810.0
# MSFT     331.0

# Multiple aggregations
agg = df.groupby('Symbol').agg({
    'Price': ['mean', 'std', 'min', 'max'],
    'Date': ['count']
})

3.10 Financial Data Processing Pipeline

python
def financial_data_pipeline(filename):
    """
    Complete pipeline for processing financial data.
    """
    # 1. Load data
    df = pd.read_csv(filename, parse_dates=['Date'])
    df.set_index('Date', inplace=True)
    
    # 2. Clean data
    df = df.dropna()
    df = df[~df.index.duplicated(keep='first')]
    
    # 3. Calculate returns
    df['Returns'] = df['Price'].pct_change()
    df['Log_Returns'] = np.log(df['Price'] / df['Price'].shift(1))
    
    # 4. Calculate rolling statistics
    df['MA_50'] = df['Price'].rolling(50).mean()
    df['MA_200'] = df['Price'].rolling(200).mean()
    df['Volatility_20'] = df['Returns'].rolling(20).std() * np.sqrt(252)
    
    # 5. Calculate indicators
    df['RSI'] = calculate_rsi(df['Price'], 14)
    
    # 6. Create signals
    df['Signal'] = 0
    df.loc[df['Price'] > df['MA_50'], 'Signal'] = 1
    df.loc[df['Price'] < df['MA_50'], 'Signal'] = -1
    
    return df

def calculate_rsi(prices, period=14):
    delta = prices.diff()
    gain = (delta.where(delta > 0, 0)).rolling(window=period).mean()
    loss = (-delta.where(delta < 0, 0)).rolling(window=period).mean()
    rs = gain / loss
    rsi = 100 - (100 / (1 + rs))
    return rsi

4. PRACTICAL IMPLEMENTATION

A. Portfolio Performance Analysis:

python
def portfolio_performance(returns, weights, risk_free_rate=0.02):
    """
    Calculate portfolio performance metrics.
    """
    # Expected return
    portfolio_return = np.sum(returns.mean() * weights) * 252
    
    # Volatility
    cov_matrix = returns.cov() * 252
    portfolio_vol = np.sqrt(weights.T @ cov_matrix @ weights)
    
    # Sharpe ratio
    sharpe = (portfolio_return - risk_free_rate) / portfolio_vol
    
    # Maximum drawdown
    cumulative = (1 + returns).cumprod()
    running_max = cumulative.expanding().max()
    drawdown = (cumulative / running_max) - 1
    max_drawdown = drawdown.min()
    
    return {
        'return': portfolio_return,
        'volatility': portfolio_vol,
        'sharpe': sharpe,
        'max_drawdown': max_drawdown
    }

B. Efficient Frontier Construction:

python
def efficient_frontier(returns, n_portfolios=1000):
    """
    Generate random portfolios and calculate the efficient frontier.
    """
    n_assets = returns.shape[1]
    
    # Random weights
    weights = np.random.random((n_portfolios, n_assets))
    weights = weights / weights.sum(axis=1, keepdims=True)
    
    # Portfolio returns and volatility
    portfolio_returns = np.sum(returns.mean() * weights, axis=1) * 252
    cov_matrix = returns.cov() * 252
    
    portfolio_vol = []
    for w in weights:
        vol = np.sqrt(w @ cov_matrix @ w)
        portfolio_vol.append(vol)
    
    portfolio_vol = np.array(portfolio_vol)
    sharpe = (portfolio_returns - 0.02) / portfolio_vol
    
    # Find optimal portfolios
    min_vol_idx = np.argmin(portfolio_vol)
    max_sharpe_idx = np.argmax(sharpe)
    
    return {
        'weights': weights,
        'returns': portfolio_returns,
        'volatility': portfolio_vol,
        'sharpe': sharpe,
        'min_vol': portfolio_vol[min_vol_idx],
        'max_sharpe': sharpe[max_sharpe_idx]
    }

C. Backtesting a Trading Strategy:

python
def backtest_strategy(df, signal_column, price_column='Price', initial_capital=10000):
    """
    Backtest a trading strategy based on signal column.
    """
    # Position: 1 = long, -1 = short, 0 = out
    positions = df[signal_column].shift(1).fillna(0)
    
    # Returns
    strategy_returns = positions * df[price_column].pct_change()
    
    # Cumulative returns
    cumulative = (1 + strategy_returns).cumprod() * initial_capital
    benchmark = (1 + df[price_column].pct_change()).cumprod() * initial_capital
    
    # Performance metrics
    total_return = cumulative.iloc[-1] - initial_capital
    annualized_return = (cumulative.iloc[-1] / initial_capital) ** (252 / len(df)) - 1
    volatility = strategy_returns.std() * np.sqrt(252)
    sharpe = annualized_return / volatility
    
    max_drawdown = (cumulative / cumulative.expanding().max()) - 1
    max_dd = max_drawdown.min()
    
    return {
        'cumulative': cumulative,
        'benchmark': benchmark,
        'total_return': total_return,
        'annualized_return': annualized_return,
        'volatility': volatility,
        'sharpe': sharpe,
        'max_drawdown': max_dd
    }
 
PreviousNext