numpy >= 1.24.0      # Core numerical computing
pandas >= 2.0.0      # Data manipulation and time series
scipy >= 1.10.0      # Scientific computing (optimisation, stats)
matplotlib >= 3.7.0  # Visualisation
seaborn >= 0.12.0    # Statistical visualisation
scikit-learn >= 1.3.0 # Machine learning
statsmodels >= 0.14.0 # Statistical models (ARIMA, GARCH)
yfinance >= 0.2.0    # Free financial data (Yahoo Finance)
pytorch >= 2.0.0     # Deep learning (CUDA support)
tensorflow >= 2.13.0 # Alternative deep learning framework

# Create a new conda environment
conda create -n finai python=3.10
conda activate finai

# Install core libraries
conda install numpy pandas scipy matplotlib seaborn scikit-learn statsmodels
pip install yfinance torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118

import numpy as np

# From Python list
prices = np.array([100.0, 101.5, 102.0, 101.0, 103.5])

# Zeros, ones, identity matrix
zeros = np.zeros((3, 4))           # 3 rows, 4 columns of zeros
ones = np.ones((3, 4))              # All ones
identity = np.eye(5)                # 5x5 identity matrix

# Random arrays
uniform = np.random.uniform(-1, 1, (1000, 10))  # 1000x10 uniform [-1,1]
normal = np.random.normal(0, 1, (1000, 10))     # 1000x10 standard normal

# Arange and linspace
sequence = np.arange(0, 100, 0.5)   # 0 to 100 step 0.5
logspace = np.logspace(0, 3, 10)    # 10 points between 10^0 and 10^3

# Element-wise arithmetic
returns = np.diff(prices) / prices[:-1]  # Simple returns
log_returns = np.log(prices[1:] / prices[:-1])  # Log returns

# Mathematical functions
r_mean = np.mean(log_returns)
r_std = np.std(log_returns, ddof=1)  # Sample standard deviation
r_skew = np.mean((log_returns - r_mean)**3) / r_std**3  # Skewness
r_kurt = np.mean((log_returns - r_mean)**4) / r_std**4  # Kurtosis

# Boolean operations
above_zero = log_returns > 0
pct_positive = np.mean(above_zero)  # Percentage of positive returns

# Add a scalar to every element
prices_scaled = prices * 1.05

# Add a row vector to every row of a matrix
returns_matrix = np.random.normal(0, 0.02, (1000, 5))  # 1000 days, 5 assets
weights = np.array([0.2, 0.3, 0.1, 0.25, 0.15])
portfolio_returns = returns_matrix @ weights  # Matrix-vector multiplication

# Covariance matrix
cov_matrix = np.cov(returns_matrix, rowvar=False)  # rowvar=False means columns are variables

# Eigen-decomposition
eigenvalues, eigenvectors = np.linalg.eig(cov_matrix)

# Cholesky decomposition
L = np.linalg.cholesky(cov_matrix)

# SVD
U, S, Vt = np.linalg.svd(returns_matrix, full_matrices=False)

# Solving linear systems Ax = b
A = np.array([[2, 1], [1, 2]])
b = np.array([3, 3])
x = np.linalg.solve(A, b)  # Uses LU decomposition internally

# Python loop (slow)
N = 1000000
result_loop = 0
for i in range(N):
    result_loop += i**2

# Vectorised (fast)
result_vec = np.sum(np.arange(N)**2)  # ~100x faster

# View vs Copy: Views share memory, copies allocate new memory
arr = np.arange(10)
view = arr[0:5]          # View (no copy)
copy = arr[0:5].copy()   # Copy (new memory)

# In-place operations
arr *= 2                 # Modifies arr in place (no new memory)
arr = arr * 2            # Creates a new array (uses more memory)

import pandas as pd

# From dictionary
data = {
    'AAPL': [150.0, 151.5, 152.0, 151.0, 153.5],
    'MSFT': [300.0, 302.0, 301.5, 303.0, 305.0],
    'GOOGL': [2800.0, 2820.0, 2810.0, 2830.0, 2840.0]
}
df = pd.DataFrame(data)

# With datetime index
dates = pd.date_range('2024-01-01', periods=5, freq='D')
df.index = dates

# Basic operations
df.head()          # First 5 rows
df.tail()          # Last 5 rows
df.info()          # Column types and memory usage
df.describe()      # Summary statistics
df.shape           # (rows, columns)

# By label (using .loc)
df.loc['2024-01-01']                 # Single row
df.loc['2024-01-01':'2024-01-03']    # Slice by date
df.loc[:, 'AAPL']                    # Single column
df.loc['2024-01-01', 'AAPL']         # Single cell

# By position (using .iloc)
df.iloc[0]            # First row
df.iloc[:, 0]         # First column
df.iloc[0:3, 0:2]     # First 3 rows, first 2 columns

# Boolean indexing
df[df['AAPL'] > 152]  # Rows where AAPL > 152

# Load financial data from Yahoo Finance
import yfinance as yf

sp500 = yf.download('^GSPC', start='2020-01-01', end='2024-01-01')
# sp500 has columns: Open, High, Low, Close, Adj Close, Volume

# Compute returns
sp500['Return'] = sp500['Adj Close'].pct_change()
sp500['Log_Return'] = np.log(sp500['Adj Close'] / sp500['Adj Close'].shift(1))

# Rolling statistics
sp500['SMA_20'] = sp500['Adj Close'].rolling(window=20).mean()
sp500['Vol_20'] = sp500['Return'].rolling(window=20).std()
sp500['EMA_12'] = sp500['Adj Close'].ewm(span=12).mean()

# Drop NaN values
sp500_clean = sp500.dropna()

# Daily data to monthly
monthly = sp500_clean['Adj Close'].resample('M').last()  # Last day of month
monthly_returns = monthly.pct_change()

# Downsampling with aggregation
weekly = sp500_clean.resample('W').agg({
    'Open': 'first',
    'High': 'max',
    'Low': 'min',
    'Close': 'last',
    'Volume': 'sum'
})

# Upsampling with interpolation
daily = monthly.resample('D').interpolate(method='linear')

# Download multiple assets
tickers = ['AAPL', 'MSFT', 'GOOGL', 'AMZN']
data = yf.download(tickers, start='2020-01-01', end='2024-01-01')['Adj Close']

# Compute pairwise correlations
corr_matrix = data.pct_change().corr()

# Merge different datasets
fundamentals = pd.read_csv('fundamental_data.csv', index_col='Date')
merged = data.merge(fundamentals, left_index=True, right_index=True, how='left')

# Check for missing data
missing_count = data.isna().sum()
missing_pct = data.isna().sum() / len(data) * 100

# Forward fill (holidays, weekends)
data_ffill = data.fillna(method='ffill')

# Interpolation
data_interp = data.interpolate(method='time')  # Time-based interpolation

# Drop rows with any NaN
data_drop = data.dropna()

# Drop columns with > 5% missing data
data_filtered = data.loc[:, data.isna().sum() / len(data) < 0.05]

# Group by year, month, day
data['Year'] = data.index.year
data['Month'] = data.index.month
data['Day'] = data.index.day

# Average monthly returns
monthly_avg = data.groupby('Month')['AAPL'].mean()

# Calculate median, std, etc.
stats = data.groupby('Year').agg(['mean', 'std', 'skew', 'kurtosis'])

# Given a returns matrix and weights
returns_matrix = data.pct_change().dropna()
weights = np.array([0.3, 0.25, 0.25, 0.2])  # AAPL, MSFT, GOOGL, AMZN

# Portfolio returns (vectorised)
portfolio_returns = returns_matrix @ weights

# Annualised return
annual_return = (1 + portfolio_returns.mean()) ** 252 - 1

# Annualised volatility
annual_vol = portfolio_returns.std() * np.sqrt(252)

# Sharpe ratio (assuming risk-free rate = 0.02)
rf = 0.02
sharpe = (annual_return - rf) / annual_vol

# Sample covariance matrix
cov_matrix = returns_matrix.cov()

# Ledoit-Wolf shrinkage (using scikit-learn)
from sklearn.covariance import LedoitWolf
lw = LedoitWolf().fit(returns_matrix)
cov_shrink = lw.covariance_

# Portfolio variance
portfolio_variance = weights @ cov_matrix @ weights
portfolio_vol = np.sqrt(portfolio_variance)

def efficient_frontier(returns, target_return, cov_matrix):
    n = len(returns)
    ones = np.ones(n)

    # Solve for weights
    A = np.block([
        [2 * cov_matrix, returns, ones],
        [returns.T, 0, 0],
        [ones.T, 0, 0]
    ])
    b = np.array([0, target_return, 1])

    # Solve: A * [w, λ1, λ2]^T = b
    solution = np.linalg.solve(A, b)
    weights = solution[:n]
    return weights

# Compute minimum variance portfolio
target_returns = np.linspace(0.05, 0.20, 50)
weights_list = []
for r in target_returns:
    w = efficient_frontier(returns_matrix.mean(), r, cov_matrix)
    weights_list.append(w)

# Check memory usage
df.info(memory_usage='deep')

# Downcast numeric types
df['Volume'] = df['Volume'].astype('int32')  # Instead of float64
df['Price'] = df['Price'].astype('float32')  # Less precision = less memory

# Read large CSV in chunks
chunk_size = 100000
chunks = []
for chunk in pd.read_csv('large_file.csv', chunksize=chunk_size):
    # Process each chunk
    chunks.append(chunk)
df = pd.concat(chunks)

# Convert sparse data to sparse DataFrame (saves memory)
df_sparse = df.to_sparse(fill_value=0)

# Save as Parquet (compressed, faster to read)
df.to_parquet('data.parquet')
df = pd.read_parquet('data.parquet')