# ===================================================================
# MODULE 6, LESSON 1: NATURAL LANGUAGE PROCESSING FOR FINANCE
# ===================================================================

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import re
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
import warnings
warnings.filterwarnings('ignore')

# Download NLTK data (only need to run once)
# nltk.download('stopwords')
# nltk.download('wordnet')
# nltk.download('punkt')
# nltk.download('punkt_tab')
# nltk.download('omw-1.4')

# Set style
sns.set_style("whitegrid")
np.random.seed(42)

print("="*70)
print("NLP FOR FINANCE – SENTIMENT ANALYSIS AND TEXT MINING")
print("="*70)

# ----------------------------------------------------------------
# PART A: SAMPLE FINANCIAL TEXT DATA
# ----------------------------------------------------------------

# Sample financial news headlines and earnings call excerpts
financial_texts = [
    "The Federal Reserve raised interest rates by 25 basis points, signalling confidence in the economy.",
    "Bank of America reported record quarterly profits, beating analyst expectations by a wide margin.",
    "Inflation fears are driving a sell-off in technology stocks, with the NASDAQ down 3% today.",
    "Goldman Sachs announces layoffs of 3,000 employees amid a challenging market environment.",
    "JPMorgan Chase's CEO warns of an impending recession, citing global economic uncertainties.",
    "Tesla's stock surged 15% after announcing better-than-expected delivery numbers.",
    "The yield curve inverted today, a key signal that a recession may be on the horizon.",
    "Citigroup is investing $2 billion in climate tech startups, focusing on renewable energy solutions.",
    "The dollar strengthened against major currencies after the Fed's hawkish comments.",
    "JP Morgan upgrades Apple to 'overweight', citing strong iPhone demand and services growth.",
    "Bearish sentiment is spreading across Wall Street as earnings season disappoints.",
    "The ECB announced a pause on rate hikes, sending European markets higher.",
    "Wells Fargo settles money laundering charges for $3 billion, agreeing to implement new controls.",
    "Bitcoin falls below $30,000 as regulatory concerns weigh on the cryptocurrency market.",
    "BlackRock recommends investors increase exposure to bonds amid market volatility."
]

# Add some earnings call excerpts (longer texts)
earnings_excerpts = [
    """We are pleased to report another quarter of strong financial performance. Revenue grew 12% year-over-year, driven by robust demand across all business segments. Our investment banking division performed particularly well, with fees increasing 18% compared to the same quarter last year. Looking forward, we remain cautiously optimistic about the macroeconomic environment.""",
    
    """The current quarter was challenging as we faced headwinds from rising interest rates and inflationary pressures. Consumer spending has softened, and we have seen higher-than-expected defaults in our credit card portfolio. We are taking proactive measures to strengthen our balance sheet and reduce costs. We expect a recovery in the second half of the year."""
]

financial_texts.extend(earnings_excerpts)

# Create DataFrame
text_df = pd.DataFrame({
    'text': financial_texts,
    'doc_id': range(1, len(financial_texts)+1)
})

print("Sample Financial Text:")
print(text_df['text'].head(3).to_string(index=False))

# ----------------------------------------------------------------
# PART B: TEXT PREPROCESSING
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART B: Text Preprocessing")
print("-"*60)

# Initialise tools
stop_words = set(stopwords.words('english'))
lemmatizer = WordNetLemmatizer()

def preprocess_text(text, lemmatize=True, remove_stopwords=True):
    """
    Clean and preprocess text.
    """
    # Convert to lowercase
    text = text.lower()
    # Remove punctuation and numbers (keep only alphabetic and spaces)
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    # Remove extra whitespace
    text = re.sub(r'\s+', ' ', text).strip()
    # Tokenise
    tokens = text.split()
    # Remove stopwords
    if remove_stopwords:
        tokens = [t for t in tokens if t not in stop_words]
    # Lemmatise
    if lemmatize:
        tokens = [lemmatizer.lemmatize(t) for t in tokens]
    # Rejoin
    return ' '.join(tokens)

# Apply preprocessing
text_df['text_clean'] = text_df['text'].apply(preprocess_text)

print("Original vs Cleaned (first document):")
print(f"Original: {text_df['text'].iloc[0]}")
print(f"Cleaned:  {text_df['text_clean'].iloc[0]}")

# ----------------------------------------------------------------
# PART C: TF-IDF VECTORISATION
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART C: TF-IDF Vectorisation")
print("-"*60)

# Create TF-IDF matrix
tfidf_vectorizer = TfidfVectorizer(max_features=50, stop_words='english')
tfidf_matrix = tfidf_vectorizer.fit_transform(text_df['text_clean'])

# Show top features
feature_names = tfidf_vectorizer.get_feature_names_out()
feature_importance = np.array(tfidf_matrix.sum(axis=0)).flatten()
feature_df = pd.DataFrame({
    'word': feature_names,
    'tfidf_sum': feature_importance
}).sort_values('tfidf_sum', ascending=False)

print("Top 10 TF-IDF Features (most important words across all documents):")
print(feature_df.head(10).to_string(index=False))

# ----------------------------------------------------------------
# PART D: VADER SENTIMENT ANALYSIS
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART D: VADER Sentiment Analysis")
print("-"*60)

# Initialise VADER
analyzer = SentimentIntensityAnalyzer()

def get_vader_sentiment(text):
    """Get VADER sentiment scores."""
    scores = analyzer.polarity_scores(text)
    return scores['compound'], scores['pos'], scores['neu'], scores['neg']

# Apply to all documents
text_df[['compound', 'pos_score', 'neu_score', 'neg_score']] = text_df['text'].apply(
    lambda x: pd.Series(get_vader_sentiment(x))
)

# Classify sentiment
def classify_sentiment(compound):
    if compound >= 0.05:
        return 'Positive'
    elif compound <= -0.05:
        return 'Negative'
    else:
        return 'Neutral'

text_df['vader_sentiment'] = text_df['compound'].apply(classify_sentiment)

print("VADER Sentiment Distribution:")
print(text_df['vader_sentiment'].value_counts())

print("\nSample Documents with VADER Scores:")
sample_docs = text_df[['text', 'compound', 'vader_sentiment']].head(10)
print(sample_docs.to_string(index=False))

# ----------------------------------------------------------------
# PART E: FINBERT SENTIMENT ANALYSIS (Hugging Face Transformers)
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART E: FinBERT Sentiment Analysis")
print("-"*60)

# Check if transformers is installed
try:
    from transformers import AutoTokenizer, AutoModelForSequenceClassification
    import torch
    
    # Load FinBERT model
    model_name = "ProsusAI/finbert"
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSequenceClassification.from_pretrained(model_name)
    
    def get_finbert_sentiment(text):
        """Get FinBERT sentiment (positive, negative, neutral)."""
        inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512, padding=True)
        outputs = model(**inputs)
        predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
        # Labels: 0 = positive, 1 = negative, 2 = neutral
        sentiment = torch.argmax(predictions, dim=1).item()
        probs = predictions.detach().numpy().flatten()
        return sentiment, probs
    
    # Test on sample texts
    test_texts = [
        "The company reported record profits and strong revenue growth.",
        "The bank faces significant challenges due to rising defaults and regulatory scrutiny.",
        "The market is experiencing volatility but remains resilient."
    ]
    
    print("FinBERT Sentiment Analysis (Sample):")
    for txt in test_texts:
        sentiment, probs = get_finbert_sentiment(txt)
        labels = ['Positive', 'Negative', 'Neutral']
        print(f"\nText: {txt}")
        print(f"Sentiment: {labels[sentiment]} (Pos: {probs[0]:.3f}, Neg: {probs[1]:.3f}, Neu: {probs[2]:.3f})")
    
    # Apply to all documents (limit for speed)
    print("\nApplying FinBERT to all documents (this may take a moment)...")
    finbert_results = []
    for txt in text_df['text']:
        sentiment, probs = get_finbert_sentiment(txt)
        finbert_results.append({
            'finbert_label': ['Positive', 'Negative', 'Neutral'][sentiment],
            'finbert_pos': probs[0],
            'finbert_neg': probs[1],
            'finbert_neu': probs[2]
        })
    
    finbert_df = pd.DataFrame(finbert_results)
    text_df = pd.concat([text_df, finbert_df], axis=1)
    
    print("\nFinBERT Sentiment Distribution:")
    print(text_df['finbert_label'].value_counts())
    
    # Compare VADER vs FinBERT
    agreement = (text_df['vader_sentiment'] == text_df['finbert_label']).mean()
    print(f"\nAgreement between VADER and FinBERT: {agreement:.1%}")
    
except ImportError:
    print("Transformers library not installed. Install with: pip install transformers torch")
    print("Skipping FinBERT demonstration.")
    # Create placeholder columns for compatibility
    text_df['finbert_label'] = text_df['vader_sentiment']
    text_df['finbert_pos'] = 0
    text_df['finbert_neg'] = 0
    text_df['finbert_neu'] = 0

# ----------------------------------------------------------------
# PART F: TOPIC MODELLING WITH LDA
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART F: Topic Modelling (Latent Dirichlet Allocation)")
print("-"*60)

# Use CountVectorizer for LDA (requires word counts)
count_vectorizer = CountVectorizer(max_features=100, stop_words='english')
count_matrix = count_vectorizer.fit_transform(text_df['text_clean'])
feature_names_lda = count_vectorizer.get_feature_names_out()

# Fit LDA model (2 topics)
n_topics = 3
lda = LatentDirichletAllocation(n_components=n_topics, random_state=42, max_iter=100)
lda.fit(count_matrix)

# Get topic-word distribution
def display_topics(model, feature_names, n_words=8):
    """Display top words for each topic."""
    for topic_idx, topic in enumerate(model.components_):
        top_words = [feature_names[i] for i in topic.argsort()[:-n_words-1:-1]]
        print(f"Topic {topic_idx+1}: {', '.join(top_words)}")

print(f"Topics (LDA, n={n_topics}):")
display_topics(lda, feature_names_lda, n_words=8)

# Assign topic to each document
doc_topic = lda.transform(count_matrix)
text_df['dominant_topic'] = np.argmax(doc_topic, axis=1) + 1

print("\nTopic Assignment per Document:")
topic_df = text_df[['doc_id', 'text_clean', 'dominant_topic']].head(10)
print(topic_df.to_string(index=False))

# ----------------------------------------------------------------
# PART G: NAMED ENTITY RECOGNITION (NER) – CONCEPTUAL
# ----------------------------------------------------------------

print("\n" + "-"*60)
print("PART G: Named Entity Recognition (NER)")
print("-"*60)

print("""
NER extracts structured information from unstructured text:

Entities typically identified:
  - ORGANISATION: Bank of America, Goldman Sachs, Federal Reserve
  - PERSON: Jamie Dimon, Janet Yellen
  - LOCATION: New York, London, United States
  - DATE: Q4 2023, January 15th
  - MONEY: $3 billion, 25 basis points
  - PERCENT: 15%, 2.5%
  - PRODUCT: iPhone, Tesla Model 3

In finance, NER is used for:
  - Extracting company names from news articles for trading signals.
  - Identifying regulatory references in filings.
  - Building knowledge graphs of financial entities.

Example using spaCy:
  import spacy
  nlp = spacy.load('en_core_web_sm')
  doc = nlp("Goldman Sachs reported profits of $5 billion in Q4 2023.")
  for ent in doc.ents:
      print(ent.text, ent.label_)
  # Output: Goldman Sachs ORG, $5 billion MONEY, Q4 2023 DATE
""")

# ----------------------------------------------------------------
# PART H: BUSINESS APPLICATIONS AND RECOMMENDATIONS
# ----------------------------------------------------------------

print("\n" + "="*70)
print("PART H: Business Applications and Recommendations")
print("="*70)

print("""
1. News-Based Trading Signals:
   - Use sentiment analysis (FinBERT) on real-time news feeds.
   - Generate buy/sell signals based on sentiment changes.
   - Backtest with historical data to validate alpha generation.

2. Earnings Call Analysis:
   - Analyse sentiment and tone of management during earnings calls.
   - Correlate sentiment with stock price movements post-announcement.
   - Identify forward-looking statements (e.g., "guidance", "outlook").

3. Credit Risk Assessment:
   - Analyse borrower's news and social media for early warning signals.
   - Use topic modelling to identify risk factors (e.g., "layoffs", "lawsuits", "regulatory").

4. Regulatory Compliance:
   - Monitor communications for potential misconduct (e.g., insider trading).
   - Use NER to identify sensitive entities and relationships.

5. Customer Service Automation:
   - Implement chatbots with intent recognition and sentiment analysis.
   - Improve response times and customer satisfaction.

6. M&A Target Identification:
   - Use topic modelling and NER to scan news and filings for potential acquisition targets.
   - Identify companies in related industries with positive sentiment.

Recommendations:
  - For high-frequency trading, use lightweight models (VADER, TF-IDF).
  - For deep analysis (e.g., regulatory compliance), use FinBERT/Transformers.
  - Always combine NLP with traditional quantitative signals.
  - Validate NLP models on financial data specific to your domain.
""")