import spacy 
spacy.cli.download("en_core_web_sm")

# make a path for the outputs
from pathlib import Path

OUTPUT_DIR = Path("outputs")
OUTPUT_DIR.mkdir(exist_ok=True)

import pandas as pd

sou = pd.read_csv("data/SOTU.csv")

sou_21cen = sou[sou["Year"] >= 2000].reset_index(drop=True)

import spacy
from tqdm import tqdm

nlp = spacy.load("en_core_web_sm")

# process speeches
processed_speech = [nlp(text) for text in tqdm(sou_21cen["Text"])]

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(stop_words="english")
tfidf_matrix = tfidf.fit_transform(sou_21cen["Text"])


tfidf_df = pd.DataFrame(tfidf_matrix.toarray(), columns=tfidf.get_feature_names_out())

# add president label
tfidf_df["President"] = sou_21cen["President"].values
tfidf_df.head()

# group by president and obtain means
president_vectors = tfidf_df.groupby("President").mean()
president_vectors

from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
import mpld3

# plot pca analysis -- which presidents had most similar speeches
pca = PCA(n_components=2)
coords = pca.fit_transform(president_vectors)

pca_df = pd.DataFrame(coords, columns=["PC1", "PC2"],
                      index=president_vectors.index)

fig, ax = plt.subplots(figsize=(8,6))

for name, row in pca_df.iterrows():
    plt.scatter(row["PC1"], row["PC2"])
    plt.text(row["PC1"] + 0.01, row["PC2"] + 0.01, name, fontsize=9)

plt.title("Presidents' Speeches (PCA on TF–IDF)")
plt.xlabel("PC1")
plt.ylabel("PC2")
plt.axhline(0, color="gray", linewidth=0.5)
plt.axvline(0, color="gray", linewidth=0.5)
plt.show()

# save as html
mpld3.save_html(fig, str(OUTPUT_DIR / "president_speeches_PCA.html"))