"""Regenerate top features excluding English stopwords for cleaner Table 1."""
import pandas as pd
import numpy as np
import re
from collections import Counter
from sklearn.feature_extraction.text import TfidfVectorizer, ENGLISH_STOP_WORDS

DEFS_FILE = 'cleaned_definitions.xlsx'
df = pd.read_excel(DEFS_FILE)

# 20 PANAS items and their column patterns
ITEMS = [
    ('Interested', 'Q1b', 'PA'), ('Excited', 'Q3b', 'PA'), ('Strong', 'Q5b', 'PA'),
    ('Enthusiastic', 'Q9b', 'PA'), ('Proud', 'Q10b', 'PA'), ('Alert', 'Q12b', 'PA'),
    ('Inspired', 'Q14b', 'PA'), ('Determined', 'Q16b', 'PA'), ('Attentive', 'Q17b', 'PA'),
    ('Active', 'Q19b', 'PA'),
    ('Distressed', 'Q2b', 'NA'), ('Upset', 'Q4b', 'NA'), ('Guilty', 'Q6b', 'NA'),
    ('Scared', 'Q7b', 'NA'), ('Hostile', 'Q8b', 'NA'), ('Irritable', 'Q11b', 'NA'),
    ('Ashamed', 'Q13b', 'NA'), ('Nervous', 'Q15b', 'NA'), ('Jittery', 'Q18b', 'NA'),
    ('Afraid', 'Q20b', 'NA'),
]

stop = set(ENGLISH_STOP_WORDS)
# Also remove very high-frequency function-y words that aren't content
stop.update({'feel', 'feeling', 'feels', 'felt', 'someone', 'something', 'one', 'thing',
             'people', 'person', 'way', 'really', 'just', 'like', 'get', 'getting',
             'go', 'going', 'come', 'coming', 'make', 'making', 'know', 'knowing',
             'mean', 'lot'})

def find_col(item, qb):
    # column names look like "Q1b_Interested" etc.
    for c in df.columns:
        if c.startswith(qb):
            return c
    return None

rows = []
for item, qb, val in ITEMS:
    col = find_col(item, qb)
    if col is None:
        # fallback: case-insensitive contains item
        for c in df.columns:
            if item.lower() in c.lower():
                col = c
                break
    series = df[col].dropna().astype(str).str.strip()
    series = series[series.str.len() > 1]
    series = series[~series.str.lower().isin({'none','na','n/a','nan','nil','no','-','--'})]
    n = len(series)

    # Tokenize, drop stopwords, count occurrences per definition (presence not count)
    presence = Counter()
    for s in series:
        toks = re.findall(r"[a-zA-Z][a-zA-Z'\-]+", s.lower())
        toks = [t for t in toks if t not in stop and len(t) >= 3]
        # bigrams too
        bigrams = [f"{toks[i]} {toks[i+1]}" for i in range(len(toks)-1)]
        seen = set(toks) | set(bigrams)
        for t in seen:
            presence[t] += 1

    # Top 5 content features by presence percentage
    top_by_pct = []
    for word, count in presence.most_common(20):
        # skip if the word is the item lemma itself? No - keep it, it's informative
        pct = 100.0 * count / n
        top_by_pct.append((word, pct))
        if len(top_by_pct) >= 5:
            break

    # Also compute TF-IDF weights as another view
    vec = TfidfVectorizer(stop_words='english', ngram_range=(1,2), min_df=2, max_features=200,
                          token_pattern=r"(?u)\b[a-zA-Z][a-zA-Z'\-]+\b")
    M = vec.fit_transform(series.tolist())
    mean_w = np.asarray(M.mean(axis=0)).ravel()
    vocab = vec.get_feature_names_out()
    idx = np.argsort(-mean_w)[:5]
    top_tfidf = [(vocab[i], mean_w[i]) for i in idx]

    pct_str = "; ".join([f"{w} ({p:.1f}%)" for w, p in top_by_pct])
    tf_str = "; ".join([f"{w} ({v:.3f})" for w, v in top_tfidf])

    rows.append({'Item': item, 'Valence': val, 'TopByPct': pct_str, 'TopByTFIDF': tf_str})

out = pd.DataFrame(rows)
out.to_csv('out_top_features.csv', index=False)
print(out.to_string(index=False))