Update app.py

app.py

@@ -1,24 +1,15 @@
-# Install datasets library
+import os, re, functools, numpy as np, pandas as pd
+import gradio as gr
+from datasets import load_dataset
+from sklearn.metrics.pairwise import cosine_similarity
 
+# -------- Config (safe defaults for CPU Spaces) --------
+SAMPLE_SIZE = int(os.getenv("SAMPLE_SIZE", "5000"))
+RANDOM_STATE = 42
+DEFAULT_INPUT = "I am so happy with this product"
 
-# Load Sentiment140 dataset
-from datasets import load_dataset
-dataset = load_dataset("sentiment140")
-
-# Convert to pandas
-import pandas as pd
-df = dataset["train"].to_pandas()
-df.head()
-# Drop null values in text and sentiment
-df.dropna(subset=["text", "sentiment"], inplace=True)
-
-# Filter tweets with reasonable length
-df["text_length"] = df["text"].apply(len)
-df = df[(df["text_length"] >= 5) & (df["text_length"] <= 280)]
-
-# Clean the text
-import re
-def clean_text(text):
+# -------- Text cleaning (yours) --------
+def clean_text(text: str) -> str:
     text = text.lower()
     text = re.sub(r"http\S+", "", text)
     text = re.sub(r"@\w+", "", text)
@@ -27,83 +18,140 @@ def clean_text(text):
     text = re.sub(r"\s+", " ", text).strip()
     return text
 
-df["clean_text"] = df["text"].apply(clean_text)
-df[["text", "clean_text"]].head()
-# Convert sentiment labels from numbers to text
-def map_sentiment(label):
-    return "negative" if label == 0 else "neutral" if label == 2 else "positive"
-
-df["sentiment_label"] = df["sentiment"].apply(map_sentiment)
-df["sentiment_label"].value_counts()
-# Save for future use
-df[["clean_text", "sentiment_label"]].to_csv("cleaned_sentiment140.csv", index=False)
-print("Cleaned data saved!")
-!pip install -U sentence-transformers
-from sentence_transformers import SentenceTransformer
-import numpy as np
-from sklearn.metrics.pairwise import cosine_similarity
-
-# Use a small sample for speed (feel free to increase)
-sample_df = df.sample(5000, random_state=42).reset_index(drop=True)
-texts = sample_df["clean_text"].tolist()
-
-# Load 3 different embedding models
-models = {
-    "MiniLM": SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2"),
-    "MPNet": SentenceTransformer("sentence-transformers/all-mpnet-base-v2"),
-    "DistilRoBERTa": SentenceTransformer("sentence-transformers/paraphrase-distilroberta-base-v1")
+# -------- Load sample data once --------
+@functools.lru_cache(maxsize=1)
+def load_sample_df():
+    ds = load_dataset("sentiment140", split="train")
+    df = ds.to_pandas()
+    df = df.dropna(subset=["text", "sentiment"]).copy()
+    df["text_length"] = df["text"].str.len()
+    df = df[(df["text_length"] >= 5) & (df["text_length"] <= 280)].copy()
+    df["clean_text"] = df["text"].apply(clean_text)
+    df = df.sample(min(SAMPLE_SIZE, len(df)), random_state=RANDOM_STATE).reset_index(drop=True)
+    return df[["text", "clean_text"]]
+
+# -------- Lazy model loaders --------
+@functools.lru_cache(maxsize=None)
+def load_sentence_model(model_id: str):
+    from sentence_transformers import SentenceTransformer
+    return SentenceTransformer(model_id)
+
+@functools.lru_cache(maxsize=None)
+def load_generator():
+    from transformers import pipeline, set_seed
+    set_seed(RANDOM_STATE)
+    return pipeline("text-generation", model="distilgpt2")
+
+# Map names → HF ids
+EMBEDDERS = {
+    "MiniLM (fast)": "sentence-transformers/all-MiniLM-L6-v2",
+    "MPNet (heavier)": "sentence-transformers/all-mpnet-base-v2",
+    "DistilRoBERTa (paraphrase)": "sentence-transformers/paraphrase-distilroberta-base-v1",
 }
-# Compute and compare similarity for one test input
-test_input = "I am so happy with this product"
-
-def get_top3_similarities(model, texts, test_input):
-    text_embeddings = model.encode(texts, show_progress_bar=True)
-    input_embedding = model.encode([test_input])
-    similarities = cosine_similarity(input_embedding, text_embeddings)[0]
-    top_indices = similarities.argsort()[-3:][::-1]
-    return [(i, texts[i], similarities[i]) for i in top_indices]
-
-# Try each model
-results = {}
-for name, model in models.items():
-    print(f"\n🔎 Top 3 results from: {name}")
-    top3 = get_top3_similarities(model, texts, test_input)
-    for rank, (idx, text, score) in enumerate(top3, start=1):
-        print(f"{rank}. [{score:.4f}] {text}")
-    results[name] = top3
-!pip install -U transformers
-from transformers import pipeline, set_seed
-
-# Load small GPT-2 model for text generation
-generator = pipeline("text-generation", model="distilgpt2")
-set_seed(42)  # reproducible results
-# Example user input
-test_input = "I'm feeling amazing about our product launch!"
-# Generate synthetic tweets
-synthetic_outputs = generator(
-    test_input,
-    max_length=50,
-    num_return_sequences=10,
-    do_sample=True,
-    temperature=0.9
-)
-
-# Extract just the generated text
-generated_tweets = [output["generated_text"].strip() for output in synthetic_outputs]
-for i, tweet in enumerate(generated_tweets, 1):
-    print(f"{i}. {tweet}\n")
-from sentence_transformers import SentenceTransformer
-
-# Load your best model again (MiniLM is a good choice)
-embedding_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
-# Embed input and generated tweets
-input_vec = embedding_model.encode([test_input])
-gen_vecs = embedding_model.encode(generated_tweets)
-
-# Compute similarity and select best
-from sklearn.metrics.pairwise import cosine_similarity
-similarities = cosine_similarity(input_vec, gen_vecs)[0]
-top_index = similarities.argmax()
-best_generated = generated_tweets[top_index]
 
-print(f"✅ Best AI-generated tweet:\n[{similarities[top_index]:.4f}] {best_generated}")
+# Cache for precomputed corpus embeddings per model
+_CORPUS_CACHE = {}
+
+def ensure_corpus_embeddings(model_name: str, texts: list[str]):
+    """Compute & cache corpus embeddings for a given model name."""
+    if model_name in _CORPUS_CACHE:
+        return _CORPUS_CACHE[model_name]
+    model_id = EMBEDDERS[model_name]
+    model = load_sentence_model(model_id)
+    # encode with no progress bar to keep logs clean on Spaces
+    emb = model.encode(texts, show_progress_bar=False, convert_to_numpy=True, normalize_embeddings=True)
+    _CORPUS_CACHE[model_name] = emb
+    return emb
+
+def top3_for_each_model(user_input: str, selected_models: list[str]):
+    df = load_sample_df()
+    texts = df["clean_text"].tolist()
+    rows = []
+    for name in selected_models:
+        try:
+            model = load_sentence_model(EMBEDDERS[name])
+            corpus_emb = ensure_corpus_embeddings(name, texts)
+            q = model.encode([clean_text(user_input)], show_progress_bar=False, normalize_embeddings=True)
+            sims = cosine_similarity(q, corpus_emb)[0]
+            top_idx = sims.argsort()[-3:][::-1]
+            for rank, i in enumerate(top_idx, start=1):
+                rows.append({
+                    "Model": name,
+                    "Rank": rank,
+                    "Similarity": float(sims[i]),
+                    "Tweet (clean)": texts[i],
+                    "Tweet (orig)": df.loc[i, "text"]
+                })
+        except Exception as e:
+            rows.append({"Model": name, "Rank": "-", "Similarity": "-", "Tweet (clean)": f"[Error: {e}]", "Tweet (orig)": ""})
+    out = pd.DataFrame(rows, columns=["Model","Rank","Similarity","Tweet (clean)","Tweet (orig)"])
+    return out
+
+def generate_and_pick_best(prompt: str, n_sequences: int, max_length: int, temperature: float, scorer_model_name: str):
+    gen = load_generator()
+    outputs = gen(prompt, max_length=max_length, num_return_sequences=n_sequences, do_sample=True, temperature=temperature)
+    candidates = [o["generated_text"].strip() for o in outputs]
+
+    scorer_id = EMBEDDERS[scorer_model_name]
+    scorer = load_sentence_model(scorer_id)
+    q = scorer.encode([prompt], show_progress_bar=False, normalize_embeddings=True)
+    cand_vecs = scorer.encode(candidates, show_progress_bar=False, normalize_embeddings=True)
+    sims = cosine_similarity(q, cand_vecs)[0]
+    best_idx = int(sims.argmax())
+    table = pd.DataFrame({
+        "Rank": np.argsort(-sims)+1,
+        "Similarity": np.sort(sims)[::-1],
+        "Generated Tweet": [c for _, c in sorted(zip(-sims, candidates))]
+    })
+    best = candidates[best_idx]
+    best_score = float(sims[best_idx])
+    return best, best_score, table
+
+with gr.Blocks(title="Sentiment140 Embeddings + Generation") as demo:
+    gr.Markdown(
+        """
+# 🧪 Sentiment140 — Embeddings & Tweet Generator
+Small, reliable demo for your final project:
+1) Compare top-3 most similar tweets from **Sentiment140** across embedding models.  
+2) Generate synthetic tweets with **DistilGPT‑2** and auto‑pick the best by semantic similarity.
+
+> Tip: Start with **MiniLM (fast)** on CPU Spaces. Add MPNet/DistilRoBERTa if you have a GPU.
+        """
+    )
+
+    with gr.Row():
+        test_input = gr.Textbox(label="Your input", value=DEFAULT_INPUT, lines=2)
+        models = gr.CheckboxGroup(
+            choices=list(EMBEDDERS.keys()),
+            value=["MiniLM (fast)"],
+            label="Embedding models to compare"
+        )
+
+    run_btn = gr.Button("🔎 Find Top‑3 Similar Tweets")
+    table_out = gr.Dataframe(interactive=False, wrap=True)
+
+    run_btn.click(top3_for_each_model, inputs=[test_input, models], outputs=table_out)
+
+    gr.Markdown("---")
+    gr.Markdown("## 📝 Generate Tweets and Pick the Best (by similarity to your input)")
+
+    with gr.Row():
+        n_seq = gr.Slider(3, 15, value=8, step=1, label="Number of candidates")
+        max_len = gr.Slider(30, 120, value=60, step=1, label="Max length")
+        temp = gr.Slider(0.5, 1.5, value=0.9, step=0.05, label="Temperature")
+        scorer_model = gr.Dropdown(list(EMBEDDERS.keys()), value="MiniLM (fast)", label="Scorer embedding")
+
+    gen_btn = gr.Button("✨ Generate & Score")
+    best_txt = gr.Textbox(label="Best generated tweet")
+    best_score = gr.Number(label="Similarity (best)")
+    gen_table = gr.Dataframe(interactive=False, wrap=True)
+
+    gen_btn.click(generate_and_pick_best,
+                  inputs=[test_input, n_seq, max_len, temp, scorer_model],
+                  outputs=[best_txt, best_score, gen_table])
+
+    gr.Markdown("---")
+    gr.Markdown("## 🖼️ Project Photo (optional, just to display it in the app)")
+    photo = gr.Image(label="Upload your project photo (jpg/png)", type="filepath")
+
+demo.queue(max_size=32).launch()