updated

app.py

@@ -48,6 +48,11 @@ row_lookup = {row.id: i for i, row in df.iterrows()}
 # 2)  Load embeddings + ids once per session  (S3) – new, cached
 # ────────────────────────────────────────────────────────────────────────
 @st.cache_resource
+def get_st_model():
+    return SentenceTransformer(
+        "sentence-transformers/all-MiniLM-L6-v2",
+        device="cpu",
+    )
 def load_embeddings():
     # credentials already in env (HF secrets) – boto3 will pick them up
     BUCKET = "cgd-embeddings-bucket"
@@ -89,31 +94,36 @@ st.sidebar.subheader("🧠 Semantic Search")
 sem_query = st.sidebar.text_input("Enter a natural-language query")
 if st.sidebar.button("Search", disabled=not sem_query.strip()):
     with st.spinner("Embedding & searching…"):
-        model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
-        q_vec = model.encode(sem_query.strip(), convert_to_tensor=True).cpu()
-        scores = util.cos_sim(q_vec, emb_tensor)[0]
-        top_vals, top_idx = torch.topk(scores, k=10)   # grab extra
-
-        results = []
-        for score, emb_row in zip(top_vals.tolist(), top_idx.tolist()):
-            db_id = ids_list[emb_row]
-            if db_id in row_lookup:
-                row = df.iloc[row_lookup[db_id]]
-                if row["question_text"] and row["answer_text"]:
-                    results.append({
-                        "Score": f"{score:.3f}",
-                        "Country": row["country"],
-                        "Year": row["year"],
-                        "Question": row["question_text"],
-                        "Answer": row["answer_text"],
-                    })
-        if results:
-            st.subheader(f"🔍 Semantic Results ({len(results)} found)")
-            st.dataframe(pd.DataFrame(results).head(5))
-        else:
-            st.info("No semantic matches found.")
-
-st.markdown("---")
+        # 1) embed query
+        model = get_st_model()                      # cached CPU model
+        q_vec = model.encode(
+            sem_query.strip(),
+            convert_to_tensor=True,
+            device="cpu"
+        ).cpu()
+
+        # 2) semantic similarity
+        sims = util.cos_sim(q_vec, emb_tensor)[0]
+        top_vals, top_idx = torch.topk(sims, k=50)
+
+        sem_ids   = [ids_list[i] for i in top_idx.tolist()]
+        sem_rows  = df.loc[df["id"].isin(sem_ids)].copy()
+        score_map = dict(zip(sem_ids, top_vals.tolist()))
+        sem_rows["Score"] = sem_rows["id"].map(score_map)
+        sem_rows = sem_rows.sort_values("Score", ascending=False)
+
+        # 3) keyword / dropdown remainder
+        remainder = filtered.loc[~filtered["id"].isin(sem_ids)].copy()
+        remainder["Score"] = ""         # blank for keyword-only rows
+
+        combined = pd.concat([sem_rows, remainder], ignore_index=True)
+
+    st.subheader(f"🔍 Combined Results ({len(combined)})")
+    st.dataframe(
+        combined[["Score", "country", "year", "question_text", "answer_text"]],
+        use_container_width=True,
+    )
+    st.stop()   # skip the old display logic below when semantic search ran
 
 # ── apply original filters ──────────────────────────────────────────────
 filtered = df[