Spaces:
Running
Running
update to tabulator
Browse files
app.py
CHANGED
@@ -1,17 +1,17 @@
|
|
1 |
-
#
|
2 |
|
3 |
-
import os, io,
|
4 |
import panel as pn
|
5 |
import pandas as pd
|
6 |
import boto3, torch
|
7 |
-
from sentence_transformers import SentenceTransformer, util
|
8 |
import psycopg2
|
|
|
9 |
|
10 |
pn.extension()
|
11 |
|
12 |
-
#
|
13 |
-
# 1)
|
14 |
-
#
|
15 |
DB_HOST = os.getenv("DB_HOST")
|
16 |
DB_PORT = os.getenv("DB_PORT", "5432")
|
17 |
DB_NAME = os.getenv("DB_NAME")
|
@@ -35,7 +35,6 @@ def get_data():
|
|
35 |
return df_
|
36 |
|
37 |
df = get_data()
|
38 |
-
row_lookup = {row.id: i for i, row in df.iterrows()}
|
39 |
|
40 |
@pn.cache()
|
41 |
def load_embeddings():
|
@@ -47,15 +46,19 @@ def load_embeddings():
|
|
47 |
buf.close(); gc.collect()
|
48 |
return ckpt["ids"], ckpt["embeddings"]
|
49 |
|
50 |
-
ids_list, emb_tensor = load_embeddings()
|
51 |
-
|
52 |
@pn.cache()
|
53 |
def get_st_model():
|
54 |
return SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2", device="cpu")
|
55 |
|
56 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
57 |
# 2) Widgets
|
58 |
-
#
|
59 |
country_opts = sorted(df["country"].dropna().unique())
|
60 |
year_opts = sorted(df["year"].dropna().unique())
|
61 |
|
@@ -64,15 +67,22 @@ w_years = pn.widgets.MultiSelect(name="Years", options=year_opts)
|
|
64 |
w_keyword = pn.widgets.TextInput(name="Keyword Search", placeholder="Search questions or answers")
|
65 |
w_group = pn.widgets.Checkbox(name="Group by Question Text", value=False)
|
66 |
|
67 |
-
# Semantic search
|
68 |
w_semquery = pn.widgets.TextInput(name="Semantic Query")
|
69 |
-
w_search_button = pn.widgets.Button(name="Search", button_type="primary"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
70 |
|
71 |
-
|
72 |
-
|
73 |
-
# βββββββββββββββββββββββββββββββββββββββββββββββ
|
74 |
-
@pn.depends(w_countries, w_years, w_keyword, w_group)
|
75 |
-
def keyword_filter(countries, years, keyword, group):
|
76 |
filt = df.copy()
|
77 |
if countries:
|
78 |
filt = filt[filt["country"].isin(countries)]
|
@@ -100,23 +110,18 @@ def keyword_filter(countries, years, keyword, group):
|
|
100 |
"answer_text": "Sample Answers"
|
101 |
})
|
102 |
)
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
filt[["country", "year", "question_text", "answer_text"]],
|
107 |
-
sizing_mode="stretch_width", height=400
|
108 |
-
)
|
109 |
|
110 |
-
# ββββββββββββοΏ½οΏ½ββββββββββββββββββββββββββββββββββ
|
111 |
-
# 4) Semantic Search Callback
|
112 |
-
# βββββββββββββββββββββββββββββββββββββββββββββββ
|
113 |
def semantic_search(event=None):
|
114 |
query = w_semquery.value.strip()
|
115 |
if not query:
|
116 |
return
|
117 |
|
118 |
-
model =
|
119 |
q_vec = model.encode(query, convert_to_tensor=True, device="cpu").cpu()
|
|
|
120 |
sims = util.cos_sim(q_vec, emb_tensor)[0]
|
121 |
top_vals, top_idx = torch.topk(sims, k=50)
|
122 |
|
@@ -126,30 +131,31 @@ def semantic_search(event=None):
|
|
126 |
sem_rows["Score"] = sem_rows["id"].map(score_map)
|
127 |
sem_rows = sem_rows.sort_values("Score", ascending=False)
|
128 |
|
129 |
-
|
130 |
-
|
131 |
-
w_countries.value
|
132 |
-
|
133 |
-
|
134 |
-
|
135 |
-
|
|
|
|
|
|
|
|
|
136 |
|
137 |
-
remainder =
|
138 |
remainder["Score"] = ""
|
139 |
|
140 |
combined = pd.concat([sem_rows, remainder], ignore_index=True)
|
141 |
-
|
142 |
-
result_pane.object = combined[["Score", "country", "year", "question_text", "answer_text"]]
|
143 |
|
144 |
w_search_button.on_click(semantic_search)
|
145 |
|
146 |
-
|
147 |
-
|
148 |
-
#
|
149 |
-
# 5) Layout
|
150 |
-
# βββββββββββββββββββββββββββββββββββββββββββββββ
|
151 |
sidebar = pn.Column(
|
152 |
-
"##
|
153 |
w_countries, w_years, w_keyword, w_group,
|
154 |
pn.Spacer(height=20),
|
155 |
"## π§ Semantic Search",
|
@@ -159,10 +165,7 @@ sidebar = pn.Column(
|
|
159 |
|
160 |
main = pn.Column(
|
161 |
pn.pane.Markdown("## π CGD Survey Explorer"),
|
162 |
-
|
163 |
-
("Filtered Results", keyword_filter),
|
164 |
-
("Semantic Search Results", result_pane),
|
165 |
-
)
|
166 |
)
|
167 |
|
168 |
pn.template.FastListTemplate(
|
|
|
1 |
+
# app.py β Unified Panel App with Semantic Search + Filterable Tabulator
|
2 |
|
3 |
+
import os, io, gc
|
4 |
import panel as pn
|
5 |
import pandas as pd
|
6 |
import boto3, torch
|
|
|
7 |
import psycopg2
|
8 |
+
from sentence_transformers import SentenceTransformer, util
|
9 |
|
10 |
pn.extension()
|
11 |
|
12 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
13 |
+
# 1) Database and Resource Loading
|
14 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
15 |
DB_HOST = os.getenv("DB_HOST")
|
16 |
DB_PORT = os.getenv("DB_PORT", "5432")
|
17 |
DB_NAME = os.getenv("DB_NAME")
|
|
|
35 |
return df_
|
36 |
|
37 |
df = get_data()
|
|
|
38 |
|
39 |
@pn.cache()
|
40 |
def load_embeddings():
|
|
|
46 |
buf.close(); gc.collect()
|
47 |
return ckpt["ids"], ckpt["embeddings"]
|
48 |
|
|
|
|
|
49 |
@pn.cache()
|
50 |
def get_st_model():
|
51 |
return SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2", device="cpu")
|
52 |
|
53 |
+
@pn.cache()
|
54 |
+
def get_semantic_resources():
|
55 |
+
model = get_st_model()
|
56 |
+
ids_list, emb_tensor = load_embeddings()
|
57 |
+
return model, ids_list, emb_tensor
|
58 |
+
|
59 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
60 |
# 2) Widgets
|
61 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
62 |
country_opts = sorted(df["country"].dropna().unique())
|
63 |
year_opts = sorted(df["year"].dropna().unique())
|
64 |
|
|
|
67 |
w_keyword = pn.widgets.TextInput(name="Keyword Search", placeholder="Search questions or answers")
|
68 |
w_group = pn.widgets.Checkbox(name="Group by Question Text", value=False)
|
69 |
|
|
|
70 |
w_semquery = pn.widgets.TextInput(name="Semantic Query")
|
71 |
+
w_search_button = pn.widgets.Button(name="Semantic Search", button_type="primary")
|
72 |
+
|
73 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
74 |
+
# 3) Unified Results Table (Tabulator)
|
75 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
76 |
+
result_table = pn.widgets.Tabulator(
|
77 |
+
pagination='remote',
|
78 |
+
page_size=15,
|
79 |
+
sizing_mode="stretch_width",
|
80 |
+
layout='fit_columns',
|
81 |
+
show_index=False,
|
82 |
+
)
|
83 |
|
84 |
+
@pn.depends(w_countries, w_years, w_keyword, w_group, watch=True)
|
85 |
+
def update_table(countries, years, keyword, group):
|
|
|
|
|
|
|
86 |
filt = df.copy()
|
87 |
if countries:
|
88 |
filt = filt[filt["country"].isin(countries)]
|
|
|
110 |
"answer_text": "Sample Answers"
|
111 |
})
|
112 |
)
|
113 |
+
result_table.value = grouped
|
114 |
+
else:
|
115 |
+
result_table.value = filt[["country", "year", "question_text", "answer_text"]]
|
|
|
|
|
|
|
116 |
|
|
|
|
|
|
|
117 |
def semantic_search(event=None):
|
118 |
query = w_semquery.value.strip()
|
119 |
if not query:
|
120 |
return
|
121 |
|
122 |
+
model, ids_list, emb_tensor = get_semantic_resources()
|
123 |
q_vec = model.encode(query, convert_to_tensor=True, device="cpu").cpu()
|
124 |
+
|
125 |
sims = util.cos_sim(q_vec, emb_tensor)[0]
|
126 |
top_vals, top_idx = torch.topk(sims, k=50)
|
127 |
|
|
|
131 |
sem_rows["Score"] = sem_rows["id"].map(score_map)
|
132 |
sem_rows = sem_rows.sort_values("Score", ascending=False)
|
133 |
|
134 |
+
filt = df.copy()
|
135 |
+
if w_countries.value:
|
136 |
+
filt = filt[filt["country"].isin(w_countries.value)]
|
137 |
+
if w_years.value:
|
138 |
+
filt = filt[filt["year"].isin(w_years.value)]
|
139 |
+
if w_keyword.value:
|
140 |
+
filt = filt[
|
141 |
+
filt["question_text"].str.contains(w_keyword.value, case=False, na=False) |
|
142 |
+
filt["answer_text"].str.contains(w_keyword.value, case=False, na=False) |
|
143 |
+
filt["question_code"].astype(str).str.contains(w_keyword.value, case=False, na=False)
|
144 |
+
]
|
145 |
|
146 |
+
remainder = filt.loc[~filt["id"].isin(sem_ids)].copy()
|
147 |
remainder["Score"] = ""
|
148 |
|
149 |
combined = pd.concat([sem_rows, remainder], ignore_index=True)
|
150 |
+
result_table.value = combined[["Score", "country", "year", "question_text", "answer_text"]]
|
|
|
151 |
|
152 |
w_search_button.on_click(semantic_search)
|
153 |
|
154 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
155 |
+
# 4) Layout
|
156 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
|
157 |
sidebar = pn.Column(
|
158 |
+
"## π Filters",
|
159 |
w_countries, w_years, w_keyword, w_group,
|
160 |
pn.Spacer(height=20),
|
161 |
"## π§ Semantic Search",
|
|
|
165 |
|
166 |
main = pn.Column(
|
167 |
pn.pane.Markdown("## π CGD Survey Explorer"),
|
168 |
+
result_table
|
|
|
|
|
|
|
169 |
)
|
170 |
|
171 |
pn.template.FastListTemplate(
|