myshirk commited on
Commit
1e66d1d
Β·
verified Β·
1 Parent(s): 630223b

update to tabulator

Browse files
Files changed (1) hide show
  1. app.py +51 -48
app.py CHANGED
@@ -1,17 +1,17 @@
1
- # app_panel.py – Panel-based CGD Survey Explorer
2
 
3
- import os, io, json, gc
4
  import panel as pn
5
  import pandas as pd
6
  import boto3, torch
7
- from sentence_transformers import SentenceTransformer, util
8
  import psycopg2
 
9
 
10
  pn.extension()
11
 
12
- # ───────────────────────────────────────────────
13
- # 1) Data / Embeddings Loaders
14
- # ───────────────────────────────────────────────
15
  DB_HOST = os.getenv("DB_HOST")
16
  DB_PORT = os.getenv("DB_PORT", "5432")
17
  DB_NAME = os.getenv("DB_NAME")
@@ -35,7 +35,6 @@ def get_data():
35
  return df_
36
 
37
  df = get_data()
38
- row_lookup = {row.id: i for i, row in df.iterrows()}
39
 
40
  @pn.cache()
41
  def load_embeddings():
@@ -47,15 +46,19 @@ def load_embeddings():
47
  buf.close(); gc.collect()
48
  return ckpt["ids"], ckpt["embeddings"]
49
 
50
- ids_list, emb_tensor = load_embeddings()
51
-
52
  @pn.cache()
53
  def get_st_model():
54
  return SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2", device="cpu")
55
 
56
- # ───────────────────────────────────────────────
 
 
 
 
 
 
57
  # 2) Widgets
58
- # ───────────────────────────────────────────────
59
  country_opts = sorted(df["country"].dropna().unique())
60
  year_opts = sorted(df["year"].dropna().unique())
61
 
@@ -64,15 +67,22 @@ w_years = pn.widgets.MultiSelect(name="Years", options=year_opts)
64
  w_keyword = pn.widgets.TextInput(name="Keyword Search", placeholder="Search questions or answers")
65
  w_group = pn.widgets.Checkbox(name="Group by Question Text", value=False)
66
 
67
- # Semantic search
68
  w_semquery = pn.widgets.TextInput(name="Semantic Query")
69
- w_search_button = pn.widgets.Button(name="Search", button_type="primary", disabled=False)
 
 
 
 
 
 
 
 
 
 
 
70
 
71
- # ───────────────────────────────────────────────
72
- # 3) Filtering Logic
73
- # ───────────────────────────────────────────────
74
- @pn.depends(w_countries, w_years, w_keyword, w_group)
75
- def keyword_filter(countries, years, keyword, group):
76
  filt = df.copy()
77
  if countries:
78
  filt = filt[filt["country"].isin(countries)]
@@ -100,23 +110,18 @@ def keyword_filter(countries, years, keyword, group):
100
  "answer_text": "Sample Answers"
101
  })
102
  )
103
- return pn.pane.DataFrame(grouped, sizing_mode="stretch_width", height=400)
104
-
105
- return pn.pane.DataFrame(
106
- filt[["country", "year", "question_text", "answer_text"]],
107
- sizing_mode="stretch_width", height=400
108
- )
109
 
110
- # ────────────��──────────────────────────────────
111
- # 4) Semantic Search Callback
112
- # ───────────────────────────────────────────────
113
  def semantic_search(event=None):
114
  query = w_semquery.value.strip()
115
  if not query:
116
  return
117
 
118
- model = get_st_model()
119
  q_vec = model.encode(query, convert_to_tensor=True, device="cpu").cpu()
 
120
  sims = util.cos_sim(q_vec, emb_tensor)[0]
121
  top_vals, top_idx = torch.topk(sims, k=50)
122
 
@@ -126,30 +131,31 @@ def semantic_search(event=None):
126
  sem_rows["Score"] = sem_rows["id"].map(score_map)
127
  sem_rows = sem_rows.sort_values("Score", ascending=False)
128
 
129
- # Get keyword-filtered data
130
- keyword_df = keyword_filter(
131
- w_countries.value,
132
- w_years.value,
133
- w_keyword.value,
134
- False
135
- ).object
 
 
 
 
136
 
137
- remainder = keyword_df.loc[~keyword_df["id"].isin(sem_ids)].copy()
138
  remainder["Score"] = ""
139
 
140
  combined = pd.concat([sem_rows, remainder], ignore_index=True)
141
-
142
- result_pane.object = combined[["Score", "country", "year", "question_text", "answer_text"]]
143
 
144
  w_search_button.on_click(semantic_search)
145
 
146
- result_pane = pn.pane.DataFrame(height=500, sizing_mode="stretch_width")
147
-
148
- # ───────────────────────────────────────────────
149
- # 5) Layout
150
- # ───────────────────────────────────────────────
151
  sidebar = pn.Column(
152
- "## πŸ” Filter Questions",
153
  w_countries, w_years, w_keyword, w_group,
154
  pn.Spacer(height=20),
155
  "## 🧠 Semantic Search",
@@ -159,10 +165,7 @@ sidebar = pn.Column(
159
 
160
  main = pn.Column(
161
  pn.pane.Markdown("## 🌍 CGD Survey Explorer"),
162
- pn.Tabs(
163
- ("Filtered Results", keyword_filter),
164
- ("Semantic Search Results", result_pane),
165
- )
166
  )
167
 
168
  pn.template.FastListTemplate(
 
1
+ # app.py – Unified Panel App with Semantic Search + Filterable Tabulator
2
 
3
+ import os, io, gc
4
  import panel as pn
5
  import pandas as pd
6
  import boto3, torch
 
7
  import psycopg2
8
+ from sentence_transformers import SentenceTransformer, util
9
 
10
  pn.extension()
11
 
12
+ # ──────────────────────────────────────────────────────────────────────
13
+ # 1) Database and Resource Loading
14
+ # ──────────────────────────────────────────────────────────────────────
15
  DB_HOST = os.getenv("DB_HOST")
16
  DB_PORT = os.getenv("DB_PORT", "5432")
17
  DB_NAME = os.getenv("DB_NAME")
 
35
  return df_
36
 
37
  df = get_data()
 
38
 
39
  @pn.cache()
40
  def load_embeddings():
 
46
  buf.close(); gc.collect()
47
  return ckpt["ids"], ckpt["embeddings"]
48
 
 
 
49
  @pn.cache()
50
  def get_st_model():
51
  return SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2", device="cpu")
52
 
53
+ @pn.cache()
54
+ def get_semantic_resources():
55
+ model = get_st_model()
56
+ ids_list, emb_tensor = load_embeddings()
57
+ return model, ids_list, emb_tensor
58
+
59
+ # ──────────────────────────────────────────────────────────────────────
60
  # 2) Widgets
61
+ # ──────────────────────────────────────────────────────────────────────
62
  country_opts = sorted(df["country"].dropna().unique())
63
  year_opts = sorted(df["year"].dropna().unique())
64
 
 
67
  w_keyword = pn.widgets.TextInput(name="Keyword Search", placeholder="Search questions or answers")
68
  w_group = pn.widgets.Checkbox(name="Group by Question Text", value=False)
69
 
 
70
  w_semquery = pn.widgets.TextInput(name="Semantic Query")
71
+ w_search_button = pn.widgets.Button(name="Semantic Search", button_type="primary")
72
+
73
+ # ──────────────────────────────────────────────────────────────────────
74
+ # 3) Unified Results Table (Tabulator)
75
+ # ──────────────────────────────────────────────────────────────────────
76
+ result_table = pn.widgets.Tabulator(
77
+ pagination='remote',
78
+ page_size=15,
79
+ sizing_mode="stretch_width",
80
+ layout='fit_columns',
81
+ show_index=False,
82
+ )
83
 
84
+ @pn.depends(w_countries, w_years, w_keyword, w_group, watch=True)
85
+ def update_table(countries, years, keyword, group):
 
 
 
86
  filt = df.copy()
87
  if countries:
88
  filt = filt[filt["country"].isin(countries)]
 
110
  "answer_text": "Sample Answers"
111
  })
112
  )
113
+ result_table.value = grouped
114
+ else:
115
+ result_table.value = filt[["country", "year", "question_text", "answer_text"]]
 
 
 
116
 
 
 
 
117
  def semantic_search(event=None):
118
  query = w_semquery.value.strip()
119
  if not query:
120
  return
121
 
122
+ model, ids_list, emb_tensor = get_semantic_resources()
123
  q_vec = model.encode(query, convert_to_tensor=True, device="cpu").cpu()
124
+
125
  sims = util.cos_sim(q_vec, emb_tensor)[0]
126
  top_vals, top_idx = torch.topk(sims, k=50)
127
 
 
131
  sem_rows["Score"] = sem_rows["id"].map(score_map)
132
  sem_rows = sem_rows.sort_values("Score", ascending=False)
133
 
134
+ filt = df.copy()
135
+ if w_countries.value:
136
+ filt = filt[filt["country"].isin(w_countries.value)]
137
+ if w_years.value:
138
+ filt = filt[filt["year"].isin(w_years.value)]
139
+ if w_keyword.value:
140
+ filt = filt[
141
+ filt["question_text"].str.contains(w_keyword.value, case=False, na=False) |
142
+ filt["answer_text"].str.contains(w_keyword.value, case=False, na=False) |
143
+ filt["question_code"].astype(str).str.contains(w_keyword.value, case=False, na=False)
144
+ ]
145
 
146
+ remainder = filt.loc[~filt["id"].isin(sem_ids)].copy()
147
  remainder["Score"] = ""
148
 
149
  combined = pd.concat([sem_rows, remainder], ignore_index=True)
150
+ result_table.value = combined[["Score", "country", "year", "question_text", "answer_text"]]
 
151
 
152
  w_search_button.on_click(semantic_search)
153
 
154
+ # ──────────────────────────────────────────────────────────────────────
155
+ # 4) Layout
156
+ # ──────────────────────────────────────────────────────────────────────
 
 
157
  sidebar = pn.Column(
158
+ "## πŸ”Ž Filters",
159
  w_countries, w_years, w_keyword, w_group,
160
  pn.Spacer(height=20),
161
  "## 🧠 Semantic Search",
 
165
 
166
  main = pn.Column(
167
  pn.pane.Markdown("## 🌍 CGD Survey Explorer"),
168
+ result_table
 
 
 
169
  )
170
 
171
  pn.template.FastListTemplate(