Initial commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,205 @@
+import os
+from datetime import datetime
+
+import gradio as gr
+import pandas as pd
+
+# -----------------------------------------------------------------------------
+# Configuration – adjust these paths to point at your data location
+# -----------------------------------------------------------------------------
+DATA_PATH = "human_judgement/selected_samples.json"  # CSV with columns: question, answer1, answer2
+RATINGS_PATH = (
+    "human_judgement/human_judgement.csv"  # File where user ratings will be appended
+)
+
+# -----------------------------------------------------------------------------
+# Helper functions
+# -----------------------------------------------------------------------------
+
+
+def load_data(path: str = DATA_PATH) -> pd.DataFrame:
+    """Load the Q/A pairs once and cache them inside gradio runtime."""
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Could not find data file at {path}.")
+    df = pd.read_json(path, lines=True)
+    expected_cols = {"question", "response1", "response2"}
+    if not expected_cols.issubset(df.columns):
+        raise ValueError(f"CSV file must contain columns: {', '.join(expected_cols)}")
+    return df
+
+
+def load_ratings(path: str = RATINGS_PATH) -> pd.DataFrame:
+    """Load the ratings file (creates an empty one if absent)."""
+    if os.path.exists(path):
+        return pd.read_csv(path)
+    return pd.DataFrame(columns=["user_id", "row_index", "choice", "timestamp"])
+
+
+def save_rating(user_id: str, row_index: int, choice: int, path: str = RATINGS_PATH):
+    """Append a single rating row to disk, avoiding accidental duplicates."""
+    ratings = load_ratings(path)
+
+    # Prevent duplicate entries for the same user/question pair
+    duplicate = (ratings.user_id == user_id) & (ratings.row_index == row_index)
+    if duplicate.any():
+        return  # already stored, nothing to do
+
+    new_entry = {
+        "user_id": user_id,
+        "row_index": row_index,
+        "choice": choice,  # 1 means answer1 preferred, 2 means answer2 preferred
+        "timestamp": datetime.utcnow().isoformat(),
+    }
+    ratings = pd.concat([ratings, pd.DataFrame([new_entry])], ignore_index=True)
+    ratings.to_csv(path, index=False)
+
+
+def get_next_unrated(df: pd.DataFrame, ratings: pd.DataFrame, user_id: str):
+    """Return (row_index, question, answer1, answer2) or None if finished."""
+    rated_indices = ratings.loc[ratings.user_id == user_id, "row_index"].tolist()
+    unrated_df = df[~df.index.isin(rated_indices)]
+    if unrated_df.empty:
+        return None
+    row = unrated_df.iloc[0]
+    return row.name, row.question, row.response1, row.response2
+
+
+# -----------------------------------------------------------------------------
+# Gradio callbacks
+# -----------------------------------------------------------------------------
+
+
+def start_or_resume(user_id: str, state_df):
+    """Initialise or resume a session for a given user id."""
+    if not user_id.strip():
+        return (
+            gr.update(visible=True),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            "",
+            "",
+            "",
+            "",
+            "Please enter a non‑empty identifier to begin.",
+        )
+
+    ratings = load_ratings()
+    record = get_next_unrated(state_df, ratings, user_id)
+    if record is None:
+        # Completed all tasks
+        return (
+            gr.update(visible=True),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            "",
+            "",
+            "",
+            "",
+            "🎉 You have evaluated every item – thank you!",
+        )
+
+    idx, q, a1, a2 = record
+    return (
+        gr.update(visible=True),  # keep user id input visible for reference
+        gr.update(visible=True),  # show evaluation section
+        gr.update(visible=True),  # enable submit button
+        "**" + q + "**",
+        a1,
+        a2,
+        str(idx),
+        "",
+    )
+
+
+def submit_preference(user_id: str, row_idx_str: str, choice: str, state_df):
+    """Handle a single preference submission and load the next question."""
+    if choice not in {"answer1", "answer2"}:
+        return gr.update(
+            value="Please choose either Answer 1 or Answer 2 before submitting."
+        )
+
+    row_idx = int(row_idx_str)
+    save_rating(user_id, row_idx, 1 if choice == "answer1" else 2)
+
+    ratings = load_ratings()
+    record = get_next_unrated(state_df, ratings, user_id)
+    if record is None:
+        return "", "", "", "", "🎉 You have evaluated every item – thank you!"
+
+    idx, q, a1, a2 = record
+    return "**" + q + "**", a1, a2, str(idx), ""
+
+
+# -----------------------------------------------------------------------------
+# Build Gradio interface
+# -----------------------------------------------------------------------------
+
+
+def build_demo():
+    df = load_data()
+
+    with gr.Blocks(title="Question/Answer Preference Rater") as demo:
+        gr.Markdown(
+            """# Q/A Preference Rater
+        Enter your identifier below to start or resume your evaluation session. For every question, select which answer you prefer. Your progress is saved automatically so you can return at any time using the **same identifier**."""
+        )
+
+        state_df = gr.State(df)  # keep dataset in memory for callbacks
+        state_row_idx = gr.State("")
+
+        # User identifier section
+        id_input = gr.Textbox(
+            label="User Identifier", placeholder="e.g. Alice", scale=3
+        )
+        start_btn = gr.Button("Start / Resume", scale=1)
+
+        # Feedback / status message
+        info_md = gr.Markdown("", visible=True)
+
+        # Evaluation section (initially hidden)
+        with gr.Column(visible=False) as eval_col:
+            question_md = gr.Markdown("", label="Question")
+            with gr.Row():
+                # answer1_box = gr.Textbox(
+                #     label="Answer\u00a01", interactive=False, lines=10
+                # )
+                # answer2_box = gr.Textbox(
+                #     label="Answer\u00a02", interactive=False, lines=10
+                # )
+                answer1_box = gr.Markdown(label="Answer 1")
+                answer2_box = gr.Markdown(label="Answer 2")
+            choice_radio = gr.Radio(
+                ["answer1", "answer2"],
+                label="Which answer do you prefer?",
+                interactive=True,
+            )
+        submit_btn = gr.Button("Submit Preference", visible=False)
+
+        # Wire callbacks
+        start_btn.click(
+            fn=start_or_resume,
+            inputs=[id_input, state_df],
+            outputs=[
+                id_input,
+                eval_col,
+                submit_btn,
+                question_md,
+                answer1_box,
+                answer2_box,
+                state_row_idx,
+                info_md,
+            ],
+        )
+
+        submit_btn.click(
+            fn=submit_preference,
+            inputs=[id_input, state_row_idx, choice_radio, state_df],
+            outputs=[question_md, answer1_box, answer2_box, state_row_idx, info_md],
+        )
+
+    return demo
+
+
+# if __name__ == "__main__":
+#     build_demo().launch()
+build_demo().launch()

human_judgement/human_judgement.csv

@@ -0,0 +1,26 @@
+user_id,row_index,choice,timestamp
+tung,0,2,2025-07-11T14:23:04.840194
+tung,1,2,2025-07-11T14:23:11.475744
+tung,2,2,2025-07-11T14:23:13.510468
+tung,3,1,2025-07-11T14:35:31.929781
+tung,4,2,2025-07-11T14:35:33.996639
+tung,5,1,2025-07-11T14:35:36.862713
+tung,6,2,2025-07-11T14:35:39.247847
+t,0,1,2025-07-11T14:48:38.266875
+1,0,2,2025-07-11T14:49:47.794730
+t,1,2,2025-07-11T14:54:17.996900
+t,2,2,2025-07-11T14:54:20.045234
+t,3,2,2025-07-11T14:54:21.926244
+t,4,2,2025-07-11T14:54:23.190934
+t,5,2,2025-07-11T14:54:24.518148
+t,6,2,2025-07-11T14:54:26.607548
+t,7,2,2025-07-11T14:54:28.998385
+t,8,1,2025-07-11T14:54:30.680462
+t,9,2,2025-07-11T14:54:32.624527
+t,10,1,2025-07-11T14:54:36.288496
+t,11,2,2025-07-11T14:54:46.931882
+t,12,2,2025-07-11T14:54:48.409522
+1,1,2,2025-07-11T14:57:30.841905
+123,0,1,2025-07-11T14:58:59.498550
+123,1,2,2025-07-11T14:59:01.489360
+123,2,2,2025-07-11T14:59:03.171541

human_judgement/selected_samples.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9138e1765027de6639319c7acf865a1b1356a58074c632e88f1bdde54840d1be
+size 91168861