Update app.py

app.py

@@ -1,9 +1,10 @@
 import json
+import os
 import pandas as pd
 import requests
 import threading
 import streamlit as st
-
+from datasets import load_dataset, load_metric
 
 MODELS = ["CodeParrot", "InCoder", "CodeGen", "PolyCoder"]
 GENERATION_MODELS = ["CodeParrot", "InCoder", "CodeGen"]
@@ -14,6 +15,16 @@ def load_examples():
     with open("utils/examples.json", "r") as f:
         examples = json.load(f)
     return examples
+    
+    
+def load_evaluation():
+    # load task 2 of HumanEval and code_eval_metric
+    os.environ["HF_ALLOW_CODE_EVAL"] = "1"
+    human_eval = load_dataset("openai_humaneval")
+    entry_point = f"check({human_eval['test'][2]['entry_point']})"
+    test_func = "\n" + human_eval["test"][2]["test"] + "\n" + entry_point
+    code_eval = load_metric("code_eval")
+    return code_eval, test_func
 
 
 def read_markdown(path):
@@ -111,7 +122,28 @@ read_markdown(f"architectures/{selected_model.lower()}.md")
 st.subheader("3 - Code model evaluation")
 read_markdown("evaluation/intro.md")
 read_markdown("evaluation/demo_humaneval.md")
-
+## quiz
+st.markdown("Below you can try solving this problem or visualize the solution of CodeParrot:")
+with open("evaluation/problem.md", "r") as f:
+    problem = f.read()
+with open("evaluation/solution.md", "r") as f:
+    solution = f.read()
+    
+candidate_solution = st.text_area(
+    "Complete the problem:",
+    value=problem,
+    height=200,
+).strip()
+if st.button("Test my solution", key=2):
+    with st.spinner("Testing..."):
+        code_eval, test_func = load_evaluation()
+        test_cases = [test_func]
+        candidates = [[candidate_solution]]
+        pass_at_k, _ = code_eval.compute(references=test_cases, predictions=candidates)
+        st.markdown(f"Your pass@1 is {int(pass_at_k['pass@1'])}")
+if st.button("Show model solution", key=3):
+    st.markdown(solution)
+    
 # Code generation
 st.subheader("4 - Code generation ✨")
 read_markdown("generation/intro.md")