support downloading updated data

app.py

@@ -173,7 +173,7 @@ def download(access_key):
     if not access_key.startswith(os.getenv('access_key')):
         chatbot_ret = [(f"Your entered Access Key:<br>{access_key}<br>is incorrect.", f"So i cannot provide you any information in this private space.")]
         file_ret = gr.File.update(value=None, visible=False)
-    elif: access_key == f"{os.getenv('access_key')}: update":
+    elif access_key == f"{os.getenv('access_key')}: update":
         chatbot_ret = [(f"Your entered Access Key is correct.", f"The file containing new processed query-responses ({len(qas)-num_read_qas} in total) can be downloaded below.")]
         filename = f"qas-{num_read_qas}-{len(qas)}.json"
         with open(filename, "w", encoding="utf-8") as f:

merge_qas.py

@@ -0,0 +1,56 @@
+import os
+import json
+import collections
+
+def read_qs():
+    qs = []
+    directory = "./dialogues_set"
+    filenames = [
+        'dialogues_film.json', 
+        'dialogues_jindong.json', 
+        'dialogues_music.json', 
+        'dialogues_natural.json', 
+        'dialogues_taobao.json', 
+        'dialogues_travel_kd.json'
+    ]
+    for filename in filenames:
+        with open(f"{directory}/{filename}", "r", encoding="utf-8") as f:
+            for idx,line in enumerate(f):
+                idx2query = json.loads(line)
+                query = idx2query[str(idx)]
+                qs.append(query)
+    print(f"read {len(qs)} queries from files")
+    return qs
+
+def read_qas():
+    qas = []
+    directory = "./dialogues_set"
+    
+    for filename in os.listdir(directory):
+        if filename.endswith(".json") and "qas" in filename:
+            with open(f"{directory}/{filename}", "r", encoding="utf-8") as f:
+                for qa in json.loads(f.read()):
+                    qas.append(qa)
+    return qas
+
+def merge(qs, qas):
+    q_to_as = collections.defaultdict(lambda:[])
+    for qa in qas:
+        q_to_as[qa["q"]].append(qa["a"])
+    
+    qas = []
+    for q in qs:
+        if len(q_to_as[q])==0:
+            continue
+        a = q_to_as[q].pop()
+        qas.append({"q":q, "a":a})
+    
+    return qas
+
+
+if __name__ == "__main__":
+    qs = read_qs()
+    qas = read_qas()
+    qas = merge(qs, qas)
+    with open("./dialogues_set/qas.json", "w", encoding="utf-8") as f:
+        f.write(json.dumps(qas, ensure_ascii=False, indent=2))