Update the second version

README.md

@@ -17,16 +17,16 @@ analysis, and testing. By interacting with a conversational agent, users can ste
 extraction of competency questions, while receiving computational support to analyse the overall requirements and test
 early versions of the resulting ontologies.
 
-## Deploy
-If you would like to deploy this demo locally,
-1. Create a python environment and install the requirements using `pip install -r requirements.txt`.
-2. Run `app.py`.
+## GitHub Copy
+- The source code will also be available soon on GitHub: [King-s-Knowledge-Graph-Lab/OntoChat](https://github.com/King-s-Knowledge-Graph-Lab/OntoChat).
+
+## Note
+- The ontology testing part has been tested with the [Music Meta Ontology](https://github.com/polifonia-project/music-meta-ontology) and works well.
 
 ## TODO
-- Hosting in Hugging Face Space
-- Add ontology testing
-- Add the evaluation panel (?)
-- Optimize clustering visualization
-- Adjust flagging
+- Improve the verbaliser (classes, named entities, and relations might be messy in some cases)
+- Optimize clustering visualization (maybe only keep LLM lcustering)
+- Add [flagging](https://www.gradio.app/docs/flagging), e.g., [`HuggingFaceDatasetSaver`](https://www.gradio.app/docs/flagging#hugging-face-dataset-saver-header)
+
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -28,82 +28,99 @@ with gr.Blocks() as user_story_interface:
 
     with gr.Row():
         with gr.Column():
-            persona = gr.Textbox(
-                label="Persona",
-                placeholder="Please input the persona of the user, including the name, occupations, skills, interests.",
-                info="Example: The user, Mark, is an experienced musicologist. He's an expert in western music, "
-                     "and plays piano and guitar."
-            )
-            goal = gr.Textbox(
-                label="Goal",
-                placeholder="Please input the goal of the user and any specific issues faced.",
-                info="Example: The goal of the user is to analyse analogies and simmetries between music scores, "
-                     "with a particular focus on harmony and the lyrics of the music piece."
-            )
-            sample_data = gr.Textbox(
-                label="Sample of Data",
-                placeholder="Please input a sample of data.",
-                info="Example: An example of data would be: - 'Let it be' by 'The Beatles' has a sequence of chords "
-                     "composed by 'F, Amin, F' that is recurring every time the lyrics say 'Let it be'; - The lyrics "
-                     "of 'Running with the Devil' by 'Van Halen' have a recurring chord sequence for the chorus and a "
-                     "recurring chord sequence for the bridge."
-            )
-            generate_btn = gr.Button(value="Generate")
-            user_story_chatbot = gr.Chatbot(
-
-            )
-            chatbot_input = gr.Textbox(
-                placeholder="Please tell me what improvements I should make to the user story :)"
+            user_story_chatbot = gr.Chatbot([
+                [None, "Hello! I am OntoChat, your conversational ontology engineering assistant."],
+                ["I am a domain expert trying to create a user story to be used by ontology engineers. You are the "
+                 "ontology expert. Only ask the following question once I have responded. Ask for the specifications "
+                 "to generate a user story as a user of the system, which should include: 1. The Persona: What are "
+                 "the name, occupation, skills and interests of the user? 2. The Goal: What is the goal of the user? "
+                 "Are they facing specific issues? 3. Example Data: Do you have examples of the specific data "
+                 "available? Make sure you have answers to all three questions before providing a user story. Only "
+                 "ask the next question once I have responded.", "Sure. Let's start with the persona. What are the "
+                 "name, occupations, skills, interests of the user?"]
+            ])
+            user_story_input = gr.Textbox(
+                label="Chatbot input",
+                placeholder="Please type your message here and press Enter to interact with the chatbot :)"
             )
         user_story = gr.TextArea(
             label="User story",
             interactive=True
         )
-    generate_btn.click(
-        fn=user_story_init_generator,
+    user_story_input.submit(
+        fn=user_story_generator,
         inputs=[
-            persona, goal, sample_data
+            user_story_input, user_story_chatbot
         ],
         outputs=[
-            user_story, user_story_chatbot
+            user_story, user_story_chatbot, user_story_input
         ]
     )
-    chatbot_input.submit(
-        fn=user_story_generator,
+
+
+with gr.Blocks() as cq_interface:
+    gr.Markdown(
+        """
+        # OntoChat 
+        This is the second step of OntoChat. Please copy the generated user story from the previous 
+        step and use it here. You can also modify the user story before using it for generating competency questions. 
+        **Recommended prompt workflow:** 
+        1. Obtain competency questions from the user story. 
+        - Zero-shot learning:
+            - Prompt template: Given the user story: {user story}, generate {number} competency questions base on it. 
+        - Few-shot learning (i.e., provide examples to give more instructions on how to generate competency questions):
+            - Prompt template: Here are some good examples of competency questions generated from example data. 
+              Formatted in {"Example data": "Competency questions"}. 
+              {"Yesterday was performed by Armando Rocca.": "Who performs the song?"},
+              {"The Church was built in 1619.": "When (what year) was the building built?"},
+              {"The Church is located in a periurban context.": "In which context is the building located?"},
+              {"The mounting system of the bells is the falling clapper.": "Which is the mounting system of the bell?"}
+        2. Clean and refine competency questions. 
+        - Obtain multiple competency questions. 
+            - Prompt template: Take the generated competency questions and check if any of them can be divided into 
+              multiple questions. If they do, split the competency question into multiple competency questions. If it 
+              does not, leave the competency question as it is. For example, the competency question "Who wrote The 
+              Hobbit and in what year was the book written?" must be split into two competency questions: "Who wrote 
+              the book?" and "In what year was the book written?". Another example is the competency question, "When 
+              was the person born?". This competency question cannot be divided into multiple questions.
+        - Remove specific named entities.
+            - Prompt template: Take the competency questions and check if they contain real-world entities, like 
+              "Freddy Mercury" or "1837". If they do, change those real-world entities from these competency questions 
+              to more general concepts. For example, the competency question "Which is the author of Harry Potter?" 
+              should be changed to "Which is the author of the book?". Similarly, the competency question "Who wrote 
+              the book in 2018?" should be changed to "Who wrote the book, and in what year was the book written?"
+        """
+    )
+
+    with gr.Row():
+        with gr.Column():
+            cq_chatbot = gr.Chatbot([
+                [None, "I am OntoChat, your conversational ontology engineering assistant. Here is the second step of "
+                 "the system. Please give me your user story and tell me how many competency questions you want."]
+            ])
+            cq_input = gr.Textbox(
+                label="Chatbot input",
+                placeholder="Please type your message here and press Enter to interact with the chatbot :)"
+            )
+        cq_output = gr.TextArea(
+            label="Competency questions",
+            interactive=True
+        )
+    cq_input.submit(
+        fn=cq_generator,
         inputs=[
-            chatbot_input, user_story_chatbot
+            cq_input, cq_chatbot
         ],
         outputs=[
-            user_story, user_story_chatbot
+            cq_output, cq_chatbot, cq_input
         ]
     )
 
-cq_interface = gr.Interface(
-    fn=cq_generator,
-    inputs=[
-        gr.Textbox(
-            label="User story",
-            info="Please copy the previously generated user story and paste it here. You can also modify the user "
-                 "story before submitting it."
-        ),
-        gr.Slider(
-            minimum=5,
-            maximum=50,
-            step=1,
-            label="Number of competency questions",
-            info="Please select the number of competency questions you want to generate."
-        )
-    ],
-    outputs=[
-        gr.Textbox(label="Competency questions")
-    ],
-    title="OntoChat",
-)
 
 clustering_interface = gr.Interface(
     fn=clustering_generator,
     inputs=[
-        gr.Textbox(
+        gr.TextArea(
             label="Competency questions",
             info="Please copy the previously generated competency questions and paste it here. You can also modify "
                  "the questions before submitting them."
@@ -131,14 +148,48 @@ clustering_interface = gr.Interface(
         )
     ],
     title="OntoChat",
+    description="This is the third step of OntoChat. Please copy the generated competency questions from the previous "
+                "step and run the clustering algorithm to group the competency questions based on their topics. From "
+                "our experience, LLM clustering has the best performance.",
+    allow_flagging="never"
 )
 
+
+with gr.Blocks() as testing_interface:
+    gr.Markdown(
+        """
+        # OntoChat 
+        This is the final part of OntoChat which performs ontology testing based on the input ontology file and CQs. 
+        """
+    )
+    ontology_file = gr.File(label="Ontology file")
+    ontology_desc = gr.Textbox(
+        label="Ontology description",
+        placeholder="Please provide a description of the ontology uploaded to provide basic information and "
+                    "additional context."
+    )
+    cq_testing_input = gr.Textbox(
+        label="Competency questions",
+        placeholder="Please provide the competency questions that you want to test with."
+    )
+    testing_btn = gr.Button(value="Test")
+    testing_output = gr.TextArea(label="Ontology testing output")
+    testing_btn.click(
+        fn=ontology_testing,
+        inputs=[
+            ontology_file, ontology_desc, cq_testing_input
+        ],
+        outputs=[
+            testing_output
+        ]
+    )
+
+
 demo = gr.TabbedInterface(
-    [user_story_interface, cq_interface, clustering_interface],
-    ["User Story Generation", "Competency Question Extraction", "Competency Question Analysis"]
+    [user_story_interface, cq_interface, clustering_interface, testing_interface],
+    ["User Story Generation", "Competency Question Extraction", "Competency Question Analysis", "Ontology Testing"]
 )
 
 
 if __name__ == "__main__":
-    # demo.launch(share=True)
     demo.launch()

ontochat/chatbot.py

@@ -1,6 +1,6 @@
 from openai import OpenAI
 
-# client = OpenAI()
+
 MODEL_NAME = "gpt-3.5-turbo"
 TEMPERATURE = 0
 SEED = 1234
@@ -17,19 +17,6 @@ def chat_completion(api_key, messages):
     return completion.choices[0].message.content
 
 
-def build_history(messages):
-    """
-    convert OpenAI client messages to gradio.Chatbot history
-    :param messages:
-    :return:
-    """
-    message_list = [None, ]
-    for item in messages:
-        message_list.append(item["content"])
-    history = [[message_list[i], message_list[i + 1]] for i in range(0, len(message_list), 2)]
-    return history
-
-
 def build_messages(history):
     """
     convert gardio.Chatbot history to OpenAI client messages

ontochat/functions.py

@@ -4,73 +4,79 @@ Interface functions
 
 import json
 
-from ontochat.chatbot import chat_completion, build_history, build_messages
+from ontochat.chatbot import chat_completion, build_messages
 from ontochat.analysis import compute_embeddings, agglomerative_clustering, hdbscan_clustering, llm_cq_clustering
+from ontochat.verbaliser import verbalise_ontology
 
 
 def set_openai_api_key(api_key: str):
     global openai_api_key
     openai_api_key = api_key
-    return "API key has been set!"
+    return "API key has been set! Now you can chat with the chatbot. Enjoy :)"
 
 
-def user_story_init_generator(persona, goal, sample_data):
-    # if os.environ.get("OPENAI_API_KEY") is None:
-    #     # openai.api_key = api_key
-    #     os.environ["OPENAI_API_KEY"] = api_key
-    messages = [{
-        "role": "system",
-        "content": "I am a conversational ontology engineering assistant, to help the user generate user stories, "
-                   "elicit requirements, and extract and analyze competency questions. In ontology engineering, "
-                   "a user story contains all the requirements from the perspective of an end user of the ontology. "
-                   "It is a way of capturing what a user needs to achieve with the ontology while also providing "
-                   "context and value. I will guide the user step-by-step to create a user story and generate "
-                   "competency questions from it."
-    }, {
-        "role": "user",
-        "content": f"The persona of the user is {persona}. The goal of the user is {goal}. A sampple of data is "
-                   f"{sample_data}. Write a user story for the ontology that fit into the information provided."
-    }]
+def user_story_generator(message, history):
+    print(history)
+    if len(history) == 1:  # initial round
+        messages = [{
+            "role": "system",
+            "content": "Hello! I am OntoChat, your conversational ontology engineering assistant."
+        }, {
+            "role": "user",
+            "content": "I am a domain expert trying to create a user story to be used by ontology engineers. You are "
+                       "the ontology expert. Only ask the following question once I have responded. Ask for the"
+                       "specifications to generate a user story as a user of the system, which should include: 1. The "
+                       "Persona: What are the name, occupation, skills and interests of the user? 2. The Goal: What is "
+                       "the goal of the user? Are they facing specific issues? 3. Example Data: Do you have examples "
+                       "of the specific data available? Make sure you have answers to all three questions before "
+                       "providing a user story. Only ask the next question once I have responded."
+        }, {
+            "role": "system",
+            "content": "Sure. Let's start with the persona. What are the name, occupations, skills, interests of the user?"
+        }, {
+            "role": "user",
+            "content": message
+        }]
+    else:
+        messages = build_messages(history)
+        messages.append({
+            "role": "user",
+            "content": message
+        })
     bot_message = chat_completion(openai_api_key, messages)
-    messages.append({
-        "role": "system",
-        "content": bot_message
-    })
-    history = build_history(messages)
-    return bot_message, history
+    history.append([message, bot_message])
+    return bot_message, history, ""
 
 
-def user_story_generator(message, history):
+def cq_generator(message, history):
     """
-
+    generate competency questions based on the user story
+    format constraint may not be necessary if we only use LLMs for clustering
     :param message:
     :param history:
     :return:
     """
-    messages = build_messages(history)
-    bot_message = chat_completion(openai_api_key, messages)
-    history.append((message, bot_message))
-    return bot_message, history
-
-
-def cq_generator(messages, numbers):
-    """
-
-    :param messages:
-    :param numbers:
-    :return:
-    """
-    messages = [
-        {
-            "role": "system",
-            "content": "You are an ontology engineer."
-        }, {
+    if (len(history)) == 1:  # initial round
+        messages = [
+            {
+                "role": "system",
+                "content": "I am OntoChat, your conversational ontology engineering assistant. Here is the second step "
+                           "of the system. Please give me your user story and tell me how many competency questions "
+                           "you want."
+            }, {
+                "role": "user",
+                "content": message
+            }
+        ]
+    else:
+        messages = build_messages(history)
+        messages.append({
             "role": "user",
-            "content": f"Please generate {numbers} competency questions based on the user story: {messages}"
-        }  # TODO: format constraint
-    ]
-    response = chat_completion(openai_api_key, messages)
-    return response
+            "content": message
+        })
+    bot_message = chat_completion(openai_api_key, messages)
+    history.append([message, bot_message])
+    return bot_message, history, ""
 
 
 def clustering_generator(cqs, cluster_method, n_clusters):
@@ -91,3 +97,26 @@ def clustering_generator(cqs, cluster_method, n_clusters):
         cq_clusters, cluster_image = llm_cq_clustering(cqs, n_clusters, openai_api_key)
 
     return cluster_image, json.dumps(cq_clusters, indent=4)
+
+
+def ontology_testing(ontology_file, ontology_desc, cqs):
+    """
+
+    :param ontology_file:
+    :param ontology_desc:
+    :param cqs:
+    :return:
+    """
+    verbalisation = verbalise_ontology(ontology_file, ontology_desc, "")
+    messages = [{
+        "role": "system",
+        "content": "Please (1) provide a description of the ontology uploaded to provide basic information and "
+                   "additional context, (2) give the competency questions (CQs) that you want to test with."
+    }, {
+        "role": "user",
+        "content": verbalisation + "\n" + f"Given the above ontology, please label each competency question: {cqs} to "
+                                          f"determine whether it is addressed properly or not. Format your response in"
+                                          f" ['yes': 'CQ1', 'no': 'CQ2', ...]."
+    }]
+    bot_message = chat_completion(openai_api_key, messages)
+    return bot_message

ontochat/queries.py

@@ -0,0 +1,17 @@
+"""
+General-purpose SPARQL queries
+
+"""
+
+NE_QUERY = """
+    PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    PREFIX owl: <http://www.w3.org/2002/07/owl#>
+
+    SELECT DISTINCT ?individual ?other
+    WHERE {
+        ?individual rdf:type owl:NamedIndividual ;
+        rdf:type ?other .
+        FILTER ( ?other not in ( owl:NamedIndividual ) )
+    }
+    """

ontochat/verbaliser.py

@@ -0,0 +1,199 @@
+"""
+Utilities for the verbalisation of an ontology.
+
+Examples of possible uses cases for ontology verbalisation:
+- Summarising the features provided by the ontology (doc)
+- Using a LM to extract competency questions from the ontology.
+- Asking a LM if the ontology can be used for certain requirements.
+
+"""
+import logging
+from typing import List
+
+import rdflib
+from rdflib import Graph
+from rdflib.namespace import RDF, RDFS, OWL
+
+from ontochat.queries import NE_QUERY
+
+logger = logging.getLogger("ontochat.verbaliser")
+
+
+def verbalise_ontology(ontology_path: str, onto_about: str, onto_desc: str):
+    """
+    A simple method to verbalise ontologies and extract requirements. This is
+    currently designed to produce a plain verbalisation.
+
+    Parameters
+    ----------
+    ontology_path : str
+        Path to the ontology encoded in a format that is readable by `rdflib`.
+    onto_about : str
+        A short description of the ontology, if documentation is missing.
+    onto_desc : str
+        An extended description of the ontology to provide more context.
+
+    Returns
+    -------
+    verbalisation : str
+        A string verbalisation of the ontology produced by the language model.
+    
+    """
+    g = Graph()
+    g.parse(ontology_path)
+
+    # Everything that has a label is mapped here, otherwise we get a URI label
+    label_dict = {s: str(o) for s, _, o in g.triples((None, RDFS.label, None))}
+    # just get the last part of the URI otw
+    label_fn = lambda x: label_dict[x] if x in label_dict else str(x).split("/")[-1]
+    comment_dict = {s: str(o) for s, _, o in g.triples((None, RDFS.comment, None))}
+
+    logger.info("Class verbalisation: start")
+    class_vrbs = verbalise_classes(g, label_fn, comment_dict)
+    logger.info(f"Class verbalisation: found {len(class_vrbs)} classes")
+
+    logger.info("Named entity verbalisation: start")
+    nament_vrbs = verbalise_named_entities(g, label_fn, comment_dict)
+    logger.info(f"Named entity verbalisation: found {len(class_vrbs)} entities")
+
+    logger.info("Relation verbalisation: start")
+    relat_vrbs = verbalise_relations(g, label_fn, comment_dict)
+    logger.info(f"Relation verbalisation: found {len(class_vrbs)} classes")
+
+    return collate_verbalisations(class_vrbs, nament_vrbs, relat_vrbs, onto_about, onto_desc)
+
+
+def create_relation_dict(graph, relation):
+    """
+    Returns all the objects appearing as tails of the given relation.
+    """
+    relation_dict = {}  # subject to all possible objects via relation
+    for s, p, o in graph.triples((None, relation, None)):
+        if isinstance(o, rdflib.term.BNode):
+            continue  # skip blank node
+        if s not in relation_dict:
+            relation_dict[s] = []
+        relation_dict[s].append(o)
+    return relation_dict
+
+
+def verbalise_classes(graph: rdflib.Graph, label_fn, comment_dict: dict):
+    # Classes are first to be extracted, subclasses follow
+    classes = [s for s, _, _ in graph.triples((None, RDF.type, OWL.Class))]
+    subclasses = create_relation_dict(graph, relation=RDFS.subClassOf)
+    logger.info(f"Found: {len(classes)} classes, {len(subclasses)} subclasses")
+    # Step 1: Verbalisation of classes, one by one
+    verbalisation_hist = []
+    class_verbalisations = []
+    for base_class in classes:
+        # The base verbalisation is the class label, if available
+        vrbn = f"{label_fn(base_class)}"
+
+        if base_class in subclasses:  # list all parent classes
+            vrbn += " (subconcept of "  # opening parenthesis
+            vrbn += ", ".join([label_fn(u) for u in subclasses[base_class]])
+            vrbn += ")"  # closing parenthesis
+
+        if base_class in comment_dict:  # include comment
+            vrbn += f": {comment_dict[base_class]}"
+
+        verbalisation_hist.append(base_class)
+        class_verbalisations.append(vrbn)
+
+    # Step 2: verbalisation of remaining subclasses
+    for subclass in subclasses:  # check remaining subclasses
+        if subclass not in verbalisation_hist:
+            raise NotImplementedError(subclass)
+
+    return class_verbalisations
+
+
+def verbalise_named_entities(graph: rdflib.Graph, label, comment_dict: dict):
+    """
+    Note: TODO append NE comment (if available) to each named entity.
+    Note: FIXME still, a named entity can have more than 1 parent class.
+    """
+    qres = graph.query(NE_QUERY)
+    named_entities = {n: c for n, c in list(qres)}
+
+    nentities_verbalisations = []
+    for named_entity, named_type in named_entities.items():
+        verbalisation = f"{label(named_entity)} is an instance of class {label(named_type)}."
+        nentities_verbalisations.append(verbalisation)
+
+    return nentities_verbalisations
+
+
+def verbalise_relations(graph: rdflib.Graph, label, comment_dict: dict):
+    properties = [s for s, _, _ in graph.triples(
+        (None, RDF.type, OWL.ObjectProperty))]
+    subprops = create_relation_dict(graph, relation=RDFS.subPropertyOf)
+    domains = create_relation_dict(graph, relation=RDFS.domain)
+    ranges = create_relation_dict(graph, relation=RDFS.range)
+
+    # Step 1: Verbalisation of classes
+    verbalisation_hist = []
+    relation_verbalisations = []
+
+    for base_prop in properties:
+
+        # The base verbalisation is the class label, if available
+        verbalisation = f"{label(base_prop)}"
+
+        if base_prop in subprops:
+            verbalisation += " (subproperty of "  # opening parenthesis
+            verbalisation += ", and".join([label(u) for u in subprops[base_prop]])
+            verbalisation += ")"  # closing parenthesis
+
+        if base_prop in comment_dict:  # include
+            verbalisation += f": {comment_dict[base_prop]}"
+
+        if base_prop in domains:
+            verbalisation += f" The domain of this relation can be: "
+            verbalisation += ", or ".join([label(u) for u in domains[base_prop]])
+            verbalisation += "."
+
+        if base_prop in ranges:
+            verbalisation += f" The range of this relation can be: "
+            verbalisation += ", or ".join([label(u) for u in ranges[base_prop]])
+            verbalisation += "."
+
+        verbalisation_hist.append(base_prop)
+        relation_verbalisations.append(verbalisation)
+
+    for subprop in subprops:  # check remaining subclasses
+        if subprop not in verbalisation_hist:
+            raise NotImplementedError(subprop)
+
+    return relation_verbalisations
+
+
+def collate_verbalisations(class_verbalisations: List[str],
+                           relation_verbalisations: List[str],
+                           nentities_verbalisations: List[str],
+                           onto_about: str, onto_desc: str,
+                           ):
+    ontoverb = ""  # This is the basic prompt with the ontology description
+    # ontoverb += f"You are given an ontology about {onto_about}. {onto_desc}\n"
+    ontoverb += f"Ontology description: {onto_about}. {onto_desc}"
+
+    ontoverb += "\n"
+
+    ontoverb += "The main classes of the ontology are listed below:\n"
+    for class_verb in class_verbalisations:
+        ontoverb += f"- {class_verb}\n"
+
+    ontoverb += "\n"
+
+    ontoverb += "The main named entities (individuals) are listed below:\n"
+
+    for ne_verb in nentities_verbalisations:
+        ontoverb += f"- {ne_verb}\n"
+
+    ontoverb += "\n"
+
+    ontoverb += "The main relations of the ontology are listed below:\n"
+    for rel_verb in relation_verbalisations:
+        ontoverb += f"- {rel_verb}\n"
+
+    return ontoverb

requirements.txt

@@ -1,4 +1,5 @@
 openai
 gradio
 scikit-learn
-sentence-transformers
+sentence-transformers
+rdflib