calculatescores.py

import formatresultshelper

#Defined as utilized documents / retrieved documents for the query
def compute_context_relevance(relevant_sentences, support_keys):
    total_relevance_score = 0
    total_relevant_sentences = len(relevant_sentences)

    for sentence in relevant_sentences:
      if sentence in support_keys:
        total_relevance_score += 1

    # To avoid division by zero in case there are no relevant sentences
    if total_relevant_sentences == 0:
        return 0

    return total_relevance_score / total_relevant_sentences

def compute_context_utilization(relevant_sentences, utilization_levels):
    total_utilization_score = 0
    total_relevant_sentences = len(relevant_sentences)
    for sentence in relevant_sentences:
      if sentence in utilization_levels:
        total_utilization_score += 1
    # To avoid division by zero in case there are no relevant sentences
    if total_relevant_sentences == 0:
        return 0
    return total_utilization_score / total_relevant_sentences


def CalculateScores():
   #compute Context Relevance
   contextrel = compute_context_relevance(formatresultshelper.relevant_sentence_keys, formatresultshelper.support_keys)
   print(f"Context Relevance = {contextrel}")

   contextutil = compute_context_utilization(formatresultshelper.relevant_sentence_keys, formatresultshelper.all_utilized_sentence_keys)
   print(f"Context Utilization = {contextutil}")

   compnum = np.intersect1d(formatresultshelper.support_keys, formatresultshelper.all_utilized_sentence_keys)
   completenes = compnum.size / len(formatresultshelper.support_keys)
   print(f"Completeness = {completenes}")

   #Adherence : whether all parts of response are grounded by context
   for val in formatresultshelper.support_level:
     prevval = 1;
     if val == False:
       adherence = 0 * prevval
       break
     else:
       adherence = 1 * prevval
     prevval = adherence

   print(f"Adherence = {adherence}")

   

def mse(actual, predicted):
    return (actual - predicted)**2