model_compare.py

from functools import cmp_to_key
from compare_significance import check_significance

class ModelCompare():

    TASKS =  ["propaganda_demonizace",
                "propaganda_vina",
                "propaganda_relativizace",
                "propaganda_argumentace",
                "propaganda_lokace",
                "propaganda_nazor",
                "propaganda_emoce",
                "propaganda_fabulace",
                "propaganda_nalepkovani",
                "propaganda_zamereni",
                "propaganda_zanr",
                "propaganda_rusko",
                "propaganda_strach",
                "benczechmark_sentiment"]

    def __init__(self, ranks:dict=None):
        self.ranks = ranks

    def compare_models(self, modelA_id, modelB_id):
        if not self.ranks:
            raise Exception("Missing model rankings")
        
        res = self.ranks[modelA_id][modelB_id][self.current_task]
        if res == True:   
            return 1
        elif res == False:
            return -1
        else:
            return -1


    def get_tasks_ranks(self, ranks:dict) -> dict:
        '''Order models based on the significance improvement'''
        
        self.ranks = ranks
        
        tasks_ranks = {}
        
        models = ranks.keys()
        for task in self.TASKS:
            self.current_task = task
            tasks_ranks[task] = sorted(models, key=cmp_to_key(self.compare_models))
        return tasks_ranks
        
    
# models = {
# model1 : {
#         task1 : order_idx
#         task2 : order_idx
#         task3 : order_idx
#     }
# }