|
|
from typing import Any, Callable, List |
|
|
from src.retrievers.base_retriever import Retriever |
|
|
|
|
|
from src.utils.string_utils import (lower, remove_articles, remove_punc, |
|
|
white_space_fix) |
|
|
|
|
|
|
|
|
def _normalize_text(inp: str, preprocessing_functions: List[Callable[[str], str]]): |
|
|
for fun in preprocessing_functions: |
|
|
inp = fun(inp) |
|
|
return inp |
|
|
|
|
|
|
|
|
def _normalize_text_default(inp: str) -> str: |
|
|
"""Preprocesses the sentence string by normalizing. |
|
|
|
|
|
Args: |
|
|
s (str): the sentence |
|
|
|
|
|
Returns: |
|
|
string: normalized with default parames |
|
|
""" |
|
|
|
|
|
steps = [remove_articles, white_space_fix, remove_punc, lower] |
|
|
|
|
|
return _normalize_text(inp, steps) |
|
|
|
|
|
|
|
|
def exact_match(prediction: str, answer: str) -> int: |
|
|
"""Computes exact match for sentences. |
|
|
|
|
|
Args: |
|
|
prediction (str): the predicted answer |
|
|
answer (str): the gold answer |
|
|
|
|
|
Returns: |
|
|
int: 1 for exact match, 0 for not |
|
|
""" |
|
|
return int(_normalize_text_default(prediction) == _normalize_text_default(answer)) |
|
|
|
|
|
|
|
|
def f1(prediction: str, answer: str) -> float: |
|
|
"""Computes F1-score on token overlap for sentences. |
|
|
|
|
|
Args: |
|
|
prediction (str): the predicted answer |
|
|
answer (str): the gold answer |
|
|
|
|
|
Returns: |
|
|
boolean: the f1 score |
|
|
""" |
|
|
pred_tokens = _normalize_text_default(prediction).split() |
|
|
answer_tokens = _normalize_text_default(answer).split() |
|
|
|
|
|
if len(pred_tokens) == 0 or len(answer_tokens) == 0: |
|
|
return int(pred_tokens == answer_tokens) |
|
|
|
|
|
common_tokens = set(pred_tokens) & set(answer_tokens) |
|
|
|
|
|
if len(common_tokens) == 0: |
|
|
return 0 |
|
|
|
|
|
prec = len(common_tokens) / len(pred_tokens) |
|
|
rec = len(common_tokens) / len(answer_tokens) |
|
|
|
|
|
return 2 * (prec * rec) / (prec + rec) |
|
|
|
|
|
|
|
|
def evaluate(retriever: Retriever, questions: Any, answers: Any): |
|
|
"""Evaluates the entire model by computing F1-score and exact match on the |
|
|
entire dataset. |
|
|
|
|
|
Returns: |
|
|
float: overall exact match |
|
|
float: overall F1-score |
|
|
""" |
|
|
|
|
|
predictions = [] |
|
|
scores = 0 |
|
|
|
|
|
|
|
|
for question in questions: |
|
|
score, result = retriever.retrieve(question, 1) |
|
|
scores += score[0] |
|
|
predictions.append(result['text'][0]) |
|
|
|
|
|
exact_matches = [exact_match( |
|
|
predictions[i], answers[i]) for i in range(len(answers))] |
|
|
f1_scores = [f1( |
|
|
predictions[i], answers[i]) for i in range(len(answers))] |
|
|
|
|
|
return sum(exact_matches) / len(exact_matches), sum(f1_scores) / len(f1_scores) |
|
|
|
