Spaces:
Sleeping
Sleeping
File size: 5,252 Bytes
44921ac c50c9d2 44921ac c50c9d2 19d19d4 44921ac 19d19d4 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 44921ac c50c9d2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
import pandas as pd
import streamlit as st
# from annotated_text import annotated_text
from annotated_text.util import get_annotated_html
from streamlit_annotation_tools import text_labeler
from constants import PREDICTION_ADDITION_INSTRUCTION
from evaluation_metrics import EVALUATION_METRICS
from predefined_example import EXAMPLES
from span_dataclass_converters import (
get_highlight_spans_from_ner_spans,
get_ner_spans_from_annotations,
)
@st.cache_resource
def get_examples_attributes(selected_example):
"Return example attributes so that they are not refreshed on every interaction"
return (
selected_example.text,
selected_example.gt_labels,
selected_example.gt_spans,
selected_example.predictions,
selected_example.tags,
)
if __name__ == "__main__":
st.set_page_config(layout="wide")
st.title("📈 NER Metrics Comparison ⚖️")
st.write(
"Evaluation for the NER task requires a ground truth and a prediction that will be evaluated. The ground truth is shown below, add predictions in the next section to compare the evaluation metrics."
)
explanation_tab, comparision_tab = st.tabs(["📙 Explanation", "⚖️ Comparision"])
with explanation_tab:
st.write("This is the place holder for explanation of all the metrics")
with comparision_tab:
# with st.container():
st.subheader("Ground Truth & Predictions") # , divider='rainbow')
selected_example = st.selectbox(
"Select an example text from the drop down below",
[example for example in EXAMPLES],
format_func=lambda ex: ex.text,
)
text, gt_labels, gt_spans, predictions, tags = get_examples_attributes(
selected_example
)
# annotated_text(
# get_highlight_spans_from_ner_spans(
# get_ner_spans_from_annotations(gt_labels), text
# )
# )
annotated_predictions = [
get_annotated_html(get_highlight_spans_from_ner_spans(ner_span, text))
for ner_span in predictions
]
predictions_df = pd.DataFrame(
{
# "ID": [f"Prediction_{index}" for index in range(len(predictions))],
"Prediction": annotated_predictions,
"ner_spans": predictions,
},
index=["Ground Truth"]
+ [f"Prediction_{index}" for index in range(len(predictions) - 1)],
)
# st.subheader("Predictions") # , divider='rainbow')
with st.expander("Click to Add Predictions"):
st.subheader("Adding predictions")
st.markdown(PREDICTION_ADDITION_INSTRUCTION)
st.write(
"Note: Only the spans of the selected label name is shown at a given instance.",
)
labels = text_labeler(text, gt_labels)
st.json(labels, expanded=False)
# if st.button("Add Prediction"):
# labels = text_labeler(text)
if st.button("Add!"):
spans = get_ner_spans_from_annotations(labels)
spans = sorted(spans, key=lambda span: span["start"])
predictions.append(spans)
annotated_predictions.append(
get_annotated_html(get_highlight_spans_from_ner_spans(spans, text))
)
predictions_df = pd.DataFrame(
{
# "ID": [f"Prediction_{index}" for index in range(len(predictions))],
"Prediction": annotated_predictions,
"ner_spans": predictions,
},
index=["Ground Truth"]
+ [f"Prediction_{index}" for index in range(len(predictions) - 1)],
)
print("added")
highlighted_predictions_df = predictions_df[["Prediction"]]
st.write(
highlighted_predictions_df.to_html(escape=False), unsafe_allow_html=True
)
st.divider()
### EVALUATION METRICS COMPARISION ###
st.subheader("Evaluation Metrics Comparision") # , divider='rainbow')
st.markdown(
"The different evaluation metrics we have for the NER task are\n"
f"{''.join(['- '+evaluation_metric.name+'\n' for evaluation_metric in EVALUATION_METRICS])}"
)
with st.expander("View Predictions Details"):
st.write(predictions_df.to_html(escape=False), unsafe_allow_html=True)
if st.button("Get Metrics!"):
for evaluation_metric in EVALUATION_METRICS:
predictions_df[evaluation_metric.name] = predictions_df.ner_spans.apply(
lambda ner_spans: evaluation_metric.get_evaluation_metric(
# metric_type=evaluation_metric_type,
gt_ner_span=gt_spans,
pred_ner_span=ner_spans,
text=text,
tags=tags,
)
)
metrics_df = predictions_df.drop(["ner_spans"], axis=1)
st.write(metrics_df.to_html(escape=False), unsafe_allow_html=True)
|