Spaces:
Runtime error
Runtime error
Commit
·
65757ca
1
Parent(s):
0c51c37
Update README.md
Browse files
README.md
CHANGED
|
@@ -47,7 +47,7 @@ evaluation_scores = module.compute(predictions=predictions, references=reference
|
|
| 47 |
- **predictions** (`list` of `list` of `dictionary`): a list of list of dictionary with every element's relation_type and their entity name
|
| 48 |
- **references** (`list` of `list` of `dictionary`): a list of list of dictionary with every element's relation_type and their entity name
|
| 49 |
- **mode** (`str`): define strict or boundaries mode for evaluation, strict mode consider "head_type" and "tail_type", boundaries mode doesn't consider "head_type" and "tail_type"
|
| 50 |
-
- **
|
| 51 |
- **relation_types** (`list`): define which relation type that need to be evaluate and show, if not given, it will construct relation_types from ground truth, default []
|
| 52 |
|
| 53 |
### Output Values
|
|
@@ -79,7 +79,7 @@ Output Example:
|
|
| 79 |
Remind : Macro_f1、Macro_p、Macro_r、p、r、f1 are always a number between 0 and 1. And tp、fp、fn depend on how many data inputs.
|
| 80 |
|
| 81 |
### Examples
|
| 82 |
-
Example1 : only one prediction and reference, mode=strict, only output total relation score
|
| 83 |
```python
|
| 84 |
metric_path = "Ikala-allen/relation_extraction"
|
| 85 |
module = evaluate.load(metric_path)
|
|
@@ -96,12 +96,12 @@ predictions = [
|
|
| 96 |
{"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
|
| 97 |
]
|
| 98 |
]
|
| 99 |
-
evaluation_scores = module.compute(predictions=predictions, references=references, mode="strict",
|
| 100 |
print(evaluation_scores)
|
| 101 |
>>> {'tp': 1, 'fp': 1, 'fn': 2, 'p': 50.0, 'r': 33.333333333333336, 'f1': 40.0, 'Macro_f1': 25.0, 'Macro_p': 25.0, 'Macro_r': 25.0}
|
| 102 |
```
|
| 103 |
|
| 104 |
-
Example2 : only one prediction and reference, mode=boundaries, only output total relation score
|
| 105 |
```python
|
| 106 |
metric_path = "Ikala-allen/relation_extraction"
|
| 107 |
module = evaluate.load(metric_path)
|
|
@@ -118,12 +118,12 @@ predictions = [
|
|
| 118 |
{"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
|
| 119 |
]
|
| 120 |
]
|
| 121 |
-
evaluation_scores = module.compute(predictions=predictions, references=references, mode="strict",
|
| 122 |
print(evaluation_scores)
|
| 123 |
-
>>> {'tp':
|
| 124 |
```
|
| 125 |
|
| 126 |
-
Example3 : two or more prediction and reference, mode=boundaries,
|
| 127 |
```python
|
| 128 |
metric_path = "Ikala-allen/relation_extraction"
|
| 129 |
module = evaluate.load(metric_path)
|
|
@@ -147,12 +147,12 @@ predictions = [
|
|
| 147 |
{'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}
|
| 148 |
]
|
| 149 |
]
|
| 150 |
-
evaluation_scores = module.compute(predictions=predictions, references=references, mode="boundaries",
|
| 151 |
print(evaluation_scores)
|
| 152 |
>>> {'sell': {'tp': 3, 'fp': 1, 'fn': 0, 'p': 75.0, 'r': 100.0, 'f1': 85.71428571428571}, 'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 3, 'fp': 1, 'fn': 1, 'p': 75.0, 'r': 75.0, 'f1': 75.0, 'Macro_f1': 42.857142857142854, 'Macro_p': 37.5, 'Macro_r': 50.0}}
|
| 153 |
```
|
| 154 |
|
| 155 |
-
Example 4 : two or more prediction and reference, mode=boundaries,
|
| 156 |
```python
|
| 157 |
metric_path = "Ikala-allen/relation_extraction"
|
| 158 |
module = evaluate.load(metric_path)
|
|
@@ -176,7 +176,7 @@ predictions = [
|
|
| 176 |
{'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}
|
| 177 |
]
|
| 178 |
]
|
| 179 |
-
evaluation_scores = module.compute(predictions=predictions, references=references, mode="boundaries",
|
| 180 |
print(evaluation_scores)
|
| 181 |
>>> {'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0, 'Macro_f1': 0.0, 'Macro_p': 0.0, 'Macro_r': 0.0}}
|
| 182 |
```
|
|
|
|
| 47 |
- **predictions** (`list` of `list` of `dictionary`): a list of list of dictionary with every element's relation_type and their entity name
|
| 48 |
- **references** (`list` of `list` of `dictionary`): a list of list of dictionary with every element's relation_type and their entity name
|
| 49 |
- **mode** (`str`): define strict or boundaries mode for evaluation, strict mode consider "head_type" and "tail_type", boundaries mode doesn't consider "head_type" and "tail_type"
|
| 50 |
+
- **show** (`bool`): False for only output all relation_type score. True for output every relation_type score, default False
|
| 51 |
- **relation_types** (`list`): define which relation type that need to be evaluate and show, if not given, it will construct relation_types from ground truth, default []
|
| 52 |
|
| 53 |
### Output Values
|
|
|
|
| 79 |
Remind : Macro_f1、Macro_p、Macro_r、p、r、f1 are always a number between 0 and 1. And tp、fp、fn depend on how many data inputs.
|
| 80 |
|
| 81 |
### Examples
|
| 82 |
+
Example1 : only one prediction and reference, mode=strict, show=False, only output total relation score
|
| 83 |
```python
|
| 84 |
metric_path = "Ikala-allen/relation_extraction"
|
| 85 |
module = evaluate.load(metric_path)
|
|
|
|
| 96 |
{"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
|
| 97 |
]
|
| 98 |
]
|
| 99 |
+
evaluation_scores = module.compute(predictions=predictions, references=references, mode="strict", show=False, relation_types=[])
|
| 100 |
print(evaluation_scores)
|
| 101 |
>>> {'tp': 1, 'fp': 1, 'fn': 2, 'p': 50.0, 'r': 33.333333333333336, 'f1': 40.0, 'Macro_f1': 25.0, 'Macro_p': 25.0, 'Macro_r': 25.0}
|
| 102 |
```
|
| 103 |
|
| 104 |
+
Example2 : only one prediction and reference, mode=boundaries, show=False, only output total relation score
|
| 105 |
```python
|
| 106 |
metric_path = "Ikala-allen/relation_extraction"
|
| 107 |
module = evaluate.load(metric_path)
|
|
|
|
| 118 |
{"head": "tinadaviespigments", "head_type": "brand", "type": "sell", "tail": "國際認證之色乳", "tail_type": "product"},
|
| 119 |
]
|
| 120 |
]
|
| 121 |
+
evaluation_scores = module.compute(predictions=predictions, references=references, mode="strict", show=False, relation_types=[])
|
| 122 |
print(evaluation_scores)
|
| 123 |
+
>>> {'tp': 1, 'fp': 1, 'fn': 2, 'p': 50.0, 'r': 33.333333333333336, 'f1': 40.0, 'Macro_f1': 25.0, 'Macro_p': 25.0, 'Macro_r': 25.0}
|
| 124 |
```
|
| 125 |
|
| 126 |
+
Example3 : two or more prediction and reference, mode=boundaries, show=True, output all relation type score
|
| 127 |
```python
|
| 128 |
metric_path = "Ikala-allen/relation_extraction"
|
| 129 |
module = evaluate.load(metric_path)
|
|
|
|
| 147 |
{'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}
|
| 148 |
]
|
| 149 |
]
|
| 150 |
+
evaluation_scores = module.compute(predictions=predictions, references=references, mode="boundaries", show=True, relation_types=[])
|
| 151 |
print(evaluation_scores)
|
| 152 |
>>> {'sell': {'tp': 3, 'fp': 1, 'fn': 0, 'p': 75.0, 'r': 100.0, 'f1': 85.71428571428571}, 'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 3, 'fp': 1, 'fn': 1, 'p': 75.0, 'r': 75.0, 'f1': 75.0, 'Macro_f1': 42.857142857142854, 'Macro_p': 37.5, 'Macro_r': 50.0}}
|
| 153 |
```
|
| 154 |
|
| 155 |
+
Example 4 : two or more prediction and reference, mode=boundaries, show=True, output all relation type score, relation_types = ["belongs_to"], only consider belongs_to type score
|
| 156 |
```python
|
| 157 |
metric_path = "Ikala-allen/relation_extraction"
|
| 158 |
module = evaluate.load(metric_path)
|
|
|
|
| 176 |
{'head': 'SNTAIWAN', 'tail': '大馬士革玫瑰有機光燦系列', 'head_type': 'brand', 'tail_type': 'product', 'type': 'sell'}
|
| 177 |
]
|
| 178 |
]
|
| 179 |
+
evaluation_scores = module.compute(predictions=predictions, references=references, mode="boundaries", show=True, relation_types=["belongs_to"])
|
| 180 |
print(evaluation_scores)
|
| 181 |
>>> {'belongs_to': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0}, 'ALL': {'tp': 0, 'fp': 0, 'fn': 1, 'p': 0, 'r': 0, 'f1': 0, 'Macro_f1': 0.0, 'Macro_p': 0.0, 'Macro_r': 0.0}}
|
| 182 |
```
|