Upload schema.py with huggingface_hub
Browse files
schema.py
CHANGED
|
@@ -34,9 +34,14 @@ class ToUnitxtGroup(StreamInstanceOperatorValidator):
|
|
| 34 |
|
| 35 |
def process(self, instance: Dict[str, Any], stream_name: str = None) -> Dict[str, Any]:
|
| 36 |
if self.remove_unnecessary_fields:
|
|
|
|
|
|
|
| 37 |
for key in instance.keys():
|
| 38 |
if key not in UNITXT_DATASET_SCHEMA:
|
| 39 |
-
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
instance["group"] = self.group
|
| 42 |
if self.metrics is not None:
|
|
|
|
| 34 |
|
| 35 |
def process(self, instance: Dict[str, Any], stream_name: str = None) -> Dict[str, Any]:
|
| 36 |
if self.remove_unnecessary_fields:
|
| 37 |
+
keys_to_delete = []
|
| 38 |
+
|
| 39 |
for key in instance.keys():
|
| 40 |
if key not in UNITXT_DATASET_SCHEMA:
|
| 41 |
+
keys_to_delete.append(key)
|
| 42 |
+
|
| 43 |
+
for key in keys_to_delete:
|
| 44 |
+
del instance[key]
|
| 45 |
|
| 46 |
instance["group"] = self.group
|
| 47 |
if self.metrics is not None:
|