Upload splitters.py with huggingface_hub

splitters.py

@@ -70,6 +70,7 @@ class SeparateSplit(Splitter):
     from_split: str
     to_split_names: List[str]
     to_split_sizes: List[int]
+    remove_targets_from_source_split: bool = True
 
     def verify(self):
         assert (
@@ -82,13 +83,14 @@ class SeparateSplit(Splitter):
         mapping = {
             key: {key: [(None, None)]}
             for key in multi_stream.keys()
-            if key != self.from_split
+            if not self.remove_targets_from_source_split or key != self.from_split
         }
         so_far = 0
         for name, size in itertools.zip_longest(
             self.to_split_names, self.to_split_sizes
         ):
-            mapping[name] = {self.from_split: [(so_far, size)]}
+            if self.remove_targets_from_source_split or name != self.from_split:
+                mapping[name] = {self.from_split: [(so_far, size)]}
             if size:
                 so_far += size
         generators = slice_streams(multi_stream, mapping)
@@ -131,6 +133,14 @@ class Sampler(Artifact):
     ) -> List[Dict[str, object]]:
         pass
 
+    def filter_source_by_instance(
+        self, instances_pool: List[Dict[str, object]], instance: Dict[str, object]
+    ) -> List[Dict[str, object]]:
+        if "inputs" not in instance:
+            raise ValueError(f"'inputs' field is missing from '{instance}'.")
+
+        return list(filter(lambda x: x["inputs"] != instance["inputs"], instances_pool))
+
 
 class RandomSampler(Sampler):
     def sample(
@@ -172,6 +182,7 @@ class DiverseLabelsSampler(Sampler):
 
     choices: str = "choices"
     labels: str = "labels"
+    include_empty_label: bool = True
 
     def prepare(self):
         super().prepare()
@@ -207,6 +218,8 @@ class DiverseLabelsSampler(Sampler):
         labels = {}
         for examplar in examplars_pool:
             label_repr = self.examplar_repr(examplar)
+            if label_repr == "[]" and not self.include_empty_label:
+                continue
             if label_repr not in labels:
                 labels[label_repr] = []
             labels[label_repr].append(examplar)
@@ -269,7 +282,9 @@ class SpreadSplit(InstanceOperatorWithMultiStreamAccess):
                 self.local_cache = list(multi_stream[self.source_stream])
 
             source_stream = self.local_cache
-
+            source_stream = self.sampler.filter_source_by_instance(
+                source_stream, instance
+            )
             sampled_instances = self.sampler.sample(source_stream)
             instance[self.target_field] = sampled_instances
             return instance