Upload standard.py with huggingface_hub
Browse files- standard.py +12 -6
standard.py
CHANGED
|
@@ -5,11 +5,7 @@ from .dataclass import Field, InternalField, OptionalField
|
|
| 5 |
from .formats import Format, SystemFormat
|
| 6 |
from .logging_utils import get_logger
|
| 7 |
from .operator import SourceSequentialOperator, StreamingOperator
|
| 8 |
-
from .operators import
|
| 9 |
-
Augmentor,
|
| 10 |
-
NullAugmentor,
|
| 11 |
-
StreamRefiner,
|
| 12 |
-
)
|
| 13 |
from .recipe import Recipe
|
| 14 |
from .schema import ToUnitxtGroup
|
| 15 |
from .splitters import Sampler, SeparateSplit, SpreadSplit
|
|
@@ -67,7 +63,7 @@ class BaseRecipe(Recipe, SourceSequentialOperator):
|
|
| 67 |
if self.num_demos > 0:
|
| 68 |
if self.demos_pool_size is None or self.demos_pool_size < 1:
|
| 69 |
raise ValueError(
|
| 70 |
-
"When using demonstrations both num_demos and demos_pool_size should be assigned with
|
| 71 |
)
|
| 72 |
if self.demos_pool_size < self.num_demos:
|
| 73 |
raise ValueError(
|
|
@@ -114,6 +110,16 @@ class BaseRecipe(Recipe, SourceSequentialOperator):
|
|
| 114 |
def prepare(self):
|
| 115 |
self.steps = [
|
| 116 |
self.card.loader,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
]
|
| 118 |
|
| 119 |
if self.loader_limit:
|
|
|
|
| 5 |
from .formats import Format, SystemFormat
|
| 6 |
from .logging_utils import get_logger
|
| 7 |
from .operator import SourceSequentialOperator, StreamingOperator
|
| 8 |
+
from .operators import AddFields, Augmentor, NullAugmentor, StreamRefiner
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
from .recipe import Recipe
|
| 10 |
from .schema import ToUnitxtGroup
|
| 11 |
from .splitters import Sampler, SeparateSplit, SpreadSplit
|
|
|
|
| 63 |
if self.num_demos > 0:
|
| 64 |
if self.demos_pool_size is None or self.demos_pool_size < 1:
|
| 65 |
raise ValueError(
|
| 66 |
+
"When using demonstrations both num_demos and demos_pool_size should be assigned with positive integers."
|
| 67 |
)
|
| 68 |
if self.demos_pool_size < self.num_demos:
|
| 69 |
raise ValueError(
|
|
|
|
| 110 |
def prepare(self):
|
| 111 |
self.steps = [
|
| 112 |
self.card.loader,
|
| 113 |
+
AddFields(
|
| 114 |
+
fields={
|
| 115 |
+
"recipe_metadata": {
|
| 116 |
+
"card": self.card,
|
| 117 |
+
"template": self.template,
|
| 118 |
+
"system_prompt": self.system_prompt,
|
| 119 |
+
"format": self.format,
|
| 120 |
+
}
|
| 121 |
+
}
|
| 122 |
+
),
|
| 123 |
]
|
| 124 |
|
| 125 |
if self.loader_limit:
|