Update model_hf.py
Browse files- model_hf.py +2 -2
model_hf.py
CHANGED
@@ -1564,8 +1564,8 @@ def prepare_datasets(tokenizer, token: str, sanity_check: bool = False, dataset_
|
|
1564 |
|
1565 |
dataset = dataset.filter(filter_func)
|
1566 |
prepare_fn = partial(extract_features, tokenizer=tokenizer, **dataset_config)
|
1567 |
-
train_dataset = dataset["train"].take(
|
1568 |
-
test_dataset = dataset["test"].take(
|
1569 |
|
1570 |
train_dataset = train_dataset.map(
|
1571 |
function=prepare_fn,
|
|
|
1564 |
|
1565 |
dataset = dataset.filter(filter_func)
|
1566 |
prepare_fn = partial(extract_features, tokenizer=tokenizer, **dataset_config)
|
1567 |
+
train_dataset = dataset["train"].take(1000)
|
1568 |
+
test_dataset = dataset["test"].take(100)
|
1569 |
|
1570 |
train_dataset = train_dataset.map(
|
1571 |
function=prepare_fn,
|