Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -8,7 +8,7 @@ import gradio as gr
|
|
| 8 |
import torch
|
| 9 |
from utils import *
|
| 10 |
from presets import *
|
| 11 |
-
from transformers import Trainer, TrainingArguments
|
| 12 |
import numpy as np
|
| 13 |
import evaluate
|
| 14 |
import pandas as pd
|
|
@@ -188,6 +188,7 @@ lm_datasets = tokenized_datasets.map(
|
|
| 188 |
num_proc=4,
|
| 189 |
)
|
| 190 |
|
|
|
|
| 191 |
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|
| 192 |
|
| 193 |
|
|
|
|
| 8 |
import torch
|
| 9 |
from utils import *
|
| 10 |
from presets import *
|
| 11 |
+
from transformers import Trainer, TrainingArguments, DataCollatorWithPadding
|
| 12 |
import numpy as np
|
| 13 |
import evaluate
|
| 14 |
import pandas as pd
|
|
|
|
| 188 |
num_proc=4,
|
| 189 |
)
|
| 190 |
|
| 191 |
+
# Batches von Daten zusammenfassen
|
| 192 |
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|
| 193 |
|
| 194 |
|