Spaces:
Runtime error
Runtime error
File size: 1,799 Bytes
cbdb918 ce79099 f5a6032 ce79099 f5a6032 cbdb918 cc5362f f5a6032 cc5362f f5a6032 cc5362f cbdb918 283d1fe cbdb918 283d1fe cbdb918 f5a6032 887ad63 cbdb918 f5a6032 cbdb918 f5a6032 cbdb918 f5a6032 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
from fastapi import FastAPI
from transformers import PreTrainedTokenizerFast
from tokenizers import ByteLevelBPETokenizer
from datasets import load_dataset
from contextlib import asynccontextmanager
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI):
logger.info("Application starting...")
await train_tokenizer()
yield
logger.info("Application shutting down...")
app = FastAPI(lifespan=lifespan)
async def train_tokenizer():
vocab_size = 50000
min_frequency = 2
#dataset_greek = load_dataset("oscar", "unshuffled_deduplicated_el", split="train", streaming=True)
dataset_greek = load_dataset("wikipedia", "20231101.el", split="train", streaming=True)
dataset_english = load_dataset("wikipedia", "20231101.en", split="train", streaming=True)
try:
dataset_code = load_dataset("bigcode/the-stack", split="train", streaming=True)
datasets_list = [dataset_greek, dataset_english]
except:
datasets_list = [dataset_greek, dataset_english]
def preprocess_data(dataset):
for item in dataset:
text = item["text"]
text = text.strip().lower()
if text:
yield text
combined_data = (preprocess_data(dataset.take(1000)) for dataset in datasets_list)
tokenizer = ByteLevelBPETokenizer()
tokenizer.train_from_iterator(
combined_data,
vocab_size=vocab_size,
min_frequency=min_frequency,
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"]
)
tokenizer.save_model(".")
logger.info("Tokenizer training completed!")
@app.get("/")
async def root():
return {"message": "Custom Tokenizer Training Completed and Saved"} |