Spaces:
Build error
Build error
File size: 1,139 Bytes
64772a4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
from dataclasses import asdict, dataclass
from typing import Any, Dict, Mapping, Optional, Sequence
@dataclass(frozen=True)
class TokenizationRequest:
"""Describes a tokenization request.
Parameters
prompt (str)
The text prompt which should be converted into tokens
tokens (bool)
True to extract text-tokens
token_ids (bool)
True to extract token-ids
Returns
TokenizationResponse
Examples:
>>> request = TokenizationRequest(prompt="This is an example.", tokens=True, token_ids=True)
"""
prompt: str
tokens: bool
token_ids: bool
def to_json(self) -> Mapping[str, Any]:
return self._asdict()
def _asdict(self) -> Mapping[str, Any]:
return asdict(self)
@dataclass(frozen=True)
class TokenizationResponse:
tokens: Optional[Sequence[str]] = None
token_ids: Optional[Sequence[int]] = None
@staticmethod
def from_json(json: Dict[str, Any]) -> "TokenizationResponse":
return TokenizationResponse(
tokens=json.get("tokens"), token_ids=json.get("token_ids")
)
|