Hugging Face's logo

llm-jp
/
llm-jp-roberta-base

Fill-Mask
Transformers
Safetensors
roberta
Model card Files
xet
Community
speed commited on
Commit
203e0f0
·
verified ·
1 Parent(s): bb8327c

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +14 -14
tokenizer.json CHANGED
@@ -101,7 +101,7 @@
101
  "single": [
102
  {
103
  "SpecialToken": {
104
- "id": "<s>",
105
  "type_id": 0
106
  }
107
  },
@@ -113,7 +113,7 @@
113
  },
114
  {
115
  "SpecialToken": {
116
- "id": "</s>",
117
  "type_id": 0
118
  }
119
  }
@@ -121,7 +121,7 @@
121
  "pair": [
122
  {
123
  "SpecialToken": {
124
- "id": "<s>",
125
  "type_id": 0
126
  }
127
  },
@@ -133,7 +133,7 @@
133
  },
134
  {
135
  "SpecialToken": {
136
- "id": "</s>",
137
  "type_id": 0
138
  }
139
  },
@@ -145,28 +145,28 @@
145
  },
146
  {
147
  "SpecialToken": {
148
- "id": "</s>",
149
- "type_id": 0
150
  }
151
  }
152
  ],
153
  "special_tokens": {
154
- "</s>": {
155
- "id": "</s>",
156
  "ids": [
157
- 2
158
  ],
159
  "tokens": [
160
- "</s>"
161
  ]
162
  },
163
- "<s>": {
164
- "id": "<s>",
165
  "ids": [
166
- 1
167
  ],
168
  "tokens": [
169
- "<s>"
170
  ]
171
  }
172
  }
 
101
  "single": [
102
  {
103
  "SpecialToken": {
104
+ "id": "<CLS|LLM-jp>",
105
  "type_id": 0
106
  }
107
  },
 
113
  },
114
  {
115
  "SpecialToken": {
116
+ "id": "<SEP|LLM-jp>",
117
  "type_id": 0
118
  }
119
  }
 
121
  "pair": [
122
  {
123
  "SpecialToken": {
124
+ "id": "<CLS|LLM-jp>",
125
  "type_id": 0
126
  }
127
  },
 
133
  },
134
  {
135
  "SpecialToken": {
136
+ "id": "<SEP|LLM-jp>",
137
  "type_id": 0
138
  }
139
  },
 
145
  },
146
  {
147
  "SpecialToken": {
148
+ "id": "<SEP|LLM-jp>",
149
+ "type_id": 1
150
  }
151
  }
152
  ],
153
  "special_tokens": {
154
+ "<CLS|LLM-jp>": {
155
+ "id": "<CLS|LLM-jp>",
156
  "ids": [
157
+ 5
158
  ],
159
  "tokens": [
160
+ "<CLS|LLM-jp>"
161
  ]
162
  },
163
+ "<SEP|LLM-jp>": {
164
+ "id": "<SEP|LLM-jp>",
165
  "ids": [
166
+ 6
167
  ],
168
  "tokens": [
169
+ "<SEP|LLM-jp>"
170
  ]
171
  }
172
  }