Transformers
undfined commited on
Commit
b303991
·
verified ·
1 Parent(s): f4dd444

Upload special_tokens_map.json

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +33 -30
special_tokens_map.json CHANGED
@@ -1,54 +1,57 @@
1
  {
2
- "additional_special_tokens": [
3
- "<|extra_id_0|>",
4
- "<|endoftext|>",
5
- "<|fim_prefix|>",
6
- "<|fim_middle|>",
7
- "<|fim_suffix|>",
8
- "|||PHONE_NUMBER|||",
9
- "|||EMAIL_ADDRESS|||",
10
- "|||IP_ADDRESS|||",
11
- "<|im_start|>",
12
- "<|im_end|>",
13
- "<|extra_id_1|>",
14
- "<|extra_id_2|>",
15
- "<|extra_id_3|>",
16
- "<|extra_id_4|>",
17
- "<|extra_id_5|>",
18
- "<|extra_id_6|>",
19
- "<|extra_id_7|>",
20
- "<|extra_id_8|>",
21
- "<|extra_id_9|>",
22
- "<|extra_id_10|>",
23
- "<|endofprompt|>",
24
- "<|pad|>"
25
- ],
26
  "bos_token": {
27
  "content": "<|endoftext|>",
28
  "lstrip": false,
29
  "normalized": false,
30
- "rstrip": false,
31
- "single_word": false
32
  },
33
  "eos_token": {
34
  "content": "<|endoftext|>",
35
  "lstrip": false,
36
  "normalized": false,
37
  "rstrip": false,
38
- "single_word": false
 
39
  },
40
  "pad_token": {
41
  "content": "<|pad|>",
42
  "lstrip": false,
43
  "normalized": false,
44
  "rstrip": false,
45
- "single_word": false
 
46
  },
47
  "unk_token": {
48
  "content": "<|endoftext|>",
49
  "lstrip": false,
50
  "normalized": false,
51
  "rstrip": false,
52
- "single_word": false
53
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
  }
 
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "bos_token": {
3
  "content": "<|endoftext|>",
4
  "lstrip": false,
5
  "normalized": false,
6
+ "single_word": false,
7
+ "special": true
8
  },
9
  "eos_token": {
10
  "content": "<|endoftext|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
14
+ "single_word": false,
15
+ "special": true
16
  },
17
  "pad_token": {
18
  "content": "<|pad|>",
19
  "lstrip": false,
20
  "normalized": false,
21
  "rstrip": false,
22
+ "single_word": false,
23
+ "special": true
24
  },
25
  "unk_token": {
26
  "content": "<|endoftext|>",
27
  "lstrip": false,
28
  "normalized": false,
29
  "rstrip": false,
30
+ "single_word": false,
31
+ "special": true
32
+ },
33
+ "additional_special_tokens": [
34
+ "<|extra_id_0|>",
35
+ "<|endoftext|>",
36
+ "<|fim_prefix|>",
37
+ "<|fim_middle|>",
38
+ "<|fim_suffix|>",
39
+ "|||PHONE_NUMBER|||",
40
+ "|||EMAIL_ADDRESS|||",
41
+ "|||IP_ADDRESS|||",
42
+ "<|im_start|>",
43
+ "<|im_end|>",
44
+ "<|extra_id_1|>",
45
+ "<|extra_id_2|>",
46
+ "<|extra_id_3|>",
47
+ "<|extra_id_4|>",
48
+ "<|extra_id_5|>",
49
+ "<|extra_id_6|>",
50
+ "<|extra_id_7|>",
51
+ "<|extra_id_8|>",
52
+ "<|extra_id_9|>",
53
+ "<|extra_id_10|>",
54
+ "<|endofprompt|>",
55
+ "<|pad|>"
56
+ ]
57
  }