AkashDataScience commited on
Commit
22095bf
·
1 Parent(s): e525413

First commit

Browse files
Files changed (3) hide show
  1. app.py +83 -0
  2. gpt2.pt +3 -0
  3. model.py +215 -0
app.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import tiktoken
3
+ import gradio as gr
4
+ import torch.nn.functional as F
5
+ from model import GPT, GPTConfig
6
+
7
+ device = 'cpu'
8
+ if torch.cuda.is_available():
9
+ device = 'cuda'
10
+ elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
11
+ device = "mps"
12
+
13
+ ckpt = torch.load("gpt2.pt", map_location=torch.device(device))
14
+ config = GPTConfig(**ckpt['model_args'])
15
+ model = GPT(config)
16
+ state_dict = ckpt['model']
17
+ model.load_state_dict(state_dict)
18
+
19
+ model.to(device)
20
+
21
+ enc = tiktoken.get_encoding('gpt2')
22
+
23
+ def inference(input_text, num_return_sequences, max_length):
24
+ input_tokens = torch.tensor(enc.encode(input_text), dtype=torch.long)
25
+ input_tokens = input_tokens.unsqueeze(0).repeat(num_return_sequences, 1)
26
+ x = input_tokens.to('cuda')
27
+
28
+ while x.size(1) < max_length:
29
+ # forward the model to get the logits
30
+ with torch.no_grad():
31
+ logits = model(x)[0] # (B, T, vocab_size)
32
+ # take the logits at the last position
33
+ logits = logits[:, -1, :] # (B, vocab_size)
34
+ # get the probabilities
35
+ probs = F.softmax(logits, dim=-1)
36
+ # do top-k sampling of 50 (huggingface pipeline default)
37
+ # topk_probs here becomes (5, 50), topk_indices is (5, 50)
38
+ topk_probs, topk_indices = torch.topk(probs, 50, dim=-1)
39
+ # select a token from the top-k probabilities
40
+ # note: multinomial does not demand the input to sum to 1
41
+ ix = torch.multinomial(topk_probs, 1) # (B, 1)
42
+ # gather the corresponding indices
43
+ xcol = torch.gather(topk_indices, -1, ix) # (B, 1)
44
+ # append to the sequence
45
+ x = torch.cat((x, xcol), dim=1)
46
+
47
+ decode_list = []
48
+ # print the generated text
49
+ for i in range(num_return_sequences):
50
+ tokens = x[i, :max_length].tolist()
51
+ decoded = enc.decode(tokens)
52
+ decode_list.append(decoded)
53
+
54
+ output = "\n======\n".join(decode_list)
55
+ return output
56
+
57
+ title = "GPT-2 trained on Shakespeare Plays dataset"
58
+ description = "A simple Gradio interface to generate text from GPT-2 model trained on Shakespeare Plays"
59
+ examples = [["Please put on these earmuffs because I can't you hear.", 2, 20],
60
+ ["Twin 4-month-olds slept in the shade of the palm tree while the mother tanned in the sun.", 2, 20],
61
+ ["Happiness can be found in the depths of chocolate pudding.", 2, 20],
62
+ ["Seek success, but always be prepared for random cats.", 2, 20],
63
+ ["This made him feel like an old-style rootbeer float smells.", 2, 20],
64
+ ["The view from the lighthouse excited even the most seasoned traveler.", 2, 20],
65
+ ["I've always wanted to go to Tajikistan, but my cat would miss me.", 2, 20],
66
+ ["He found rain fascinating yet unpleasant.", 2, 20],
67
+ ["Plans for this weekend include turning wine into water.", 2, 20],
68
+ ["Iron pyrite is the most foolish of all minerals.", 2, 20],
69
+ ]
70
+ demo = gr.Interface(
71
+ inference,
72
+ inputs = [
73
+ gr.Textbox(label="Enter some text", type="text"),
74
+ gr.Slider(minimum=1, maximum=5, step=1, value=5, label="Number of outputs"),
75
+ gr.Slider(minimum=10, maximum=30, step=1, value=20, label="Maximum lenght of a sequence")
76
+ ],
77
+ outputs = [
78
+ gr.Textbox(label="Output", type="text")
79
+ ],
80
+ title = title,
81
+ description = description,
82
+ examples = examples,
83
+ )
gpt2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e335dfabae121d1024b71b09cb77ae0ebefe64a40ed06feba7f099115b346a7c
3
+ size 548285034
model.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # torch.compile
2
+ import os
3
+ import math
4
+ import time
5
+ import inspect
6
+ from dataclasses import dataclass
7
+ import torch
8
+ import torch.nn as nn
9
+ from torch.nn import functional as F
10
+ torch._dynamo.reset()
11
+
12
+ class CausalSelfAttention(nn.Module):
13
+
14
+ def __init__(self, config):
15
+ super().__init__()
16
+ assert config.n_embd % config.n_head == 0
17
+ # key, query, value projections for all heads, but in a batch
18
+ self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
19
+ # output projection
20
+ self.c_proj = nn.Linear(config.n_embd, config.n_embd)
21
+ self.c_proj.NANGPT_SCALE_INIT = 1
22
+ # regularization
23
+ self.n_head = config.n_head
24
+ self.n_embd = config.n_embd
25
+ self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size)).view(1, 1, config.block_size, config.block_size))
26
+
27
+ def forward(self, x):
28
+ B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
29
+ # calculate query, key, values for all heads in batch and move head forward to be the batch dim
30
+ # nh is "number of heads", hs is "head size", and C (number of channels) = nh * hs
31
+ # e.g. in GPT-2 (124M), n_head=12, hs=64, so nh*hs=C=768 channels in the Transformer
32
+ qkv = self.c_attn(x)
33
+ q, k, v = qkv.split(self.n_embd, dim=2)
34
+ k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
35
+ q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
36
+ v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
37
+
38
+ # att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))
39
+ # att = att.masked_fill(self.bias[:, :, :T, :T] == 0, float('-inf'))
40
+ # att = F.softmax(att, dim=-1)
41
+ # y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)
42
+
43
+ y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
44
+
45
+ y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side
46
+ # output projection
47
+ y = self.c_proj(y)
48
+ return y
49
+
50
+
51
+ class MLP(nn.Module):
52
+
53
+ def __init__(self, config):
54
+ super().__init__()
55
+ self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd)
56
+ self.gelu = nn.GELU(approximate='tanh')
57
+ self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd)
58
+ self.c_proj.NANOGPT_SCALE_INIT = 1
59
+
60
+ def forward(self, x):
61
+ x = self.c_fc(x)
62
+ x = self.gelu(x)
63
+ x = self.c_proj(x)
64
+ return x
65
+
66
+ class Block(nn.Module):
67
+
68
+ def __init__(self, config):
69
+ super().__init__()
70
+ self.ln_1 = nn.LayerNorm(config.n_embd)
71
+ self.attn = CausalSelfAttention(config)
72
+ self.ln_2 = nn.LayerNorm(config.n_embd)
73
+ self.mlp = MLP(config)
74
+
75
+ def forward(self, x):
76
+ x = x + self.attn(self.ln_1(x))
77
+ x = x + self.mlp(self.ln_2(x))
78
+ return x
79
+
80
+
81
+ @dataclass
82
+ class GPTConfig:
83
+ block_size: int = 1024 # max sequence length
84
+ vocab_size: int = 50304 # number of tokens: 50,000 BPE merges + 256 bytes tokens + 1 <|endoftext|> token
85
+ n_layer: int = 12 # number of layers
86
+ n_head: int = 12 # number of heads
87
+ n_embd: int = 768 # embedding dimension
88
+
89
+
90
+ class GPT(nn.Module):
91
+
92
+ def __init__(self, config):
93
+ super().__init__()
94
+ self.config = config
95
+
96
+ self.transformer = nn.ModuleDict(dict(
97
+ wte = nn.Embedding(config.vocab_size, config.n_embd),
98
+ wpe = nn.Embedding(config.block_size, config.n_embd),
99
+ h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
100
+ ln_f = nn.LayerNorm(config.n_embd),
101
+ ))
102
+ self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
103
+
104
+ # weight sharing
105
+ self.transformer.wte.weight = self.lm_head.weight
106
+
107
+ # weight initialization
108
+ self.apply(self._init_weights)
109
+
110
+ def _init_weights(self, module):
111
+ if isinstance(module, nn.Linear):
112
+ std = 0.02
113
+ if hasattr(module, 'NANGPT_SCALE_INIT'):
114
+ std *= (2 * self.config.n_layer) ** -0.5
115
+ torch.nn.init.normal_(module.weight, mean = 0.0, std = std)
116
+ if module.bias is not None:
117
+ torch.nn.init.zeros_(module.bias)
118
+ elif isinstance(module, nn.Embedding):
119
+ torch.nn.init.normal_(module.weight, mean=0.0, std = 0.02)
120
+
121
+
122
+
123
+ def forward(self, idx, targets=None):
124
+ # idx is of shape (B, T)
125
+ B, T = idx.size()
126
+ assert T <= self.config.block_size, f"Cannot forward sequence of length {T}, block size is only {self.config.block_size}"
127
+ # forward the token and posisition embeddings
128
+ pos = torch.arange(0, T, dtype=torch.long, device=idx.device) # shape (T)
129
+ pos_emb = self.transformer.wpe(pos) # position embeddings of shape (T, n_embd)
130
+ tok_emb = self.transformer.wte(idx) # token embeddings of shape (B, T, n_embd)
131
+ x = tok_emb + pos_emb
132
+ # forward the blocks of the transformer
133
+ for block in self.transformer.h:
134
+ x = block(x)
135
+ # forward the final layernorm and the classifier
136
+ x = self.transformer.ln_f(x)
137
+ logits = self.lm_head(x) # (B, T, vocab_size)
138
+ loss = None
139
+ if targets is not None:
140
+ loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
141
+ return logits, loss
142
+
143
+ @classmethod
144
+ def from_pretrained(cls, model_type):
145
+ """Loads pretrained GPT-2 model weights from huggingface"""
146
+ assert model_type in {'gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl'}
147
+ from transformers import GPT2LMHeadModel
148
+ print("loading weights from pretrained gpt: %s" % model_type)
149
+
150
+ # n_layer, n_head and n_embd are determined from model_type
151
+ config_args = {
152
+ 'gpt2': dict(n_layer=12, n_head=12, n_embd=768), # 124M params
153
+ 'gpt2-medium': dict(n_layer=24, n_head=16, n_embd=1024), # 350M params
154
+ 'gpt2-large': dict(n_layer=36, n_head=20, n_embd=1280), # 774M params
155
+ 'gpt2-xl': dict(n_layer=48, n_head=25, n_embd=1600), # 1558M params
156
+ }[model_type]
157
+ config_args['vocab_size'] = 50257 # always 50257 for GPT model checkpoints
158
+ config_args['block_size'] = 1024 # always 1024 for GPT model checkpoints
159
+ # create a from-scratch initialized minGPT model
160
+ config = GPTConfig(**config_args)
161
+ model = GPT(config)
162
+ sd = model.state_dict()
163
+ sd_keys = sd.keys()
164
+ sd_keys = [k for k in sd_keys if not k.endswith('.attn.bias')] # discard this mask / buffer, not a param
165
+
166
+ # init a huggingface/transformers model
167
+ model_hf = GPT2LMHeadModel.from_pretrained(model_type)
168
+ sd_hf = model_hf.state_dict()
169
+
170
+ # copy while ensuring all of the parameters are aligned and match in names and shapes
171
+ sd_keys_hf = sd_hf.keys()
172
+ sd_keys_hf = [k for k in sd_keys_hf if not k.endswith('.attn.masked_bias')] # ignore these, just a buffer
173
+ sd_keys_hf = [k for k in sd_keys_hf if not k.endswith('.attn.bias')] # same, just the mask (buffer)
174
+ transposed = ['attn.c_attn.weight', 'attn.c_proj.weight', 'mlp.c_fc.weight', 'mlp.c_proj.weight']
175
+ # basically the openai checkpoints use a "Conv1D" module, but we only want to use a vanilla Linear
176
+ # this means that we have to transpose these weights when we import them
177
+ assert len(sd_keys_hf) == len(sd_keys), f"mismatched keys: {len(sd_keys_hf)} != {len(sd_keys)}"
178
+ for k in sd_keys_hf:
179
+ if any(k.endswith(w) for w in transposed):
180
+ # special treatment for the Conv1D weights we need to transpose
181
+ assert sd_hf[k].shape[::-1] == sd[k].shape
182
+ with torch.no_grad():
183
+ sd[k].copy_(sd_hf[k].t())
184
+ else:
185
+ # vanilla copy over the other parameters
186
+ assert sd_hf[k].shape == sd[k].shape
187
+ with torch.no_grad():
188
+ sd[k].copy_(sd_hf[k])
189
+
190
+ return model
191
+
192
+ def configure_optimizers(self, weight_decay, learning_rate, device_type):
193
+ # start with all of the candidate parameters (that require grad)
194
+ param_dict = {pn: p for pn, p in self.named_parameters()}
195
+ param_dict = {pn: p for pn, p in param_dict.items() if p.requires_grad}
196
+ # create optim groups. Any parameters that is 2D will be weight decayed, otherwise no.
197
+ # i.e. all weight tensors in matmuls + embeddings decay, all biases and layernorms don't.
198
+ decay_params = [p for n, p in param_dict.items() if p.dim() >= 2]
199
+ nodecay_params = [p for n, p in param_dict.items() if p.dim() < 2]
200
+ optim_groups = [
201
+ {'params': decay_params, 'weight_decay': weight_decay},
202
+ {'params': nodecay_params, 'weight_decay': 0.0}
203
+ ]
204
+ num_decay_params = sum(p.numel() for p in decay_params)
205
+ num_nodecay_params = sum(p.numel() for p in nodecay_params)
206
+
207
+ print(f"num decayed parameter tensors: {len(decay_params)}, with {num_decay_params:,} parameters")
208
+ print(f"num non-decayed parameter tensors: {len(nodecay_params)}, with {num_nodecay_params:,} parameters")
209
+ # Create AdamW optimizer and use the fused version if it is available
210
+ fused_available = 'fused' in inspect.signature(torch.optim.AdamW).parameters
211
+ use_fused = fused_available and device_type == "cuda"
212
+
213
+ print(f"using fused AdamW: {use_fused}")
214
+ optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=(0.9, 0.95), eps=1e-8, fused=use_fused)
215
+ return optimizer