don't resize embeddings if it's already large enough (#577)
Browse files* don't resize embeddings if it's already large enough
* make sure to tie weights, even if we aren't resizing
src/axolotl/utils/models.py
CHANGED
|
@@ -302,7 +302,10 @@ def load_model(
|
|
| 302 |
if cfg.resize_token_embeddings_to_32x
|
| 303 |
else len(tokenizer)
|
| 304 |
)
|
| 305 |
-
model.
|
|
|
|
|
|
|
|
|
|
| 306 |
|
| 307 |
if (
|
| 308 |
hasattr(model.config, "max_position_embeddings")
|
|
|
|
| 302 |
if cfg.resize_token_embeddings_to_32x
|
| 303 |
else len(tokenizer)
|
| 304 |
)
|
| 305 |
+
if model.get_input_embeddings().num_embeddings < embeddings_len:
|
| 306 |
+
model.resize_token_embeddings(embeddings_len)
|
| 307 |
+
else:
|
| 308 |
+
model.tie_weights()
|
| 309 |
|
| 310 |
if (
|
| 311 |
hasattr(model.config, "max_position_embeddings")
|