Upload model weights
Browse filesCo-authored-by: myturing <myturing@users.noreply.huggingface.co>
- .gitattributes +8 -0
- .gitignore +6 -0
- README.md +194 -0
- assets/MemT-1.0.png +3 -0
- assets/application.png +3 -0
- assets/badges/arxiv-coming-soon.svg +1 -0
- assets/badges/github.svg +1 -0
- assets/badges/huggingface-models.svg +1 -0
- assets/badges/tech-report.svg +11 -0
- assets/figure_performance.png +3 -0
- assets/memt-arc.jpg +3 -0
- chat_template.jinja +156 -0
- config.json +125 -0
- configuration_memt.py +103 -0
- generation_config.json +14 -0
- memt_block.safetensors +3 -0
- merges.txt +0 -0
- model-00001-of-00026.safetensors +3 -0
- model-00002-of-00026.safetensors +3 -0
- model-00003-of-00026.safetensors +3 -0
- model-00004-of-00026.safetensors +3 -0
- model-00005-of-00026.safetensors +3 -0
- model-00006-of-00026.safetensors +3 -0
- model-00007-of-00026.safetensors +3 -0
- model-00008-of-00026.safetensors +3 -0
- model-00009-of-00026.safetensors +3 -0
- model-00010-of-00026.safetensors +3 -0
- model-00011-of-00026.safetensors +3 -0
- model-00012-of-00026.safetensors +3 -0
- model-00013-of-00026.safetensors +3 -0
- model-00014-of-00026.safetensors +3 -0
- model-00015-of-00026.safetensors +3 -0
- model-00016-of-00026.safetensors +3 -0
- model-00017-of-00026.safetensors +3 -0
- model-00018-of-00026.safetensors +3 -0
- model-00019-of-00026.safetensors +3 -0
- model-00020-of-00026.safetensors +3 -0
- model-00021-of-00026.safetensors +3 -0
- model-00022-of-00026.safetensors +3 -0
- model-00023-of-00026.safetensors +3 -0
- model-00024-of-00026.safetensors +3 -0
- model-00025-of-00026.safetensors +3 -0
- model-00026-of-00026.safetensors +3 -0
- model.safetensors.index.json +760 -0
- modeling_memt.py +978 -0
- tech_report.pdf +3 -0
- tokenizer.json +3 -0
- tokenizer_config.json +306 -0
- vocab.json +0 -0
.gitattributes
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
assets/MemT-1.0.png filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
assets/application.jpg filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
assets/figure_performance.png filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
assets/memt-arc.jpg filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
tech_report.pdf filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
assets/application.png filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.safetensors
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.pyc
|
| 4 |
+
.venv/
|
| 5 |
+
.pytest_cache/
|
| 6 |
+
.coverage
|
README.md
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: transformers
|
| 3 |
+
pipeline_tag: text-generation
|
| 4 |
+
tags:
|
| 5 |
+
- custom_code
|
| 6 |
+
- memt
|
| 7 |
+
- text-generation
|
| 8 |
+
base_model: Qwen/Qwen3.6-35B-A3B
|
| 9 |
+
license: apache-2.0
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# **MemT-1.0-Nano**
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
<p align="center" style="line-height: 0;">
|
| 17 |
+
<a href="https://huggingface.co/collections/MemTensor/memt-1" style="display: inline-block; height: 20px; line-height: 0; vertical-align: middle;"><img src="assets/badges/huggingface-models.svg" alt="Hugging Face Models" width="151" height="20" style="display: inline-block; height: 20px; vertical-align: middle; margin: 0;"></a>
|
| 18 |
+
<a href="https://huggingface.co/MemTensor/MemT-1.0-Nano/resolve/main/tech_report.pdf" style="display: inline-block; height: 20px; line-height: 0; vertical-align: middle;"><img src="assets/badges/tech-report.svg" alt="Tech Report: PDF" width="140" height="20" style="display: inline-block; height: 20px; vertical-align: middle; margin: 0;"></a>
|
| 19 |
+
<a href="https://github.com/MemTensor/MemT-1.0" style="display: inline-block; height: 20px; line-height: 0; vertical-align: middle;"><img src="assets/badges/github.svg" alt="GitHub: MemT-1.0" width="147" height="20" style="display: inline-block; height: 20px; vertical-align: middle; margin: 0;"></a>
|
| 20 |
+
</p>
|
| 21 |
+
|
| 22 |
+
## **Overview**
|
| 23 |
+
|
| 24 |
+
**MemT-1.0 is a new generation of memory foundation models that makes persistent, dynamically evolving memory an intrinsic part of model computation.**
|
| 25 |
+
|
| 26 |
+
MemT-1.0 integrates dynamic local memory and trainable hyper memory to retain and leverage historical information without accessing original context at inference. Its persistent state remains constant in size, reducing storage by approximately 66× versus full-context KV cache at 128K history length.
|
| 27 |
+
|
| 28 |
+
The project makes three main contributions:
|
| 29 |
+
|
| 30 |
+
* **Memory architecture**. Introduce MemT-1.0 with improved memory storage and utilization, and Mixture-of-Storage (MoS) for scalable memory capacity.
|
| 31 |
+
|
| 32 |
+
* **Training and evaluation**. Develop backbone preparation and a two-stage training pipeline, supported by 34B tokens of data and dedicated infrastructure. Evaluate memory and general capabilities.
|
| 33 |
+
|
| 34 |
+
* **Scaling and applications**. Study memory scaling and demonstrate native memory for experience-guided agentic coding.
|
| 35 |
+
|
| 36 |
+
### Model Family
|
| 37 |
+
|
| 38 |
+
| Model | Backbone | Total Parameters | Activated Parameters | Trainable Memory Parameters | Dynamic Memory State |
|
| 39 |
+
| --------------------------- | -------------------------- | ---------------- | -------------------- | --------------------------- | -------------------- |
|
| 40 |
+
| MemT-1.0-Nano | Qwen3.6-35B-A3B | 36B | 3B | 451M | 10M |
|
| 41 |
+
| MemT-1.0-Flash | DeepSeek-V4-Flash | 288B | 17B | 3.61B | 336M |
|
| 42 |
+
|
| 43 |
+
### Architecture
|
| 44 |
+
|
| 45 |
+
A **Native Memory Block** runs alongside the backbone attention and contains two components:
|
| 46 |
+
|
| 47 |
+
* The **Local Memory Block** maintains a dynamic, multi-channel memory matrix and evolves across interaction steps.
|
| 48 |
+
|
| 49 |
+
* The **Hyper Memory Block** contains trainable memory query, key, and value projections, erase, write, and forget gates, as well as an output gate.
|
| 50 |
+
|
| 51 |
+
During generation, memory attention reads from the current memory state, normalizes and gates the readout, and adds it to the backbone attention output. At the end of each turn or segment, a stable GDN-2 performs token-level memory updates the memory state for subsequent steps.
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
### Evaluation Results
|
| 56 |
+
|
| 57 |
+
Across 11 memory benchmarks, MemT-1.0-Flash and MemT-1.0-Nano rank first and second among the **evaluated parametric-memory methods**, respectively. Strong textual-memory systems remain ahead overall, particularly on long histories that require detailed evidence retrieval.
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
### Experience-Guided Agentic Coding
|
| 62 |
+
|
| 63 |
+
MemT-1.0 introduces an agentic framework for reusing procedural experience in long-horizon coding tasks. Retrieved experience is stored in native memory and accessed through memory attention during execution, improving task performance without adding experience text to the task context.
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
## **Quick Start**
|
| 68 |
+
|
| 69 |
+
The examples below use MemT-1.0 with Hugging Face Transformers.
|
| 70 |
+
|
| 71 |
+
### Installation
|
| 72 |
+
|
| 73 |
+
Use Python 3.11 or newer and install a CUDA 13 build of PyTorch 2.11.0 for your GPU, then:
|
| 74 |
+
|
| 75 |
+
```bash
|
| 76 |
+
pip install transformers==5.12.1 accelerate==1.15.0 safetensors==0.8.0
|
| 77 |
+
```
|
| 78 |
+
|
| 79 |
+
### Load the Model
|
| 80 |
+
|
| 81 |
+
```python
|
| 82 |
+
import torch
|
| 83 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
|
| 84 |
+
|
| 85 |
+
model_id = "MemTensor/MemT-1.0-Nano" # Or a local model directory.
|
| 86 |
+
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
|
| 87 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 88 |
+
model_id,
|
| 89 |
+
trust_remote_code=True,
|
| 90 |
+
dtype=torch.bfloat16,
|
| 91 |
+
device_map="auto",
|
| 92 |
+
attn_implementation="sdpa",
|
| 93 |
+
)
|
| 94 |
+
```
|
| 95 |
+
|
| 96 |
+
### A Conversation with Memory
|
| 97 |
+
|
| 98 |
+
```python
|
| 99 |
+
session = model.create_session(tokenizer)
|
| 100 |
+
|
| 101 |
+
# Set enable_memory=True to save past and current turns information to latent
|
| 102 |
+
# local memory and clear the plaintext conversation context.
|
| 103 |
+
session.chat(
|
| 104 |
+
"I have a three-month-old kitten named memmy who loves eating "
|
| 105 |
+
"small shrimp and is excellent at remembering things.",
|
| 106 |
+
enable_memory=True,
|
| 107 |
+
enable_thinking=False,
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
# Answer using the latent local memory.
|
| 111 |
+
answer = session.chat(
|
| 112 |
+
"What are my kitten's name, age, favorite food, and special trait?",
|
| 113 |
+
enable_thinking=False,
|
| 114 |
+
)
|
| 115 |
+
print(answer)
|
| 116 |
+
# Based on the information you provided earlier:
|
| 117 |
+
#
|
| 118 |
+
# * **Name:** Memmy
|
| 119 |
+
# * **Age:** Three months old
|
| 120 |
+
# * **Favorite Food:** Small shrimp
|
| 121 |
+
# * **Special Trait:** An excellent memory
|
| 122 |
+
```
|
| 123 |
+
|
| 124 |
+
### Save and Resume
|
| 125 |
+
|
| 126 |
+
```python
|
| 127 |
+
# Save local memory, including any unsaved conversation turns.
|
| 128 |
+
state_path = "memmy_memory.safetensors"
|
| 129 |
+
session.save_state(state_path)
|
| 130 |
+
session.close()
|
| 131 |
+
|
| 132 |
+
# Restore local memory without loading the plaintext conversation.
|
| 133 |
+
restored = model.create_session(tokenizer, state=state_path)
|
| 134 |
+
|
| 135 |
+
print(len(restored.messages))
|
| 136 |
+
# 0
|
| 137 |
+
|
| 138 |
+
answer = restored.chat(
|
| 139 |
+
"How old is my kitten?",
|
| 140 |
+
enable_thinking=False,
|
| 141 |
+
)
|
| 142 |
+
print(answer)
|
| 143 |
+
# Your kitten is **3 months old**.
|
| 144 |
+
|
| 145 |
+
restored.close()
|
| 146 |
+
```
|
| 147 |
+
|
| 148 |
+
### Standard Chat
|
| 149 |
+
|
| 150 |
+
```python
|
| 151 |
+
# Use the standard Transformers pipeline for chat generation with plaintext context, without enable_memory.
|
| 152 |
+
generator = pipeline("text-generation", model=model, tokenizer=tokenizer)
|
| 153 |
+
messages = [{
|
| 154 |
+
"role": "user",
|
| 155 |
+
"content": "Type 'I love MemT' exactly.",
|
| 156 |
+
}]
|
| 157 |
+
result = generator(
|
| 158 |
+
messages,
|
| 159 |
+
generation_config=model.generation_config,
|
| 160 |
+
tokenizer_encode_kwargs={"enable_thinking": False},
|
| 161 |
+
)
|
| 162 |
+
print(result[0]["generated_text"][-1]["content"])
|
| 163 |
+
# I love MemT
|
| 164 |
+
```
|
| 165 |
+
|
| 166 |
+
## **Citation**
|
| 167 |
+
|
| 168 |
+
For the MemT-1.0 technical report:
|
| 169 |
+
|
| 170 |
+
```bibtex
|
| 171 |
+
@misc{memtensor2026memt1,
|
| 172 |
+
title = {MemT-1.0: Towards Memory-Native Evolution},
|
| 173 |
+
author = {{MemTensor Team}},
|
| 174 |
+
year = {2026},
|
| 175 |
+
howpublished = {Technical report}
|
| 176 |
+
}
|
| 177 |
+
```
|
| 178 |
+
|
| 179 |
+
## **License**
|
| 180 |
+
|
| 181 |
+
The MemT-1.0-Nano model weights are released under the
|
| 182 |
+
**[Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0)**.
|
| 183 |
+
|
| 184 |
+
We thank the Qwen team for the backbone model.
|
| 185 |
+
The backbone and third-party components retain their original licenses and
|
| 186 |
+
copyright notices; the MemT license does not replace their terms.
|
| 187 |
+
|
| 188 |
+
Portions of the inference code are adapted from Hugging Face Transformers
|
| 189 |
+
and retain their upstream attribution:
|
| 190 |
+
Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved. Apache License 2.0.
|
| 191 |
+
|
| 192 |
+
## **Contact**
|
| 193 |
+
|
| 194 |
+
Correspondence: [lizy@memtensor.cn](mailto:lizy@memtensor.cn).
|
assets/MemT-1.0.png
ADDED
|
Git LFS Details
|
assets/application.png
ADDED
|
Git LFS Details
|
assets/badges/arxiv-coming-soon.svg
ADDED
|
assets/badges/github.svg
ADDED
|
assets/badges/huggingface-models.svg
ADDED
|
assets/badges/tech-report.svg
ADDED
|
assets/figure_performance.png
ADDED
|
Git LFS Details
|
assets/memt-arc.jpg
ADDED
|
Git LFS Details
|
chat_template.jinja
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- set image_count = namespace(value=0) %}
|
| 2 |
+
{%- set video_count = namespace(value=0) %}
|
| 3 |
+
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
|
| 4 |
+
{%- if content is string %}
|
| 5 |
+
{{- content }}
|
| 6 |
+
{%- elif content is iterable and content is not mapping %}
|
| 7 |
+
{%- for item in content %}
|
| 8 |
+
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
|
| 9 |
+
{%- if is_system_content %}
|
| 10 |
+
{{- raise_exception('System message cannot contain images.') }}
|
| 11 |
+
{%- endif %}
|
| 12 |
+
{%- if do_vision_count %}
|
| 13 |
+
{%- set image_count.value = image_count.value + 1 %}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{%- if add_vision_id %}
|
| 16 |
+
{{- 'Picture ' ~ image_count.value ~ ': ' }}
|
| 17 |
+
{%- endif %}
|
| 18 |
+
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
|
| 19 |
+
{%- elif 'video' in item or item.type == 'video' %}
|
| 20 |
+
{%- if is_system_content %}
|
| 21 |
+
{{- raise_exception('System message cannot contain videos.') }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- if do_vision_count %}
|
| 24 |
+
{%- set video_count.value = video_count.value + 1 %}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- if add_vision_id %}
|
| 27 |
+
{{- 'Video ' ~ video_count.value ~ ': ' }}
|
| 28 |
+
{%- endif %}
|
| 29 |
+
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
|
| 30 |
+
{%- elif 'text' in item %}
|
| 31 |
+
{{- item.text }}
|
| 32 |
+
{%- else %}
|
| 33 |
+
{{- raise_exception('Unexpected item type in content.') }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{%- endfor %}
|
| 36 |
+
{%- elif content is none or content is undefined %}
|
| 37 |
+
{{- '' }}
|
| 38 |
+
{%- else %}
|
| 39 |
+
{{- raise_exception('Unexpected content type.') }}
|
| 40 |
+
{%- endif %}
|
| 41 |
+
{%- endmacro %}
|
| 42 |
+
{%- if not messages %}
|
| 43 |
+
{{- raise_exception('No messages provided.') }}
|
| 44 |
+
{%- endif %}
|
| 45 |
+
{%- if tools and tools is iterable and tools is not mapping %}
|
| 46 |
+
{{- '<|im_start|>system\n' }}
|
| 47 |
+
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
|
| 48 |
+
{%- for tool in tools %}
|
| 49 |
+
{{- "\n" }}
|
| 50 |
+
{{- tool | tojson }}
|
| 51 |
+
{%- endfor %}
|
| 52 |
+
{{- "\n</tools>" }}
|
| 53 |
+
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
| 54 |
+
{%- if messages[0].role == 'system' %}
|
| 55 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 56 |
+
{%- if content %}
|
| 57 |
+
{{- '\n\n' + content }}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{{- '<|im_end|>\n' }}
|
| 61 |
+
{%- else %}
|
| 62 |
+
{%- if messages[0].role == 'system' %}
|
| 63 |
+
{%- set content = render_content(messages[0].content, false, true)|trim %}
|
| 64 |
+
{{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
|
| 65 |
+
{%- endif %}
|
| 66 |
+
{%- endif %}
|
| 67 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 68 |
+
{%- for message in messages[::-1] %}
|
| 69 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 70 |
+
{%- if ns.multi_step_tool and message.role == "user" %}
|
| 71 |
+
{%- set content = render_content(message.content, false)|trim %}
|
| 72 |
+
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
|
| 73 |
+
{%- set ns.multi_step_tool = false %}
|
| 74 |
+
{%- set ns.last_query_index = index %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- if ns.multi_step_tool %}
|
| 79 |
+
{{- raise_exception('No user query found in messages.') }}
|
| 80 |
+
{%- endif %}
|
| 81 |
+
{%- for message in messages %}
|
| 82 |
+
{%- set content = render_content(message.content, true)|trim %}
|
| 83 |
+
{%- if message.role == "system" %}
|
| 84 |
+
{%- if not loop.first %}
|
| 85 |
+
{{- raise_exception('System message must be at the beginning.') }}
|
| 86 |
+
{%- endif %}
|
| 87 |
+
{%- elif message.role == "user" %}
|
| 88 |
+
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
| 89 |
+
{%- elif message.role == "assistant" %}
|
| 90 |
+
{%- set reasoning_content = '' %}
|
| 91 |
+
{%- if message.reasoning_content is string %}
|
| 92 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 93 |
+
{%- else %}
|
| 94 |
+
{%- if '</think>' in content %}
|
| 95 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 96 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{%- endif %}
|
| 99 |
+
{%- set reasoning_content = reasoning_content|trim %}
|
| 100 |
+
{%- if preserve_thinking is defined and preserve_thinking is true and content.startswith('<think>') and '</think>' not in content %}
|
| 101 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 102 |
+
{%- elif (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %}
|
| 103 |
+
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
|
| 104 |
+
{%- else %}
|
| 105 |
+
{{- '<|im_start|>' + message.role + '\n' + content }}
|
| 106 |
+
{%- endif %}
|
| 107 |
+
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
|
| 108 |
+
{%- for tool_call in message.tool_calls %}
|
| 109 |
+
{%- if tool_call.function is defined %}
|
| 110 |
+
{%- set tool_call = tool_call.function %}
|
| 111 |
+
{%- endif %}
|
| 112 |
+
{%- if loop.first %}
|
| 113 |
+
{%- if content|trim %}
|
| 114 |
+
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 115 |
+
{%- else %}
|
| 116 |
+
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 117 |
+
{%- endif %}
|
| 118 |
+
{%- else %}
|
| 119 |
+
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
|
| 120 |
+
{%- endif %}
|
| 121 |
+
{%- if tool_call.arguments is defined %}
|
| 122 |
+
{%- for args_name, args_value in tool_call.arguments|items %}
|
| 123 |
+
{{- '<parameter=' + args_name + '>\n' }}
|
| 124 |
+
{%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
|
| 125 |
+
{{- args_value }}
|
| 126 |
+
{{- '\n</parameter>\n' }}
|
| 127 |
+
{%- endfor %}
|
| 128 |
+
{%- endif %}
|
| 129 |
+
{{- '</function>\n</tool_call>' }}
|
| 130 |
+
{%- endfor %}
|
| 131 |
+
{%- endif %}
|
| 132 |
+
{{- '<|im_end|>\n' }}
|
| 133 |
+
{%- elif message.role == "tool" %}
|
| 134 |
+
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
| 135 |
+
{{- '<|im_start|>user' }}
|
| 136 |
+
{%- endif %}
|
| 137 |
+
{{- '\n<tool_response>\n' }}
|
| 138 |
+
{{- content }}
|
| 139 |
+
{{- '\n</tool_response>' }}
|
| 140 |
+
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
| 141 |
+
{{- '<|im_end|>\n' }}
|
| 142 |
+
{%- elif loop.last %}
|
| 143 |
+
{{- '<|im_end|>\n' }}
|
| 144 |
+
{%- endif %}
|
| 145 |
+
{%- else %}
|
| 146 |
+
{{- raise_exception('Unexpected message role.') }}
|
| 147 |
+
{%- endif %}
|
| 148 |
+
{%- endfor %}
|
| 149 |
+
{%- if add_generation_prompt %}
|
| 150 |
+
{{- '<|im_start|>assistant\n' }}
|
| 151 |
+
{%- if enable_thinking is defined and enable_thinking is false %}
|
| 152 |
+
{{- '<think>\n\n</think>\n\n' }}
|
| 153 |
+
{%- else %}
|
| 154 |
+
{{- '<think>\n' }}
|
| 155 |
+
{%- endif %}
|
| 156 |
+
{%- endif %}
|
config.json
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "memt_qwen3_5_moe",
|
| 3 |
+
"pad_token_id": 248044,
|
| 4 |
+
"architectures": [
|
| 5 |
+
"MemTQwen3_5MoeForCausalLM"
|
| 6 |
+
],
|
| 7 |
+
"torch_dtype": "bfloat16",
|
| 8 |
+
"tie_word_embeddings": false,
|
| 9 |
+
"backbone_configs": {
|
| 10 |
+
"model_type": "qwen3_5_moe",
|
| 11 |
+
"text_config": {
|
| 12 |
+
"attention_bias": false,
|
| 13 |
+
"attention_dropout": 0.0,
|
| 14 |
+
"attn_output_gate": true,
|
| 15 |
+
"bos_token_id": 248044,
|
| 16 |
+
"dtype": "bfloat16",
|
| 17 |
+
"eos_token_id": 248044,
|
| 18 |
+
"full_attention_interval": 4,
|
| 19 |
+
"head_dim": 256,
|
| 20 |
+
"hidden_act": "silu",
|
| 21 |
+
"hidden_size": 2048,
|
| 22 |
+
"initializer_range": 0.02,
|
| 23 |
+
"layer_types": [
|
| 24 |
+
"linear_attention",
|
| 25 |
+
"linear_attention",
|
| 26 |
+
"linear_attention",
|
| 27 |
+
"full_attention",
|
| 28 |
+
"linear_attention",
|
| 29 |
+
"linear_attention",
|
| 30 |
+
"linear_attention",
|
| 31 |
+
"full_attention",
|
| 32 |
+
"linear_attention",
|
| 33 |
+
"linear_attention",
|
| 34 |
+
"linear_attention",
|
| 35 |
+
"full_attention",
|
| 36 |
+
"linear_attention",
|
| 37 |
+
"linear_attention",
|
| 38 |
+
"linear_attention",
|
| 39 |
+
"full_attention",
|
| 40 |
+
"linear_attention",
|
| 41 |
+
"linear_attention",
|
| 42 |
+
"linear_attention",
|
| 43 |
+
"full_attention",
|
| 44 |
+
"linear_attention",
|
| 45 |
+
"linear_attention",
|
| 46 |
+
"linear_attention",
|
| 47 |
+
"full_attention",
|
| 48 |
+
"linear_attention",
|
| 49 |
+
"linear_attention",
|
| 50 |
+
"linear_attention",
|
| 51 |
+
"full_attention",
|
| 52 |
+
"linear_attention",
|
| 53 |
+
"linear_attention",
|
| 54 |
+
"linear_attention",
|
| 55 |
+
"full_attention",
|
| 56 |
+
"linear_attention",
|
| 57 |
+
"linear_attention",
|
| 58 |
+
"linear_attention",
|
| 59 |
+
"full_attention",
|
| 60 |
+
"linear_attention",
|
| 61 |
+
"linear_attention",
|
| 62 |
+
"linear_attention",
|
| 63 |
+
"full_attention"
|
| 64 |
+
],
|
| 65 |
+
"linear_conv_kernel_dim": 4,
|
| 66 |
+
"linear_key_head_dim": 128,
|
| 67 |
+
"linear_num_key_heads": 16,
|
| 68 |
+
"linear_num_value_heads": 32,
|
| 69 |
+
"linear_value_head_dim": 128,
|
| 70 |
+
"mamba_ssm_dtype": "float32",
|
| 71 |
+
"max_position_embeddings": 262144,
|
| 72 |
+
"model_type": "qwen3_5_moe_text",
|
| 73 |
+
"moe_intermediate_size": 512,
|
| 74 |
+
"mtp_num_hidden_layers": 1,
|
| 75 |
+
"mtp_use_dedicated_embeddings": false,
|
| 76 |
+
"num_attention_heads": 16,
|
| 77 |
+
"num_experts": 256,
|
| 78 |
+
"num_experts_per_tok": 8,
|
| 79 |
+
"num_hidden_layers": 40,
|
| 80 |
+
"num_key_value_heads": 2,
|
| 81 |
+
"output_router_logits": false,
|
| 82 |
+
"pad_token_id": null,
|
| 83 |
+
"partial_rotary_factor": 0.25,
|
| 84 |
+
"rms_norm_eps": 1e-06,
|
| 85 |
+
"rope_parameters": {
|
| 86 |
+
"mrope_interleaved": true,
|
| 87 |
+
"mrope_section": [
|
| 88 |
+
11,
|
| 89 |
+
11,
|
| 90 |
+
10
|
| 91 |
+
],
|
| 92 |
+
"partial_rotary_factor": 0.25,
|
| 93 |
+
"rope_theta": 10000000,
|
| 94 |
+
"rope_type": "default"
|
| 95 |
+
},
|
| 96 |
+
"router_aux_loss_coef": 0.001,
|
| 97 |
+
"shared_expert_intermediate_size": 512,
|
| 98 |
+
"tie_word_embeddings": false,
|
| 99 |
+
"use_cache": true,
|
| 100 |
+
"vocab_size": 248320
|
| 101 |
+
}
|
| 102 |
+
},
|
| 103 |
+
"memt": {
|
| 104 |
+
"memory_update_mode": "token",
|
| 105 |
+
"gdn_version": "stable_gdn2",
|
| 106 |
+
"memory_use_short_conv": false,
|
| 107 |
+
"memory_head_mode": "mha",
|
| 108 |
+
"memory_read_scale": "k_inv_sqrt",
|
| 109 |
+
"num_memory_heads": 16,
|
| 110 |
+
"num_kv_heads": 16,
|
| 111 |
+
"linear_attention_state_policy": "reset_per_chunk",
|
| 112 |
+
"chunk_tokens": 2048,
|
| 113 |
+
"tail_policy": "commit",
|
| 114 |
+
"output_gate_type": "silu",
|
| 115 |
+
"output_gate_bias": false,
|
| 116 |
+
"output_gate_softcap_enabled": true,
|
| 117 |
+
"output_gate_softcap_limit": 2.0,
|
| 118 |
+
"stable_gdn2_sqrt_erase_gate": true,
|
| 119 |
+
"backbone_gdn_override": false
|
| 120 |
+
},
|
| 121 |
+
"auto_map": {
|
| 122 |
+
"AutoConfig": "configuration_memt.MemTQwen3_5MoeConfig",
|
| 123 |
+
"AutoModelForCausalLM": "modeling_memt.MemTQwen3_5MoeForCausalLM"
|
| 124 |
+
}
|
| 125 |
+
}
|
configuration_memt.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Standalone configuration for the published Qwen MoE + MemT model."""
|
| 2 |
+
|
| 3 |
+
import math
|
| 4 |
+
from copy import deepcopy
|
| 5 |
+
|
| 6 |
+
from transformers import PretrainedConfig
|
| 7 |
+
from transformers.models.qwen3_5_moe.configuration_qwen3_5_moe import (
|
| 8 |
+
Qwen3_5MoeTextConfig,
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
_REQUIRED = {
|
| 12 |
+
"memory_update_mode": "token",
|
| 13 |
+
"gdn_version": "stable_gdn2",
|
| 14 |
+
"memory_use_short_conv": False,
|
| 15 |
+
"memory_head_mode": "mha",
|
| 16 |
+
"memory_read_scale": "k_inv_sqrt",
|
| 17 |
+
"linear_attention_state_policy": "reset_per_chunk",
|
| 18 |
+
"chunk_tokens": 2048,
|
| 19 |
+
"tail_policy": "commit",
|
| 20 |
+
"output_gate_type": "silu",
|
| 21 |
+
"stable_gdn2_sqrt_erase_gate": True,
|
| 22 |
+
"backbone_gdn_override": False,
|
| 23 |
+
}
|
| 24 |
+
_EXTRA = {
|
| 25 |
+
"num_memory_heads",
|
| 26 |
+
"num_kv_heads",
|
| 27 |
+
"output_gate_bias",
|
| 28 |
+
"output_gate_softcap_enabled",
|
| 29 |
+
"output_gate_softcap_limit",
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class MemTQwen3_5MoeConfig(PretrainedConfig):
|
| 34 |
+
model_type = "memt_qwen3_5_moe"
|
| 35 |
+
has_no_defaults_at_init = True
|
| 36 |
+
|
| 37 |
+
def __init__(self, backbone_configs=None, memt=None, **kwargs):
|
| 38 |
+
if not isinstance(backbone_configs, dict) or not isinstance(memt, dict):
|
| 39 |
+
raise TypeError("backbone_configs and memt are required objects")
|
| 40 |
+
unknown = set(memt) - set(_REQUIRED) - _EXTRA
|
| 41 |
+
missing = (set(_REQUIRED) | _EXTRA) - set(memt)
|
| 42 |
+
if unknown or missing:
|
| 43 |
+
raise ValueError(f"unknown MemT fields {sorted(unknown)}; missing {sorted(missing)}")
|
| 44 |
+
for key, expected in _REQUIRED.items():
|
| 45 |
+
if type(memt[key]) is not type(expected) or memt[key] != expected:
|
| 46 |
+
raise ValueError(f"Unsupported memt.{key}: {memt[key]!r}")
|
| 47 |
+
for key in ("output_gate_bias", "output_gate_softcap_enabled"):
|
| 48 |
+
if type(memt[key]) is not bool:
|
| 49 |
+
raise ValueError(f"memt.{key} must be bool")
|
| 50 |
+
value = memt["output_gate_softcap_limit"]
|
| 51 |
+
if (
|
| 52 |
+
isinstance(value, bool)
|
| 53 |
+
or not isinstance(value, (int, float))
|
| 54 |
+
or not math.isfinite(value)
|
| 55 |
+
or value <= 0
|
| 56 |
+
):
|
| 57 |
+
raise ValueError("memt.output_gate_softcap_limit must be positive and finite")
|
| 58 |
+
text = deepcopy(backbone_configs.get("text_config", backbone_configs))
|
| 59 |
+
if text.get("model_type") != "qwen3_5_moe_text":
|
| 60 |
+
raise ValueError("This release requires qwen3_5_moe_text backbone")
|
| 61 |
+
self.backbone_configs = deepcopy(backbone_configs)
|
| 62 |
+
self.memt = deepcopy(memt)
|
| 63 |
+
for key in ("num_memory_heads", "num_kv_heads"):
|
| 64 |
+
if type(memt[key]) is not int or memt[key] != text["num_attention_heads"]:
|
| 65 |
+
raise ValueError(f"memt.{key} must match backbone query heads")
|
| 66 |
+
for key in kwargs:
|
| 67 |
+
if key in (
|
| 68 |
+
"memory_configs",
|
| 69 |
+
"metis_settings",
|
| 70 |
+
"sglang_metis_native_overlay",
|
| 71 |
+
):
|
| 72 |
+
raise ValueError(f"Legacy configuration field {key}")
|
| 73 |
+
kwargs.setdefault("bos_token_id", text.get("bos_token_id"))
|
| 74 |
+
kwargs.setdefault("eos_token_id", text.get("eos_token_id"))
|
| 75 |
+
kwargs.setdefault("pad_token_id", text.get("pad_token_id"))
|
| 76 |
+
kwargs.setdefault("tie_word_embeddings", False)
|
| 77 |
+
super().__init__(**kwargs)
|
| 78 |
+
self.vocab_size = text["vocab_size"]
|
| 79 |
+
self.hidden_size = text["hidden_size"]
|
| 80 |
+
self.num_hidden_layers = text["num_hidden_layers"]
|
| 81 |
+
self.initializer_range = text.get("initializer_range", 0.02)
|
| 82 |
+
self.use_cache = kwargs.get("use_cache", True)
|
| 83 |
+
self.is_encoder_decoder = False
|
| 84 |
+
|
| 85 |
+
@classmethod
|
| 86 |
+
def from_dict(cls, config_dict, **kwargs):
|
| 87 |
+
# PretrainedConfig applies kwargs after construction. Reconstruct the
|
| 88 |
+
# merged config so semantic overrides cannot bypass validation.
|
| 89 |
+
result = super().from_dict(deepcopy(config_dict), **kwargs)
|
| 90 |
+
config = result[0] if isinstance(result, tuple) else result
|
| 91 |
+
cls(**config.to_dict())
|
| 92 |
+
return result
|
| 93 |
+
|
| 94 |
+
def get_text_config(self, decoder=False, encoder=False):
|
| 95 |
+
text = Qwen3_5MoeTextConfig(
|
| 96 |
+
**deepcopy(self.backbone_configs.get("text_config", self.backbone_configs))
|
| 97 |
+
)
|
| 98 |
+
text._attn_implementation = self._attn_implementation or "sdpa"
|
| 99 |
+
text._experts_implementation = self._experts_implementation
|
| 100 |
+
return text
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
MemTQwen3_5MoeConfig.register_for_auto_class()
|
generation_config.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bos_token_id": 248044,
|
| 3 |
+
"pad_token_id": 248044,
|
| 4 |
+
"eos_token_id": [
|
| 5 |
+
248046,
|
| 6 |
+
248044
|
| 7 |
+
],
|
| 8 |
+
"max_new_tokens": 4096,
|
| 9 |
+
"temperature": 0.7,
|
| 10 |
+
"top_p": 0.95,
|
| 11 |
+
"top_k": 40,
|
| 12 |
+
"num_beams": 1,
|
| 13 |
+
"do_sample": true
|
| 14 |
+
}
|
memt_block.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7bd0a677391c225767d0a268d3aa23ae3f83c2fb3781b3336e17da6ba8309d10
|
| 3 |
+
size 901870672
|
merges.txt
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
model-00001-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a7e2d472aa223fb8ac427cbf31f52954a4d80d22c894ef204918e2924b08cfb
|
| 3 |
+
size 3835692392
|
model-00002-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:50577d889396c8543c19486db53a51e8be2ad45837122e54993a10b43e306d1e
|
| 3 |
+
size 552235168
|
model-00003-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8f7d72178d3f4431864978e5bcfa4c6cb1c204bc00590644d90bb19d6d522eeb
|
| 3 |
+
size 3357898360
|
model-00004-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:12d7db38689ba3c8af74b23ef8523eca41e0cd95db870583d0663a3ee8a6bd60
|
| 3 |
+
size 3370808712
|
model-00005-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a836047305d0f7a7b50f0815d09d5c03ec03d59ec2c763fcdc4bf7e9936bf902
|
| 3 |
+
size 3357898360
|
model-00006-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c9080d718e9c5f9e337443225aa417d4c24d00ae7995d76ee3f1cc296b557d15
|
| 3 |
+
size 3959424904
|
model-00007-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e8c05e23131b1dd45a455ec38cfac7db14667358268623c3938d00cf3e959a68
|
| 3 |
+
size 1096788232
|
model-00008-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4b6a6d495053089f4a80e7cbc82e848fba44e2c0c60122233d8fdff79fa7b296
|
| 3 |
+
size 3946842008
|
model-00009-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a31a954bb72d1c714e751bf0aabf2ff533f5a509693ebf7dd22ad6e90be46f67
|
| 3 |
+
size 1096460848
|
model-00010-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:246560e66570fe746653b8443e245dc334c9b8b831ea43d2d9f1b7d98623994e
|
| 3 |
+
size 3946841992
|
model-00011-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7180392817fe3ecb3a27a1da43b7ff22c1a94806bac49975f9f122c3126df675
|
| 3 |
+
size 1096460752
|
model-00012-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:043fb525f6625c2f2acb75e65a9959ee3fa7b6e3fdd2034b5cfe1859b01d3cfb
|
| 3 |
+
size 3409971080
|
model-00013-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:33a20fb20a21379bf43c84a43105f9c0cc35bd50d740b1c302dcbe4b700f5425
|
| 3 |
+
size 1633331664
|
model-00014-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:be823e33c5cb6120ad3769d081f34a2449dc2358041fca7c29d636c1ba19130d
|
| 3 |
+
size 3422553872
|
model-00015-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a89d547c6f9d0b535ee5ea2f2478f163089539f3f0dd330cb23d278a19d76123
|
| 3 |
+
size 1633659224
|
model-00016-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:69fc3ae0316482288afdcdd0b9eb7d626703ae26f7567e89aa3fc8d1ffd4ff5b
|
| 3 |
+
size 3946842136
|
model-00017-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e356e3943cf3852b76bb8992e674f3256013e27d54b78e8250514151cdc29637
|
| 3 |
+
size 1096460608
|
model-00018-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9e5e63fd1cc7d6848330c1fa363dfcb661bbc2ac87e672d0e28b71c9cb7f3c7f
|
| 3 |
+
size 3946841992
|
model-00019-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:708644ad34f1de727bf484f396944d8ec628645d52c183e9a992e65671685e21
|
| 3 |
+
size 1096460808
|
model-00020-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ca083a1d1aa64f8e8a785998f543a43374f13436dc85d396eee4e72c7a84e1ae
|
| 3 |
+
size 3409971072
|
model-00021-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ada4ae48f3d48fe01b4c53f2f82bce25e798a9631fd33959c881156fef2ccbce
|
| 3 |
+
size 1633331744
|
model-00022-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:def207fb42d7db31efb512755557763c23233c6e4d4c433027cb5102a7bce2f7
|
| 3 |
+
size 3370808752
|
model-00023-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:864d52ca7768a36f514069222e8de8626264ae124097ba8fcce5b5da2c6e2ed7
|
| 3 |
+
size 3357898392
|
model-00024-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:391acd27420cdce5935ff18152423c70620d19dac3c39a5ef1a81d369f82d737
|
| 3 |
+
size 3370808752
|
model-00025-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1860f20d0f301e6c9e8b7797328cb3de244ef1b6d2d682f27a6193650035fc4d
|
| 3 |
+
size 2759146312
|
model-00026-of-00026.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9cc7fc893a4f3826496a50df75e99befcd56861be7a677c6b901c97e02dd3b2b
|
| 3 |
+
size 1615875128
|
model.safetensors.index.json
ADDED
|
@@ -0,0 +1,760 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata": {
|
| 3 |
+
"total_size": 70223084096
|
| 4 |
+
},
|
| 5 |
+
"weight_map": {
|
| 6 |
+
"model.language_model.layers.11.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 7 |
+
"model.language_model.layers.11.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 8 |
+
"model.language_model.layers.11.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 9 |
+
"model.language_model.layers.11.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 10 |
+
"model.language_model.layers.11.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 11 |
+
"model.language_model.layers.11.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 12 |
+
"model.language_model.layers.15.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 13 |
+
"model.language_model.layers.15.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 14 |
+
"model.language_model.layers.15.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 15 |
+
"model.language_model.layers.15.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 16 |
+
"model.language_model.layers.15.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 17 |
+
"model.language_model.layers.15.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 18 |
+
"model.language_model.layers.19.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 19 |
+
"model.language_model.layers.19.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 20 |
+
"model.language_model.layers.19.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 21 |
+
"model.language_model.layers.19.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 22 |
+
"model.language_model.layers.19.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 23 |
+
"model.language_model.layers.19.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 24 |
+
"model.language_model.layers.23.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 25 |
+
"model.language_model.layers.23.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 26 |
+
"model.language_model.layers.23.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 27 |
+
"model.language_model.layers.23.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 28 |
+
"model.language_model.layers.23.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 29 |
+
"model.language_model.layers.23.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 30 |
+
"model.language_model.layers.27.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 31 |
+
"model.language_model.layers.27.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 32 |
+
"model.language_model.layers.27.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 33 |
+
"model.language_model.layers.27.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 34 |
+
"model.language_model.layers.27.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 35 |
+
"model.language_model.layers.27.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 36 |
+
"model.language_model.layers.3.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 37 |
+
"model.language_model.layers.3.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 38 |
+
"model.language_model.layers.3.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 39 |
+
"model.language_model.layers.3.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 40 |
+
"model.language_model.layers.3.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 41 |
+
"model.language_model.layers.3.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 42 |
+
"model.language_model.layers.31.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 43 |
+
"model.language_model.layers.31.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 44 |
+
"model.language_model.layers.31.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 45 |
+
"model.language_model.layers.31.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 46 |
+
"model.language_model.layers.31.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 47 |
+
"model.language_model.layers.31.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 48 |
+
"model.language_model.layers.35.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 49 |
+
"model.language_model.layers.35.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 50 |
+
"model.language_model.layers.35.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 51 |
+
"model.language_model.layers.35.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 52 |
+
"model.language_model.layers.35.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 53 |
+
"model.language_model.layers.35.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 54 |
+
"model.language_model.layers.39.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 55 |
+
"model.language_model.layers.39.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 56 |
+
"model.language_model.layers.39.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 57 |
+
"model.language_model.layers.39.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 58 |
+
"model.language_model.layers.39.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 59 |
+
"model.language_model.layers.39.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 60 |
+
"model.language_model.layers.7.self_attn.memt.decay_bias": "memt_block.safetensors",
|
| 61 |
+
"model.language_model.layers.7.self_attn.memt.decay_scale_log": "memt_block.safetensors",
|
| 62 |
+
"model.language_model.layers.7.self_attn.memt.f_down.weight": "memt_block.safetensors",
|
| 63 |
+
"model.language_model.layers.7.self_attn.memt.g_down.weight": "memt_block.safetensors",
|
| 64 |
+
"model.language_model.layers.7.self_attn.memt.in_proj.weight": "memt_block.safetensors",
|
| 65 |
+
"model.language_model.layers.7.self_attn.memt.o_norm.weight": "memt_block.safetensors",
|
| 66 |
+
"model.language_model.embed_tokens.weight": "model-00001-of-00026.safetensors",
|
| 67 |
+
"model.language_model.layers.0.linear_attn.in_proj_qkv.weight": "model-00001-of-00026.safetensors",
|
| 68 |
+
"model.language_model.layers.0.linear_attn.in_proj_z.weight": "model-00001-of-00026.safetensors",
|
| 69 |
+
"model.language_model.layers.0.linear_attn.out_proj.weight": "model-00001-of-00026.safetensors",
|
| 70 |
+
"model.language_model.layers.0.mlp.experts.gate_up_proj": "model-00001-of-00026.safetensors",
|
| 71 |
+
"model.language_model.layers.1.linear_attn.in_proj_qkv.weight": "model-00001-of-00026.safetensors",
|
| 72 |
+
"model.language_model.layers.1.linear_attn.in_proj_z.weight": "model-00001-of-00026.safetensors",
|
| 73 |
+
"model.language_model.layers.1.linear_attn.out_proj.weight": "model-00001-of-00026.safetensors",
|
| 74 |
+
"model.language_model.layers.1.mlp.experts.down_proj": "model-00001-of-00026.safetensors",
|
| 75 |
+
"model.language_model.layers.1.mlp.experts.gate_up_proj": "model-00001-of-00026.safetensors",
|
| 76 |
+
"model.language_model.layers.0.input_layernorm.weight": "model-00002-of-00026.safetensors",
|
| 77 |
+
"model.language_model.layers.0.linear_attn.A_log": "model-00002-of-00026.safetensors",
|
| 78 |
+
"model.language_model.layers.0.linear_attn.conv1d.weight": "model-00002-of-00026.safetensors",
|
| 79 |
+
"model.language_model.layers.0.linear_attn.dt_bias": "model-00002-of-00026.safetensors",
|
| 80 |
+
"model.language_model.layers.0.linear_attn.in_proj_a.weight": "model-00002-of-00026.safetensors",
|
| 81 |
+
"model.language_model.layers.0.linear_attn.in_proj_b.weight": "model-00002-of-00026.safetensors",
|
| 82 |
+
"model.language_model.layers.0.linear_attn.norm.weight": "model-00002-of-00026.safetensors",
|
| 83 |
+
"model.language_model.layers.0.mlp.experts.down_proj": "model-00002-of-00026.safetensors",
|
| 84 |
+
"model.language_model.layers.0.mlp.gate.weight": "model-00002-of-00026.safetensors",
|
| 85 |
+
"model.language_model.layers.0.mlp.shared_expert.down_proj.weight": "model-00002-of-00026.safetensors",
|
| 86 |
+
"model.language_model.layers.0.mlp.shared_expert.gate_proj.weight": "model-00002-of-00026.safetensors",
|
| 87 |
+
"model.language_model.layers.0.mlp.shared_expert.up_proj.weight": "model-00002-of-00026.safetensors",
|
| 88 |
+
"model.language_model.layers.0.mlp.shared_expert_gate.weight": "model-00002-of-00026.safetensors",
|
| 89 |
+
"model.language_model.layers.0.post_attention_layernorm.weight": "model-00002-of-00026.safetensors",
|
| 90 |
+
"model.language_model.layers.1.input_layernorm.weight": "model-00002-of-00026.safetensors",
|
| 91 |
+
"model.language_model.layers.1.linear_attn.A_log": "model-00002-of-00026.safetensors",
|
| 92 |
+
"model.language_model.layers.1.linear_attn.conv1d.weight": "model-00002-of-00026.safetensors",
|
| 93 |
+
"model.language_model.layers.1.linear_attn.dt_bias": "model-00002-of-00026.safetensors",
|
| 94 |
+
"model.language_model.layers.1.linear_attn.in_proj_a.weight": "model-00002-of-00026.safetensors",
|
| 95 |
+
"model.language_model.layers.1.linear_attn.in_proj_b.weight": "model-00002-of-00026.safetensors",
|
| 96 |
+
"model.language_model.layers.1.linear_attn.norm.weight": "model-00002-of-00026.safetensors",
|
| 97 |
+
"model.language_model.layers.1.mlp.gate.weight": "model-00002-of-00026.safetensors",
|
| 98 |
+
"model.language_model.layers.1.mlp.shared_expert.down_proj.weight": "model-00002-of-00026.safetensors",
|
| 99 |
+
"model.language_model.layers.1.mlp.shared_expert.gate_proj.weight": "model-00002-of-00026.safetensors",
|
| 100 |
+
"model.language_model.layers.1.mlp.shared_expert.up_proj.weight": "model-00002-of-00026.safetensors",
|
| 101 |
+
"model.language_model.layers.1.mlp.shared_expert_gate.weight": "model-00002-of-00026.safetensors",
|
| 102 |
+
"model.language_model.layers.1.post_attention_layernorm.weight": "model-00002-of-00026.safetensors",
|
| 103 |
+
"model.language_model.layers.2.input_layernorm.weight": "model-00003-of-00026.safetensors",
|
| 104 |
+
"model.language_model.layers.2.linear_attn.A_log": "model-00003-of-00026.safetensors",
|
| 105 |
+
"model.language_model.layers.2.linear_attn.conv1d.weight": "model-00003-of-00026.safetensors",
|
| 106 |
+
"model.language_model.layers.2.linear_attn.dt_bias": "model-00003-of-00026.safetensors",
|
| 107 |
+
"model.language_model.layers.2.linear_attn.in_proj_a.weight": "model-00003-of-00026.safetensors",
|
| 108 |
+
"model.language_model.layers.2.linear_attn.in_proj_b.weight": "model-00003-of-00026.safetensors",
|
| 109 |
+
"model.language_model.layers.2.linear_attn.in_proj_qkv.weight": "model-00003-of-00026.safetensors",
|
| 110 |
+
"model.language_model.layers.2.linear_attn.in_proj_z.weight": "model-00003-of-00026.safetensors",
|
| 111 |
+
"model.language_model.layers.2.linear_attn.norm.weight": "model-00003-of-00026.safetensors",
|
| 112 |
+
"model.language_model.layers.2.linear_attn.out_proj.weight": "model-00003-of-00026.safetensors",
|
| 113 |
+
"model.language_model.layers.2.mlp.experts.down_proj": "model-00003-of-00026.safetensors",
|
| 114 |
+
"model.language_model.layers.2.mlp.experts.gate_up_proj": "model-00003-of-00026.safetensors",
|
| 115 |
+
"model.language_model.layers.2.mlp.gate.weight": "model-00003-of-00026.safetensors",
|
| 116 |
+
"model.language_model.layers.2.mlp.shared_expert.down_proj.weight": "model-00003-of-00026.safetensors",
|
| 117 |
+
"model.language_model.layers.2.mlp.shared_expert.gate_proj.weight": "model-00003-of-00026.safetensors",
|
| 118 |
+
"model.language_model.layers.2.mlp.shared_expert.up_proj.weight": "model-00003-of-00026.safetensors",
|
| 119 |
+
"model.language_model.layers.2.mlp.shared_expert_gate.weight": "model-00003-of-00026.safetensors",
|
| 120 |
+
"model.language_model.layers.2.post_attention_layernorm.weight": "model-00003-of-00026.safetensors",
|
| 121 |
+
"model.language_model.layers.3.input_layernorm.weight": "model-00003-of-00026.safetensors",
|
| 122 |
+
"model.language_model.layers.3.mlp.experts.down_proj": "model-00003-of-00026.safetensors",
|
| 123 |
+
"model.language_model.layers.3.mlp.experts.gate_up_proj": "model-00003-of-00026.safetensors",
|
| 124 |
+
"model.language_model.layers.3.mlp.gate.weight": "model-00003-of-00026.safetensors",
|
| 125 |
+
"model.language_model.layers.3.mlp.shared_expert.down_proj.weight": "model-00003-of-00026.safetensors",
|
| 126 |
+
"model.language_model.layers.3.mlp.shared_expert.gate_proj.weight": "model-00003-of-00026.safetensors",
|
| 127 |
+
"model.language_model.layers.3.mlp.shared_expert.up_proj.weight": "model-00003-of-00026.safetensors",
|
| 128 |
+
"model.language_model.layers.3.mlp.shared_expert_gate.weight": "model-00003-of-00026.safetensors",
|
| 129 |
+
"model.language_model.layers.3.post_attention_layernorm.weight": "model-00003-of-00026.safetensors",
|
| 130 |
+
"model.language_model.layers.3.self_attn.k_norm.weight": "model-00003-of-00026.safetensors",
|
| 131 |
+
"model.language_model.layers.3.self_attn.k_proj.weight": "model-00003-of-00026.safetensors",
|
| 132 |
+
"model.language_model.layers.3.self_attn.o_proj.weight": "model-00003-of-00026.safetensors",
|
| 133 |
+
"model.language_model.layers.3.self_attn.q_norm.weight": "model-00003-of-00026.safetensors",
|
| 134 |
+
"model.language_model.layers.3.self_attn.q_proj.weight": "model-00003-of-00026.safetensors",
|
| 135 |
+
"model.language_model.layers.3.self_attn.v_proj.weight": "model-00003-of-00026.safetensors",
|
| 136 |
+
"model.language_model.layers.4.input_layernorm.weight": "model-00004-of-00026.safetensors",
|
| 137 |
+
"model.language_model.layers.4.linear_attn.A_log": "model-00004-of-00026.safetensors",
|
| 138 |
+
"model.language_model.layers.4.linear_attn.conv1d.weight": "model-00004-of-00026.safetensors",
|
| 139 |
+
"model.language_model.layers.4.linear_attn.dt_bias": "model-00004-of-00026.safetensors",
|
| 140 |
+
"model.language_model.layers.4.linear_attn.in_proj_a.weight": "model-00004-of-00026.safetensors",
|
| 141 |
+
"model.language_model.layers.4.linear_attn.in_proj_b.weight": "model-00004-of-00026.safetensors",
|
| 142 |
+
"model.language_model.layers.4.linear_attn.in_proj_qkv.weight": "model-00004-of-00026.safetensors",
|
| 143 |
+
"model.language_model.layers.4.linear_attn.in_proj_z.weight": "model-00004-of-00026.safetensors",
|
| 144 |
+
"model.language_model.layers.4.linear_attn.norm.weight": "model-00004-of-00026.safetensors",
|
| 145 |
+
"model.language_model.layers.4.linear_attn.out_proj.weight": "model-00004-of-00026.safetensors",
|
| 146 |
+
"model.language_model.layers.4.mlp.experts.down_proj": "model-00004-of-00026.safetensors",
|
| 147 |
+
"model.language_model.layers.4.mlp.experts.gate_up_proj": "model-00004-of-00026.safetensors",
|
| 148 |
+
"model.language_model.layers.4.mlp.gate.weight": "model-00004-of-00026.safetensors",
|
| 149 |
+
"model.language_model.layers.4.mlp.shared_expert.down_proj.weight": "model-00004-of-00026.safetensors",
|
| 150 |
+
"model.language_model.layers.4.mlp.shared_expert.gate_proj.weight": "model-00004-of-00026.safetensors",
|
| 151 |
+
"model.language_model.layers.4.mlp.shared_expert.up_proj.weight": "model-00004-of-00026.safetensors",
|
| 152 |
+
"model.language_model.layers.4.mlp.shared_expert_gate.weight": "model-00004-of-00026.safetensors",
|
| 153 |
+
"model.language_model.layers.4.post_attention_layernorm.weight": "model-00004-of-00026.safetensors",
|
| 154 |
+
"model.language_model.layers.5.input_layernorm.weight": "model-00004-of-00026.safetensors",
|
| 155 |
+
"model.language_model.layers.5.linear_attn.A_log": "model-00004-of-00026.safetensors",
|
| 156 |
+
"model.language_model.layers.5.linear_attn.conv1d.weight": "model-00004-of-00026.safetensors",
|
| 157 |
+
"model.language_model.layers.5.linear_attn.dt_bias": "model-00004-of-00026.safetensors",
|
| 158 |
+
"model.language_model.layers.5.linear_attn.in_proj_a.weight": "model-00004-of-00026.safetensors",
|
| 159 |
+
"model.language_model.layers.5.linear_attn.in_proj_b.weight": "model-00004-of-00026.safetensors",
|
| 160 |
+
"model.language_model.layers.5.linear_attn.in_proj_qkv.weight": "model-00004-of-00026.safetensors",
|
| 161 |
+
"model.language_model.layers.5.linear_attn.in_proj_z.weight": "model-00004-of-00026.safetensors",
|
| 162 |
+
"model.language_model.layers.5.linear_attn.norm.weight": "model-00004-of-00026.safetensors",
|
| 163 |
+
"model.language_model.layers.5.linear_attn.out_proj.weight": "model-00004-of-00026.safetensors",
|
| 164 |
+
"model.language_model.layers.5.mlp.experts.down_proj": "model-00004-of-00026.safetensors",
|
| 165 |
+
"model.language_model.layers.5.mlp.experts.gate_up_proj": "model-00004-of-00026.safetensors",
|
| 166 |
+
"model.language_model.layers.5.mlp.gate.weight": "model-00004-of-00026.safetensors",
|
| 167 |
+
"model.language_model.layers.5.mlp.shared_expert.down_proj.weight": "model-00004-of-00026.safetensors",
|
| 168 |
+
"model.language_model.layers.5.mlp.shared_expert.gate_proj.weight": "model-00004-of-00026.safetensors",
|
| 169 |
+
"model.language_model.layers.5.mlp.shared_expert.up_proj.weight": "model-00004-of-00026.safetensors",
|
| 170 |
+
"model.language_model.layers.5.mlp.shared_expert_gate.weight": "model-00004-of-00026.safetensors",
|
| 171 |
+
"model.language_model.layers.5.post_attention_layernorm.weight": "model-00004-of-00026.safetensors",
|
| 172 |
+
"model.language_model.layers.6.input_layernorm.weight": "model-00005-of-00026.safetensors",
|
| 173 |
+
"model.language_model.layers.6.linear_attn.A_log": "model-00005-of-00026.safetensors",
|
| 174 |
+
"model.language_model.layers.6.linear_attn.conv1d.weight": "model-00005-of-00026.safetensors",
|
| 175 |
+
"model.language_model.layers.6.linear_attn.dt_bias": "model-00005-of-00026.safetensors",
|
| 176 |
+
"model.language_model.layers.6.linear_attn.in_proj_a.weight": "model-00005-of-00026.safetensors",
|
| 177 |
+
"model.language_model.layers.6.linear_attn.in_proj_b.weight": "model-00005-of-00026.safetensors",
|
| 178 |
+
"model.language_model.layers.6.linear_attn.in_proj_qkv.weight": "model-00005-of-00026.safetensors",
|
| 179 |
+
"model.language_model.layers.6.linear_attn.in_proj_z.weight": "model-00005-of-00026.safetensors",
|
| 180 |
+
"model.language_model.layers.6.linear_attn.norm.weight": "model-00005-of-00026.safetensors",
|
| 181 |
+
"model.language_model.layers.6.linear_attn.out_proj.weight": "model-00005-of-00026.safetensors",
|
| 182 |
+
"model.language_model.layers.6.mlp.experts.down_proj": "model-00005-of-00026.safetensors",
|
| 183 |
+
"model.language_model.layers.6.mlp.experts.gate_up_proj": "model-00005-of-00026.safetensors",
|
| 184 |
+
"model.language_model.layers.6.mlp.gate.weight": "model-00005-of-00026.safetensors",
|
| 185 |
+
"model.language_model.layers.6.mlp.shared_expert.down_proj.weight": "model-00005-of-00026.safetensors",
|
| 186 |
+
"model.language_model.layers.6.mlp.shared_expert.gate_proj.weight": "model-00005-of-00026.safetensors",
|
| 187 |
+
"model.language_model.layers.6.mlp.shared_expert.up_proj.weight": "model-00005-of-00026.safetensors",
|
| 188 |
+
"model.language_model.layers.6.mlp.shared_expert_gate.weight": "model-00005-of-00026.safetensors",
|
| 189 |
+
"model.language_model.layers.6.post_attention_layernorm.weight": "model-00005-of-00026.safetensors",
|
| 190 |
+
"model.language_model.layers.7.input_layernorm.weight": "model-00005-of-00026.safetensors",
|
| 191 |
+
"model.language_model.layers.7.mlp.experts.down_proj": "model-00005-of-00026.safetensors",
|
| 192 |
+
"model.language_model.layers.7.mlp.experts.gate_up_proj": "model-00005-of-00026.safetensors",
|
| 193 |
+
"model.language_model.layers.7.mlp.gate.weight": "model-00005-of-00026.safetensors",
|
| 194 |
+
"model.language_model.layers.7.mlp.shared_expert.down_proj.weight": "model-00005-of-00026.safetensors",
|
| 195 |
+
"model.language_model.layers.7.mlp.shared_expert.gate_proj.weight": "model-00005-of-00026.safetensors",
|
| 196 |
+
"model.language_model.layers.7.mlp.shared_expert.up_proj.weight": "model-00005-of-00026.safetensors",
|
| 197 |
+
"model.language_model.layers.7.mlp.shared_expert_gate.weight": "model-00005-of-00026.safetensors",
|
| 198 |
+
"model.language_model.layers.7.post_attention_layernorm.weight": "model-00005-of-00026.safetensors",
|
| 199 |
+
"model.language_model.layers.7.self_attn.k_norm.weight": "model-00005-of-00026.safetensors",
|
| 200 |
+
"model.language_model.layers.7.self_attn.k_proj.weight": "model-00005-of-00026.safetensors",
|
| 201 |
+
"model.language_model.layers.7.self_attn.o_proj.weight": "model-00005-of-00026.safetensors",
|
| 202 |
+
"model.language_model.layers.7.self_attn.q_norm.weight": "model-00005-of-00026.safetensors",
|
| 203 |
+
"model.language_model.layers.7.self_attn.q_proj.weight": "model-00005-of-00026.safetensors",
|
| 204 |
+
"model.language_model.layers.7.self_attn.v_proj.weight": "model-00005-of-00026.safetensors",
|
| 205 |
+
"model.language_model.layers.10.linear_attn.in_proj_qkv.weight": "model-00006-of-00026.safetensors",
|
| 206 |
+
"model.language_model.layers.10.linear_attn.in_proj_z.weight": "model-00006-of-00026.safetensors",
|
| 207 |
+
"model.language_model.layers.10.linear_attn.out_proj.weight": "model-00006-of-00026.safetensors",
|
| 208 |
+
"model.language_model.layers.10.mlp.experts.down_proj": "model-00006-of-00026.safetensors",
|
| 209 |
+
"model.language_model.layers.8.linear_attn.in_proj_qkv.weight": "model-00006-of-00026.safetensors",
|
| 210 |
+
"model.language_model.layers.8.linear_attn.in_proj_z.weight": "model-00006-of-00026.safetensors",
|
| 211 |
+
"model.language_model.layers.8.linear_attn.out_proj.weight": "model-00006-of-00026.safetensors",
|
| 212 |
+
"model.language_model.layers.8.mlp.experts.down_proj": "model-00006-of-00026.safetensors",
|
| 213 |
+
"model.language_model.layers.8.mlp.experts.gate_up_proj": "model-00006-of-00026.safetensors",
|
| 214 |
+
"model.language_model.layers.9.linear_attn.in_proj_qkv.weight": "model-00006-of-00026.safetensors",
|
| 215 |
+
"model.language_model.layers.9.linear_attn.in_proj_z.weight": "model-00006-of-00026.safetensors",
|
| 216 |
+
"model.language_model.layers.9.linear_attn.out_proj.weight": "model-00006-of-00026.safetensors",
|
| 217 |
+
"model.language_model.layers.9.mlp.experts.down_proj": "model-00006-of-00026.safetensors",
|
| 218 |
+
"model.language_model.layers.9.mlp.experts.gate_up_proj": "model-00006-of-00026.safetensors",
|
| 219 |
+
"model.language_model.layers.10.input_layernorm.weight": "model-00007-of-00026.safetensors",
|
| 220 |
+
"model.language_model.layers.10.linear_attn.A_log": "model-00007-of-00026.safetensors",
|
| 221 |
+
"model.language_model.layers.10.linear_attn.conv1d.weight": "model-00007-of-00026.safetensors",
|
| 222 |
+
"model.language_model.layers.10.linear_attn.dt_bias": "model-00007-of-00026.safetensors",
|
| 223 |
+
"model.language_model.layers.10.linear_attn.in_proj_a.weight": "model-00007-of-00026.safetensors",
|
| 224 |
+
"model.language_model.layers.10.linear_attn.in_proj_b.weight": "model-00007-of-00026.safetensors",
|
| 225 |
+
"model.language_model.layers.10.linear_attn.norm.weight": "model-00007-of-00026.safetensors",
|
| 226 |
+
"model.language_model.layers.10.mlp.experts.gate_up_proj": "model-00007-of-00026.safetensors",
|
| 227 |
+
"model.language_model.layers.10.mlp.gate.weight": "model-00007-of-00026.safetensors",
|
| 228 |
+
"model.language_model.layers.10.mlp.shared_expert.down_proj.weight": "model-00007-of-00026.safetensors",
|
| 229 |
+
"model.language_model.layers.10.mlp.shared_expert.gate_proj.weight": "model-00007-of-00026.safetensors",
|
| 230 |
+
"model.language_model.layers.10.mlp.shared_expert.up_proj.weight": "model-00007-of-00026.safetensors",
|
| 231 |
+
"model.language_model.layers.10.mlp.shared_expert_gate.weight": "model-00007-of-00026.safetensors",
|
| 232 |
+
"model.language_model.layers.10.post_attention_layernorm.weight": "model-00007-of-00026.safetensors",
|
| 233 |
+
"model.language_model.layers.8.input_layernorm.weight": "model-00007-of-00026.safetensors",
|
| 234 |
+
"model.language_model.layers.8.linear_attn.A_log": "model-00007-of-00026.safetensors",
|
| 235 |
+
"model.language_model.layers.8.linear_attn.conv1d.weight": "model-00007-of-00026.safetensors",
|
| 236 |
+
"model.language_model.layers.8.linear_attn.dt_bias": "model-00007-of-00026.safetensors",
|
| 237 |
+
"model.language_model.layers.8.linear_attn.in_proj_a.weight": "model-00007-of-00026.safetensors",
|
| 238 |
+
"model.language_model.layers.8.linear_attn.in_proj_b.weight": "model-00007-of-00026.safetensors",
|
| 239 |
+
"model.language_model.layers.8.linear_attn.norm.weight": "model-00007-of-00026.safetensors",
|
| 240 |
+
"model.language_model.layers.8.mlp.gate.weight": "model-00007-of-00026.safetensors",
|
| 241 |
+
"model.language_model.layers.8.mlp.shared_expert.down_proj.weight": "model-00007-of-00026.safetensors",
|
| 242 |
+
"model.language_model.layers.8.mlp.shared_expert.gate_proj.weight": "model-00007-of-00026.safetensors",
|
| 243 |
+
"model.language_model.layers.8.mlp.shared_expert.up_proj.weight": "model-00007-of-00026.safetensors",
|
| 244 |
+
"model.language_model.layers.8.mlp.shared_expert_gate.weight": "model-00007-of-00026.safetensors",
|
| 245 |
+
"model.language_model.layers.8.post_attention_layernorm.weight": "model-00007-of-00026.safetensors",
|
| 246 |
+
"model.language_model.layers.9.input_layernorm.weight": "model-00007-of-00026.safetensors",
|
| 247 |
+
"model.language_model.layers.9.linear_attn.A_log": "model-00007-of-00026.safetensors",
|
| 248 |
+
"model.language_model.layers.9.linear_attn.conv1d.weight": "model-00007-of-00026.safetensors",
|
| 249 |
+
"model.language_model.layers.9.linear_attn.dt_bias": "model-00007-of-00026.safetensors",
|
| 250 |
+
"model.language_model.layers.9.linear_attn.in_proj_a.weight": "model-00007-of-00026.safetensors",
|
| 251 |
+
"model.language_model.layers.9.linear_attn.in_proj_b.weight": "model-00007-of-00026.safetensors",
|
| 252 |
+
"model.language_model.layers.9.linear_attn.norm.weight": "model-00007-of-00026.safetensors",
|
| 253 |
+
"model.language_model.layers.9.mlp.gate.weight": "model-00007-of-00026.safetensors",
|
| 254 |
+
"model.language_model.layers.9.mlp.shared_expert.down_proj.weight": "model-00007-of-00026.safetensors",
|
| 255 |
+
"model.language_model.layers.9.mlp.shared_expert.gate_proj.weight": "model-00007-of-00026.safetensors",
|
| 256 |
+
"model.language_model.layers.9.mlp.shared_expert.up_proj.weight": "model-00007-of-00026.safetensors",
|
| 257 |
+
"model.language_model.layers.9.mlp.shared_expert_gate.weight": "model-00007-of-00026.safetensors",
|
| 258 |
+
"model.language_model.layers.9.post_attention_layernorm.weight": "model-00007-of-00026.safetensors",
|
| 259 |
+
"model.language_model.layers.11.mlp.experts.gate_up_proj": "model-00008-of-00026.safetensors",
|
| 260 |
+
"model.language_model.layers.11.self_attn.k_proj.weight": "model-00008-of-00026.safetensors",
|
| 261 |
+
"model.language_model.layers.11.self_attn.o_proj.weight": "model-00008-of-00026.safetensors",
|
| 262 |
+
"model.language_model.layers.11.self_attn.q_proj.weight": "model-00008-of-00026.safetensors",
|
| 263 |
+
"model.language_model.layers.11.self_attn.v_proj.weight": "model-00008-of-00026.safetensors",
|
| 264 |
+
"model.language_model.layers.12.linear_attn.in_proj_qkv.weight": "model-00008-of-00026.safetensors",
|
| 265 |
+
"model.language_model.layers.12.linear_attn.in_proj_z.weight": "model-00008-of-00026.safetensors",
|
| 266 |
+
"model.language_model.layers.12.linear_attn.out_proj.weight": "model-00008-of-00026.safetensors",
|
| 267 |
+
"model.language_model.layers.12.mlp.experts.gate_up_proj": "model-00008-of-00026.safetensors",
|
| 268 |
+
"model.language_model.layers.13.linear_attn.in_proj_qkv.weight": "model-00008-of-00026.safetensors",
|
| 269 |
+
"model.language_model.layers.13.linear_attn.in_proj_z.weight": "model-00008-of-00026.safetensors",
|
| 270 |
+
"model.language_model.layers.13.linear_attn.out_proj.weight": "model-00008-of-00026.safetensors",
|
| 271 |
+
"model.language_model.layers.13.mlp.experts.down_proj": "model-00008-of-00026.safetensors",
|
| 272 |
+
"model.language_model.layers.13.mlp.experts.gate_up_proj": "model-00008-of-00026.safetensors",
|
| 273 |
+
"model.language_model.layers.11.input_layernorm.weight": "model-00009-of-00026.safetensors",
|
| 274 |
+
"model.language_model.layers.11.mlp.experts.down_proj": "model-00009-of-00026.safetensors",
|
| 275 |
+
"model.language_model.layers.11.mlp.gate.weight": "model-00009-of-00026.safetensors",
|
| 276 |
+
"model.language_model.layers.11.mlp.shared_expert.down_proj.weight": "model-00009-of-00026.safetensors",
|
| 277 |
+
"model.language_model.layers.11.mlp.shared_expert.gate_proj.weight": "model-00009-of-00026.safetensors",
|
| 278 |
+
"model.language_model.layers.11.mlp.shared_expert.up_proj.weight": "model-00009-of-00026.safetensors",
|
| 279 |
+
"model.language_model.layers.11.mlp.shared_expert_gate.weight": "model-00009-of-00026.safetensors",
|
| 280 |
+
"model.language_model.layers.11.post_attention_layernorm.weight": "model-00009-of-00026.safetensors",
|
| 281 |
+
"model.language_model.layers.11.self_attn.k_norm.weight": "model-00009-of-00026.safetensors",
|
| 282 |
+
"model.language_model.layers.11.self_attn.q_norm.weight": "model-00009-of-00026.safetensors",
|
| 283 |
+
"model.language_model.layers.12.input_layernorm.weight": "model-00009-of-00026.safetensors",
|
| 284 |
+
"model.language_model.layers.12.linear_attn.A_log": "model-00009-of-00026.safetensors",
|
| 285 |
+
"model.language_model.layers.12.linear_attn.conv1d.weight": "model-00009-of-00026.safetensors",
|
| 286 |
+
"model.language_model.layers.12.linear_attn.dt_bias": "model-00009-of-00026.safetensors",
|
| 287 |
+
"model.language_model.layers.12.linear_attn.in_proj_a.weight": "model-00009-of-00026.safetensors",
|
| 288 |
+
"model.language_model.layers.12.linear_attn.in_proj_b.weight": "model-00009-of-00026.safetensors",
|
| 289 |
+
"model.language_model.layers.12.linear_attn.norm.weight": "model-00009-of-00026.safetensors",
|
| 290 |
+
"model.language_model.layers.12.mlp.experts.down_proj": "model-00009-of-00026.safetensors",
|
| 291 |
+
"model.language_model.layers.12.mlp.gate.weight": "model-00009-of-00026.safetensors",
|
| 292 |
+
"model.language_model.layers.12.mlp.shared_expert.down_proj.weight": "model-00009-of-00026.safetensors",
|
| 293 |
+
"model.language_model.layers.12.mlp.shared_expert.gate_proj.weight": "model-00009-of-00026.safetensors",
|
| 294 |
+
"model.language_model.layers.12.mlp.shared_expert.up_proj.weight": "model-00009-of-00026.safetensors",
|
| 295 |
+
"model.language_model.layers.12.mlp.shared_expert_gate.weight": "model-00009-of-00026.safetensors",
|
| 296 |
+
"model.language_model.layers.12.post_attention_layernorm.weight": "model-00009-of-00026.safetensors",
|
| 297 |
+
"model.language_model.layers.13.input_layernorm.weight": "model-00009-of-00026.safetensors",
|
| 298 |
+
"model.language_model.layers.13.linear_attn.A_log": "model-00009-of-00026.safetensors",
|
| 299 |
+
"model.language_model.layers.13.linear_attn.conv1d.weight": "model-00009-of-00026.safetensors",
|
| 300 |
+
"model.language_model.layers.13.linear_attn.dt_bias": "model-00009-of-00026.safetensors",
|
| 301 |
+
"model.language_model.layers.13.linear_attn.in_proj_a.weight": "model-00009-of-00026.safetensors",
|
| 302 |
+
"model.language_model.layers.13.linear_attn.in_proj_b.weight": "model-00009-of-00026.safetensors",
|
| 303 |
+
"model.language_model.layers.13.linear_attn.norm.weight": "model-00009-of-00026.safetensors",
|
| 304 |
+
"model.language_model.layers.13.mlp.gate.weight": "model-00009-of-00026.safetensors",
|
| 305 |
+
"model.language_model.layers.13.mlp.shared_expert.down_proj.weight": "model-00009-of-00026.safetensors",
|
| 306 |
+
"model.language_model.layers.13.mlp.shared_expert.gate_proj.weight": "model-00009-of-00026.safetensors",
|
| 307 |
+
"model.language_model.layers.13.mlp.shared_expert.up_proj.weight": "model-00009-of-00026.safetensors",
|
| 308 |
+
"model.language_model.layers.13.mlp.shared_expert_gate.weight": "model-00009-of-00026.safetensors",
|
| 309 |
+
"model.language_model.layers.13.post_attention_layernorm.weight": "model-00009-of-00026.safetensors",
|
| 310 |
+
"model.language_model.layers.14.linear_attn.in_proj_qkv.weight": "model-00010-of-00026.safetensors",
|
| 311 |
+
"model.language_model.layers.14.linear_attn.in_proj_z.weight": "model-00010-of-00026.safetensors",
|
| 312 |
+
"model.language_model.layers.14.linear_attn.out_proj.weight": "model-00010-of-00026.safetensors",
|
| 313 |
+
"model.language_model.layers.14.mlp.experts.down_proj": "model-00010-of-00026.safetensors",
|
| 314 |
+
"model.language_model.layers.14.mlp.experts.gate_up_proj": "model-00010-of-00026.safetensors",
|
| 315 |
+
"model.language_model.layers.15.mlp.experts.gate_up_proj": "model-00010-of-00026.safetensors",
|
| 316 |
+
"model.language_model.layers.15.self_attn.k_proj.weight": "model-00010-of-00026.safetensors",
|
| 317 |
+
"model.language_model.layers.15.self_attn.o_proj.weight": "model-00010-of-00026.safetensors",
|
| 318 |
+
"model.language_model.layers.15.self_attn.q_proj.weight": "model-00010-of-00026.safetensors",
|
| 319 |
+
"model.language_model.layers.15.self_attn.v_proj.weight": "model-00010-of-00026.safetensors",
|
| 320 |
+
"model.language_model.layers.16.linear_attn.in_proj_qkv.weight": "model-00010-of-00026.safetensors",
|
| 321 |
+
"model.language_model.layers.16.linear_attn.in_proj_z.weight": "model-00010-of-00026.safetensors",
|
| 322 |
+
"model.language_model.layers.16.linear_attn.out_proj.weight": "model-00010-of-00026.safetensors",
|
| 323 |
+
"model.language_model.layers.16.mlp.experts.gate_up_proj": "model-00010-of-00026.safetensors",
|
| 324 |
+
"model.language_model.layers.14.input_layernorm.weight": "model-00011-of-00026.safetensors",
|
| 325 |
+
"model.language_model.layers.14.linear_attn.A_log": "model-00011-of-00026.safetensors",
|
| 326 |
+
"model.language_model.layers.14.linear_attn.conv1d.weight": "model-00011-of-00026.safetensors",
|
| 327 |
+
"model.language_model.layers.14.linear_attn.dt_bias": "model-00011-of-00026.safetensors",
|
| 328 |
+
"model.language_model.layers.14.linear_attn.in_proj_a.weight": "model-00011-of-00026.safetensors",
|
| 329 |
+
"model.language_model.layers.14.linear_attn.in_proj_b.weight": "model-00011-of-00026.safetensors",
|
| 330 |
+
"model.language_model.layers.14.linear_attn.norm.weight": "model-00011-of-00026.safetensors",
|
| 331 |
+
"model.language_model.layers.14.mlp.gate.weight": "model-00011-of-00026.safetensors",
|
| 332 |
+
"model.language_model.layers.14.mlp.shared_expert.down_proj.weight": "model-00011-of-00026.safetensors",
|
| 333 |
+
"model.language_model.layers.14.mlp.shared_expert.gate_proj.weight": "model-00011-of-00026.safetensors",
|
| 334 |
+
"model.language_model.layers.14.mlp.shared_expert.up_proj.weight": "model-00011-of-00026.safetensors",
|
| 335 |
+
"model.language_model.layers.14.mlp.shared_expert_gate.weight": "model-00011-of-00026.safetensors",
|
| 336 |
+
"model.language_model.layers.14.post_attention_layernorm.weight": "model-00011-of-00026.safetensors",
|
| 337 |
+
"model.language_model.layers.15.input_layernorm.weight": "model-00011-of-00026.safetensors",
|
| 338 |
+
"model.language_model.layers.15.mlp.experts.down_proj": "model-00011-of-00026.safetensors",
|
| 339 |
+
"model.language_model.layers.15.mlp.gate.weight": "model-00011-of-00026.safetensors",
|
| 340 |
+
"model.language_model.layers.15.mlp.shared_expert.down_proj.weight": "model-00011-of-00026.safetensors",
|
| 341 |
+
"model.language_model.layers.15.mlp.shared_expert.gate_proj.weight": "model-00011-of-00026.safetensors",
|
| 342 |
+
"model.language_model.layers.15.mlp.shared_expert.up_proj.weight": "model-00011-of-00026.safetensors",
|
| 343 |
+
"model.language_model.layers.15.mlp.shared_expert_gate.weight": "model-00011-of-00026.safetensors",
|
| 344 |
+
"model.language_model.layers.15.post_attention_layernorm.weight": "model-00011-of-00026.safetensors",
|
| 345 |
+
"model.language_model.layers.15.self_attn.k_norm.weight": "model-00011-of-00026.safetensors",
|
| 346 |
+
"model.language_model.layers.15.self_attn.q_norm.weight": "model-00011-of-00026.safetensors",
|
| 347 |
+
"model.language_model.layers.16.input_layernorm.weight": "model-00011-of-00026.safetensors",
|
| 348 |
+
"model.language_model.layers.16.linear_attn.A_log": "model-00011-of-00026.safetensors",
|
| 349 |
+
"model.language_model.layers.16.linear_attn.conv1d.weight": "model-00011-of-00026.safetensors",
|
| 350 |
+
"model.language_model.layers.16.linear_attn.dt_bias": "model-00011-of-00026.safetensors",
|
| 351 |
+
"model.language_model.layers.16.linear_attn.in_proj_a.weight": "model-00011-of-00026.safetensors",
|
| 352 |
+
"model.language_model.layers.16.linear_attn.in_proj_b.weight": "model-00011-of-00026.safetensors",
|
| 353 |
+
"model.language_model.layers.16.linear_attn.norm.weight": "model-00011-of-00026.safetensors",
|
| 354 |
+
"model.language_model.layers.16.mlp.experts.down_proj": "model-00011-of-00026.safetensors",
|
| 355 |
+
"model.language_model.layers.16.mlp.gate.weight": "model-00011-of-00026.safetensors",
|
| 356 |
+
"model.language_model.layers.16.mlp.shared_expert.down_proj.weight": "model-00011-of-00026.safetensors",
|
| 357 |
+
"model.language_model.layers.16.mlp.shared_expert.gate_proj.weight": "model-00011-of-00026.safetensors",
|
| 358 |
+
"model.language_model.layers.16.mlp.shared_expert.up_proj.weight": "model-00011-of-00026.safetensors",
|
| 359 |
+
"model.language_model.layers.16.mlp.shared_expert_gate.weight": "model-00011-of-00026.safetensors",
|
| 360 |
+
"model.language_model.layers.16.post_attention_layernorm.weight": "model-00011-of-00026.safetensors",
|
| 361 |
+
"model.language_model.layers.17.linear_attn.in_proj_qkv.weight": "model-00012-of-00026.safetensors",
|
| 362 |
+
"model.language_model.layers.17.linear_attn.in_proj_z.weight": "model-00012-of-00026.safetensors",
|
| 363 |
+
"model.language_model.layers.17.linear_attn.out_proj.weight": "model-00012-of-00026.safetensors",
|
| 364 |
+
"model.language_model.layers.17.mlp.experts.down_proj": "model-00012-of-00026.safetensors",
|
| 365 |
+
"model.language_model.layers.17.mlp.experts.gate_up_proj": "model-00012-of-00026.safetensors",
|
| 366 |
+
"model.language_model.layers.18.linear_attn.in_proj_qkv.weight": "model-00012-of-00026.safetensors",
|
| 367 |
+
"model.language_model.layers.18.linear_attn.in_proj_z.weight": "model-00012-of-00026.safetensors",
|
| 368 |
+
"model.language_model.layers.18.linear_attn.out_proj.weight": "model-00012-of-00026.safetensors",
|
| 369 |
+
"model.language_model.layers.18.mlp.experts.down_proj": "model-00012-of-00026.safetensors",
|
| 370 |
+
"model.language_model.layers.19.mlp.experts.gate_up_proj": "model-00012-of-00026.safetensors",
|
| 371 |
+
"model.language_model.layers.19.self_attn.k_proj.weight": "model-00012-of-00026.safetensors",
|
| 372 |
+
"model.language_model.layers.19.self_attn.o_proj.weight": "model-00012-of-00026.safetensors",
|
| 373 |
+
"model.language_model.layers.19.self_attn.q_proj.weight": "model-00012-of-00026.safetensors",
|
| 374 |
+
"model.language_model.layers.19.self_attn.v_proj.weight": "model-00012-of-00026.safetensors",
|
| 375 |
+
"model.language_model.layers.17.input_layernorm.weight": "model-00013-of-00026.safetensors",
|
| 376 |
+
"model.language_model.layers.17.linear_attn.A_log": "model-00013-of-00026.safetensors",
|
| 377 |
+
"model.language_model.layers.17.linear_attn.conv1d.weight": "model-00013-of-00026.safetensors",
|
| 378 |
+
"model.language_model.layers.17.linear_attn.dt_bias": "model-00013-of-00026.safetensors",
|
| 379 |
+
"model.language_model.layers.17.linear_attn.in_proj_a.weight": "model-00013-of-00026.safetensors",
|
| 380 |
+
"model.language_model.layers.17.linear_attn.in_proj_b.weight": "model-00013-of-00026.safetensors",
|
| 381 |
+
"model.language_model.layers.17.linear_attn.norm.weight": "model-00013-of-00026.safetensors",
|
| 382 |
+
"model.language_model.layers.17.mlp.gate.weight": "model-00013-of-00026.safetensors",
|
| 383 |
+
"model.language_model.layers.17.mlp.shared_expert.down_proj.weight": "model-00013-of-00026.safetensors",
|
| 384 |
+
"model.language_model.layers.17.mlp.shared_expert.gate_proj.weight": "model-00013-of-00026.safetensors",
|
| 385 |
+
"model.language_model.layers.17.mlp.shared_expert.up_proj.weight": "model-00013-of-00026.safetensors",
|
| 386 |
+
"model.language_model.layers.17.mlp.shared_expert_gate.weight": "model-00013-of-00026.safetensors",
|
| 387 |
+
"model.language_model.layers.17.post_attention_layernorm.weight": "model-00013-of-00026.safetensors",
|
| 388 |
+
"model.language_model.layers.18.input_layernorm.weight": "model-00013-of-00026.safetensors",
|
| 389 |
+
"model.language_model.layers.18.linear_attn.A_log": "model-00013-of-00026.safetensors",
|
| 390 |
+
"model.language_model.layers.18.linear_attn.conv1d.weight": "model-00013-of-00026.safetensors",
|
| 391 |
+
"model.language_model.layers.18.linear_attn.dt_bias": "model-00013-of-00026.safetensors",
|
| 392 |
+
"model.language_model.layers.18.linear_attn.in_proj_a.weight": "model-00013-of-00026.safetensors",
|
| 393 |
+
"model.language_model.layers.18.linear_attn.in_proj_b.weight": "model-00013-of-00026.safetensors",
|
| 394 |
+
"model.language_model.layers.18.linear_attn.norm.weight": "model-00013-of-00026.safetensors",
|
| 395 |
+
"model.language_model.layers.18.mlp.experts.gate_up_proj": "model-00013-of-00026.safetensors",
|
| 396 |
+
"model.language_model.layers.18.mlp.gate.weight": "model-00013-of-00026.safetensors",
|
| 397 |
+
"model.language_model.layers.18.mlp.shared_expert.down_proj.weight": "model-00013-of-00026.safetensors",
|
| 398 |
+
"model.language_model.layers.18.mlp.shared_expert.gate_proj.weight": "model-00013-of-00026.safetensors",
|
| 399 |
+
"model.language_model.layers.18.mlp.shared_expert.up_proj.weight": "model-00013-of-00026.safetensors",
|
| 400 |
+
"model.language_model.layers.18.mlp.shared_expert_gate.weight": "model-00013-of-00026.safetensors",
|
| 401 |
+
"model.language_model.layers.18.post_attention_layernorm.weight": "model-00013-of-00026.safetensors",
|
| 402 |
+
"model.language_model.layers.19.input_layernorm.weight": "model-00013-of-00026.safetensors",
|
| 403 |
+
"model.language_model.layers.19.mlp.experts.down_proj": "model-00013-of-00026.safetensors",
|
| 404 |
+
"model.language_model.layers.19.mlp.gate.weight": "model-00013-of-00026.safetensors",
|
| 405 |
+
"model.language_model.layers.19.mlp.shared_expert.down_proj.weight": "model-00013-of-00026.safetensors",
|
| 406 |
+
"model.language_model.layers.19.mlp.shared_expert.gate_proj.weight": "model-00013-of-00026.safetensors",
|
| 407 |
+
"model.language_model.layers.19.mlp.shared_expert.up_proj.weight": "model-00013-of-00026.safetensors",
|
| 408 |
+
"model.language_model.layers.19.mlp.shared_expert_gate.weight": "model-00013-of-00026.safetensors",
|
| 409 |
+
"model.language_model.layers.19.post_attention_layernorm.weight": "model-00013-of-00026.safetensors",
|
| 410 |
+
"model.language_model.layers.19.self_attn.k_norm.weight": "model-00013-of-00026.safetensors",
|
| 411 |
+
"model.language_model.layers.19.self_attn.q_norm.weight": "model-00013-of-00026.safetensors",
|
| 412 |
+
"model.language_model.layers.20.linear_attn.in_proj_qkv.weight": "model-00014-of-00026.safetensors",
|
| 413 |
+
"model.language_model.layers.20.linear_attn.in_proj_z.weight": "model-00014-of-00026.safetensors",
|
| 414 |
+
"model.language_model.layers.20.linear_attn.out_proj.weight": "model-00014-of-00026.safetensors",
|
| 415 |
+
"model.language_model.layers.20.mlp.experts.down_proj": "model-00014-of-00026.safetensors",
|
| 416 |
+
"model.language_model.layers.20.mlp.experts.gate_up_proj": "model-00014-of-00026.safetensors",
|
| 417 |
+
"model.language_model.layers.21.linear_attn.in_proj_qkv.weight": "model-00014-of-00026.safetensors",
|
| 418 |
+
"model.language_model.layers.21.linear_attn.in_proj_z.weight": "model-00014-of-00026.safetensors",
|
| 419 |
+
"model.language_model.layers.21.linear_attn.out_proj.weight": "model-00014-of-00026.safetensors",
|
| 420 |
+
"model.language_model.layers.21.mlp.experts.down_proj": "model-00014-of-00026.safetensors",
|
| 421 |
+
"model.language_model.layers.22.linear_attn.in_proj_qkv.weight": "model-00014-of-00026.safetensors",
|
| 422 |
+
"model.language_model.layers.22.linear_attn.in_proj_z.weight": "model-00014-of-00026.safetensors",
|
| 423 |
+
"model.language_model.layers.22.linear_attn.out_proj.weight": "model-00014-of-00026.safetensors",
|
| 424 |
+
"model.language_model.layers.22.mlp.experts.gate_up_proj": "model-00014-of-00026.safetensors",
|
| 425 |
+
"model.language_model.layers.20.input_layernorm.weight": "model-00015-of-00026.safetensors",
|
| 426 |
+
"model.language_model.layers.20.linear_attn.A_log": "model-00015-of-00026.safetensors",
|
| 427 |
+
"model.language_model.layers.20.linear_attn.conv1d.weight": "model-00015-of-00026.safetensors",
|
| 428 |
+
"model.language_model.layers.20.linear_attn.dt_bias": "model-00015-of-00026.safetensors",
|
| 429 |
+
"model.language_model.layers.20.linear_attn.in_proj_a.weight": "model-00015-of-00026.safetensors",
|
| 430 |
+
"model.language_model.layers.20.linear_attn.in_proj_b.weight": "model-00015-of-00026.safetensors",
|
| 431 |
+
"model.language_model.layers.20.linear_attn.norm.weight": "model-00015-of-00026.safetensors",
|
| 432 |
+
"model.language_model.layers.20.mlp.gate.weight": "model-00015-of-00026.safetensors",
|
| 433 |
+
"model.language_model.layers.20.mlp.shared_expert.down_proj.weight": "model-00015-of-00026.safetensors",
|
| 434 |
+
"model.language_model.layers.20.mlp.shared_expert.gate_proj.weight": "model-00015-of-00026.safetensors",
|
| 435 |
+
"model.language_model.layers.20.mlp.shared_expert.up_proj.weight": "model-00015-of-00026.safetensors",
|
| 436 |
+
"model.language_model.layers.20.mlp.shared_expert_gate.weight": "model-00015-of-00026.safetensors",
|
| 437 |
+
"model.language_model.layers.20.post_attention_layernorm.weight": "model-00015-of-00026.safetensors",
|
| 438 |
+
"model.language_model.layers.21.input_layernorm.weight": "model-00015-of-00026.safetensors",
|
| 439 |
+
"model.language_model.layers.21.linear_attn.A_log": "model-00015-of-00026.safetensors",
|
| 440 |
+
"model.language_model.layers.21.linear_attn.conv1d.weight": "model-00015-of-00026.safetensors",
|
| 441 |
+
"model.language_model.layers.21.linear_attn.dt_bias": "model-00015-of-00026.safetensors",
|
| 442 |
+
"model.language_model.layers.21.linear_attn.in_proj_a.weight": "model-00015-of-00026.safetensors",
|
| 443 |
+
"model.language_model.layers.21.linear_attn.in_proj_b.weight": "model-00015-of-00026.safetensors",
|
| 444 |
+
"model.language_model.layers.21.linear_attn.norm.weight": "model-00015-of-00026.safetensors",
|
| 445 |
+
"model.language_model.layers.21.mlp.experts.gate_up_proj": "model-00015-of-00026.safetensors",
|
| 446 |
+
"model.language_model.layers.21.mlp.gate.weight": "model-00015-of-00026.safetensors",
|
| 447 |
+
"model.language_model.layers.21.mlp.shared_expert.down_proj.weight": "model-00015-of-00026.safetensors",
|
| 448 |
+
"model.language_model.layers.21.mlp.shared_expert.gate_proj.weight": "model-00015-of-00026.safetensors",
|
| 449 |
+
"model.language_model.layers.21.mlp.shared_expert.up_proj.weight": "model-00015-of-00026.safetensors",
|
| 450 |
+
"model.language_model.layers.21.mlp.shared_expert_gate.weight": "model-00015-of-00026.safetensors",
|
| 451 |
+
"model.language_model.layers.21.post_attention_layernorm.weight": "model-00015-of-00026.safetensors",
|
| 452 |
+
"model.language_model.layers.22.input_layernorm.weight": "model-00015-of-00026.safetensors",
|
| 453 |
+
"model.language_model.layers.22.linear_attn.A_log": "model-00015-of-00026.safetensors",
|
| 454 |
+
"model.language_model.layers.22.linear_attn.conv1d.weight": "model-00015-of-00026.safetensors",
|
| 455 |
+
"model.language_model.layers.22.linear_attn.dt_bias": "model-00015-of-00026.safetensors",
|
| 456 |
+
"model.language_model.layers.22.linear_attn.in_proj_a.weight": "model-00015-of-00026.safetensors",
|
| 457 |
+
"model.language_model.layers.22.linear_attn.in_proj_b.weight": "model-00015-of-00026.safetensors",
|
| 458 |
+
"model.language_model.layers.22.linear_attn.norm.weight": "model-00015-of-00026.safetensors",
|
| 459 |
+
"model.language_model.layers.22.mlp.experts.down_proj": "model-00015-of-00026.safetensors",
|
| 460 |
+
"model.language_model.layers.22.mlp.gate.weight": "model-00015-of-00026.safetensors",
|
| 461 |
+
"model.language_model.layers.22.mlp.shared_expert.down_proj.weight": "model-00015-of-00026.safetensors",
|
| 462 |
+
"model.language_model.layers.22.mlp.shared_expert.gate_proj.weight": "model-00015-of-00026.safetensors",
|
| 463 |
+
"model.language_model.layers.22.mlp.shared_expert.up_proj.weight": "model-00015-of-00026.safetensors",
|
| 464 |
+
"model.language_model.layers.22.mlp.shared_expert_gate.weight": "model-00015-of-00026.safetensors",
|
| 465 |
+
"model.language_model.layers.22.post_attention_layernorm.weight": "model-00015-of-00026.safetensors",
|
| 466 |
+
"model.language_model.layers.23.mlp.experts.down_proj": "model-00016-of-00026.safetensors",
|
| 467 |
+
"model.language_model.layers.23.mlp.experts.gate_up_proj": "model-00016-of-00026.safetensors",
|
| 468 |
+
"model.language_model.layers.23.self_attn.k_proj.weight": "model-00016-of-00026.safetensors",
|
| 469 |
+
"model.language_model.layers.23.self_attn.o_proj.weight": "model-00016-of-00026.safetensors",
|
| 470 |
+
"model.language_model.layers.23.self_attn.q_proj.weight": "model-00016-of-00026.safetensors",
|
| 471 |
+
"model.language_model.layers.23.self_attn.v_proj.weight": "model-00016-of-00026.safetensors",
|
| 472 |
+
"model.language_model.layers.24.linear_attn.in_proj_qkv.weight": "model-00016-of-00026.safetensors",
|
| 473 |
+
"model.language_model.layers.24.linear_attn.in_proj_z.weight": "model-00016-of-00026.safetensors",
|
| 474 |
+
"model.language_model.layers.24.linear_attn.out_proj.weight": "model-00016-of-00026.safetensors",
|
| 475 |
+
"model.language_model.layers.24.mlp.experts.down_proj": "model-00016-of-00026.safetensors",
|
| 476 |
+
"model.language_model.layers.24.mlp.experts.gate_up_proj": "model-00016-of-00026.safetensors",
|
| 477 |
+
"model.language_model.layers.25.linear_attn.in_proj_qkv.weight": "model-00016-of-00026.safetensors",
|
| 478 |
+
"model.language_model.layers.25.linear_attn.in_proj_z.weight": "model-00016-of-00026.safetensors",
|
| 479 |
+
"model.language_model.layers.25.linear_attn.out_proj.weight": "model-00016-of-00026.safetensors",
|
| 480 |
+
"model.language_model.layers.25.mlp.experts.down_proj": "model-00016-of-00026.safetensors",
|
| 481 |
+
"model.language_model.layers.23.input_layernorm.weight": "model-00017-of-00026.safetensors",
|
| 482 |
+
"model.language_model.layers.23.mlp.gate.weight": "model-00017-of-00026.safetensors",
|
| 483 |
+
"model.language_model.layers.23.mlp.shared_expert.down_proj.weight": "model-00017-of-00026.safetensors",
|
| 484 |
+
"model.language_model.layers.23.mlp.shared_expert.gate_proj.weight": "model-00017-of-00026.safetensors",
|
| 485 |
+
"model.language_model.layers.23.mlp.shared_expert.up_proj.weight": "model-00017-of-00026.safetensors",
|
| 486 |
+
"model.language_model.layers.23.mlp.shared_expert_gate.weight": "model-00017-of-00026.safetensors",
|
| 487 |
+
"model.language_model.layers.23.post_attention_layernorm.weight": "model-00017-of-00026.safetensors",
|
| 488 |
+
"model.language_model.layers.23.self_attn.k_norm.weight": "model-00017-of-00026.safetensors",
|
| 489 |
+
"model.language_model.layers.23.self_attn.q_norm.weight": "model-00017-of-00026.safetensors",
|
| 490 |
+
"model.language_model.layers.24.input_layernorm.weight": "model-00017-of-00026.safetensors",
|
| 491 |
+
"model.language_model.layers.24.linear_attn.A_log": "model-00017-of-00026.safetensors",
|
| 492 |
+
"model.language_model.layers.24.linear_attn.conv1d.weight": "model-00017-of-00026.safetensors",
|
| 493 |
+
"model.language_model.layers.24.linear_attn.dt_bias": "model-00017-of-00026.safetensors",
|
| 494 |
+
"model.language_model.layers.24.linear_attn.in_proj_a.weight": "model-00017-of-00026.safetensors",
|
| 495 |
+
"model.language_model.layers.24.linear_attn.in_proj_b.weight": "model-00017-of-00026.safetensors",
|
| 496 |
+
"model.language_model.layers.24.linear_attn.norm.weight": "model-00017-of-00026.safetensors",
|
| 497 |
+
"model.language_model.layers.24.mlp.gate.weight": "model-00017-of-00026.safetensors",
|
| 498 |
+
"model.language_model.layers.24.mlp.shared_expert.down_proj.weight": "model-00017-of-00026.safetensors",
|
| 499 |
+
"model.language_model.layers.24.mlp.shared_expert.gate_proj.weight": "model-00017-of-00026.safetensors",
|
| 500 |
+
"model.language_model.layers.24.mlp.shared_expert.up_proj.weight": "model-00017-of-00026.safetensors",
|
| 501 |
+
"model.language_model.layers.24.mlp.shared_expert_gate.weight": "model-00017-of-00026.safetensors",
|
| 502 |
+
"model.language_model.layers.24.post_attention_layernorm.weight": "model-00017-of-00026.safetensors",
|
| 503 |
+
"model.language_model.layers.25.input_layernorm.weight": "model-00017-of-00026.safetensors",
|
| 504 |
+
"model.language_model.layers.25.linear_attn.A_log": "model-00017-of-00026.safetensors",
|
| 505 |
+
"model.language_model.layers.25.linear_attn.conv1d.weight": "model-00017-of-00026.safetensors",
|
| 506 |
+
"model.language_model.layers.25.linear_attn.dt_bias": "model-00017-of-00026.safetensors",
|
| 507 |
+
"model.language_model.layers.25.linear_attn.in_proj_a.weight": "model-00017-of-00026.safetensors",
|
| 508 |
+
"model.language_model.layers.25.linear_attn.in_proj_b.weight": "model-00017-of-00026.safetensors",
|
| 509 |
+
"model.language_model.layers.25.linear_attn.norm.weight": "model-00017-of-00026.safetensors",
|
| 510 |
+
"model.language_model.layers.25.mlp.experts.gate_up_proj": "model-00017-of-00026.safetensors",
|
| 511 |
+
"model.language_model.layers.25.mlp.gate.weight": "model-00017-of-00026.safetensors",
|
| 512 |
+
"model.language_model.layers.25.mlp.shared_expert.down_proj.weight": "model-00017-of-00026.safetensors",
|
| 513 |
+
"model.language_model.layers.25.mlp.shared_expert.gate_proj.weight": "model-00017-of-00026.safetensors",
|
| 514 |
+
"model.language_model.layers.25.mlp.shared_expert.up_proj.weight": "model-00017-of-00026.safetensors",
|
| 515 |
+
"model.language_model.layers.25.mlp.shared_expert_gate.weight": "model-00017-of-00026.safetensors",
|
| 516 |
+
"model.language_model.layers.25.post_attention_layernorm.weight": "model-00017-of-00026.safetensors",
|
| 517 |
+
"model.language_model.layers.26.linear_attn.in_proj_qkv.weight": "model-00018-of-00026.safetensors",
|
| 518 |
+
"model.language_model.layers.26.linear_attn.in_proj_z.weight": "model-00018-of-00026.safetensors",
|
| 519 |
+
"model.language_model.layers.26.linear_attn.out_proj.weight": "model-00018-of-00026.safetensors",
|
| 520 |
+
"model.language_model.layers.26.mlp.experts.gate_up_proj": "model-00018-of-00026.safetensors",
|
| 521 |
+
"model.language_model.layers.27.mlp.experts.gate_up_proj": "model-00018-of-00026.safetensors",
|
| 522 |
+
"model.language_model.layers.27.self_attn.k_proj.weight": "model-00018-of-00026.safetensors",
|
| 523 |
+
"model.language_model.layers.27.self_attn.o_proj.weight": "model-00018-of-00026.safetensors",
|
| 524 |
+
"model.language_model.layers.27.self_attn.q_proj.weight": "model-00018-of-00026.safetensors",
|
| 525 |
+
"model.language_model.layers.27.self_attn.v_proj.weight": "model-00018-of-00026.safetensors",
|
| 526 |
+
"model.language_model.layers.28.linear_attn.in_proj_qkv.weight": "model-00018-of-00026.safetensors",
|
| 527 |
+
"model.language_model.layers.28.linear_attn.in_proj_z.weight": "model-00018-of-00026.safetensors",
|
| 528 |
+
"model.language_model.layers.28.linear_attn.out_proj.weight": "model-00018-of-00026.safetensors",
|
| 529 |
+
"model.language_model.layers.28.mlp.experts.down_proj": "model-00018-of-00026.safetensors",
|
| 530 |
+
"model.language_model.layers.28.mlp.experts.gate_up_proj": "model-00018-of-00026.safetensors",
|
| 531 |
+
"model.language_model.layers.26.input_layernorm.weight": "model-00019-of-00026.safetensors",
|
| 532 |
+
"model.language_model.layers.26.linear_attn.A_log": "model-00019-of-00026.safetensors",
|
| 533 |
+
"model.language_model.layers.26.linear_attn.conv1d.weight": "model-00019-of-00026.safetensors",
|
| 534 |
+
"model.language_model.layers.26.linear_attn.dt_bias": "model-00019-of-00026.safetensors",
|
| 535 |
+
"model.language_model.layers.26.linear_attn.in_proj_a.weight": "model-00019-of-00026.safetensors",
|
| 536 |
+
"model.language_model.layers.26.linear_attn.in_proj_b.weight": "model-00019-of-00026.safetensors",
|
| 537 |
+
"model.language_model.layers.26.linear_attn.norm.weight": "model-00019-of-00026.safetensors",
|
| 538 |
+
"model.language_model.layers.26.mlp.experts.down_proj": "model-00019-of-00026.safetensors",
|
| 539 |
+
"model.language_model.layers.26.mlp.gate.weight": "model-00019-of-00026.safetensors",
|
| 540 |
+
"model.language_model.layers.26.mlp.shared_expert.down_proj.weight": "model-00019-of-00026.safetensors",
|
| 541 |
+
"model.language_model.layers.26.mlp.shared_expert.gate_proj.weight": "model-00019-of-00026.safetensors",
|
| 542 |
+
"model.language_model.layers.26.mlp.shared_expert.up_proj.weight": "model-00019-of-00026.safetensors",
|
| 543 |
+
"model.language_model.layers.26.mlp.shared_expert_gate.weight": "model-00019-of-00026.safetensors",
|
| 544 |
+
"model.language_model.layers.26.post_attention_layernorm.weight": "model-00019-of-00026.safetensors",
|
| 545 |
+
"model.language_model.layers.27.input_layernorm.weight": "model-00019-of-00026.safetensors",
|
| 546 |
+
"model.language_model.layers.27.mlp.experts.down_proj": "model-00019-of-00026.safetensors",
|
| 547 |
+
"model.language_model.layers.27.mlp.gate.weight": "model-00019-of-00026.safetensors",
|
| 548 |
+
"model.language_model.layers.27.mlp.shared_expert.down_proj.weight": "model-00019-of-00026.safetensors",
|
| 549 |
+
"model.language_model.layers.27.mlp.shared_expert.gate_proj.weight": "model-00019-of-00026.safetensors",
|
| 550 |
+
"model.language_model.layers.27.mlp.shared_expert.up_proj.weight": "model-00019-of-00026.safetensors",
|
| 551 |
+
"model.language_model.layers.27.mlp.shared_expert_gate.weight": "model-00019-of-00026.safetensors",
|
| 552 |
+
"model.language_model.layers.27.post_attention_layernorm.weight": "model-00019-of-00026.safetensors",
|
| 553 |
+
"model.language_model.layers.27.self_attn.k_norm.weight": "model-00019-of-00026.safetensors",
|
| 554 |
+
"model.language_model.layers.27.self_attn.q_norm.weight": "model-00019-of-00026.safetensors",
|
| 555 |
+
"model.language_model.layers.28.input_layernorm.weight": "model-00019-of-00026.safetensors",
|
| 556 |
+
"model.language_model.layers.28.linear_attn.A_log": "model-00019-of-00026.safetensors",
|
| 557 |
+
"model.language_model.layers.28.linear_attn.conv1d.weight": "model-00019-of-00026.safetensors",
|
| 558 |
+
"model.language_model.layers.28.linear_attn.dt_bias": "model-00019-of-00026.safetensors",
|
| 559 |
+
"model.language_model.layers.28.linear_attn.in_proj_a.weight": "model-00019-of-00026.safetensors",
|
| 560 |
+
"model.language_model.layers.28.linear_attn.in_proj_b.weight": "model-00019-of-00026.safetensors",
|
| 561 |
+
"model.language_model.layers.28.linear_attn.norm.weight": "model-00019-of-00026.safetensors",
|
| 562 |
+
"model.language_model.layers.28.mlp.gate.weight": "model-00019-of-00026.safetensors",
|
| 563 |
+
"model.language_model.layers.28.mlp.shared_expert.down_proj.weight": "model-00019-of-00026.safetensors",
|
| 564 |
+
"model.language_model.layers.28.mlp.shared_expert.gate_proj.weight": "model-00019-of-00026.safetensors",
|
| 565 |
+
"model.language_model.layers.28.mlp.shared_expert.up_proj.weight": "model-00019-of-00026.safetensors",
|
| 566 |
+
"model.language_model.layers.28.mlp.shared_expert_gate.weight": "model-00019-of-00026.safetensors",
|
| 567 |
+
"model.language_model.layers.28.post_attention_layernorm.weight": "model-00019-of-00026.safetensors",
|
| 568 |
+
"model.language_model.layers.29.linear_attn.in_proj_qkv.weight": "model-00020-of-00026.safetensors",
|
| 569 |
+
"model.language_model.layers.29.linear_attn.in_proj_z.weight": "model-00020-of-00026.safetensors",
|
| 570 |
+
"model.language_model.layers.29.linear_attn.out_proj.weight": "model-00020-of-00026.safetensors",
|
| 571 |
+
"model.language_model.layers.29.mlp.experts.down_proj": "model-00020-of-00026.safetensors",
|
| 572 |
+
"model.language_model.layers.30.linear_attn.in_proj_qkv.weight": "model-00020-of-00026.safetensors",
|
| 573 |
+
"model.language_model.layers.30.linear_attn.in_proj_z.weight": "model-00020-of-00026.safetensors",
|
| 574 |
+
"model.language_model.layers.30.linear_attn.out_proj.weight": "model-00020-of-00026.safetensors",
|
| 575 |
+
"model.language_model.layers.30.mlp.experts.gate_up_proj": "model-00020-of-00026.safetensors",
|
| 576 |
+
"model.language_model.layers.31.mlp.experts.down_proj": "model-00020-of-00026.safetensors",
|
| 577 |
+
"model.language_model.layers.31.mlp.experts.gate_up_proj": "model-00020-of-00026.safetensors",
|
| 578 |
+
"model.language_model.layers.31.self_attn.k_proj.weight": "model-00020-of-00026.safetensors",
|
| 579 |
+
"model.language_model.layers.31.self_attn.o_proj.weight": "model-00020-of-00026.safetensors",
|
| 580 |
+
"model.language_model.layers.31.self_attn.q_proj.weight": "model-00020-of-00026.safetensors",
|
| 581 |
+
"model.language_model.layers.31.self_attn.v_proj.weight": "model-00020-of-00026.safetensors",
|
| 582 |
+
"model.language_model.layers.29.input_layernorm.weight": "model-00021-of-00026.safetensors",
|
| 583 |
+
"model.language_model.layers.29.linear_attn.A_log": "model-00021-of-00026.safetensors",
|
| 584 |
+
"model.language_model.layers.29.linear_attn.conv1d.weight": "model-00021-of-00026.safetensors",
|
| 585 |
+
"model.language_model.layers.29.linear_attn.dt_bias": "model-00021-of-00026.safetensors",
|
| 586 |
+
"model.language_model.layers.29.linear_attn.in_proj_a.weight": "model-00021-of-00026.safetensors",
|
| 587 |
+
"model.language_model.layers.29.linear_attn.in_proj_b.weight": "model-00021-of-00026.safetensors",
|
| 588 |
+
"model.language_model.layers.29.linear_attn.norm.weight": "model-00021-of-00026.safetensors",
|
| 589 |
+
"model.language_model.layers.29.mlp.experts.gate_up_proj": "model-00021-of-00026.safetensors",
|
| 590 |
+
"model.language_model.layers.29.mlp.gate.weight": "model-00021-of-00026.safetensors",
|
| 591 |
+
"model.language_model.layers.29.mlp.shared_expert.down_proj.weight": "model-00021-of-00026.safetensors",
|
| 592 |
+
"model.language_model.layers.29.mlp.shared_expert.gate_proj.weight": "model-00021-of-00026.safetensors",
|
| 593 |
+
"model.language_model.layers.29.mlp.shared_expert.up_proj.weight": "model-00021-of-00026.safetensors",
|
| 594 |
+
"model.language_model.layers.29.mlp.shared_expert_gate.weight": "model-00021-of-00026.safetensors",
|
| 595 |
+
"model.language_model.layers.29.post_attention_layernorm.weight": "model-00021-of-00026.safetensors",
|
| 596 |
+
"model.language_model.layers.30.input_layernorm.weight": "model-00021-of-00026.safetensors",
|
| 597 |
+
"model.language_model.layers.30.linear_attn.A_log": "model-00021-of-00026.safetensors",
|
| 598 |
+
"model.language_model.layers.30.linear_attn.conv1d.weight": "model-00021-of-00026.safetensors",
|
| 599 |
+
"model.language_model.layers.30.linear_attn.dt_bias": "model-00021-of-00026.safetensors",
|
| 600 |
+
"model.language_model.layers.30.linear_attn.in_proj_a.weight": "model-00021-of-00026.safetensors",
|
| 601 |
+
"model.language_model.layers.30.linear_attn.in_proj_b.weight": "model-00021-of-00026.safetensors",
|
| 602 |
+
"model.language_model.layers.30.linear_attn.norm.weight": "model-00021-of-00026.safetensors",
|
| 603 |
+
"model.language_model.layers.30.mlp.experts.down_proj": "model-00021-of-00026.safetensors",
|
| 604 |
+
"model.language_model.layers.30.mlp.gate.weight": "model-00021-of-00026.safetensors",
|
| 605 |
+
"model.language_model.layers.30.mlp.shared_expert.down_proj.weight": "model-00021-of-00026.safetensors",
|
| 606 |
+
"model.language_model.layers.30.mlp.shared_expert.gate_proj.weight": "model-00021-of-00026.safetensors",
|
| 607 |
+
"model.language_model.layers.30.mlp.shared_expert.up_proj.weight": "model-00021-of-00026.safetensors",
|
| 608 |
+
"model.language_model.layers.30.mlp.shared_expert_gate.weight": "model-00021-of-00026.safetensors",
|
| 609 |
+
"model.language_model.layers.30.post_attention_layernorm.weight": "model-00021-of-00026.safetensors",
|
| 610 |
+
"model.language_model.layers.31.input_layernorm.weight": "model-00021-of-00026.safetensors",
|
| 611 |
+
"model.language_model.layers.31.mlp.gate.weight": "model-00021-of-00026.safetensors",
|
| 612 |
+
"model.language_model.layers.31.mlp.shared_expert.down_proj.weight": "model-00021-of-00026.safetensors",
|
| 613 |
+
"model.language_model.layers.31.mlp.shared_expert.gate_proj.weight": "model-00021-of-00026.safetensors",
|
| 614 |
+
"model.language_model.layers.31.mlp.shared_expert.up_proj.weight": "model-00021-of-00026.safetensors",
|
| 615 |
+
"model.language_model.layers.31.mlp.shared_expert_gate.weight": "model-00021-of-00026.safetensors",
|
| 616 |
+
"model.language_model.layers.31.post_attention_layernorm.weight": "model-00021-of-00026.safetensors",
|
| 617 |
+
"model.language_model.layers.31.self_attn.k_norm.weight": "model-00021-of-00026.safetensors",
|
| 618 |
+
"model.language_model.layers.31.self_attn.q_norm.weight": "model-00021-of-00026.safetensors",
|
| 619 |
+
"model.language_model.layers.32.input_layernorm.weight": "model-00022-of-00026.safetensors",
|
| 620 |
+
"model.language_model.layers.32.linear_attn.A_log": "model-00022-of-00026.safetensors",
|
| 621 |
+
"model.language_model.layers.32.linear_attn.conv1d.weight": "model-00022-of-00026.safetensors",
|
| 622 |
+
"model.language_model.layers.32.linear_attn.dt_bias": "model-00022-of-00026.safetensors",
|
| 623 |
+
"model.language_model.layers.32.linear_attn.in_proj_a.weight": "model-00022-of-00026.safetensors",
|
| 624 |
+
"model.language_model.layers.32.linear_attn.in_proj_b.weight": "model-00022-of-00026.safetensors",
|
| 625 |
+
"model.language_model.layers.32.linear_attn.in_proj_qkv.weight": "model-00022-of-00026.safetensors",
|
| 626 |
+
"model.language_model.layers.32.linear_attn.in_proj_z.weight": "model-00022-of-00026.safetensors",
|
| 627 |
+
"model.language_model.layers.32.linear_attn.norm.weight": "model-00022-of-00026.safetensors",
|
| 628 |
+
"model.language_model.layers.32.linear_attn.out_proj.weight": "model-00022-of-00026.safetensors",
|
| 629 |
+
"model.language_model.layers.32.mlp.experts.down_proj": "model-00022-of-00026.safetensors",
|
| 630 |
+
"model.language_model.layers.32.mlp.experts.gate_up_proj": "model-00022-of-00026.safetensors",
|
| 631 |
+
"model.language_model.layers.32.mlp.gate.weight": "model-00022-of-00026.safetensors",
|
| 632 |
+
"model.language_model.layers.32.mlp.shared_expert.down_proj.weight": "model-00022-of-00026.safetensors",
|
| 633 |
+
"model.language_model.layers.32.mlp.shared_expert.gate_proj.weight": "model-00022-of-00026.safetensors",
|
| 634 |
+
"model.language_model.layers.32.mlp.shared_expert.up_proj.weight": "model-00022-of-00026.safetensors",
|
| 635 |
+
"model.language_model.layers.32.mlp.shared_expert_gate.weight": "model-00022-of-00026.safetensors",
|
| 636 |
+
"model.language_model.layers.32.post_attention_layernorm.weight": "model-00022-of-00026.safetensors",
|
| 637 |
+
"model.language_model.layers.33.input_layernorm.weight": "model-00022-of-00026.safetensors",
|
| 638 |
+
"model.language_model.layers.33.linear_attn.A_log": "model-00022-of-00026.safetensors",
|
| 639 |
+
"model.language_model.layers.33.linear_attn.conv1d.weight": "model-00022-of-00026.safetensors",
|
| 640 |
+
"model.language_model.layers.33.linear_attn.dt_bias": "model-00022-of-00026.safetensors",
|
| 641 |
+
"model.language_model.layers.33.linear_attn.in_proj_a.weight": "model-00022-of-00026.safetensors",
|
| 642 |
+
"model.language_model.layers.33.linear_attn.in_proj_b.weight": "model-00022-of-00026.safetensors",
|
| 643 |
+
"model.language_model.layers.33.linear_attn.in_proj_qkv.weight": "model-00022-of-00026.safetensors",
|
| 644 |
+
"model.language_model.layers.33.linear_attn.in_proj_z.weight": "model-00022-of-00026.safetensors",
|
| 645 |
+
"model.language_model.layers.33.linear_attn.norm.weight": "model-00022-of-00026.safetensors",
|
| 646 |
+
"model.language_model.layers.33.linear_attn.out_proj.weight": "model-00022-of-00026.safetensors",
|
| 647 |
+
"model.language_model.layers.33.mlp.experts.down_proj": "model-00022-of-00026.safetensors",
|
| 648 |
+
"model.language_model.layers.33.mlp.experts.gate_up_proj": "model-00022-of-00026.safetensors",
|
| 649 |
+
"model.language_model.layers.33.mlp.gate.weight": "model-00022-of-00026.safetensors",
|
| 650 |
+
"model.language_model.layers.33.mlp.shared_expert.down_proj.weight": "model-00022-of-00026.safetensors",
|
| 651 |
+
"model.language_model.layers.33.mlp.shared_expert.gate_proj.weight": "model-00022-of-00026.safetensors",
|
| 652 |
+
"model.language_model.layers.33.mlp.shared_expert.up_proj.weight": "model-00022-of-00026.safetensors",
|
| 653 |
+
"model.language_model.layers.33.mlp.shared_expert_gate.weight": "model-00022-of-00026.safetensors",
|
| 654 |
+
"model.language_model.layers.33.post_attention_layernorm.weight": "model-00022-of-00026.safetensors",
|
| 655 |
+
"model.language_model.layers.34.input_layernorm.weight": "model-00023-of-00026.safetensors",
|
| 656 |
+
"model.language_model.layers.34.linear_attn.A_log": "model-00023-of-00026.safetensors",
|
| 657 |
+
"model.language_model.layers.34.linear_attn.conv1d.weight": "model-00023-of-00026.safetensors",
|
| 658 |
+
"model.language_model.layers.34.linear_attn.dt_bias": "model-00023-of-00026.safetensors",
|
| 659 |
+
"model.language_model.layers.34.linear_attn.in_proj_a.weight": "model-00023-of-00026.safetensors",
|
| 660 |
+
"model.language_model.layers.34.linear_attn.in_proj_b.weight": "model-00023-of-00026.safetensors",
|
| 661 |
+
"model.language_model.layers.34.linear_attn.in_proj_qkv.weight": "model-00023-of-00026.safetensors",
|
| 662 |
+
"model.language_model.layers.34.linear_attn.in_proj_z.weight": "model-00023-of-00026.safetensors",
|
| 663 |
+
"model.language_model.layers.34.linear_attn.norm.weight": "model-00023-of-00026.safetensors",
|
| 664 |
+
"model.language_model.layers.34.linear_attn.out_proj.weight": "model-00023-of-00026.safetensors",
|
| 665 |
+
"model.language_model.layers.34.mlp.experts.down_proj": "model-00023-of-00026.safetensors",
|
| 666 |
+
"model.language_model.layers.34.mlp.experts.gate_up_proj": "model-00023-of-00026.safetensors",
|
| 667 |
+
"model.language_model.layers.34.mlp.gate.weight": "model-00023-of-00026.safetensors",
|
| 668 |
+
"model.language_model.layers.34.mlp.shared_expert.down_proj.weight": "model-00023-of-00026.safetensors",
|
| 669 |
+
"model.language_model.layers.34.mlp.shared_expert.gate_proj.weight": "model-00023-of-00026.safetensors",
|
| 670 |
+
"model.language_model.layers.34.mlp.shared_expert.up_proj.weight": "model-00023-of-00026.safetensors",
|
| 671 |
+
"model.language_model.layers.34.mlp.shared_expert_gate.weight": "model-00023-of-00026.safetensors",
|
| 672 |
+
"model.language_model.layers.34.post_attention_layernorm.weight": "model-00023-of-00026.safetensors",
|
| 673 |
+
"model.language_model.layers.35.input_layernorm.weight": "model-00023-of-00026.safetensors",
|
| 674 |
+
"model.language_model.layers.35.mlp.experts.down_proj": "model-00023-of-00026.safetensors",
|
| 675 |
+
"model.language_model.layers.35.mlp.experts.gate_up_proj": "model-00023-of-00026.safetensors",
|
| 676 |
+
"model.language_model.layers.35.mlp.gate.weight": "model-00023-of-00026.safetensors",
|
| 677 |
+
"model.language_model.layers.35.mlp.shared_expert.down_proj.weight": "model-00023-of-00026.safetensors",
|
| 678 |
+
"model.language_model.layers.35.mlp.shared_expert.gate_proj.weight": "model-00023-of-00026.safetensors",
|
| 679 |
+
"model.language_model.layers.35.mlp.shared_expert.up_proj.weight": "model-00023-of-00026.safetensors",
|
| 680 |
+
"model.language_model.layers.35.mlp.shared_expert_gate.weight": "model-00023-of-00026.safetensors",
|
| 681 |
+
"model.language_model.layers.35.post_attention_layernorm.weight": "model-00023-of-00026.safetensors",
|
| 682 |
+
"model.language_model.layers.35.self_attn.k_norm.weight": "model-00023-of-00026.safetensors",
|
| 683 |
+
"model.language_model.layers.35.self_attn.k_proj.weight": "model-00023-of-00026.safetensors",
|
| 684 |
+
"model.language_model.layers.35.self_attn.o_proj.weight": "model-00023-of-00026.safetensors",
|
| 685 |
+
"model.language_model.layers.35.self_attn.q_norm.weight": "model-00023-of-00026.safetensors",
|
| 686 |
+
"model.language_model.layers.35.self_attn.q_proj.weight": "model-00023-of-00026.safetensors",
|
| 687 |
+
"model.language_model.layers.35.self_attn.v_proj.weight": "model-00023-of-00026.safetensors",
|
| 688 |
+
"model.language_model.layers.36.input_layernorm.weight": "model-00024-of-00026.safetensors",
|
| 689 |
+
"model.language_model.layers.36.linear_attn.A_log": "model-00024-of-00026.safetensors",
|
| 690 |
+
"model.language_model.layers.36.linear_attn.conv1d.weight": "model-00024-of-00026.safetensors",
|
| 691 |
+
"model.language_model.layers.36.linear_attn.dt_bias": "model-00024-of-00026.safetensors",
|
| 692 |
+
"model.language_model.layers.36.linear_attn.in_proj_a.weight": "model-00024-of-00026.safetensors",
|
| 693 |
+
"model.language_model.layers.36.linear_attn.in_proj_b.weight": "model-00024-of-00026.safetensors",
|
| 694 |
+
"model.language_model.layers.36.linear_attn.in_proj_qkv.weight": "model-00024-of-00026.safetensors",
|
| 695 |
+
"model.language_model.layers.36.linear_attn.in_proj_z.weight": "model-00024-of-00026.safetensors",
|
| 696 |
+
"model.language_model.layers.36.linear_attn.norm.weight": "model-00024-of-00026.safetensors",
|
| 697 |
+
"model.language_model.layers.36.linear_attn.out_proj.weight": "model-00024-of-00026.safetensors",
|
| 698 |
+
"model.language_model.layers.36.mlp.experts.down_proj": "model-00024-of-00026.safetensors",
|
| 699 |
+
"model.language_model.layers.36.mlp.experts.gate_up_proj": "model-00024-of-00026.safetensors",
|
| 700 |
+
"model.language_model.layers.36.mlp.gate.weight": "model-00024-of-00026.safetensors",
|
| 701 |
+
"model.language_model.layers.36.mlp.shared_expert.down_proj.weight": "model-00024-of-00026.safetensors",
|
| 702 |
+
"model.language_model.layers.36.mlp.shared_expert.gate_proj.weight": "model-00024-of-00026.safetensors",
|
| 703 |
+
"model.language_model.layers.36.mlp.shared_expert.up_proj.weight": "model-00024-of-00026.safetensors",
|
| 704 |
+
"model.language_model.layers.36.mlp.shared_expert_gate.weight": "model-00024-of-00026.safetensors",
|
| 705 |
+
"model.language_model.layers.36.post_attention_layernorm.weight": "model-00024-of-00026.safetensors",
|
| 706 |
+
"model.language_model.layers.37.input_layernorm.weight": "model-00024-of-00026.safetensors",
|
| 707 |
+
"model.language_model.layers.37.linear_attn.A_log": "model-00024-of-00026.safetensors",
|
| 708 |
+
"model.language_model.layers.37.linear_attn.conv1d.weight": "model-00024-of-00026.safetensors",
|
| 709 |
+
"model.language_model.layers.37.linear_attn.dt_bias": "model-00024-of-00026.safetensors",
|
| 710 |
+
"model.language_model.layers.37.linear_attn.in_proj_a.weight": "model-00024-of-00026.safetensors",
|
| 711 |
+
"model.language_model.layers.37.linear_attn.in_proj_b.weight": "model-00024-of-00026.safetensors",
|
| 712 |
+
"model.language_model.layers.37.linear_attn.in_proj_qkv.weight": "model-00024-of-00026.safetensors",
|
| 713 |
+
"model.language_model.layers.37.linear_attn.in_proj_z.weight": "model-00024-of-00026.safetensors",
|
| 714 |
+
"model.language_model.layers.37.linear_attn.norm.weight": "model-00024-of-00026.safetensors",
|
| 715 |
+
"model.language_model.layers.37.linear_attn.out_proj.weight": "model-00024-of-00026.safetensors",
|
| 716 |
+
"model.language_model.layers.37.mlp.experts.down_proj": "model-00024-of-00026.safetensors",
|
| 717 |
+
"model.language_model.layers.37.mlp.experts.gate_up_proj": "model-00024-of-00026.safetensors",
|
| 718 |
+
"model.language_model.layers.37.mlp.gate.weight": "model-00024-of-00026.safetensors",
|
| 719 |
+
"model.language_model.layers.37.mlp.shared_expert.down_proj.weight": "model-00024-of-00026.safetensors",
|
| 720 |
+
"model.language_model.layers.37.mlp.shared_expert.gate_proj.weight": "model-00024-of-00026.safetensors",
|
| 721 |
+
"model.language_model.layers.37.mlp.shared_expert.up_proj.weight": "model-00024-of-00026.safetensors",
|
| 722 |
+
"model.language_model.layers.37.mlp.shared_expert_gate.weight": "model-00024-of-00026.safetensors",
|
| 723 |
+
"model.language_model.layers.37.post_attention_layernorm.weight": "model-00024-of-00026.safetensors",
|
| 724 |
+
"model.language_model.layers.38.input_layernorm.weight": "model-00025-of-00026.safetensors",
|
| 725 |
+
"model.language_model.layers.38.linear_attn.A_log": "model-00025-of-00026.safetensors",
|
| 726 |
+
"model.language_model.layers.38.linear_attn.conv1d.weight": "model-00025-of-00026.safetensors",
|
| 727 |
+
"model.language_model.layers.38.linear_attn.dt_bias": "model-00025-of-00026.safetensors",
|
| 728 |
+
"model.language_model.layers.38.linear_attn.in_proj_a.weight": "model-00025-of-00026.safetensors",
|
| 729 |
+
"model.language_model.layers.38.linear_attn.in_proj_b.weight": "model-00025-of-00026.safetensors",
|
| 730 |
+
"model.language_model.layers.38.linear_attn.in_proj_qkv.weight": "model-00025-of-00026.safetensors",
|
| 731 |
+
"model.language_model.layers.38.linear_attn.in_proj_z.weight": "model-00025-of-00026.safetensors",
|
| 732 |
+
"model.language_model.layers.38.linear_attn.norm.weight": "model-00025-of-00026.safetensors",
|
| 733 |
+
"model.language_model.layers.38.linear_attn.out_proj.weight": "model-00025-of-00026.safetensors",
|
| 734 |
+
"model.language_model.layers.38.mlp.experts.down_proj": "model-00025-of-00026.safetensors",
|
| 735 |
+
"model.language_model.layers.38.mlp.experts.gate_up_proj": "model-00025-of-00026.safetensors",
|
| 736 |
+
"model.language_model.layers.38.mlp.gate.weight": "model-00025-of-00026.safetensors",
|
| 737 |
+
"model.language_model.layers.38.mlp.shared_expert.down_proj.weight": "model-00025-of-00026.safetensors",
|
| 738 |
+
"model.language_model.layers.38.mlp.shared_expert.gate_proj.weight": "model-00025-of-00026.safetensors",
|
| 739 |
+
"model.language_model.layers.38.mlp.shared_expert.up_proj.weight": "model-00025-of-00026.safetensors",
|
| 740 |
+
"model.language_model.layers.38.mlp.shared_expert_gate.weight": "model-00025-of-00026.safetensors",
|
| 741 |
+
"model.language_model.layers.38.post_attention_layernorm.weight": "model-00025-of-00026.safetensors",
|
| 742 |
+
"model.language_model.layers.39.mlp.experts.gate_up_proj": "model-00025-of-00026.safetensors",
|
| 743 |
+
"lm_head.weight": "model-00026-of-00026.safetensors",
|
| 744 |
+
"model.language_model.layers.39.input_layernorm.weight": "model-00026-of-00026.safetensors",
|
| 745 |
+
"model.language_model.layers.39.mlp.experts.down_proj": "model-00026-of-00026.safetensors",
|
| 746 |
+
"model.language_model.layers.39.mlp.gate.weight": "model-00026-of-00026.safetensors",
|
| 747 |
+
"model.language_model.layers.39.mlp.shared_expert.down_proj.weight": "model-00026-of-00026.safetensors",
|
| 748 |
+
"model.language_model.layers.39.mlp.shared_expert.gate_proj.weight": "model-00026-of-00026.safetensors",
|
| 749 |
+
"model.language_model.layers.39.mlp.shared_expert.up_proj.weight": "model-00026-of-00026.safetensors",
|
| 750 |
+
"model.language_model.layers.39.mlp.shared_expert_gate.weight": "model-00026-of-00026.safetensors",
|
| 751 |
+
"model.language_model.layers.39.post_attention_layernorm.weight": "model-00026-of-00026.safetensors",
|
| 752 |
+
"model.language_model.layers.39.self_attn.k_norm.weight": "model-00026-of-00026.safetensors",
|
| 753 |
+
"model.language_model.layers.39.self_attn.k_proj.weight": "model-00026-of-00026.safetensors",
|
| 754 |
+
"model.language_model.layers.39.self_attn.o_proj.weight": "model-00026-of-00026.safetensors",
|
| 755 |
+
"model.language_model.layers.39.self_attn.q_norm.weight": "model-00026-of-00026.safetensors",
|
| 756 |
+
"model.language_model.layers.39.self_attn.q_proj.weight": "model-00026-of-00026.safetensors",
|
| 757 |
+
"model.language_model.layers.39.self_attn.v_proj.weight": "model-00026-of-00026.safetensors",
|
| 758 |
+
"model.language_model.norm.weight": "model-00026-of-00026.safetensors"
|
| 759 |
+
}
|
| 760 |
+
}
|
modeling_memt.py
ADDED
|
@@ -0,0 +1,978 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Qwen MoE with explicit MemT memory and independent text sessions.
|
| 2 |
+
|
| 3 |
+
Backbone computation and generation use Transformers 5.12.1. Only the trained
|
| 4 |
+
memory branch and its lifecycle are implemented here.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import hashlib
|
| 10 |
+
import json
|
| 11 |
+
import math
|
| 12 |
+
import os
|
| 13 |
+
import re
|
| 14 |
+
import tempfile
|
| 15 |
+
import threading
|
| 16 |
+
import uuid
|
| 17 |
+
from copy import deepcopy
|
| 18 |
+
from dataclasses import dataclass, field
|
| 19 |
+
from pathlib import Path
|
| 20 |
+
from typing import ClassVar
|
| 21 |
+
|
| 22 |
+
import torch
|
| 23 |
+
import torch.nn.functional as F
|
| 24 |
+
from safetensors import safe_open
|
| 25 |
+
from safetensors.torch import save_file
|
| 26 |
+
from torch import nn
|
| 27 |
+
from transformers import GenerationMixin
|
| 28 |
+
from transformers.activations import ACT2FN
|
| 29 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 30 |
+
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
| 31 |
+
from transformers.models.qwen3_5_moe import modeling_qwen3_5_moe as qwen
|
| 32 |
+
|
| 33 |
+
from .configuration_memt import MemTQwen3_5MoeConfig
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class MemTRMSNormGated(nn.Module):
|
| 37 |
+
def __init__(self, head_dim, eps, config):
|
| 38 |
+
super().__init__()
|
| 39 |
+
self.weight = nn.Parameter(torch.ones(head_dim))
|
| 40 |
+
self.eps = eps
|
| 41 |
+
self.softcap = config["output_gate_softcap_enabled"]
|
| 42 |
+
self.limit = config["output_gate_softcap_limit"]
|
| 43 |
+
|
| 44 |
+
def forward(self, value, gate):
|
| 45 |
+
normalized = value.float() * torch.rsqrt(
|
| 46 |
+
value.float().square().mean(-1, keepdim=True) + self.eps
|
| 47 |
+
)
|
| 48 |
+
normalized = self.weight * normalized.to(value.dtype)
|
| 49 |
+
activated = F.silu(gate.float())
|
| 50 |
+
if self.softcap:
|
| 51 |
+
activated = self.limit * torch.tanh(activated / self.limit)
|
| 52 |
+
return (normalized * activated).to(value.dtype)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class MemTMemory(nn.Module):
|
| 56 |
+
"""MHA memory with sqrt erase gating; reads never modify caller-owned FP32 states."""
|
| 57 |
+
|
| 58 |
+
def __init__(self, config, hidden_size, num_heads, head_dim, rms_norm_eps=1e-6):
|
| 59 |
+
super().__init__()
|
| 60 |
+
self.num_heads, self.head_dim = num_heads, head_dim
|
| 61 |
+
self.value_dim = num_heads * head_dim
|
| 62 |
+
self.read_scale = head_dim**-0.5
|
| 63 |
+
self.in_proj_rows = (self.value_dim,) * 5 + (head_dim, head_dim)
|
| 64 |
+
self.in_proj = nn.Linear(hidden_size, sum(self.in_proj_rows), bias=False)
|
| 65 |
+
self.f_down = nn.Linear(head_dim, self.value_dim, bias=False)
|
| 66 |
+
self.g_down = nn.Linear(head_dim, self.value_dim, bias=config["output_gate_bias"])
|
| 67 |
+
self.decay_bias = nn.Parameter(torch.empty(self.value_dim))
|
| 68 |
+
self.decay_scale_log = nn.Parameter(torch.zeros(num_heads))
|
| 69 |
+
self.o_norm = MemTRMSNormGated(head_dim, rms_norm_eps, config)
|
| 70 |
+
self.reset_parameters()
|
| 71 |
+
|
| 72 |
+
@torch.no_grad()
|
| 73 |
+
def reset_parameters(self):
|
| 74 |
+
for block in self.in_proj.weight.split(self.in_proj_rows, dim=0):
|
| 75 |
+
nn.init.xavier_uniform_(block, gain=2**-2.5)
|
| 76 |
+
nn.init.xavier_uniform_(self.f_down.weight, gain=2**-2.5)
|
| 77 |
+
nn.init.xavier_uniform_(self.g_down.weight, gain=2**-2.5)
|
| 78 |
+
if self.g_down.bias is not None:
|
| 79 |
+
nn.init.zeros_(self.g_down.bias)
|
| 80 |
+
nn.init.zeros_(self.decay_scale_log)
|
| 81 |
+
half_life = (
|
| 82 |
+
torch.empty_like(self.decay_bias).uniform_(math.log(2048), math.log(32768)).exp()
|
| 83 |
+
)
|
| 84 |
+
rate = math.log(2) / half_life
|
| 85 |
+
self.decay_bias.copy_(rate + torch.log(-torch.expm1(-rate)))
|
| 86 |
+
|
| 87 |
+
@staticmethod
|
| 88 |
+
def _normalize(value, rounded):
|
| 89 |
+
result = value.float() * torch.rsqrt(value.float().square().sum(-1, keepdim=True) + 1e-6)
|
| 90 |
+
return result.to(value.dtype) if rounded else result
|
| 91 |
+
|
| 92 |
+
def forward(self, hidden, state=None, commit=False, *, write_state=None):
|
| 93 |
+
batch, length, _ = hidden.shape
|
| 94 |
+
dim, width = self.head_dim, self.value_dim
|
| 95 |
+
shape = (batch, length, self.num_heads, dim)
|
| 96 |
+
if state is None and not commit:
|
| 97 |
+
return hidden.new_zeros(batch, length, width), None
|
| 98 |
+
snapshot = (
|
| 99 |
+
hidden.new_zeros((batch, self.num_heads, dim, dim), dtype=torch.float32)
|
| 100 |
+
if state is None
|
| 101 |
+
else state
|
| 102 |
+
)
|
| 103 |
+
projected = F.linear(hidden, self.in_proj.weight[: 3 * width if commit else width])
|
| 104 |
+
query = self._normalize(projected[..., :width].reshape(shape), commit)
|
| 105 |
+
with torch.autocast(device_type=hidden.device.type, enabled=False):
|
| 106 |
+
read = self.read_scale * torch.einsum(
|
| 107 |
+
"blhk,bhkv->blhv", query.float(), snapshot.float()
|
| 108 |
+
)
|
| 109 |
+
gate = self.g_down(F.linear(hidden, self.in_proj.weight[-dim:])).reshape(shape)
|
| 110 |
+
read = self.o_norm(read.to(gate.dtype), gate).reshape(batch, length, width)
|
| 111 |
+
if not commit:
|
| 112 |
+
return read, state
|
| 113 |
+
key, value = (item.reshape(shape) for item in projected[..., width:].split(width, -1))
|
| 114 |
+
key = self._normalize(key, True)
|
| 115 |
+
b, w, f = F.linear(hidden, self.in_proj.weight[3 * width : -dim]).split(
|
| 116 |
+
(width, width, dim), -1
|
| 117 |
+
)
|
| 118 |
+
sqrt_b = (0.5 * F.logsigmoid(b.float())).exp().to(b.dtype).reshape(shape)
|
| 119 |
+
key = (key.float() * sqrt_b.float()).to(value.dtype).float()
|
| 120 |
+
write_value = w.sigmoid().reshape(shape).float() * value.float()
|
| 121 |
+
decay = (
|
| 122 |
+
-self.decay_scale_log.float().exp().reshape(self.num_heads, 1)
|
| 123 |
+
* F.softplus(self.f_down(f).float() + self.decay_bias.float()).reshape(shape)
|
| 124 |
+
).exp()
|
| 125 |
+
updated = snapshot.float() if write_state is None else write_state.float()
|
| 126 |
+
# The writer advances per event; every read in a QA keeps its frozen snapshot.
|
| 127 |
+
for index in range(length):
|
| 128 |
+
u = key[:, index]
|
| 129 |
+
updated = updated * decay[:, index].unsqueeze(-1)
|
| 130 |
+
erase = (u.unsqueeze(-1) * updated).sum(-2)
|
| 131 |
+
updated = updated + u.unsqueeze(-1) * (write_value[:, index] - erase).unsqueeze(-2)
|
| 132 |
+
return read, updated
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def _device_dispatch(reference, accelerated):
|
| 136 |
+
def operation(*args, **kwargs):
|
| 137 |
+
tensor = args[0] if args else kwargs.get("q", kwargs.get("x"))
|
| 138 |
+
implementation = accelerated if accelerated is not None and tensor.is_cuda else reference
|
| 139 |
+
return implementation(*args, **kwargs)
|
| 140 |
+
|
| 141 |
+
return operation
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
def _causal_convolution(x, weight, bias=None, activation=None, seq_idx=None):
|
| 145 |
+
if x.is_cuda and qwen.causal_conv1d_fn is not None:
|
| 146 |
+
return qwen.causal_conv1d_fn(x, weight, bias, activation=activation, seq_idx=seq_idx)
|
| 147 |
+
result = F.conv1d(
|
| 148 |
+
x, weight.unsqueeze(1), bias, padding=weight.shape[-1] - 1, groups=x.shape[1]
|
| 149 |
+
)[..., : x.shape[-1]]
|
| 150 |
+
return F.silu(result) if activation in ("silu", "swish") else result
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
class MemTGatedDeltaNet(qwen.Qwen3_5MoeGatedDeltaNet):
|
| 154 |
+
"""Device-safe construction; forward and GDN equations remain native.
|
| 155 |
+
|
| 156 |
+
The optional upstream fused norm queries CUDA even during CPU/meta loading.
|
| 157 |
+
Use the native PyTorch norm and select kernels on the actual tensor device.
|
| 158 |
+
"""
|
| 159 |
+
|
| 160 |
+
def __init__(self, config, layer_idx):
|
| 161 |
+
nn.Module.__init__(self)
|
| 162 |
+
self.hidden_size = config.hidden_size
|
| 163 |
+
self.num_v_heads = config.linear_num_value_heads
|
| 164 |
+
self.num_k_heads = config.linear_num_key_heads
|
| 165 |
+
self.head_k_dim = config.linear_key_head_dim
|
| 166 |
+
self.head_v_dim = config.linear_value_head_dim
|
| 167 |
+
self.key_dim = self.head_k_dim * self.num_k_heads
|
| 168 |
+
self.value_dim = self.head_v_dim * self.num_v_heads
|
| 169 |
+
self.conv_kernel_size = config.linear_conv_kernel_dim
|
| 170 |
+
self.layer_idx = layer_idx
|
| 171 |
+
self.activation = config.hidden_act
|
| 172 |
+
self.act = ACT2FN[config.hidden_act]
|
| 173 |
+
self.layer_norm_epsilon = config.rms_norm_eps
|
| 174 |
+
self.conv_dim = self.key_dim * 2 + self.value_dim
|
| 175 |
+
self.conv1d = nn.Conv1d(
|
| 176 |
+
self.conv_dim,
|
| 177 |
+
self.conv_dim,
|
| 178 |
+
self.conv_kernel_size,
|
| 179 |
+
groups=self.conv_dim,
|
| 180 |
+
padding=self.conv_kernel_size - 1,
|
| 181 |
+
bias=False,
|
| 182 |
+
)
|
| 183 |
+
self.dt_bias = nn.Parameter(torch.ones(self.num_v_heads))
|
| 184 |
+
self.A_log = nn.Parameter(torch.empty(self.num_v_heads).uniform_(0, 16).log_())
|
| 185 |
+
self.norm = qwen.Qwen3_5MoeRMSNormGated(self.head_v_dim, eps=self.layer_norm_epsilon)
|
| 186 |
+
self.out_proj = nn.Linear(self.value_dim, self.hidden_size, bias=False)
|
| 187 |
+
self.causal_conv1d_fn = _causal_convolution
|
| 188 |
+
self.causal_conv1d_update = _device_dispatch(
|
| 189 |
+
qwen.torch_causal_conv1d_update, qwen.causal_conv1d_update
|
| 190 |
+
)
|
| 191 |
+
self.chunk_gated_delta_rule = _device_dispatch(
|
| 192 |
+
qwen.torch_chunk_gated_delta_rule, qwen.chunk_gated_delta_rule
|
| 193 |
+
)
|
| 194 |
+
self.recurrent_gated_delta_rule = _device_dispatch(
|
| 195 |
+
qwen.torch_recurrent_gated_delta_rule, qwen.fused_recurrent_gated_delta_rule
|
| 196 |
+
)
|
| 197 |
+
self.in_proj_qkv = nn.Linear(
|
| 198 |
+
self.hidden_size, 2 * self.key_dim + self.value_dim, bias=False
|
| 199 |
+
)
|
| 200 |
+
self.in_proj_z = nn.Linear(self.hidden_size, self.value_dim, bias=False)
|
| 201 |
+
self.in_proj_b = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)
|
| 202 |
+
self.in_proj_a = nn.Linear(self.hidden_size, self.num_v_heads, bias=False)
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
@dataclass
|
| 206 |
+
class _MemoryCall:
|
| 207 |
+
read: dict
|
| 208 |
+
write: dict | None = None
|
| 209 |
+
results: dict = field(default_factory=dict)
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
class MemTAttention(qwen.Qwen3_5MoeAttention):
|
| 213 |
+
def __init__(self, text, layer_idx, memory_config):
|
| 214 |
+
super().__init__(text, layer_idx)
|
| 215 |
+
self.memt = MemTMemory(
|
| 216 |
+
memory_config,
|
| 217 |
+
text.hidden_size,
|
| 218 |
+
text.num_attention_heads,
|
| 219 |
+
text.head_dim,
|
| 220 |
+
text.rms_norm_eps,
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
def forward(
|
| 224 |
+
self,
|
| 225 |
+
hidden_states,
|
| 226 |
+
position_embeddings,
|
| 227 |
+
attention_mask,
|
| 228 |
+
past_key_values=None,
|
| 229 |
+
memory_call=None,
|
| 230 |
+
**kwargs,
|
| 231 |
+
):
|
| 232 |
+
shape = hidden_states.shape[:-1]
|
| 233 |
+
heads = (*shape, -1, self.head_dim)
|
| 234 |
+
query, gate = (
|
| 235 |
+
self.q_proj(hidden_states).view(*shape, -1, 2 * self.head_dim).chunk(2, dim=-1)
|
| 236 |
+
)
|
| 237 |
+
query = self.q_norm(query.reshape(heads)).transpose(1, 2)
|
| 238 |
+
key = self.k_norm(self.k_proj(hidden_states).view(heads)).transpose(1, 2)
|
| 239 |
+
value = self.v_proj(hidden_states).view(heads).transpose(1, 2)
|
| 240 |
+
query, key = qwen.apply_rotary_pos_emb(query, key, *position_embeddings)
|
| 241 |
+
if past_key_values is not None:
|
| 242 |
+
key, value = past_key_values.update(key, value, self.layer_idx)
|
| 243 |
+
attention = ALL_ATTENTION_FUNCTIONS.get_interface(
|
| 244 |
+
self.config._attn_implementation, qwen.eager_attention_forward
|
| 245 |
+
)
|
| 246 |
+
output, weights = attention(
|
| 247 |
+
self,
|
| 248 |
+
query,
|
| 249 |
+
key,
|
| 250 |
+
value,
|
| 251 |
+
attention_mask,
|
| 252 |
+
dropout=self.attention_dropout if self.training else 0.0,
|
| 253 |
+
scaling=self.scaling,
|
| 254 |
+
**kwargs,
|
| 255 |
+
)
|
| 256 |
+
output = output.reshape(*shape, -1).contiguous() * torch.sigmoid(gate.reshape(*shape, -1))
|
| 257 |
+
if memory_call is not None:
|
| 258 |
+
state = memory_call.read.get(self.layer_idx)
|
| 259 |
+
state = state.to(hidden_states.device) if state is not None else None
|
| 260 |
+
commit = memory_call.write is not None
|
| 261 |
+
writer = memory_call.write.get(self.layer_idx) if commit else None
|
| 262 |
+
writer = writer.to(hidden_states.device) if writer is not None else None
|
| 263 |
+
read, updated = self.memt(hidden_states, state, commit, write_state=writer)
|
| 264 |
+
output = output + read.to(output.dtype)
|
| 265 |
+
if commit:
|
| 266 |
+
memory_call.results[self.layer_idx] = updated
|
| 267 |
+
return self.o_proj(output), weights
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
class MemTDecoderLayer(qwen.Qwen3_5MoeDecoderLayer):
|
| 271 |
+
def __init__(self, config, layer_idx, memory_config):
|
| 272 |
+
nn.Module.__init__(self)
|
| 273 |
+
self.hidden_size = config.hidden_size
|
| 274 |
+
self.layer_type = config.layer_types[layer_idx]
|
| 275 |
+
if self.layer_type == "linear_attention":
|
| 276 |
+
self.linear_attn = MemTGatedDeltaNet(config, layer_idx)
|
| 277 |
+
else:
|
| 278 |
+
self.self_attn = MemTAttention(config, layer_idx, memory_config)
|
| 279 |
+
self.mlp = qwen.Qwen3_5MoeSparseMoeBlock(config)
|
| 280 |
+
self.input_layernorm = qwen.Qwen3_5MoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 281 |
+
self.post_attention_layernorm = qwen.Qwen3_5MoeRMSNorm(
|
| 282 |
+
config.hidden_size, eps=config.rms_norm_eps
|
| 283 |
+
)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
class MemTTextModel(qwen.Qwen3_5MoeTextModel):
|
| 287 |
+
@classmethod
|
| 288 |
+
def _can_set_experts_implementation(cls):
|
| 289 |
+
# Upstream source-file detection cannot see imported native experts.
|
| 290 |
+
return True
|
| 291 |
+
|
| 292 |
+
def __init__(self, config, memory_config):
|
| 293 |
+
qwen.Qwen3_5MoePreTrainedModel.__init__(self, config)
|
| 294 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, config.pad_token_id)
|
| 295 |
+
self.layers = nn.ModuleList(
|
| 296 |
+
[MemTDecoderLayer(config, i, memory_config) for i in range(config.num_hidden_layers)]
|
| 297 |
+
)
|
| 298 |
+
self.norm = qwen.Qwen3_5MoeRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 299 |
+
self.rotary_emb = qwen.Qwen3_5MoeTextRotaryEmbedding(config=config)
|
| 300 |
+
self.gradient_checkpointing = False
|
| 301 |
+
self.post_init()
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
class MemTBackbone(nn.Module):
|
| 305 |
+
def __init__(self, text, memory_config):
|
| 306 |
+
super().__init__()
|
| 307 |
+
self.language_model = MemTTextModel(text, memory_config)
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
class MemTQwen3_5MoeForCausalLM(qwen.Qwen3_5MoePreTrainedModel, GenerationMixin):
|
| 311 |
+
config_class = MemTQwen3_5MoeConfig
|
| 312 |
+
base_model_prefix = "model"
|
| 313 |
+
_no_split_modules: ClassVar[list[str]] = ["MemTDecoderLayer"]
|
| 314 |
+
_tied_weights_keys: ClassVar[dict[str, str]] = {
|
| 315 |
+
"lm_head.weight": "model.language_model.embed_tokens.weight"
|
| 316 |
+
}
|
| 317 |
+
|
| 318 |
+
@classmethod
|
| 319 |
+
def _can_set_experts_implementation(cls):
|
| 320 |
+
return True
|
| 321 |
+
|
| 322 |
+
def __init__(self, config):
|
| 323 |
+
super().__init__(config)
|
| 324 |
+
text = config.get_text_config()
|
| 325 |
+
self.model = MemTBackbone(text, config.memt)
|
| 326 |
+
self.lm_head = nn.Linear(text.hidden_size, text.vocab_size, bias=False)
|
| 327 |
+
self.post_init()
|
| 328 |
+
|
| 329 |
+
def _init_weights(self, module):
|
| 330 |
+
super()._init_weights(module)
|
| 331 |
+
if isinstance(module, MemTMemory):
|
| 332 |
+
module.reset_parameters()
|
| 333 |
+
|
| 334 |
+
def get_input_embeddings(self):
|
| 335 |
+
return self.model.language_model.embed_tokens
|
| 336 |
+
|
| 337 |
+
def set_input_embeddings(self, value):
|
| 338 |
+
self.model.language_model.embed_tokens = value
|
| 339 |
+
|
| 340 |
+
def get_output_embeddings(self):
|
| 341 |
+
return self.lm_head
|
| 342 |
+
|
| 343 |
+
def set_output_embeddings(self, value):
|
| 344 |
+
self.lm_head = value
|
| 345 |
+
|
| 346 |
+
def forward(
|
| 347 |
+
self,
|
| 348 |
+
input_ids=None,
|
| 349 |
+
attention_mask=None,
|
| 350 |
+
position_ids=None,
|
| 351 |
+
past_key_values=None,
|
| 352 |
+
inputs_embeds=None,
|
| 353 |
+
labels=None,
|
| 354 |
+
use_cache=None,
|
| 355 |
+
memory_state=None,
|
| 356 |
+
logits_to_keep=0,
|
| 357 |
+
return_dict=None,
|
| 358 |
+
**kwargs,
|
| 359 |
+
):
|
| 360 |
+
outputs = self.model.language_model(
|
| 361 |
+
input_ids=input_ids,
|
| 362 |
+
attention_mask=attention_mask,
|
| 363 |
+
position_ids=position_ids,
|
| 364 |
+
past_key_values=past_key_values,
|
| 365 |
+
inputs_embeds=inputs_embeds,
|
| 366 |
+
use_cache=self.config.use_cache if use_cache is None else use_cache,
|
| 367 |
+
memory_call=_MemoryCall(memory_state) if memory_state else None,
|
| 368 |
+
return_dict=True,
|
| 369 |
+
**kwargs,
|
| 370 |
+
)
|
| 371 |
+
sliced = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
|
| 372 |
+
logits = self.lm_head(outputs.last_hidden_state[:, sliced, :])
|
| 373 |
+
loss = (
|
| 374 |
+
None
|
| 375 |
+
if labels is None
|
| 376 |
+
else self.loss_function(logits, labels, self.config.vocab_size, **kwargs)
|
| 377 |
+
)
|
| 378 |
+
result = CausalLMOutputWithPast(
|
| 379 |
+
loss=loss,
|
| 380 |
+
logits=logits,
|
| 381 |
+
past_key_values=outputs.past_key_values,
|
| 382 |
+
hidden_states=outputs.hidden_states,
|
| 383 |
+
attentions=outputs.attentions,
|
| 384 |
+
)
|
| 385 |
+
return (
|
| 386 |
+
result
|
| 387 |
+
if (self.config.return_dict if return_dict is None else return_dict)
|
| 388 |
+
else result.to_tuple()
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
+
@torch.no_grad()
|
| 392 |
+
def commit_memory(self, input_ids, memory_state=None, *, document=False):
|
| 393 |
+
"""Replay one unpadded QA or Document and return a new state dictionary.
|
| 394 |
+
|
| 395 |
+
QA events retain one read snapshot and one native KV/GDN cache. Document
|
| 396 |
+
chunks reset both domains. No input memory tensor is mutated.
|
| 397 |
+
"""
|
| 398 |
+
if input_ids.ndim != 2 or input_ids.shape[0] != 1 or input_ids.shape[1] == 0:
|
| 399 |
+
raise ValueError("commit_memory requires a nonempty [1, tokens] tensor")
|
| 400 |
+
if input_ids.dtype != torch.long:
|
| 401 |
+
raise TypeError("commit_memory requires integer token IDs (torch.long)")
|
| 402 |
+
snapshot = {} if memory_state is None else memory_state
|
| 403 |
+
if snapshot:
|
| 404 |
+
text = self.config.get_text_config()
|
| 405 |
+
layers = {i for i, kind in enumerate(text.layer_types) if kind == "full_attention"}
|
| 406 |
+
if set(snapshot) != layers:
|
| 407 |
+
raise ValueError("memory must contain every full-attention layer")
|
| 408 |
+
shape = (1, text.num_attention_heads, text.head_dim, text.head_dim)
|
| 409 |
+
if any(
|
| 410 |
+
tuple(value.shape) != shape or value.dtype != torch.float32
|
| 411 |
+
for value in snapshot.values()
|
| 412 |
+
):
|
| 413 |
+
raise ValueError(
|
| 414 |
+
"memory states must have the model's [1, heads, K, V] shape and FP32 dtype"
|
| 415 |
+
)
|
| 416 |
+
candidate, cache = snapshot, None
|
| 417 |
+
for event in input_ids.split(self.config.memt["chunk_tokens"], dim=1):
|
| 418 |
+
if document:
|
| 419 |
+
snapshot, cache = candidate, None
|
| 420 |
+
call = _MemoryCall(snapshot, candidate)
|
| 421 |
+
output = self.model.language_model(
|
| 422 |
+
input_ids=event,
|
| 423 |
+
past_key_values=cache,
|
| 424 |
+
use_cache=True,
|
| 425 |
+
memory_call=call,
|
| 426 |
+
return_dict=True,
|
| 427 |
+
)
|
| 428 |
+
candidate, cache = call.results, output.past_key_values
|
| 429 |
+
return candidate
|
| 430 |
+
|
| 431 |
+
def create_session(self, tokenizer, *, system_prompt=None, tools=None, state=None):
|
| 432 |
+
return MemTSession(self, tokenizer, system_prompt=system_prompt, tools=tools, state=state)
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
class MemTSessionError(RuntimeError):
|
| 436 |
+
"""A failed session operation, including any already-returnable answer."""
|
| 437 |
+
|
| 438 |
+
def __init__(
|
| 439 |
+
self,
|
| 440 |
+
stage,
|
| 441 |
+
reason,
|
| 442 |
+
*,
|
| 443 |
+
result=None,
|
| 444 |
+
turn_retained=False,
|
| 445 |
+
memory_updated=False,
|
| 446 |
+
publication="not_published",
|
| 447 |
+
):
|
| 448 |
+
super().__init__(f"{stage}: {reason}")
|
| 449 |
+
self.stage = stage
|
| 450 |
+
self.reason = str(reason)
|
| 451 |
+
self.result = deepcopy(result or {})
|
| 452 |
+
self.answer = self.result.get("answer")
|
| 453 |
+
self.generation_completed = self.result.get("generation_completed", False)
|
| 454 |
+
self.memory_updated = memory_updated
|
| 455 |
+
self.turn_retained = turn_retained
|
| 456 |
+
self.publication = publication
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
@dataclass
|
| 460 |
+
class _Turn:
|
| 461 |
+
messages: list
|
| 462 |
+
replay_ids: list
|
| 463 |
+
enable_memory: bool
|
| 464 |
+
enable_thinking: bool | None
|
| 465 |
+
structured: bool
|
| 466 |
+
generation_options: dict
|
| 467 |
+
tool_calls: list = field(default_factory=list)
|
| 468 |
+
|
| 469 |
+
|
| 470 |
+
_TOOL_PATTERN = (
|
| 471 |
+
r"<tool_call>\s*<function=(?P<name>[^<>\s]+)>\s*(?P<arguments>.*?)\s*</function>\s*</tool_call>"
|
| 472 |
+
)
|
| 473 |
+
_PARAMETER_PATTERN = r"<parameter=(?P<name>[^<>\s]+)>\n?(?P<value>.*?)\n?</parameter>"
|
| 474 |
+
_TOOL_SCHEMA = {
|
| 475 |
+
"type": "array",
|
| 476 |
+
"x-regex-iterator": _TOOL_PATTERN,
|
| 477 |
+
"items": {
|
| 478 |
+
"type": "object",
|
| 479 |
+
"properties": {
|
| 480 |
+
"name": {"type": "string"},
|
| 481 |
+
"arguments": {
|
| 482 |
+
"type": "array",
|
| 483 |
+
"x-regex-iterator": _PARAMETER_PATTERN,
|
| 484 |
+
"items": {
|
| 485 |
+
"type": "object",
|
| 486 |
+
"properties": {
|
| 487 |
+
"name": {"type": "string"},
|
| 488 |
+
"value": {"type": "string"},
|
| 489 |
+
},
|
| 490 |
+
},
|
| 491 |
+
},
|
| 492 |
+
},
|
| 493 |
+
},
|
| 494 |
+
}
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
class MemTSession:
|
| 498 |
+
"""Single-text-sequence session. Tools execute exclusively in caller code.
|
| 499 |
+
|
| 500 |
+
Each operation is serialized within this session. Different sessions may use
|
| 501 |
+
the same model sequentially; concurrent model execution is not coordinated.
|
| 502 |
+
"""
|
| 503 |
+
|
| 504 |
+
def __init__(self, model, tokenizer, *, system_prompt=None, tools=None, state=None):
|
| 505 |
+
if system_prompt is not None and not isinstance(system_prompt, str):
|
| 506 |
+
raise TypeError("system_prompt must be text or None")
|
| 507 |
+
if tools is not None and not isinstance(tools, list):
|
| 508 |
+
raise TypeError("tools must be a list of function schemas")
|
| 509 |
+
self.model, self.tokenizer = model, tokenizer
|
| 510 |
+
self._system_prompt, self._tools = system_prompt, deepcopy(tools or [])
|
| 511 |
+
self._functions = {}
|
| 512 |
+
for tool in self._tools:
|
| 513 |
+
function = tool["function"]
|
| 514 |
+
name = function["name"]
|
| 515 |
+
if tool.get("type", "function") != "function" or name in self._functions:
|
| 516 |
+
raise ValueError("tools must have unique function names")
|
| 517 |
+
self._functions[name] = function
|
| 518 |
+
self._lock = threading.RLock()
|
| 519 |
+
self._history, self._memory, self._pending = [], {}, None
|
| 520 |
+
self._first_turn, self._closed = state is None, False
|
| 521 |
+
if state is not None:
|
| 522 |
+
self._memory = self._load_state(state)
|
| 523 |
+
|
| 524 |
+
def _check_open(self):
|
| 525 |
+
if self._closed:
|
| 526 |
+
raise MemTSessionError("session", "session is closed; create a new session")
|
| 527 |
+
|
| 528 |
+
@property
|
| 529 |
+
def messages(self):
|
| 530 |
+
"""A detached view of the uncompressed conversation, excluding system."""
|
| 531 |
+
with self._lock:
|
| 532 |
+
turns = self._history + ([self._pending] if self._pending else [])
|
| 533 |
+
return deepcopy([message for turn in turns for message in turn.messages])
|
| 534 |
+
|
| 535 |
+
def _encode(self, text):
|
| 536 |
+
return self.tokenizer.encode(text, add_special_tokens=False)
|
| 537 |
+
|
| 538 |
+
def _tensor(self, ids):
|
| 539 |
+
return torch.tensor(
|
| 540 |
+
[ids],
|
| 541 |
+
dtype=torch.long,
|
| 542 |
+
device=self.model.get_input_embeddings().weight.device,
|
| 543 |
+
)
|
| 544 |
+
|
| 545 |
+
def _render(self, messages, *, generation=False, thinking=None, context=False):
|
| 546 |
+
options = {"preserve_thinking": True}
|
| 547 |
+
if thinking is not None:
|
| 548 |
+
options["enable_thinking"] = thinking
|
| 549 |
+
messages = list(messages)
|
| 550 |
+
if context and self._system_prompt is not None:
|
| 551 |
+
messages.insert(0, {"role": "system", "content": self._system_prompt})
|
| 552 |
+
return self.tokenizer.apply_chat_template(
|
| 553 |
+
messages,
|
| 554 |
+
tokenize=False,
|
| 555 |
+
add_generation_prompt=generation,
|
| 556 |
+
tools=self._tools if context else None,
|
| 557 |
+
**options,
|
| 558 |
+
)
|
| 559 |
+
|
| 560 |
+
def _validate_options(self, options):
|
| 561 |
+
reserved = {
|
| 562 |
+
"memory",
|
| 563 |
+
"enable_memory",
|
| 564 |
+
"memory_state",
|
| 565 |
+
"past_key_values",
|
| 566 |
+
"inputs",
|
| 567 |
+
"input_ids",
|
| 568 |
+
"inputs_embeds",
|
| 569 |
+
"attention_mask",
|
| 570 |
+
"position_ids",
|
| 571 |
+
"streamer",
|
| 572 |
+
"assistant_prefill",
|
| 573 |
+
"continue_final_message",
|
| 574 |
+
"enable_thinking",
|
| 575 |
+
"return_dict",
|
| 576 |
+
"preserve_thinking",
|
| 577 |
+
}
|
| 578 |
+
invalid = reserved.intersection(options)
|
| 579 |
+
if invalid:
|
| 580 |
+
raise ValueError(f"Session does not accept generation options: {sorted(invalid)}")
|
| 581 |
+
config = options.get("generation_config") or self.model.generation_config
|
| 582 |
+
if (options.get("num_return_sequences", config.num_return_sequences) or 1) != 1:
|
| 583 |
+
raise ValueError("Session requires one returned sequence")
|
| 584 |
+
|
| 585 |
+
def _parse_tools(self, content):
|
| 586 |
+
if not self._tools or "<tool_call>" not in content:
|
| 587 |
+
return []
|
| 588 |
+
matches = list(re.finditer(_TOOL_PATTERN, content, flags=re.DOTALL))
|
| 589 |
+
if not matches or re.sub(_TOOL_PATTERN, "", content, flags=re.DOTALL).count("<tool_call>"):
|
| 590 |
+
raise ValueError("incomplete or malformed tool request")
|
| 591 |
+
parsed = self.tokenizer.parse_response(content, schema=_TOOL_SCHEMA)
|
| 592 |
+
calls = []
|
| 593 |
+
for item, match in zip(parsed, matches):
|
| 594 |
+
function = self._functions.get(item["name"])
|
| 595 |
+
if function is None:
|
| 596 |
+
raise ValueError(f"unknown tool: {item['name']}")
|
| 597 |
+
if re.sub(_PARAMETER_PATTERN, "", match["arguments"], flags=re.DOTALL).strip():
|
| 598 |
+
raise ValueError("malformed tool parameters")
|
| 599 |
+
parameters = function.get("parameters", {})
|
| 600 |
+
properties = parameters.get("properties", {})
|
| 601 |
+
arguments = {}
|
| 602 |
+
for argument in item.get("arguments") or []:
|
| 603 |
+
name, value = argument["name"], argument["value"]
|
| 604 |
+
if name in arguments:
|
| 605 |
+
raise ValueError(f"duplicate tool argument: {name}")
|
| 606 |
+
if name not in properties and parameters.get("additionalProperties") is False:
|
| 607 |
+
raise ValueError(f"unknown tool argument: {name}")
|
| 608 |
+
if properties.get(name, {}).get("type") != "string":
|
| 609 |
+
try:
|
| 610 |
+
value = json.loads(value)
|
| 611 |
+
except json.JSONDecodeError:
|
| 612 |
+
if properties.get(name, {}).get("type") is not None:
|
| 613 |
+
raise ValueError(f"invalid JSON tool argument: {name}") from None
|
| 614 |
+
arguments[name] = value
|
| 615 |
+
if set(parameters.get("required", [])) - arguments.keys():
|
| 616 |
+
raise ValueError("missing required tool arguments")
|
| 617 |
+
calls.append(
|
| 618 |
+
{
|
| 619 |
+
"id": "call_" + uuid.uuid4().hex,
|
| 620 |
+
"type": "function",
|
| 621 |
+
"function": {"name": item["name"], "arguments": arguments},
|
| 622 |
+
}
|
| 623 |
+
)
|
| 624 |
+
return calls
|
| 625 |
+
|
| 626 |
+
@staticmethod
|
| 627 |
+
def _without_thinking(text):
|
| 628 |
+
if "</think>" in text:
|
| 629 |
+
text = text.split("</think>", 1)[1].lstrip("\n")
|
| 630 |
+
return re.sub(r"<think>.*?(?:</think>|$)", "", text, flags=re.DOTALL).lstrip("\n")
|
| 631 |
+
|
| 632 |
+
def _generate(self, turn, memory):
|
| 633 |
+
messages = [
|
| 634 |
+
message for previous in self._history for message in previous.messages
|
| 635 |
+
] + turn.messages
|
| 636 |
+
prompt = self._render(
|
| 637 |
+
messages, generation=True, thinking=turn.enable_thinking, context=True
|
| 638 |
+
)
|
| 639 |
+
ids = self._encode(prompt)
|
| 640 |
+
if len(ids) >= self.model.config.get_text_config().max_position_embeddings:
|
| 641 |
+
raise ValueError("prompt exceeds the model context; no history was truncated")
|
| 642 |
+
options = dict(turn.generation_options)
|
| 643 |
+
options["return_dict_in_generate"] = True
|
| 644 |
+
inputs = self._tensor(ids)
|
| 645 |
+
requested_config = options.get("generation_config")
|
| 646 |
+
# Match generate's unset-length checks before defaults are filled.
|
| 647 |
+
default_lengths = {
|
| 648 |
+
name: options.get(name) is None
|
| 649 |
+
and (requested_config is None or getattr(requested_config, name) is None)
|
| 650 |
+
and getattr(self.model.generation_config, name) is None
|
| 651 |
+
for name in ("max_length", "min_length")
|
| 652 |
+
}
|
| 653 |
+
generation_config, _ = self.model._prepare_generation_config(
|
| 654 |
+
requested_config,
|
| 655 |
+
**{key: value for key, value in options.items() if key != "generation_config"},
|
| 656 |
+
)
|
| 657 |
+
generation_config = self.model._prepare_generated_length(
|
| 658 |
+
generation_config=generation_config,
|
| 659 |
+
has_default_max_length=default_lengths["max_length"],
|
| 660 |
+
has_default_min_length=default_lengths["min_length"],
|
| 661 |
+
model_input_name="input_ids",
|
| 662 |
+
input_ids_length=len(ids),
|
| 663 |
+
inputs_tensor=inputs,
|
| 664 |
+
)
|
| 665 |
+
# Keep generate's original inputs; the resolved copy is only for reporting.
|
| 666 |
+
output = self.model.generate(
|
| 667 |
+
inputs,
|
| 668 |
+
attention_mask=torch.ones_like(inputs),
|
| 669 |
+
memory_state=memory or None,
|
| 670 |
+
**options,
|
| 671 |
+
)
|
| 672 |
+
generated = output.sequences[0, len(ids) :].tolist()
|
| 673 |
+
if not generated:
|
| 674 |
+
raise ValueError("generation returned no tokens")
|
| 675 |
+
eos = generation_config.eos_token_id
|
| 676 |
+
eos = [eos] if isinstance(eos, int) else list(eos or [])
|
| 677 |
+
completed = generated[-1] in eos
|
| 678 |
+
visible_ids = generated[:-1] if completed else generated
|
| 679 |
+
raw = self.tokenizer.decode(visible_ids, skip_special_tokens=False)
|
| 680 |
+
# The template supplies this opening tag, not the decoder output.
|
| 681 |
+
body = ("<think>\n" + raw) if prompt.endswith("<think>\n") else raw
|
| 682 |
+
answer = self._without_thinking(body) if turn.enable_thinking is False else body
|
| 683 |
+
limit = generation_config.max_length - len(ids)
|
| 684 |
+
finish_reason = "eos" if completed else ("length" if len(generated) >= limit else "stopped")
|
| 685 |
+
result = {
|
| 686 |
+
"answer": answer,
|
| 687 |
+
"finish_reason": finish_reason,
|
| 688 |
+
"generation_completed": completed,
|
| 689 |
+
"turn_completed": completed,
|
| 690 |
+
"memory_updated": False,
|
| 691 |
+
"committed_turns": 0,
|
| 692 |
+
"documents_imported": 0,
|
| 693 |
+
"input_tokens": len(ids),
|
| 694 |
+
"output_tokens": len(generated),
|
| 695 |
+
"tool_calls": [],
|
| 696 |
+
"requires_action": False,
|
| 697 |
+
}
|
| 698 |
+
calls = []
|
| 699 |
+
if self._tools and "<tool_call>" in self._without_thinking(body):
|
| 700 |
+
try:
|
| 701 |
+
calls = self._parse_tools(self._without_thinking(body))
|
| 702 |
+
except ValueError as error:
|
| 703 |
+
if completed:
|
| 704 |
+
raise MemTSessionError("tool_parse", error, result=result) from error
|
| 705 |
+
if calls:
|
| 706 |
+
result.update(
|
| 707 |
+
finish_reason="tool_calls",
|
| 708 |
+
turn_completed=False,
|
| 709 |
+
tool_calls=calls,
|
| 710 |
+
requires_action=True,
|
| 711 |
+
)
|
| 712 |
+
# Prompt views are re-templated; replay keeps the exact generated IDs.
|
| 713 |
+
turn.messages.append({"role": "assistant", "content": body})
|
| 714 |
+
turn.replay_ids.extend(generated)
|
| 715 |
+
turn.tool_calls = calls
|
| 716 |
+
return result
|
| 717 |
+
|
| 718 |
+
def _replay(self, turns, memory):
|
| 719 |
+
candidate = memory
|
| 720 |
+
for turn in turns:
|
| 721 |
+
candidate = self.model.commit_memory(self._tensor(turn.replay_ids), candidate)
|
| 722 |
+
return candidate
|
| 723 |
+
|
| 724 |
+
def _finish(self, turn, result, memory):
|
| 725 |
+
self._memory, self._first_turn = memory, False
|
| 726 |
+
if turn.tool_calls:
|
| 727 |
+
self._pending = turn
|
| 728 |
+
else:
|
| 729 |
+
self._pending = None
|
| 730 |
+
self._history = self._history + [turn]
|
| 731 |
+
if turn.enable_memory and result["generation_completed"]:
|
| 732 |
+
turns = self._history
|
| 733 |
+
try:
|
| 734 |
+
candidate = self._replay(turns, memory)
|
| 735 |
+
except Exception as error:
|
| 736 |
+
raise MemTSessionError(
|
| 737 |
+
"replay", error, result=result, turn_retained=True
|
| 738 |
+
) from error
|
| 739 |
+
self._memory, self._history = candidate, []
|
| 740 |
+
result.update(memory_updated=True, committed_turns=len(turns))
|
| 741 |
+
return deepcopy(result) if turn.structured else result["answer"]
|
| 742 |
+
|
| 743 |
+
def chat(
|
| 744 |
+
self,
|
| 745 |
+
query,
|
| 746 |
+
*,
|
| 747 |
+
memory=None,
|
| 748 |
+
enable_memory=False,
|
| 749 |
+
enable_thinking=None,
|
| 750 |
+
return_dict=False,
|
| 751 |
+
**generation_options,
|
| 752 |
+
):
|
| 753 |
+
"""Generate first, then optionally commit uncompressed turns including this one."""
|
| 754 |
+
with self._lock:
|
| 755 |
+
self._check_open()
|
| 756 |
+
if self._pending is not None:
|
| 757 |
+
raise MemTSessionError("input", "submit the pending tool results before a new chat")
|
| 758 |
+
if not isinstance(query, str):
|
| 759 |
+
raise TypeError(
|
| 760 |
+
"query must be text; multimodal messages and assistant prefill are unsupported"
|
| 761 |
+
)
|
| 762 |
+
if type(enable_memory) is not bool or type(return_dict) is not bool:
|
| 763 |
+
raise TypeError("enable_memory and return_dict must be bool")
|
| 764 |
+
if enable_thinking is not None and type(enable_thinking) is not bool:
|
| 765 |
+
raise TypeError("enable_thinking must be bool or None")
|
| 766 |
+
self._validate_options(generation_options)
|
| 767 |
+
documents = [] if memory is None else ([memory] if isinstance(memory, str) else memory)
|
| 768 |
+
if memory is not None and not self._first_turn:
|
| 769 |
+
raise MemTSessionError("input", "documents are accepted only in the first chat")
|
| 770 |
+
if not isinstance(documents, list) or not all(
|
| 771 |
+
isinstance(item, str) for item in documents
|
| 772 |
+
):
|
| 773 |
+
raise TypeError("memory must be text or a list of document strings")
|
| 774 |
+
messages = [{"role": "user", "content": query}]
|
| 775 |
+
turn = _Turn(
|
| 776 |
+
messages,
|
| 777 |
+
self._encode(self._render(messages, generation=True, thinking=enable_thinking)),
|
| 778 |
+
enable_memory,
|
| 779 |
+
enable_thinking,
|
| 780 |
+
bool(self._tools) or return_dict,
|
| 781 |
+
dict(generation_options),
|
| 782 |
+
)
|
| 783 |
+
candidate = self._memory
|
| 784 |
+
try:
|
| 785 |
+
for document in documents:
|
| 786 |
+
tokens = self._encode(self._render([{"role": "user", "content": document}]))
|
| 787 |
+
candidate = self.model.commit_memory(
|
| 788 |
+
self._tensor(tokens), candidate, document=True
|
| 789 |
+
)
|
| 790 |
+
except Exception as error:
|
| 791 |
+
raise MemTSessionError("document", error) from error
|
| 792 |
+
try:
|
| 793 |
+
result = self._generate(turn, candidate)
|
| 794 |
+
except MemTSessionError:
|
| 795 |
+
raise
|
| 796 |
+
except Exception as error:
|
| 797 |
+
raise MemTSessionError("generation", error) from error
|
| 798 |
+
result["documents_imported"] = len(documents)
|
| 799 |
+
return self._finish(turn, result, candidate)
|
| 800 |
+
|
| 801 |
+
def submit_tool_results(self, results, **generation_options):
|
| 802 |
+
"""Accept all results of the pending assistant message in a single batch."""
|
| 803 |
+
with self._lock:
|
| 804 |
+
self._check_open()
|
| 805 |
+
if self._pending is None:
|
| 806 |
+
raise MemTSessionError("input", "there are no pending tool requests")
|
| 807 |
+
self._validate_options(generation_options)
|
| 808 |
+
calls = self._pending.tool_calls
|
| 809 |
+
if not isinstance(results, list) or len(results) != len(calls):
|
| 810 |
+
raise ValueError("supply exactly one result for every pending tool call")
|
| 811 |
+
by_id = {}
|
| 812 |
+
for result in results:
|
| 813 |
+
if not isinstance(result, dict) or not isinstance(result.get("content"), str):
|
| 814 |
+
raise TypeError("tool results require tool_call_id and text content")
|
| 815 |
+
identifier = result.get("tool_call_id")
|
| 816 |
+
if identifier in by_id or identifier not in {call["id"] for call in calls}:
|
| 817 |
+
raise ValueError("duplicate or unknown tool_call_id")
|
| 818 |
+
by_id[identifier] = result["content"]
|
| 819 |
+
turn = deepcopy(self._pending)
|
| 820 |
+
tool_messages = [
|
| 821 |
+
{
|
| 822 |
+
"role": "tool",
|
| 823 |
+
"tool_call_id": call["id"],
|
| 824 |
+
"content": by_id[call["id"]],
|
| 825 |
+
}
|
| 826 |
+
for call in calls
|
| 827 |
+
]
|
| 828 |
+
# Render only the newly supplied result group and assistant prefix.
|
| 829 |
+
dummy = [{"role": "user", "content": ""}]
|
| 830 |
+
prefix = self._render(dummy)
|
| 831 |
+
suffix = self._render(
|
| 832 |
+
dummy + tool_messages, generation=True, thinking=turn.enable_thinking
|
| 833 |
+
)[len(prefix) :]
|
| 834 |
+
turn.replay_ids.extend(self._encode("\n" + suffix))
|
| 835 |
+
turn.messages.extend(tool_messages)
|
| 836 |
+
turn.generation_options.update(generation_options)
|
| 837 |
+
try:
|
| 838 |
+
result = self._generate(turn, self._memory)
|
| 839 |
+
except MemTSessionError as error:
|
| 840 |
+
error.turn_retained = True
|
| 841 |
+
raise
|
| 842 |
+
except Exception as error:
|
| 843 |
+
# Nothing is consumed: the caller may resubmit already computed results.
|
| 844 |
+
raise MemTSessionError("generation", error, turn_retained=True) from error
|
| 845 |
+
return self._finish(turn, result, self._memory)
|
| 846 |
+
|
| 847 |
+
def _state_identity(self):
|
| 848 |
+
identity = {
|
| 849 |
+
"backbone": self.model.config.backbone_configs,
|
| 850 |
+
"memt": self.model.config.memt,
|
| 851 |
+
"codec": "qwen-text-raw-v1",
|
| 852 |
+
"template": self.tokenizer.chat_template,
|
| 853 |
+
"vocabulary": self.tokenizer.get_vocab(),
|
| 854 |
+
}
|
| 855 |
+
return hashlib.sha256(
|
| 856 |
+
json.dumps(identity, sort_keys=True, ensure_ascii=True).encode()
|
| 857 |
+
).hexdigest()
|
| 858 |
+
|
| 859 |
+
def _load_state(self, path):
|
| 860 |
+
try:
|
| 861 |
+
path = Path(path)
|
| 862 |
+
text = self.model.config.get_text_config()
|
| 863 |
+
layers = [
|
| 864 |
+
index for index, kind in enumerate(text.layer_types) if kind == "full_attention"
|
| 865 |
+
]
|
| 866 |
+
expected = {f"memory.{index}" for index in layers}
|
| 867 |
+
memory = {}
|
| 868 |
+
with safe_open(path, framework="pt", device="cpu") as source:
|
| 869 |
+
metadata = source.metadata() or {}
|
| 870 |
+
if (
|
| 871 |
+
metadata.get("format") != "memt-memory-v1"
|
| 872 |
+
or metadata.get("identity") != self._state_identity()
|
| 873 |
+
):
|
| 874 |
+
raise ValueError("incompatible memory format, model or tokenizer")
|
| 875 |
+
keys = set(source.keys())
|
| 876 |
+
if keys and keys != expected:
|
| 877 |
+
raise ValueError("memory layer set does not match the model")
|
| 878 |
+
for index in layers if keys else []:
|
| 879 |
+
tensor = source.get_tensor(f"memory.{index}")
|
| 880 |
+
shape = (1, text.num_attention_heads, text.head_dim, text.head_dim)
|
| 881 |
+
if (
|
| 882 |
+
tuple(tensor.shape) != shape
|
| 883 |
+
or tensor.dtype != torch.float32
|
| 884 |
+
or not torch.isfinite(tensor).all()
|
| 885 |
+
):
|
| 886 |
+
raise ValueError(f"invalid FP32 memory tensor for layer {index}")
|
| 887 |
+
device = self.model.model.language_model.layers[
|
| 888 |
+
index
|
| 889 |
+
].self_attn.memt.in_proj.weight.device
|
| 890 |
+
memory[index] = tensor.to(device=device, copy=True)
|
| 891 |
+
return memory
|
| 892 |
+
except Exception as error:
|
| 893 |
+
raise MemTSessionError("load", error) from error
|
| 894 |
+
|
| 895 |
+
def save_state(self, path, *, overwrite=False):
|
| 896 |
+
"""Flush all plaintext turns, then atomically publish a memory-only file."""
|
| 897 |
+
if type(overwrite) is not bool:
|
| 898 |
+
raise TypeError("overwrite must be bool")
|
| 899 |
+
with self._lock:
|
| 900 |
+
self._check_open()
|
| 901 |
+
if self._pending is not None:
|
| 902 |
+
raise MemTSessionError("save", "cannot save while tool results are pending")
|
| 903 |
+
path = Path(path)
|
| 904 |
+
if not overwrite and os.path.lexists(path):
|
| 905 |
+
raise MemTSessionError("save", f"file already exists: {path}")
|
| 906 |
+
temporary, published, publishing = None, False, False
|
| 907 |
+
try:
|
| 908 |
+
candidate = self._replay(self._history, self._memory)
|
| 909 |
+
tensors = {
|
| 910 |
+
f"memory.{index}": tensor.detach()
|
| 911 |
+
.to(device="cpu", dtype=torch.float32)
|
| 912 |
+
.contiguous()
|
| 913 |
+
for index, tensor in candidate.items()
|
| 914 |
+
}
|
| 915 |
+
descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
| 916 |
+
os.close(descriptor)
|
| 917 |
+
save_file(
|
| 918 |
+
tensors,
|
| 919 |
+
temporary,
|
| 920 |
+
metadata={
|
| 921 |
+
"format": "memt-memory-v1",
|
| 922 |
+
"identity": self._state_identity(),
|
| 923 |
+
},
|
| 924 |
+
)
|
| 925 |
+
with open(temporary, "rb") as handle:
|
| 926 |
+
os.fsync(handle.fileno())
|
| 927 |
+
publishing = True
|
| 928 |
+
if overwrite:
|
| 929 |
+
os.replace(temporary, path)
|
| 930 |
+
else:
|
| 931 |
+
os.link(temporary, path)
|
| 932 |
+
published = True
|
| 933 |
+
# Publication precedes RAM adoption. A later failure closes this session.
|
| 934 |
+
self._memory, self._history = candidate, []
|
| 935 |
+
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
| 936 |
+
try:
|
| 937 |
+
os.fsync(directory)
|
| 938 |
+
finally:
|
| 939 |
+
os.close(directory)
|
| 940 |
+
if os.path.exists(temporary):
|
| 941 |
+
os.unlink(temporary)
|
| 942 |
+
temporary = None
|
| 943 |
+
except Exception as error:
|
| 944 |
+
uncertain = publishing and not published and not isinstance(error, FileExistsError)
|
| 945 |
+
if published or uncertain:
|
| 946 |
+
self._closed = True
|
| 947 |
+
failure = MemTSessionError(
|
| 948 |
+
"save_publish" if publishing else "save",
|
| 949 |
+
error,
|
| 950 |
+
turn_retained=not published and not uncertain,
|
| 951 |
+
memory_updated=published,
|
| 952 |
+
publication="published"
|
| 953 |
+
if published
|
| 954 |
+
else ("uncertain" if uncertain else "not_published"),
|
| 955 |
+
)
|
| 956 |
+
if temporary is not None and os.path.exists(temporary):
|
| 957 |
+
try:
|
| 958 |
+
os.unlink(temporary)
|
| 959 |
+
except OSError as cleanup_error:
|
| 960 |
+
failure.add_note(
|
| 961 |
+
f"Temporary snapshot retained at {temporary}: {cleanup_error}"
|
| 962 |
+
)
|
| 963 |
+
raise failure from error
|
| 964 |
+
return {"path": str(path), "publication": "published"}
|
| 965 |
+
|
| 966 |
+
def reset(self):
|
| 967 |
+
with self._lock:
|
| 968 |
+
self._check_open()
|
| 969 |
+
self._history, self._memory, self._pending = [], {}, None
|
| 970 |
+
self._first_turn = True
|
| 971 |
+
|
| 972 |
+
def close(self):
|
| 973 |
+
with self._lock:
|
| 974 |
+
self._history, self._memory, self._pending = [], {}, None
|
| 975 |
+
self._closed = True
|
| 976 |
+
|
| 977 |
+
|
| 978 |
+
MemTQwen3_5MoeForCausalLM.register_for_auto_class("AutoModelForCausalLM")
|
tech_report.pdf
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:775fb706e9b789fdc7077d1bd68b8ffb9d5f805f64fe28be2458af456df41f51
|
| 3 |
+
size 7093967
|
tokenizer.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42
|
| 3 |
+
size 12807982
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_prefix_space": false,
|
| 3 |
+
"added_tokens_decoder": {
|
| 4 |
+
"248044": {
|
| 5 |
+
"content": "<|endoftext|>",
|
| 6 |
+
"lstrip": false,
|
| 7 |
+
"normalized": false,
|
| 8 |
+
"rstrip": false,
|
| 9 |
+
"single_word": false,
|
| 10 |
+
"special": true
|
| 11 |
+
},
|
| 12 |
+
"248045": {
|
| 13 |
+
"content": "<|im_start|>",
|
| 14 |
+
"lstrip": false,
|
| 15 |
+
"normalized": false,
|
| 16 |
+
"rstrip": false,
|
| 17 |
+
"single_word": false,
|
| 18 |
+
"special": true
|
| 19 |
+
},
|
| 20 |
+
"248046": {
|
| 21 |
+
"content": "<|im_end|>",
|
| 22 |
+
"lstrip": false,
|
| 23 |
+
"normalized": false,
|
| 24 |
+
"rstrip": false,
|
| 25 |
+
"single_word": false,
|
| 26 |
+
"special": true
|
| 27 |
+
},
|
| 28 |
+
"248047": {
|
| 29 |
+
"content": "<|object_ref_start|>",
|
| 30 |
+
"lstrip": false,
|
| 31 |
+
"normalized": false,
|
| 32 |
+
"rstrip": false,
|
| 33 |
+
"single_word": false,
|
| 34 |
+
"special": true
|
| 35 |
+
},
|
| 36 |
+
"248048": {
|
| 37 |
+
"content": "<|object_ref_end|>",
|
| 38 |
+
"lstrip": false,
|
| 39 |
+
"normalized": false,
|
| 40 |
+
"rstrip": false,
|
| 41 |
+
"single_word": false,
|
| 42 |
+
"special": true
|
| 43 |
+
},
|
| 44 |
+
"248049": {
|
| 45 |
+
"content": "<|box_start|>",
|
| 46 |
+
"lstrip": false,
|
| 47 |
+
"normalized": false,
|
| 48 |
+
"rstrip": false,
|
| 49 |
+
"single_word": false,
|
| 50 |
+
"special": true
|
| 51 |
+
},
|
| 52 |
+
"248050": {
|
| 53 |
+
"content": "<|box_end|>",
|
| 54 |
+
"lstrip": false,
|
| 55 |
+
"normalized": false,
|
| 56 |
+
"rstrip": false,
|
| 57 |
+
"single_word": false,
|
| 58 |
+
"special": true
|
| 59 |
+
},
|
| 60 |
+
"248051": {
|
| 61 |
+
"content": "<|quad_start|>",
|
| 62 |
+
"lstrip": false,
|
| 63 |
+
"normalized": false,
|
| 64 |
+
"rstrip": false,
|
| 65 |
+
"single_word": false,
|
| 66 |
+
"special": true
|
| 67 |
+
},
|
| 68 |
+
"248052": {
|
| 69 |
+
"content": "<|quad_end|>",
|
| 70 |
+
"lstrip": false,
|
| 71 |
+
"normalized": false,
|
| 72 |
+
"rstrip": false,
|
| 73 |
+
"single_word": false,
|
| 74 |
+
"special": true
|
| 75 |
+
},
|
| 76 |
+
"248053": {
|
| 77 |
+
"content": "<|vision_start|>",
|
| 78 |
+
"lstrip": false,
|
| 79 |
+
"normalized": false,
|
| 80 |
+
"rstrip": false,
|
| 81 |
+
"single_word": false,
|
| 82 |
+
"special": true
|
| 83 |
+
},
|
| 84 |
+
"248054": {
|
| 85 |
+
"content": "<|vision_end|>",
|
| 86 |
+
"lstrip": false,
|
| 87 |
+
"normalized": false,
|
| 88 |
+
"rstrip": false,
|
| 89 |
+
"single_word": false,
|
| 90 |
+
"special": true
|
| 91 |
+
},
|
| 92 |
+
"248055": {
|
| 93 |
+
"content": "<|vision_pad|>",
|
| 94 |
+
"lstrip": false,
|
| 95 |
+
"normalized": false,
|
| 96 |
+
"rstrip": false,
|
| 97 |
+
"single_word": false,
|
| 98 |
+
"special": true
|
| 99 |
+
},
|
| 100 |
+
"248056": {
|
| 101 |
+
"content": "<|image_pad|>",
|
| 102 |
+
"lstrip": false,
|
| 103 |
+
"normalized": false,
|
| 104 |
+
"rstrip": false,
|
| 105 |
+
"single_word": false,
|
| 106 |
+
"special": true
|
| 107 |
+
},
|
| 108 |
+
"248057": {
|
| 109 |
+
"content": "<|video_pad|>",
|
| 110 |
+
"lstrip": false,
|
| 111 |
+
"normalized": false,
|
| 112 |
+
"rstrip": false,
|
| 113 |
+
"single_word": false,
|
| 114 |
+
"special": true
|
| 115 |
+
},
|
| 116 |
+
"248058": {
|
| 117 |
+
"content": "<tool_call>",
|
| 118 |
+
"lstrip": false,
|
| 119 |
+
"normalized": false,
|
| 120 |
+
"rstrip": false,
|
| 121 |
+
"single_word": false,
|
| 122 |
+
"special": false
|
| 123 |
+
},
|
| 124 |
+
"248059": {
|
| 125 |
+
"content": "</tool_call>",
|
| 126 |
+
"lstrip": false,
|
| 127 |
+
"normalized": false,
|
| 128 |
+
"rstrip": false,
|
| 129 |
+
"single_word": false,
|
| 130 |
+
"special": false
|
| 131 |
+
},
|
| 132 |
+
"248060": {
|
| 133 |
+
"content": "<|fim_prefix|>",
|
| 134 |
+
"lstrip": false,
|
| 135 |
+
"normalized": false,
|
| 136 |
+
"rstrip": false,
|
| 137 |
+
"single_word": false,
|
| 138 |
+
"special": false
|
| 139 |
+
},
|
| 140 |
+
"248061": {
|
| 141 |
+
"content": "<|fim_middle|>",
|
| 142 |
+
"lstrip": false,
|
| 143 |
+
"normalized": false,
|
| 144 |
+
"rstrip": false,
|
| 145 |
+
"single_word": false,
|
| 146 |
+
"special": false
|
| 147 |
+
},
|
| 148 |
+
"248062": {
|
| 149 |
+
"content": "<|fim_suffix|>",
|
| 150 |
+
"lstrip": false,
|
| 151 |
+
"normalized": false,
|
| 152 |
+
"rstrip": false,
|
| 153 |
+
"single_word": false,
|
| 154 |
+
"special": false
|
| 155 |
+
},
|
| 156 |
+
"248063": {
|
| 157 |
+
"content": "<|fim_pad|>",
|
| 158 |
+
"lstrip": false,
|
| 159 |
+
"normalized": false,
|
| 160 |
+
"rstrip": false,
|
| 161 |
+
"single_word": false,
|
| 162 |
+
"special": false
|
| 163 |
+
},
|
| 164 |
+
"248064": {
|
| 165 |
+
"content": "<|repo_name|>",
|
| 166 |
+
"lstrip": false,
|
| 167 |
+
"normalized": false,
|
| 168 |
+
"rstrip": false,
|
| 169 |
+
"single_word": false,
|
| 170 |
+
"special": false
|
| 171 |
+
},
|
| 172 |
+
"248065": {
|
| 173 |
+
"content": "<|file_sep|>",
|
| 174 |
+
"lstrip": false,
|
| 175 |
+
"normalized": false,
|
| 176 |
+
"rstrip": false,
|
| 177 |
+
"single_word": false,
|
| 178 |
+
"special": false
|
| 179 |
+
},
|
| 180 |
+
"248066": {
|
| 181 |
+
"content": "<tool_response>",
|
| 182 |
+
"lstrip": false,
|
| 183 |
+
"normalized": false,
|
| 184 |
+
"rstrip": false,
|
| 185 |
+
"single_word": false,
|
| 186 |
+
"special": false
|
| 187 |
+
},
|
| 188 |
+
"248067": {
|
| 189 |
+
"content": "</tool_response>",
|
| 190 |
+
"lstrip": false,
|
| 191 |
+
"normalized": false,
|
| 192 |
+
"rstrip": false,
|
| 193 |
+
"single_word": false,
|
| 194 |
+
"special": false
|
| 195 |
+
},
|
| 196 |
+
"248068": {
|
| 197 |
+
"content": "<think>",
|
| 198 |
+
"lstrip": false,
|
| 199 |
+
"normalized": false,
|
| 200 |
+
"rstrip": false,
|
| 201 |
+
"single_word": false,
|
| 202 |
+
"special": false
|
| 203 |
+
},
|
| 204 |
+
"248069": {
|
| 205 |
+
"content": "</think>",
|
| 206 |
+
"lstrip": false,
|
| 207 |
+
"normalized": false,
|
| 208 |
+
"rstrip": false,
|
| 209 |
+
"single_word": false,
|
| 210 |
+
"special": false
|
| 211 |
+
},
|
| 212 |
+
"248070": {
|
| 213 |
+
"content": "<|audio_start|>",
|
| 214 |
+
"lstrip": false,
|
| 215 |
+
"normalized": false,
|
| 216 |
+
"rstrip": false,
|
| 217 |
+
"single_word": false,
|
| 218 |
+
"special": true
|
| 219 |
+
},
|
| 220 |
+
"248071": {
|
| 221 |
+
"content": "<|audio_end|>",
|
| 222 |
+
"lstrip": false,
|
| 223 |
+
"normalized": false,
|
| 224 |
+
"rstrip": false,
|
| 225 |
+
"single_word": false,
|
| 226 |
+
"special": true
|
| 227 |
+
},
|
| 228 |
+
"248072": {
|
| 229 |
+
"content": "<tts_pad>",
|
| 230 |
+
"lstrip": false,
|
| 231 |
+
"normalized": false,
|
| 232 |
+
"rstrip": false,
|
| 233 |
+
"single_word": false,
|
| 234 |
+
"special": true
|
| 235 |
+
},
|
| 236 |
+
"248073": {
|
| 237 |
+
"content": "<tts_text_bos>",
|
| 238 |
+
"lstrip": false,
|
| 239 |
+
"normalized": false,
|
| 240 |
+
"rstrip": false,
|
| 241 |
+
"single_word": false,
|
| 242 |
+
"special": true
|
| 243 |
+
},
|
| 244 |
+
"248074": {
|
| 245 |
+
"content": "<tts_text_eod>",
|
| 246 |
+
"lstrip": false,
|
| 247 |
+
"normalized": false,
|
| 248 |
+
"rstrip": false,
|
| 249 |
+
"single_word": false,
|
| 250 |
+
"special": true
|
| 251 |
+
},
|
| 252 |
+
"248075": {
|
| 253 |
+
"content": "<tts_text_bos_single>",
|
| 254 |
+
"lstrip": false,
|
| 255 |
+
"normalized": false,
|
| 256 |
+
"rstrip": false,
|
| 257 |
+
"single_word": false,
|
| 258 |
+
"special": true
|
| 259 |
+
},
|
| 260 |
+
"248076": {
|
| 261 |
+
"content": "<|audio_pad|>",
|
| 262 |
+
"lstrip": false,
|
| 263 |
+
"normalized": false,
|
| 264 |
+
"rstrip": false,
|
| 265 |
+
"single_word": false,
|
| 266 |
+
"special": true
|
| 267 |
+
}
|
| 268 |
+
},
|
| 269 |
+
"additional_special_tokens": [
|
| 270 |
+
"<|im_start|>",
|
| 271 |
+
"<|im_end|>",
|
| 272 |
+
"<|object_ref_start|>",
|
| 273 |
+
"<|object_ref_end|>",
|
| 274 |
+
"<|box_start|>",
|
| 275 |
+
"<|box_end|>",
|
| 276 |
+
"<|quad_start|>",
|
| 277 |
+
"<|quad_end|>",
|
| 278 |
+
"<|vision_start|>",
|
| 279 |
+
"<|vision_end|>",
|
| 280 |
+
"<|vision_pad|>",
|
| 281 |
+
"<|image_pad|>",
|
| 282 |
+
"<|video_pad|>"
|
| 283 |
+
],
|
| 284 |
+
"bos_token": null,
|
| 285 |
+
"chat_template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '</think>' in content %}\n {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if preserve_thinking is defined and preserve_thinking is true and content.startswith('<think>') and '</think>' not in content %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- elif (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- else %}\n {{- '<think>\\n' }}\n {%- endif %}\n{%- endif %}",
|
| 286 |
+
"clean_up_tokenization_spaces": false,
|
| 287 |
+
"eos_token": "<|im_end|>",
|
| 288 |
+
"errors": "replace",
|
| 289 |
+
"model_max_length": 262144,
|
| 290 |
+
"pad_token": "<|endoftext|>",
|
| 291 |
+
"split_special_tokens": false,
|
| 292 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
| 293 |
+
"fix_mistral_regex": false,
|
| 294 |
+
"unk_token": null,
|
| 295 |
+
"add_bos_token": false,
|
| 296 |
+
"pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
|
| 297 |
+
"extra_special_tokens": {
|
| 298 |
+
"audio_bos_token": "<|audio_start|>",
|
| 299 |
+
"audio_eos_token": "<|audio_end|>",
|
| 300 |
+
"audio_token": "<|audio_pad|>",
|
| 301 |
+
"image_token": "<|image_pad|>",
|
| 302 |
+
"video_token": "<|video_pad|>",
|
| 303 |
+
"vision_bos_token": "<|vision_start|>",
|
| 304 |
+
"vision_eos_token": "<|vision_end|>"
|
| 305 |
+
}
|
| 306 |
+
}
|
vocab.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|