Update evo_model.py

evo_model.py

@@ -0,0 +1,32 @@
+import torch
+import torch.nn as nn
+from torch.nn import TransformerEncoder, TransformerEncoderLayer
+
+class EvoTransformer(nn.Module):
+    def __init__(self, vocab_size=30522, d_model=512, nhead=8, num_layers=6, dim_feedforward=1024, dropout=0.1):
+        super(EvoTransformer, self).__init__()
+        self.embedding = nn.Embedding(vocab_size, d_model)
+        self.memory_token = nn.Parameter(torch.zeros(1, 1, d_model))
+        
+        encoder_layer = TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward, dropout=dropout)
+        self.transformer = TransformerEncoder(encoder_layer, num_layers=num_layers)
+        
+        self.memory_proj = nn.Linear(d_model, d_model)
+        self.norm = nn.LayerNorm(d_model)
+
+    def forward(self, input_ids):
+        # Embed tokens
+        x = self.embedding(input_ids)
+
+        # Concatenate memory token
+        memory_token = self.memory_token.expand(x.size(0), -1, -1)
+        x = torch.cat([memory_token, x], dim=1)
+
+        # Transformer encoding
+        x = self.transformer(x)
+        x = self.norm(x)
+
+        # Extract memory representation (first token)
+        memory_output = self.memory_proj(x[:, 0])
+
+        return memory_output