Spaces:
Sleeping
Sleeping
File size: 1,343 Bytes
9dce9f3 d6397a3 9dce9f3 d6397a3 9dce9f3 d6397a3 9dce9f3 a41d201 d6397a3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 |
import random
import copy
class EvoTransformer:
def __init__(self):
self.history = []
self.base_config = {
"layers": 4,
"attention_heads": 4,
"ffn_dim": 1024,
"dropout": 0.1,
"memory": False
}
def reset(self):
self.history = []
def mutate(self, config):
new_config = copy.deepcopy(config)
if random.random() < 0.5:
new_config["layers"] = min(12, max(1, new_config["layers"] + random.choice([-1, 1])))
if random.random() < 0.5:
new_config["attention_heads"] = min(12, max(1, new_config["attention_heads"] + random.choice([-1, 1])))
if random.random() < 0.5:
new_config["ffn_dim"] = min(4096, max(128, new_config["ffn_dim"] + random.choice([-512, 512])))
if random.random() < 0.5:
new_config["dropout"] = round(min(0.5, max(0.0, new_config["dropout"] + random.choice([-0.02, 0.02]))), 2)
if random.random() < 0.3:
new_config["memory"] = not new_config["memory"]
return new_config
def run_evolution(self, generations=5):
current = self.base_config
self.history.append(current)
for _ in range(generations - 1):
current = self.mutate(current)
self.history.append(current)
|