Update model.py

model.py

@@ -337,14 +337,14 @@ class rotary(nn.Module):
 
         self.theta = nn.Parameter(torch.tensor(theta, device=device, dtype=dtype), requires_grad=True)
         
-        freqs = 1. / (theta ** (torch.arange(0, dim, 2, device=device, dtype=dtype)[:(dim // 2)].float() / dims))
-        self.freqs = nn.Parameter(torch.tensor(freqs, device=device, dtype=dtype), requires_grad=True)
-        self.radius = nn.Parameter(torch.ones(radius, device=device, dtype=dtype), requires_grad=True)
-        freq_data = 1.0 / (theta ** (torch.arange(start=0, end=dim, step=2).float() / dim))
-        self.inv_freq = nn.Parameter(freq_data, requires_grad=True)
+        # freqs = 1. / (theta ** (torch.arange(0, dim, 2, device=device, dtype=dtype)[:(dim // 2)].float() / dims))
+        # self.freqs = nn.Parameter(torch.tensor(freqs, device=device, dtype=dtype), requires_grad=True)
+        # self.radius = nn.Parameter(torch.ones(radius, device=device, dtype=dtype), requires_grad=True)
+        # freq_data = 1.0 / (theta ** (torch.arange(start=0, end=dim, step=2).float() / dim))
+        # self.inv_freq = nn.Parameter(freq_data, requires_grad=True)
         
         freqb = 700 * (torch.pow(10, torch.linspace(0, 2595 * torch.log10(torch.tensor(1 + 8000/700)), dim // 2, device=device, dtype=dtype) / 2595) - 1) / 1000 
-        self.freqb = nn.Parameter(torch.tensor(freqb, device=device, dtype=dtype), requires_grad=True)
+        self.inv_freq = nn.Parameter(torch.tensor(freqb, device=device, dtype=dtype), requires_grad=True)
 
     def update_base(self, pitch):
         theta = pitch.squeeze(0).to(device, dtype)
@@ -403,11 +403,11 @@ class rotary(nn.Module):
         if f0 is not None:
             f0_mean = f0.mean()
             theta = f0_mean + 1e-8
-            freqs = self.freqb
+            freqs = self.inv_freq
             if "rotary1" in self.debug:
                 print(f"{layer}: {theta:.2f} : {f0_mean:.2f} : {ctx} ")
         else:
-            freqs = self.freqb
+            freqs = self.inv_freq
             
         freqs = t[:, None] * freqs[None, :]
         if self.radii: