Update model.py

model.py

@@ -342,6 +342,9 @@ class rotary(nn.Module):
         self.radius = nn.Parameter(torch.ones(radius, device=device, dtype=dtype), requires_grad=True)
         freq_data = 1.0 / (theta ** (torch.arange(start=0, end=dim, step=2).float() / dim))
         self.inv_freq = nn.Parameter(freq_data, requires_grad=True)
+        
+        freqb = 700 * (torch.pow(10, torch.linspace(0, 2595 * torch.log10(torch.tensor(1 + 8000/700)), dim // 2, device=device, dtype=dtype) / 2595) - 1) / 1000 
+        self.freqb = nn.Parameter(torch.tensor(freqb, device=device, dtype=dtype), requires_grad=True)
 
     def update_base(self, pitch):
         theta = pitch.squeeze(0).to(device, dtype)
@@ -400,11 +403,12 @@ class rotary(nn.Module):
         if f0 is not None:
             f0_mean = f0.mean()
             theta = f0_mean + 1e-8
-            freqs = 1.0 / (theta ** (torch.arange(0, self.dim, 2, device=device, dtype=dtype)[:(self.dim // 2)].float() / self.dim))
+            freqs = self.freqb
             if "rotary1" in self.debug:
                 print(f"{layer}: {theta:.2f} : {f0_mean:.2f} : {ctx} ")
         else:
-            freqs = self.freqs
+            freqs = self.freqb
+            
         freqs = t[:, None] * freqs[None, :]
         if self.radii:
             if f0 is not None: