Update app.py

app.py

@@ -50,7 +50,11 @@ class AdvancedModel(nn.Module):
         self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers=num_layers, batch_first=True, dropout=dropout)
         self.gru = nn.GRU(input_dim, hidden_dim, num_layers=num_layers, batch_first=True, dropout=dropout)
         
-        encoder_layers = TransformerEncoderLayer(d_model=input_dim, nhead=nhead, dim_feedforward=hidden_dim, dropout=dropout)
+        # Adjust input_dim for transformer if it's not divisible by nhead
+        transformer_dim = (input_dim // nhead) * nhead
+        self.input_proj = nn.Linear(input_dim, transformer_dim) if input_dim != transformer_dim else nn.Identity()
+        
+        encoder_layers = TransformerEncoderLayer(d_model=transformer_dim, nhead=nhead, dim_feedforward=hidden_dim, dropout=dropout)
         self.transformer = TransformerEncoder(encoder_layers, num_layers=num_layers)
         
         self.attention = nn.MultiheadAttention(hidden_dim, num_heads=nhead, dropout=dropout)
@@ -65,7 +69,10 @@ class AdvancedModel(nn.Module):
     def forward(self, x):
         lstm_out, _ = self.lstm(x)
         gru_out, _ = self.gru(x)
-        transformer_out = self.transformer(x.transpose(0, 1)).transpose(0, 1)
+        
+        # Project input for transformer if necessary
+        transformer_input = self.input_proj(x)
+        transformer_out = self.transformer(transformer_input.transpose(0, 1)).transpose(0, 1)
         
         combined = torch.cat((lstm_out[:, -1, :], gru_out[:, -1, :], transformer_out[:, -1, :]), dim=1)
         
@@ -77,7 +84,12 @@ def objective(trial):
     hidden_dim = trial.suggest_int("hidden_dim", 64, 256)
     output_dim = len(target_cols)
     num_layers = trial.suggest_int("num_layers", 1, 4)
-    nhead = trial.suggest_int("nhead", 2, 8)
+    
+    # Ensure that hidden_dim is divisible by nhead
+    max_nhead = min(8, hidden_dim // 8)  # Ensure at least 8 dimensions per head
+    nhead = trial.suggest_int("nhead", 2, max_nhead)
+    hidden_dim = (hidden_dim // nhead) * nhead  # Adjust hidden_dim to be divisible by nhead
+    
     dropout = trial.suggest_float("dropout", 0.1, 0.5)
     lr = trial.suggest_loguniform("lr", 1e-5, 1e-2)