Update app.py

app.py

@@ -19,7 +19,8 @@ def generate_embeddings(texts):
 
     for text in texts:
         # Tokenize the text with truncation set to False
-        tokens = bert_tokenizer.encode(text, truncation=False)  # Do not truncate here
+        # We are using the BertTokenizer directly without using the pipeline
+        tokens = bert_tokenizer.encode(text, add_special_tokens=True, truncation=False, padding=False)
 
         # Split the tokens into chunks of size 512 (maximum length)
         chunked_texts = [tokens[i:i + 512] for i in range(0, len(tokens), 512)]
@@ -27,11 +28,11 @@ def generate_embeddings(texts):
         poem_embeddings = []
 
         for chunk in chunked_texts:
-            # Prepare the input tensor for the model
+            # Convert the chunk to a tensor and prepare the input for BERT model
             inputs = torch.tensor(chunk).unsqueeze(0)  # Adding batch dimension
             with torch.no_grad():
                 outputs = bert_model(inputs)
-            # Get the embeddings from the last hidden state
+            # Get the embeddings from the last hidden state (mean of all token embeddings)
             chunk_embedding = outputs.last_hidden_state.mean(dim=1).numpy()
 
             poem_embeddings.append(chunk_embedding)