Update app.py

app.py

@@ -2,11 +2,21 @@ from sentence_transformers import SentenceTransformer
 import streamlit as st
 import pandas as pd
 from PyPDF2 import PdfReader
+import nltk
+nltk.download('punkt')
+from nltk.corpus import stopwords
+nltk.download('stopwords')
+from nltk.tokenize import word_tokenize
 
 model = SentenceTransformer("all-mpnet-base-v2")
 st.title("AI Resume Analysis based on Keywords App")
 st.divider()
 job_desc = st.text_area("Paste the job description and then press Ctrl + Enter", key="job_desc")
+text_tokens = []
+for sentence in job_desc:
+    text_tokens.extend(word_tokenize(job_desc))  
+    job_desc = [word for word in text_tokens if not word in stopwords.words()]
+    st.write(job_desc)
 
 if 'applicant_data' not in st.session_state:
     st.session_state['applicant_data'] = {}
@@ -30,8 +40,13 @@ for i in range(1, 51):  # Looping for 50 applicants
                 for page in pdf_reader.pages:
                     text_data += page.extract_text()
                 with st.expander(f"See Applicant's {i} resume"):
-                    st.write(text_data)
-
+                    text_tokens = []
+                    for sentence in text_data:
+                        text_tokens.extend(word_tokenize(text_data))  
+                        text_data = [word for word in text_tokens if not word in stopwords.words()]
+                        st.write(text_data)
+                
+                
                 # Encode the job description and resume text separately
                 job_embedding = model.encode([job_desc])
                 resume_embedding = model.encode([text_data])