Upload 3 files

app.py

@@ -0,0 +1,19 @@
+import streamlit as st
+import seaborn as sns
+from transformers import pipeline
+
+
+sentiment_model = pipeline(model="sentiment-analysis")
+
+st.write('Hi')
+
+sentiments = []
+for text in df['clean_text']:
+    if list(sentiment_model(text)[0].values())[0] == 'LABEL_1':
+        output = 'Positive'
+    else:
+        output = 'Negative'
+    sentiments.append(output)
+
+df['sentiments'] = sentiments
+sns.countplot(df['sentiments'])

data_cleaning.py

@@ -0,0 +1,33 @@
+import re
+import nltk
+nltk.download('stopwords')
+from nltk.corpus import stopwords
+nltk.download('punkt')
+from nltk import sent_tokenize,word_tokenize
+from nltk.stem.snowball import SnowballStemmer
+
+
+def normalize(text):
+    return(text.lower())
+
+def remove_stopwords(text):
+  list_stopwords =  stopwords.words("english")
+  finalText=' '.join(a for a in word_tokenize(text) if (a not in list_stopwords and a.isalnum()))
+  return finalText
+
+def removenumbers(text):
+    re_num = "\d+" ###COMPLETE THE REGULAR EXPRESSION
+    text = re.sub(re_num, "", text)
+    return text
+
+def stem_text(text):
+  stemmer = SnowballStemmer("english")
+  t=' '.join(stemmer.stem(a) for a in word_tokenize(text))
+  return t
+
+def preprocess(text):
+  text = normalize(text)
+  text = remove_stopwords(text)
+  text = removenumbers(text)
+  text = stem_text(text)
+  return(text)

data_integration.py

@@ -0,0 +1,88 @@
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+
+custom_headers = {
+    "Accept-language": "en-GB,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Cache-Control": "max-age=0",
+    "Connection": "keep-alive",
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
+}
+
+def get_soup(url):
+    response = requests.get(url, headers=custom_headers)
+
+    if response.status_code != 200:
+        print("Error in getting webpage")
+        print(f"Error: {response.status_code} - {response.reason}")
+        exit(-1)
+
+    soup = BeautifulSoup(response.text, "lxml")
+    return soup
+
+def get_reviews(soup):
+    review_elements = soup.select("div.review")
+
+    scraped_reviews = []
+
+    for review in review_elements:
+        r_author_element = review.select_one("span.a-profile-name")
+        r_author = r_author_element.text if r_author_element else None
+
+        r_rating_element = review.select_one("i.review-rating")
+        r_rating = r_rating_element.text.replace("out of 5 stars", "") if r_rating_element else None
+
+        r_title_element = review.select_one("a.review-title")
+        r_title_span_element = r_title_element.select_one("span:not([class])") if r_title_element else None
+        r_title = r_title_span_element.text if r_title_span_element else None
+
+        r_content_element = review.select_one("span.review-text")
+        r_content = r_content_element.text if r_content_element else None
+
+        r_date_element = review.select_one("span.review-date")
+        r_date = r_date_element.text if r_date_element else None
+
+        r_verified_element = review.select_one("span.a-size-mini")
+        r_verified = r_verified_element.text if r_verified_element else None
+
+        r_image_element = review.select_one("img.review-image-tile")
+        r_image = r_image_element.attrs["src"] if r_image_element else None
+
+        r = {
+            "author": r_author,
+            "rating": r_rating,
+            "title": r_title,
+            "content": r_content,
+            "date": r_date,
+            "verified": r_verified,
+            "image_url": r_image
+        }
+
+        scraped_reviews.append(r)
+
+    return scraped_reviews
+
+def scrape_all_pages(url):
+    all_reviews = []
+
+    page_number = 1
+    while True:
+        soup = get_soup(f"{url}&pageNumber={page_number}")
+        reviews = get_reviews(soup)
+
+        if not reviews:  # Break the loop if no reviews found on this page
+            break
+
+        all_reviews.extend(reviews)
+        page_number += 1
+
+    return all_reviews
+
+# # Example usage:
+# url = "https://www.amazon.in/OnePlus-Nord-Pastel-128GB-Storage/product-reviews/B0BY8JZ22K/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews"
+# all_reviews = scrape_all_pages(url)
+
+# # Convert to DataFrame for further analysis
+# df = pd.DataFrame(all_reviews)
+# df