Hasitha16 commited on
Commit
46cd389
·
verified ·
1 Parent(s): db263fd

Update model.py

Browse files
Files changed (1) hide show
  1. model.py +44 -33
model.py CHANGED
@@ -11,74 +11,84 @@ from sklearn.metrics.pairwise import cosine_similarity
11
  from nltk.tokenize import sent_tokenize
12
  from transformers import pipeline
13
  import numpy as np
 
14
 
15
  # === Pipelines ===
16
  summarizer = pipeline("summarization", model="csebuetnlp/mT5_multilingual_XLSum")
17
  qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad")
18
  emotion_pipeline = pipeline("text-classification", model="bhadresh-savani/distilbert-base-uncased-emotion", top_k=1)
19
 
20
- # === Summarization (Brief) ===
21
  def summarize_review(text, max_len=60, min_len=10):
22
- return summarizer(text, max_length=max_len, min_length=min_len, do_sample=False)[0]["summary_text"]
 
 
 
 
23
 
24
- # === Smart Summarization ===
25
  def smart_summarize(text, n_clusters=1):
26
- sentences = sent_tokenize(text)
27
- if len(sentences) <= 1:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  return text
29
- tfidf = TfidfVectorizer(stop_words="english")
30
- tfidf_matrix = tfidf.fit_transform(sentences)
31
- if len(sentences) <= n_clusters:
32
- return " ".join(sentences)
33
- kmeans = KMeans(n_clusters=n_clusters, random_state=42).fit(tfidf_matrix)
34
- summary_sentences = []
35
- for i in range(n_clusters):
36
- idx = np.where(kmeans.labels_ == i)[0]
37
- if not len(idx):
38
- continue
39
- avg_vector = np.asarray(tfidf_matrix[idx].mean(axis=0))
40
- sim = cosine_similarity(avg_vector, tfidf_matrix[idx].toarray())
41
- most_representative = sentences[idx[np.argmax(sim)]]
42
- summary_sentences.append(most_representative)
43
- return " ".join(sorted(summary_sentences, key=sentences.index))
44
 
45
  # === Emotion Detection ===
46
  def detect_emotion(text):
47
  try:
48
  result = emotion_pipeline(text)[0]
49
  return result["label"]
50
- except Exception:
 
51
  return "neutral"
52
 
53
- # === Follow-up Q&A (single or multi) ===
54
  def answer_followup(text, question, verbosity="brief"):
55
  try:
56
  if isinstance(question, list):
57
  answers = []
58
  for q in question:
59
  response = qa_pipeline({"question": q, "context": text})
60
- answer = response.get("answer", "")
61
  if verbosity.lower() == "detailed":
62
- answers.append(f"**{q}** → {answer}")
63
  else:
64
- answers.append(answer)
65
  return answers
66
  else:
67
  response = qa_pipeline({"question": question, "context": text})
68
- answer = response.get("answer", "")
69
- if verbosity.lower() == "detailed":
70
- return f"Based on the review, the answer is: **{answer}**"
71
- return answer
72
- except Exception:
73
  return "Sorry, I couldn't generate a follow-up answer."
74
 
75
- # === Fast follow-up (no formatting, for /followup) ===
76
  def answer_only(text, question):
77
  try:
78
  if not question:
79
  return "No question provided."
80
  return qa_pipeline({"question": question, "context": text}).get("answer", "No answer found.")
81
- except Exception:
 
82
  return "Q&A failed."
83
 
84
  # === Optional Explanation Generator ===
@@ -86,7 +96,8 @@ def generate_explanation(text):
86
  try:
87
  explanation = summarizer(text, max_length=60, min_length=20, do_sample=False)[0]["summary_text"]
88
  return f"🧠 This review can be explained as: {explanation}"
89
- except Exception:
 
90
  return "⚠️ Explanation could not be generated."
91
 
92
  # === Industry Detector ===
 
11
  from nltk.tokenize import sent_tokenize
12
  from transformers import pipeline
13
  import numpy as np
14
+ import logging
15
 
16
  # === Pipelines ===
17
  summarizer = pipeline("summarization", model="csebuetnlp/mT5_multilingual_XLSum")
18
  qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad")
19
  emotion_pipeline = pipeline("text-classification", model="bhadresh-savani/distilbert-base-uncased-emotion", top_k=1)
20
 
21
+ # === Brief Summarization ===
22
  def summarize_review(text, max_len=60, min_len=10):
23
+ try:
24
+ return summarizer(text, max_length=max_len, min_length=min_len, do_sample=False)[0]["summary_text"]
25
+ except Exception as e:
26
+ logging.warning(f"Summarization fallback used: {e}")
27
+ return text
28
 
29
+ # === Smart Summarization with Clustering ===
30
  def smart_summarize(text, n_clusters=1):
31
+ try:
32
+ sentences = sent_tokenize(text)
33
+ if len(sentences) <= 1:
34
+ return text
35
+ tfidf = TfidfVectorizer(stop_words="english")
36
+ tfidf_matrix = tfidf.fit_transform(sentences)
37
+ if len(sentences) <= n_clusters:
38
+ return " ".join(sentences)
39
+ kmeans = KMeans(n_clusters=n_clusters, random_state=42).fit(tfidf_matrix)
40
+ summary_sentences = []
41
+ for i in range(n_clusters):
42
+ idx = np.where(kmeans.labels_ == i)[0]
43
+ if not len(idx):
44
+ continue
45
+ avg_vector = np.asarray(tfidf_matrix[idx].mean(axis=0))
46
+ sim = cosine_similarity(avg_vector, tfidf_matrix[idx].toarray())
47
+ most_representative = sentences[idx[np.argmax(sim)]]
48
+ summary_sentences.append(most_representative)
49
+ return " ".join(sorted(summary_sentences, key=sentences.index))
50
+ except Exception as e:
51
+ logging.error(f"Smart summarize error: {e}")
52
  return text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
 
54
  # === Emotion Detection ===
55
  def detect_emotion(text):
56
  try:
57
  result = emotion_pipeline(text)[0]
58
  return result["label"]
59
+ except Exception as e:
60
+ logging.warning(f"Emotion detection failed: {e}")
61
  return "neutral"
62
 
63
+ # === Follow-up Q&A (Flexible for list or str) ===
64
  def answer_followup(text, question, verbosity="brief"):
65
  try:
66
  if isinstance(question, list):
67
  answers = []
68
  for q in question:
69
  response = qa_pipeline({"question": q, "context": text})
70
+ ans = response.get("answer", "")
71
  if verbosity.lower() == "detailed":
72
+ answers.append(f"**{q}** → {ans}")
73
  else:
74
+ answers.append(ans)
75
  return answers
76
  else:
77
  response = qa_pipeline({"question": question, "context": text})
78
+ ans = response.get("answer", "")
79
+ return f"**{question}** → {ans}" if verbosity.lower() == "detailed" else ans
80
+ except Exception as e:
81
+ logging.warning(f"Follow-up error: {e}")
 
82
  return "Sorry, I couldn't generate a follow-up answer."
83
 
84
+ # === Fast follow-up (used for direct /followup route) ===
85
  def answer_only(text, question):
86
  try:
87
  if not question:
88
  return "No question provided."
89
  return qa_pipeline({"question": question, "context": text}).get("answer", "No answer found.")
90
+ except Exception as e:
91
+ logging.warning(f"Answer-only failed: {e}")
92
  return "Q&A failed."
93
 
94
  # === Optional Explanation Generator ===
 
96
  try:
97
  explanation = summarizer(text, max_length=60, min_length=20, do_sample=False)[0]["summary_text"]
98
  return f"🧠 This review can be explained as: {explanation}"
99
+ except Exception as e:
100
+ logging.warning(f"Explanation failed: {e}")
101
  return "⚠️ Explanation could not be generated."
102
 
103
  # === Industry Detector ===