data_preprocess.py

# In[1]:


from transformers import pipeline
from transformers import TrainingArguments, Trainer, AutoModelForSeq2SeqLM


# In[2]:


import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
import nltk
from nltk.stem.porter import PorterStemmer
from nltk.stem import WordNetLemmatizer
import re
from sklearn.metrics.pairwise import cosine_similarity
from fuzzywuzzy import fuzz
from sklearn.feature_extraction.text import TfidfVectorizer


# In[47]:


data3 = pd.read_csv('final2.csv')


# In[5]:


data3.info()


# In[6]:


data3.head()


# In[9]:


data3['topic'] = data3.topic.astype("string")
data3['discription'] = data3.discription.astype("string")
data3['keyword'] = data3.keyword.astype("string")
data3['level'] = data3.level.astype("string")
data3.info()


# # Data Cleaning Process
# '
# '
# 

# In[10]:


data3['tag'] = data3['discription'] + " " + data3['keyword'] +" " + data3['level']


# In[11]:


def remove_symbols(text):
  # Create a regular expression pattern to match unwanted symbols
    pattern = r'[^\w\s]'  # Matches characters that are not alphanumeric or whitespace
  # Substitute matched symbols with an empty string
    return re.sub(pattern, '', text.lower()) 


# In[12]:


data3['tag'] = data3['tag'].fillna('')
data3['tag'] = data3['tag'].apply(remove_symbols)
data3['level'] = data3['level'].apply(lambda x: x.replace(" ",""))
data3['keyword'] = data3['keyword'].fillna('')
data3.head()


# In[13]:


data3['tag'][0]


# # Convert tag columns into vector 

# In[14]:


cv = CountVectorizer( max_features = 5000, stop_words = 'english')
vector = cv.fit_transform(data3['tag']).toarray()


# In[15]:


vector[0]


# In[16]:


cv.get_feature_names_out()


# # Stemming And Lemmitization Process

# In[18]:


ps = PorterStemmer()


# In[30]:


def preprocess_query(query):
    
    # Lowercase the query
    cleaned_query = query.lower()

    # Remove punctuation (adjust as needed)
    import string
    punctuation = string.punctuation
    cleaned_query = ''.join([char for char in cleaned_query if char not in punctuation])

    # Remove stop words (optional, replace with your stop word list)
    stop_words = ["the", "a", "is", "in", "of"]
    cleaned_query = ' '.join([word for word in cleaned_query.split() if word not in stop_words])

    # Stemming
    ps = PorterStemmer()
    cleaned_query = ' '.join([ps.stem(word) for word in cleaned_query.split()])

    # Lemmatization
    wnl = WordNetLemmatizer()
    cleaned_query = ' '.join([wnl.lemmatize(word) for word in cleaned_query.split()])

    return cleaned_query


# In[32]:


preprocess_query('talked')


# In[31]:


preprocess_query('java james gosling website wikipedia document united states beginnertoadvance')


# In[23]:


data3['tag'].apply(stem)   # apply on tag columns 


# # Find Similarity score for finding most related topic from dataset

# In[24]:


similar = cosine_similarity(vector)


# In[27]:


sorted(list(enumerate(similar[1])),reverse = True, key = lambda x: x[1])[0:5]


# In[29]:


summarizer = pipeline("summarization", model="facebook/bart-base")
text_generator = pipeline("text-generation", model="gpt2")


# In[34]:


documents = []
for index, row in data3.iterrows():
    topic_description = preprocess_query(row["topic"]) 
    keywords = preprocess_query(row["keyword"])  
    combined_text = f"{topic_description} {keywords}"  # Combine for TF-IDF
    documents.append(combined_text)


# In[35]:


# Create TF-IDF vectorizer
vectorizer = TfidfVectorizer()

# Fit the vectorizer on the documents
document_vectors = vectorizer.fit_transform(documents)

def recommend_from_dataset(query):
    
    cleaned_query = preprocess_query(query)
    query_vector = vectorizer.transform([cleaned_query])

    # Calculate cosine similarity between query and documents
    cosine_similarities = cosine_similarity(query_vector, document_vectors)
    similarity_scores = cosine_similarities.flatten()

    # Sort documents based on similarity scores
    sorted_results = sorted(zip(similarity_scores, data3.index, range(len(documents))), reverse=True)

    # Return top N recommendations with scores, topic names, and links (if available)
    top_n_results = sorted_results[:5]  
    recommendations = []
    for result in top_n_results:
        score = result[0]
        document_id = result[1]
        topic_name = data3.loc[document_id, "topic"]  
        link = data3.loc[document_id, "Links"] if "Links" in data3.columns else "No link available" 
        if score >= 0.3:
            recommendations.append({"topic_name": topic_name, "link": link, "score": score})
    return recommendations


# In[36]:


def fine_tune_model(model_name, train_dataset, validation_dataset, epochs=3):
    # Load model and tokenizer
    model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
    tokenizer = AutoTokenizer.from_pretrained(model_name)

    # Define training arguments (adjust parameters as needed)
    training_args = TrainingArguments(
        output_dir="./results",  # Adjust output directory
        per_device_train_batch_size=8,
        per_device_eval_batch_size=8,
        num_train_epochs=epochs,
        save_steps=10_000,
    )

    # Create a Trainer instance for fine-tuning
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=validation_dataset,
        tokenizer=tokenizer,
    )

    # Train the model
    trainer.train()

    return model


# In[39]:


# train_dataset = ...  # Prepare your training dataset
# validation_dataset = ...  # Prepare your validation dataset

# Fine-tune the model (replace model name if needed)
# fine_tuned_model = fine_tune_model("facebook/bart-base", train_dataset, validation_dataset)

# Update summarization pipeline with the fine-tuned model
# summarizer1 = pipeline("text-generation", model=fine_tuned_model, tokenizer=fine_tuned_model.tokenizer)


# In[45]:


def summarize_and_generate(user_query, recommendations):
    
    # Summarize the user query
    query_summary = summarizer(user_query, max_length=100, truncation=True)[0]["summary_text"]

    # Generate creative text related to the query
    generated_text = text_generator(f"Exploring the concept of {user_query}", max_length=100, num_return_sequences=1)[0]["generated_text"]

    # Extract related links with scores
    related_links = []
    for recommendation in recommendations:
        related_links.append({"topic": recommendation["topic_name"], "link": recommendation["link"], "score": recommendation["score"]})

    return {
        "query_summary": query_summary.strip(),
        "generated_text": generated_text.strip(),
        "related_links": related_links
      }


# In[46]:


user_query = "java by james goslin"
recommendations = recommend_from_dataset(user_query)

# Get the summary, generated text, and related links
results = summarize_and_generate(user_query, recommendations)

print(f"Query Summary: {results['query_summary']}")
print(f"Creative Text: {results['generated_text']}")
print("Some Related Links for your query:")
for link in results["related_links"]:
    print(f"- {link['topic']}:\n {link['link']} : \n Score: {link['score']}") #(Score: {link['score']})


# In[ ]: