tweet_analyzer.py

import os
from PyPDF2 import PdfReader
import pandas as pd
from dotenv import load_dotenv
import groq
import json
from datetime import datetime
from sklearn.decomposition import NMF
from sklearn.feature_extraction.text import TfidfVectorizer

class TweetDatasetProcessor:
    def __init__(self):
        load_dotenv()
        self.groq_client = groq.Groq(api_key=os.getenv('Groq_api'))
        self.tweets = []
        self.personality_profile = {}

    def extract_text_from_pdf(self, pdf_path):
        """Extract text content from PDF file"""
        reader = PdfReader(pdf_path)
        text = ""
        for page in reader.pages:
            text += page.extract_text()
        return text

    def process_pdf_content(self, text):
        """Process PDF content and extract tweets with metadata"""
        lines = text.split('\n')
        for line in lines:
            if line.strip():
                self.tweets.append({
                    'content': line.strip(),
                    'timestamp': self._extract_timestamp(line) if self._extract_timestamp(line) else datetime.now(),
                    'mentions': self._extract_mentions(line),
                    'hashtags': self._extract_hashtags(line)
                })
        
        df = pd.DataFrame(self.tweets)
        df.to_csv('processed_tweets.csv', index=False)
        return df

    def _extract_timestamp(self, text):
        """Extract timestamp if present in tweet"""
        return None  # Implement timestamp extraction logic if needed

    def _extract_mentions(self, text):
        """Extract mentioned users from tweet"""
        return [word for word in text.split() if word.startswith('@')]

    def _extract_hashtags(self, text):
        """Extract hashtags from tweet"""
        return [word for word in text.split() if word.startswith('#')]

    def analyze_personality(self):
        """Comprehensive personality analysis"""
        all_tweets = [tweet['content'] for tweet in self.tweets]
        analysis_prompt = f"""Perform a deep psychological analysis of the author based on these tweets. Analyze:
        Core beliefs, emotional tendencies, cognitive patterns, etc.
        Tweets for analysis:
        {json.dumps(all_tweets[:30], indent=2)}
        """
        
        response = self.groq_client.chat.completions.create(
            messages=[
                {"role": "system", "content": "You are an expert psychologist."},
                {"role": "user", "content": analysis_prompt},
            ],
            model="mixtral-8x7b-32768",
            temperature=0.1,
        )
        self.personality_profile = response.choices[0].message.content
        return self.personality_profile

    def analyze_topics(self, n_topics=5):
        """Extract and identify different topics the author has tweeted about"""
        all_tweets = [tweet['content'] for tweet in self.tweets]
        vectorizer = TfidfVectorizer(stop_words='english')
        tfidf_matrix = vectorizer.fit_transform(all_tweets)
        nmf_model = NMF(n_components=n_topics, random_state=1)
        nmf_model.fit(tfidf_matrix)
        
        topics = []
        for topic_idx, topic in enumerate(nmf_model.components_):
            topic_words = [vectorizer.get_feature_names_out()[i] for i in topic.argsort()[:-n_topics - 1:-1]]
            topics.append(" ".join(topic_words))
        return topics

    def generate_tweet(self, context=""):
        """Generate a new tweet based on personality profile and optional context"""
        additional_contexts = [
            "Comment on a recent technological advancement.",
            "Share a motivational thought.",
            "Discuss a current trending topic.",
            "Reflect on a past experience.",
            "Provide advice to followers."
        ]

        # Include historical topics in the context
        historical_topics = self.analyze_topics()
        additional_contexts.extend(historical_topics)
    
        # Randomly choose an additional context to diversify tweets
        import random
        random_context = random.choice(additional_contexts)
    
        generation_prompt = f"""Based on this personality profile:
        {self.personality_profile}
        Current context or topic (if any):
        {context}
        Additionally, consider this specific context:
        {random_context}
        Generate a tweet that this person would write right now."""
    
        response = self.groq_client.chat.completions.create(
            messages=[
                {"role": "system", "content": "You are an expert in replicating writing patterns."},
                {"role": "user", "content": generation_prompt},
            ],
            model="mixtral-8x7b-32768",
            temperature=0.8,
            max_tokens=150,
        )
        return response.choices[0].message.content