Spaces:
Sleeping
Sleeping
| """ | |
| Updated bow_analysis.py to include similarity metrics. | |
| Preprocessing here is more advanced than n-gram version. | |
| Lowercase, tokenize, remove stopwords, non-alphabetic characters removal, short words removal, lemmatization. | |
| """ | |
| from sklearn.feature_extraction.text import CountVectorizer | |
| from nltk.corpus import stopwords | |
| from nltk.stem import WordNetLemmatizer | |
| from nltk.tokenize import word_tokenize | |
| from processors.metrics import calculate_similarity | |
| # not used currently imports, but left in case I start using them again | |
| import numpy as np | |
| from collections import Counter | |
| import re | |
| import nltk | |
| # Define the compare_bow_across_texts function directly in this file | |
| def compare_bow_across_texts(texts, model_names, top_n=25): | |
| """ | |
| Compare bag of words representations across multiple texts. | |
| Args: | |
| texts (list): List of text responses to compare | |
| model_names (list): Names of models corresponding to responses | |
| top_n (int): Number of top words to consider | |
| Returns: | |
| dict: Bag of words analysis results | |
| """ | |
| # Initialize the results dictionary | |
| result = { | |
| "models": model_names, | |
| "important_words": {}, | |
| "word_count_matrix": {}, | |
| "differential_words": [] | |
| } | |
| # Make sure we have texts to analyze | |
| if not texts or len(texts) < 1: | |
| return result | |
| # Preprocess texts (tokenize, remove stopwords, etc.) | |
| preprocessed_texts = [] | |
| stop_words = set(stopwords.words('english')) | |
| lemmatizer = WordNetLemmatizer() | |
| for text in texts: | |
| # Convert to lowercase and tokenize | |
| tokens = word_tokenize(text.lower()) | |
| # Remove stopwords, punctuation, and lemmatize | |
| filtered_tokens = [] | |
| for token in tokens: | |
| if token.isalpha() and token not in stop_words and len(token) > 2: | |
| filtered_tokens.append(lemmatizer.lemmatize(token)) | |
| preprocessed_texts.append(" ".join(filtered_tokens)) | |
| # Create bag of words representations using CountVectorizer | |
| vectorizer = CountVectorizer(max_features=1000) | |
| X = vectorizer.fit_transform(preprocessed_texts) | |
| # Get feature names (words) | |
| feature_names = vectorizer.get_feature_names_out() | |
| # Create word count matrix | |
| word_counts = {} | |
| for i, model in enumerate(model_names): | |
| counts = X[i].toarray()[0] | |
| word_counts[model] = {} | |
| # Store word frequencies for this model | |
| for j, word in enumerate(feature_names): | |
| if counts[j] > 0: # Only store words that appear | |
| word_counts[model][word] = int(counts[j]) | |
| # Add to word count matrix | |
| if word not in result["word_count_matrix"]: | |
| result["word_count_matrix"][word] = {} | |
| result["word_count_matrix"][word][model] = int(counts[j]) | |
| # Find important words for each model | |
| for model, word_freq in word_counts.items(): | |
| # Sort by frequency | |
| sorted_words = sorted(word_freq.items(), key=lambda x: x[1], reverse=True) | |
| # Store top N words | |
| result["important_words"][model] = [ | |
| {"word": word, "count": count} | |
| for word, count in sorted_words[:top_n] | |
| ] | |
| # Calculate differential words (words with biggest frequency difference between models) | |
| if len(model_names) >= 2: | |
| model1, model2 = model_names[0], model_names[1] | |
| # Calculate differences | |
| diff_scores = {} | |
| for word in result["word_count_matrix"]: | |
| count1 = result["word_count_matrix"][word].get(model1, 0) | |
| count2 = result["word_count_matrix"][word].get(model2, 0) | |
| # Absolute difference | |
| diff_scores[word] = abs(count1 - count2) | |
| # Sort by difference | |
| sorted_diffs = sorted(diff_scores.items(), key=lambda x: x[1], reverse=True) | |
| result["differential_words"] = [word for word, _ in sorted_diffs[:top_n]] | |
| # Calculate overlap statistics | |
| model1_words = set(word_counts.get(model1, {}).keys()) | |
| model2_words = set(word_counts.get(model2, {}).keys()) | |
| common_words = model1_words.intersection(model2_words) | |
| # Initialize comparisons if needed | |
| if "comparisons" not in result: | |
| result["comparisons"] = {} | |
| comparison_key = f"{model1} vs {model2}" | |
| result["comparisons"][comparison_key] = { | |
| "common_word_count": len(common_words) | |
| } | |
| return result | |
| def add_similarity_metrics(bow_results, response_texts, model_names): | |
| """ | |
| Add similarity metrics to the bag of words analysis results | |
| Args: | |
| bow_results (dict): The bag of words analysis results | |
| response_texts (list): List of response texts to compare | |
| model_names (list): List of model names corresponding to responses | |
| Returns: | |
| dict: Updated bag of words results with similarity metrics | |
| """ | |
| # Make sure we have at least two responses to compare | |
| if len(response_texts) < 2 or len(model_names) < 2: | |
| print("Need at least two responses to calculate similarity metrics") | |
| return bow_results | |
| # Get the first two responses (current implementation only handles two-way comparisons) | |
| text1, text2 = response_texts[0], response_texts[1] | |
| model1, model2 = model_names[0], model_names[1] | |
| # Generate the comparison key | |
| comparison_key = f"{model1} vs {model2}" | |
| # Initialize comparisons if needed | |
| if "comparisons" not in bow_results: | |
| bow_results["comparisons"] = {} | |
| # Initialize the comparison entry if needed | |
| if comparison_key not in bow_results["comparisons"]: | |
| bow_results["comparisons"][comparison_key] = {} | |
| # Calculate similarity metrics | |
| metrics = calculate_similarity(text1, text2) | |
| # Add metrics to the comparison | |
| bow_results["comparisons"][comparison_key].update({ | |
| "cosine_similarity": metrics.get("cosine_similarity", 0), | |
| "jaccard_similarity": metrics.get("jaccard_similarity", 0), | |
| "semantic_similarity": metrics.get("semantic_similarity", 0) | |
| }) | |
| # If we have common_word_count from BOW analysis, keep it | |
| if "common_word_count" not in bow_results["comparisons"][comparison_key]: | |
| # Calculate from bow data as a fallback | |
| if "important_words" in bow_results: | |
| words1 = set([item["word"] for item in bow_results["important_words"].get(model1, [])]) | |
| words2 = set([item["word"] for item in bow_results["important_words"].get(model2, [])]) | |
| common_words = words1.intersection(words2) | |
| bow_results["comparisons"][comparison_key]["common_word_count"] = len(common_words) | |
| return bow_results | |
| def compare_bow(texts, model_names, top_n=25): | |
| """ | |
| Compare bag of words between different texts | |
| Args: | |
| texts (list): List of text responses to compare | |
| model_names (list): Names of models corresponding to responses | |
| top_n (int): Number of top words to consider | |
| Returns: | |
| dict: Comparative analysis | |
| """ | |
| bow_results = compare_bow_across_texts(texts, model_names, top_n) | |
| # Add similarity metrics to the results | |
| if len(texts) >= 2 and len(model_names) >= 2: | |
| bow_results = add_similarity_metrics(bow_results, texts, model_names) | |
| return bow_results |