File size: 3,778 Bytes
e4d5155 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
"""
Text processing utilities for the efficient-context library.
"""
import re
from typing import List, Dict, Any
import logging
# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def split_into_sentences(text: str) -> List[str]:
"""
Split text into sentences.
Args:
text: Text to split
Returns:
sentences: List of sentences
"""
# Simple but effective sentence splitting
# This handles most common sentence endings while preserving common abbreviations
text = text.replace('\n', ' ')
# Try to use NLTK if available for better sentence splitting
try:
import nltk
try:
return nltk.sent_tokenize(text)
except Exception as e:
logger.warning(f"NLTK sentence tokenizer error: {e}. Using fallback.")
return _simple_sentence_split(text)
except ImportError:
logger.warning("NLTK not available, using fallback sentence splitter")
return _simple_sentence_split(text)
def _simple_sentence_split(text: str) -> List[str]:
"""Fallback sentence splitter without dependencies."""
# This is a simplified version, not as accurate as NLTK but works without dependencies
# Handle common abbreviations to avoid splitting them
for abbr in ['Mr.', 'Mrs.', 'Dr.', 'vs.', 'e.g.', 'i.e.', 'etc.']:
text = text.replace(abbr, abbr.replace('.', '<POINT>'))
# Split on sentence endings
sentences = re.split(r'(?<=[.!?])\s+', text)
# Restore abbreviations
sentences = [s.replace('<POINT>', '.') for s in sentences]
# Remove empty sentences
return [s for s in sentences if s.strip()]
def get_sentence_importance(sentences: List[str]) -> List[float]:
"""
Calculate importance scores for sentences based on heuristics.
Args:
sentences: List of sentences to score
Returns:
importances: List of importance scores (0.0 to 1.0)
"""
# Simple heuristics for scoring sentence importance
importances = []
for sentence in sentences:
score = 0.0
words = sentence.split()
# Longer sentences tend to be more informative (up to a point)
length_score = min(len(words) / 20, 1.0)
# Keywords suggest important content
keyword_score = 0.0
keywords = ['important', 'significant', 'key', 'critical', 'crucial',
'essential', 'main', 'major', 'primary', 'central',
'result', 'conclusion', 'finding', 'discovered', 'shows']
for word in words:
if word.lower() in keywords:
keyword_score += 0.2
keyword_score = min(keyword_score, 0.6) # Cap keyword importance
# Presence of numbers often indicates factual content
number_score = 0.0
if re.search(r'\d', sentence):
number_score = 0.2
# Combine scores
score = 0.5 * length_score + 0.3 * keyword_score + 0.2 * number_score
# Cap at 1.0
importances.append(min(score, 1.0))
return importances
def calculate_text_overlap(text1: str, text2: str) -> float:
"""
Calculate simple text overlap between two strings.
Args:
text1: First text
text2: Second text
Returns:
overlap_ratio: Ratio of shared tokens (0.0 to 1.0)
"""
# Convert to sets of tokens
tokens1 = set(text1.lower().split())
tokens2 = set(text2.lower().split())
# Calculate overlap
if not tokens1 or not tokens2:
return 0.0
overlap = tokens1.intersection(tokens2)
return len(overlap) / min(len(tokens1), len(tokens2))
|