urld / app.py
acecalisto3's picture
Update app.py
890dba9 verified
raw
history blame
1.83 kB
def process_all_inputs(urls, file, text):
"""Process all input types with progress tracking"""
try:
processor = URLProcessor()
file_processor = FileProcessor()
results = []
# Process URLs
if urls:
url_list = re.split(r'[,\n]', urls)
url_list = [url.strip() for url in url_list if url.strip()]
for url in url_list:
validation = processor.validate_url(url)
if validation.get('is_valid'):
content = processor.fetch_content(url)
if content:
results.append({
'source': 'url',
'url': url,
'content': content,
'timestamp': time.strftime("%Y-%m-%d %H:%M:%S")
})
# Process files
if file:
results.extend(file_processor.process_file(file))
# Process text input
if text:
cleaned_text = processor.advanced_text_cleaning(text)
results.append({
'source': 'direct_input',
'content': cleaned_text,
'timestamp': time.strftime("%Y-%m-%d %H:%M:%S")
})
# Generate output
if results:
output_dir = Path('output') / datetime.now().strftime('%Y-%m-%d')
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / f'processed_{int(time.time())}.json'
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(results, f, ensure_ascii=False, indent=2)
summary = f"Processed {len(results)} items successfully!"
return output