Spaces:
Running
Running
Update __init__.py
Browse files- __init__.py +164 -0
__init__.py
CHANGED
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
Advanced URL & Text Processing Suite
|
3 |
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
4 |
+
|
5 |
+
A sophisticated, enterprise-grade toolkit for URL processing, file manipulation, and QR operations.
|
6 |
+
Designed with performance, scalability, and reliability in mind.
|
7 |
+
|
8 |
+
Key Features:
|
9 |
+
- Intelligent URL Processing with adaptive rate limiting
|
10 |
+
- Universal File Processing with smart content extraction
|
11 |
+
- Advanced QR Code operations with custom styling
|
12 |
+
- Modern, responsive UI with real-time feedback
|
13 |
+
- Enterprise-grade security and performance
|
14 |
+
|
15 |
+
Example:
|
16 |
+
>>> from urld import URLProcessor, FileProcessor, QRProcessor
|
17 |
+
>>> url_proc = URLProcessor(request_delay=1.0)
|
18 |
+
>>> result = url_proc.process_urls(['https://example.com'])
|
19 |
+
>>> print(result)
|
20 |
+
"""
|
21 |
+
|
22 |
+
import logging
|
23 |
+
import sys
|
24 |
+
from typing import Dict, List, Optional, Union, Any
|
25 |
+
from pathlib import Path
|
26 |
+
|
27 |
+
__version__ = "1.0.0"
|
28 |
+
__author__ = "Advanced URL Processing Team"
|
29 |
+
__license__ = "MIT"
|
30 |
+
__copyright__ = "Copyright 2024 Advanced URL Processing Team"
|
31 |
+
|
32 |
+
# Configure advanced logging
|
33 |
+
logging.basicConfig(
|
34 |
+
level=logging.INFO,
|
35 |
+
format='%(asctime)s.%(msecs)03d [%(levelname)s] %(name)s - %(message)s',
|
36 |
+
datefmt='%Y-%m-%d %H:%M:%S',
|
37 |
+
handlers=[
|
38 |
+
logging.StreamHandler(sys.stdout),
|
39 |
+
logging.FileHandler('urld.log')
|
40 |
+
]
|
41 |
+
)
|
42 |
+
|
43 |
+
logger = logging.getLogger(__name__)
|
44 |
+
|
45 |
+
# Import core components
|
46 |
+
try:
|
47 |
+
from .url_processor import URLProcessor
|
48 |
+
from .file_processor import FileProcessor
|
49 |
+
from .qr_processor import QRProcessor
|
50 |
+
from .interface import create_interface
|
51 |
+
except ImportError as e:
|
52 |
+
logger.error(f"Failed to import core component: {e}")
|
53 |
+
raise
|
54 |
+
|
55 |
+
# Type definitions for enhanced type checking
|
56 |
+
ProcessingResult = Dict[str, Any]
|
57 |
+
URLList = List[str]
|
58 |
+
ProcessingMode = Literal['basic', 'interactive', 'deep']
|
59 |
+
|
60 |
+
class ProcessingError(Exception):
|
61 |
+
"""Custom exception for processing errors"""
|
62 |
+
def __init__(self, message: str, details: Optional[Dict] = None):
|
63 |
+
super().__init__(message)
|
64 |
+
self.details = details or {}
|
65 |
+
|
66 |
+
# Version compatibility check
|
67 |
+
if sys.version_info < (3, 8):
|
68 |
+
logger.warning("Python 3.8+ is recommended for optimal performance")
|
69 |
+
|
70 |
+
# Feature detection
|
71 |
+
def check_features() -> Dict[str, bool]:
|
72 |
+
"""Check availability of optional features"""
|
73 |
+
features = {
|
74 |
+
'selenium': False,
|
75 |
+
'gpu_acceleration': False,
|
76 |
+
'ocr_support': False,
|
77 |
+
'advanced_qr': False
|
78 |
+
}
|
79 |
+
|
80 |
+
try:
|
81 |
+
import selenium
|
82 |
+
features['selenium'] = True
|
83 |
+
except ImportError:
|
84 |
+
pass
|
85 |
+
|
86 |
+
try:
|
87 |
+
import torch
|
88 |
+
features['gpu_acceleration'] = torch.cuda.is_available()
|
89 |
+
except ImportError:
|
90 |
+
pass
|
91 |
+
|
92 |
+
try:
|
93 |
+
import pytesseract
|
94 |
+
features['ocr_support'] = True
|
95 |
+
except ImportError:
|
96 |
+
pass
|
97 |
+
|
98 |
+
try:
|
99 |
+
import qrcode
|
100 |
+
features['advanced_qr'] = True
|
101 |
+
except ImportError:
|
102 |
+
pass
|
103 |
+
|
104 |
+
return features
|
105 |
+
|
106 |
+
# Initialize feature detection
|
107 |
+
AVAILABLE_FEATURES = check_features()
|
108 |
+
|
109 |
+
# Export public interface
|
110 |
+
__all__ = [
|
111 |
+
'URLProcessor',
|
112 |
+
'FileProcessor',
|
113 |
+
'QRProcessor',
|
114 |
+
'create_interface',
|
115 |
+
'ProcessingError',
|
116 |
+
'ProcessingResult',
|
117 |
+
'URLList',
|
118 |
+
'ProcessingMode',
|
119 |
+
'AVAILABLE_FEATURES',
|
120 |
+
'__version__',
|
121 |
+
]
|
122 |
+
|
123 |
+
# Startup information
|
124 |
+
logger.info(f"Advanced URL Processing Suite v{__version__}")
|
125 |
+
logger.info(f"Available features: {', '.join(k for k, v in AVAILABLE_FEATURES.items() if v)}")
|
126 |
+
|
127 |
+
def get_config_path() -> Path:
|
128 |
+
"""Get the configuration file path"""
|
129 |
+
return Path.home() / '.urld' / 'config.json'
|
130 |
+
|
131 |
+
def initialize():
|
132 |
+
"""Initialize the module with advanced setup"""
|
133 |
+
config_path = get_config_path()
|
134 |
+
|
135 |
+
if not config_path.parent.exists():
|
136 |
+
config_path.parent.mkdir(parents=True)
|
137 |
+
logger.info(f"Created configuration directory: {config_path.parent}")
|
138 |
+
|
139 |
+
if not config_path.exists():
|
140 |
+
import json
|
141 |
+
default_config = {
|
142 |
+
'url_processor': {
|
143 |
+
'request_delay': 1.0,
|
144 |
+
'timeout': 30,
|
145 |
+
'max_retries': 3,
|
146 |
+
'respect_robots': True
|
147 |
+
},
|
148 |
+
'file_processor': {
|
149 |
+
'max_file_size': 2 * 1024 * 1024 * 1024,
|
150 |
+
'supported_formats': ['pdf', 'docx', 'xlsx', 'zip', 'tar.gz']
|
151 |
+
},
|
152 |
+
'qr_processor': {
|
153 |
+
'error_correction': 'H',
|
154 |
+
'box_size': 10,
|
155 |
+
'border': 4
|
156 |
+
}
|
157 |
+
}
|
158 |
+
|
159 |
+
with open(config_path, 'w') as f:
|
160 |
+
json.dump(default_config, f, indent=4)
|
161 |
+
logger.info(f"Created default configuration file: {config_path}")
|
162 |
+
|
163 |
+
# Run initialization
|
164 |
+
initialize()
|