Update app.py

app.py

@@ -13,19 +13,30 @@ from cachetools import TTLCache
 cache = TTLCache(maxsize=1000, ttl=3600)
 
 async def fetch_url(url, session, max_retries=3, timeout=180):
+    total_content = b""
     for attempt in range(max_retries):
         try:
             async with session.get(url, timeout=ClientTimeout(total=timeout)) as response:
                 response.raise_for_status()
-                content = await response.read()
-                return content.decode('utf-8', errors='ignore')
+                while True:
+                    chunk = await response.content.read(8192)  # 每次读取8KB
+                    if not chunk:
+                        break
+                    total_content += chunk
+                
+                if response.content_length is not None and len(total_content) < response.content_length:
+                    print(f"Warning: Received content length ({len(total_content)}) is less than expected ({response.content_length})")
+                    if attempt == max_retries - 1:
+                        print("This was the last attempt. Returning partial content.")
+                        return total_content.decode('utf-8', errors='ignore')
+                    else:
+                        print("Retrying...")
+                        await asyncio.sleep(5)
+                        continue
+                
+                return total_content.decode('utf-8', errors='ignore')
         except asyncio.TimeoutError:
             print(f"Attempt {attempt + 1} timed out after {timeout} seconds", flush=True)
-        except aiohttp.ClientPayloadError as e:
-            print(f"Payload error on attempt {attempt + 1}: {str(e)}", flush=True)
-            if response.content_length and len(content) >= response.content_length:
-                print(f"Received data length: {len(content)}, expected: {response.content_length}", flush=True)
-                return content.decode('utf-8', errors='ignore')
         except aiohttp.ClientError as e:
             print(f"Attempt {attempt + 1} failed: {str(e)}", flush=True)