Update stream_files_to_cos.py

stream_files_to_cos.py

@@ -165,24 +165,24 @@ def stream_file_to_cos():
                     session = requests.Session()
                     response = session.request(http_method, source_url, stream=True)
                     response.raise_for_status()
-
                     # Extract actual filename from response
                     filename = extract_filename_from_headers(response)
-                    
                     # Combine prefix with filename for the full COS key
                     target_key = f"{prefix}{filename}" if prefix else filename
                     
+                    # Create a BytesIO buffer and write decompressed content
+                    file_buffer = io.BytesIO()
+                    for chunk in response.iter_content(chunk_size=8192):
+                        if chunk:
+                            file_buffer.write(chunk)
+                    file_buffer.seek(0)  # Reset to beginning for upload
+                    
                     # Upload file to COS
                     conf = ibm_boto3.s3.transfer.TransferConfig(
-                        multipart_threshold=1024**2,  # 1MB
-                        max_concurrency=100
+                        multipart_threshold=1024**2, max_concurrency=100  # 1MB
                     )
-                    
                     cos_client.upload_fileobj(
-                        response.raw,
-                        cos_config['bucket_name'],
-                        target_key,
-                        Config=conf
+                        file_buffer, cos_config["bucket_name"], target_key, Config=conf
                     )
 
                     results.append({