using remove bg to add background to the output image

app.py

@@ -261,7 +261,7 @@ def start_tryon(dict,garm_img,garment_des, background_img, is_checked,is_checked
     # apply background to final image
     if background_img:
        logging.info("Adding background")
-       final_image = BackgroundProcessor.add_background_v3(final_image, background_img) 
+       final_image = BackgroundProcessor.replace_background_with_removebg(final_image, background_img) 
     return final_image, mask_gray
     # return images[0], mask_gray
 

src/background_processor.py

@@ -1,11 +1,18 @@
+import os
+import requests
+import logging
 from PIL import Image, ImageEnhance
 import cv2
 import numpy as np
 from preprocess.humanparsing.run_parsing import Parsing
+from src.image_format_convertor import ImageFormatConvertor
+
+REMOVE_BG_KEY = os.getenv('REMOVE_BG_KEY', "8XHtXvvhWFBpAA6jVt3yzVmh")
 
 parsing_model = Parsing(0)
 
 class BackgroundProcessor:
+    DeprecationWarning("Created only for testing. Not in use")
     @classmethod
     def add_background(cls, human_img: Image, background_img: Image):
         
@@ -38,59 +45,7 @@ class BackgroundProcessor:
         # Return or save the result
         return result_img
 
-    @classmethod    
-    def temp_v2(cls, human_img_path, background_img_path, mask_img_path):
-        # Load the images
-        foreground_img = cv2.imread(human_img_path).resize((768,1024))  # The segmented person image
-        background_img = cv2.imread(background_img_path)  # The new background image
-        mask_img = cv2.imread(mask_img_path, cv2.IMREAD_GRAYSCALE)  # The mask image from the human parser model
-
-        # Ensure the foreground image and the mask are the same size
-        if foreground_img.shape[:2] != mask_img.shape[:2]:
-            raise ValueError("Foreground image and mask must be the same size")
-
-        # Resize background image to match the size of the foreground image
-        background_img = cv2.resize(background_img, (foreground_img.shape[1], foreground_img.shape[0]))
-
-        # Create an inverted mask
-        mask_inv = cv2.bitwise_not(mask_img)
-
-        # Convert mask to 3 channels
-        mask_3ch = cv2.cvtColor(mask_img, cv2.COLOR_GRAY2BGR)
-        mask_inv_3ch = cv2.cvtColor(mask_inv, cv2.COLOR_GRAY2BGR)
-
-        # Extract the person from the foreground image using the mask
-        person = cv2.bitwise_and(foreground_img, mask_3ch)
-
-        # Extract the background where the person is not present
-        background = cv2.bitwise_and(background_img, mask_inv_3ch)
-
-        # Combine the person and the new background
-        combined_img = cv2.add(person, background)
-
-        # Refine edges using Gaussian Blur (feathering technique)
-        blurred_combined_img = cv2.GaussianBlur(combined_img, (5, 5), 0)
-
-        # Post-processing: Adjust brightness, contrast, etc. (optional)
-        alpha = 1.2  # Contrast control (1.0-3.0)
-        beta = 20    # Brightness control (0-100)
-
-        post_processed_img = cv2.convertScaleAbs(blurred_combined_img, alpha=alpha, beta=beta)
-
-        # Save the final image
-        # cv2.imwrite('path_to_save_final_image.png', post_processed_img)
-
-        # Display the images (optional)
-        cv2.imshow('Foreground', foreground_img)
-        cv2.imshow('Background', background_img)
-        cv2.imshow('Mask', mask_img)
-        cv2.imshow('Combined', combined_img)
-        cv2.imshow('Post Processed', post_processed_img)
-        cv2.waitKey(0)
-        cv2.destroyAllWindows()
-        return post_processed_img
-    
-
+    DeprecationWarning("Created only for testing. Not in use")
     @classmethod
     def add_background_v3(cls, foreground_pil: Image, background_pil: Image):
         foreground_pil= foreground_pil.convert("RGB")
@@ -120,8 +75,8 @@ class BackgroundProcessor:
         #mask_pil = mask_pil.resize(foreground_pil.size)
 
         # Convert PIL images to OpenCV format
-        foreground_cv2 = cls.pil_to_cv2(foreground_pil)
-        background_cv2 = cls.pil_to_cv2(background_pil)
+        foreground_cv2 = ImageFormatConvertor.pil_to_cv2(foreground_pil)
+        background_cv2 = ImageFormatConvertor.pil_to_cv2(background_pil)
         #mask_cv2 = pil_to_cv2(mask_pil)
         mask_cv2 = np.array(mask_pil)  # Directly convert to NumPy array without color conversion
 
@@ -156,7 +111,7 @@ class BackgroundProcessor:
         blurred_combined_cv2 = cv2.GaussianBlur(combined_cv2, (5, 5), 0)
 
         # Convert the result back to PIL format
-        combined_pil = cls.cv2_to_pil(blurred_combined_cv2)
+        combined_pil = ImageFormatConvertor.cv2_to_pil(blurred_combined_cv2)
         
 
         """
@@ -180,12 +135,20 @@ class BackgroundProcessor:
 
         return combined_pil
     
+    DeprecationWarning("Created only for testing. Not in use")
     @classmethod
     def replace_background(cls, foreground_img_path: str, background_img_path: str):
         # Load the input image (with alpha channel) and the background image
-        #input_image = cv2.imread(foreground_img_path, cv2.IMREAD_UNCHANGED)
-        input_image = cv2.imread(foreground_img_path)
-        background_image = cv2.imread(background_img_path)
+        #input_image = cv2.imread(foreground_img_path, cv2.IMREAD_UNCHANGED)        
+        # background_image = cv2.imread(background_img_path)
+        foreground_img_pil = Image.open(foreground_img_path)
+        width = foreground_img_pil.width
+        height = foreground_img_pil.height
+        background_image_pil = Image.open(background_img_path)
+        background_image_pil = background_image_pil.resize((width, height))
+        input_image = ImageFormatConvertor.pil_to_cv2(foreground_img_pil)
+        background_image = ImageFormatConvertor.pil_to_cv2(background_image_pil)
+        
 
         # Ensure the input image has an alpha channel
         if input_image.shape[2] != 4:
@@ -203,34 +166,63 @@ class BackgroundProcessor:
 
         # Extract the BGR channels of the input image
         input_bgr = input_image[:, :, :3]
-
+        background_bgr = background_image[:,:,:3]
         # Blend the images using the alpha channel
         foreground = cv2.multiply(alpha_channel_3ch, input_bgr.astype(float))
-        background = cv2.multiply(1.0 - alpha_channel_3ch, background_image.astype(float))
+        background = cv2.multiply(1.0 - alpha_channel_3ch, background_bgr.astype(float))
         combined_image = cv2.add(foreground, background).astype(np.uint8)
 
         # Save and display the result
         cv2.imwrite('path_to_save_combined_image.png', combined_image)
         cv2.imshow('Combined Image', combined_image)
         cv2.waitKey(0)
+
         cv2.destroyAllWindows()
     
+    @classmethod
+    def replace_background_with_removebg(cls, foreground_img_pil: Image, background_image_pil: Image):
+        foreground_img_pil= foreground_img_pil.convert("RGB")
+        width = foreground_img_pil.width
+        height = foreground_img_pil.height
+
+        # Resize background image
+        background_image_pil = background_image_pil.convert("RGB")
+        background_image_pil = background_image_pil.resize((width, height))
+
+        #foreground_img_pil = Image.open(foreground_img_path)
+        #width = foreground_img_pil.width
+        #height = foreground_img_pil.height
+        #background_image_pil = Image.open(background_img_path)
+        #background_image_pil = background_image_pil.resize((width, height)) 
+
+        foreground_binary = ImageFormatConvertor.pil_image_to_binary_data(foreground_img_pil)
+        background_binary = ImageFormatConvertor.pil_image_to_binary_data(background_image_pil)
+        combined_img_pil = cls.remove_bg(foreground_binary, background_binary)
+        combined_img_pil.show()
+        return combined_img_pil
 
 
-    # Function to convert PIL Image to OpenCV format
-    @classmethod
-    def pil_to_cv2(cls, pil_image):
-        open_cv_image = np.array(pil_image)
-        # Convert RGB to BGR if it's a 3-channel image
-        if len(open_cv_image.shape) == 3:        
-            open_cv_image = open_cv_image[:, :, ::-1].copy()
-        return open_cv_image
-
-    # Function to convert OpenCV format to PIL Image
     @classmethod
-    def cv2_to_pil(cls, cv2_image):
-        # Convert BGR to RGB if it's a 3-channel image
-        if len(cv2_image.shape) == 3:        
-            cv2_image = cv2.cvtColor(cv2_image, cv2.COLOR_BGR2RGB)
-        pil_image = Image.fromarray(cv2_image)
-        return pil_image        
+    def remove_bg(cls, foreground_binary: str, background_binary: str):
+        # ref: https://www.remove.bg/api#api-reference
+        url = "https://api.remove.bg/v1.0/removebg"        
+        
+        # using form-data as passing binary data is not supported in application/json
+        files = {
+            "image_file": ('foreground.png', foreground_binary, 'image/png'),
+            "bg_image_file": ('background.png', background_binary,  'image/png')
+        }
+                
+        headers = {
+            "accept": "image/*",
+            'X-Api-Key': REMOVE_BG_KEY
+        } 
+        remove_bg_request = requests.post(url, files=files,headers=headers, timeout=20)
+        if remove_bg_request.status_code == 200:
+            image_content = remove_bg_request.content
+            pil_image = ImageFormatConvertor.binary_data_to_pil_image(image_content)
+            return pil_image
+        logging.error(f"failed to use remove bg. Status: {remove_bg_request.status_code}. Resp: {remove_bg_request.content}")
+        return None
+    
+                    

src/image_format_convertor.py

@@ -0,0 +1,50 @@
+from io import BytesIO
+from PIL import Image
+import numpy as np
+import cv2
+
+class ImageFormatConvertor:
+    # Function to convert PIL Image to Binary Data
+    @classmethod
+    def pil_image_to_binary_data(cls, pil_image, format='PNG'):
+        # Create a buffer to hold the image data
+        buffer = BytesIO()
+        # Save the PIL image to the buffer in the specified format
+        pil_image.save(buffer, format=format)
+        # Get the byte data from the buffer
+        binary_data = buffer.getvalue()
+        return binary_data        
+    
+    # Function to convert Binary Format to PIL Image
+    @classmethod
+    def binary_data_to_pil_image(cls, binary_data):
+        # Create a BytesIO object from the binary data
+        buffer = BytesIO(binary_data)
+        # Open the image from the buffer
+        pil_image = Image.open(buffer)
+        return pil_image
+    
+    # Function to convert PIL Image to OpenCV format
+    @classmethod
+    def pil_to_cv2(cls, pil_image):
+        open_cv_image = np.array(pil_image)
+        # Convert RGB to BGR if it's a 3-channel image
+        if len(open_cv_image.shape) == 3 and open_cv_image.shape[2] == 3:
+            open_cv_image = open_cv_image[:, :, ::-1].copy()
+        # Convert RGBA to BGRA if it's a 4-channel image
+        elif len(open_cv_image.shape) == 3 and open_cv_image.shape[2] == 4:
+            open_cv_image = open_cv_image[:, :, [2, 1, 0, 3]].copy()
+        
+        return open_cv_image
+
+    # Function to convert OpenCV format to PIL Image
+    @classmethod
+    def cv2_to_pil(cls, cv2_image):
+        # Convert BGR to RGB if it's a 3-channel image
+        if len(cv2_image.shape) == 3 and cv2_image.shape[2] == 3:
+            cv2_image = cv2.cvtColor(cv2_image, cv2.COLOR_BGR2RGB)
+        # Convert BGRA to RGBA if it's a 4-channel image
+        elif len(cv2_image.shape) == 3 and cv2_image.shape[2] == 4:
+            cv2_image = cv2.cvtColor(cv2_image, cv2.COLOR_BGRA2RGBA)            
+        pil_image = Image.fromarray(cv2_image)
+        return pil_image