search fashiondb

app_fashionDB.py

@@ -0,0 +1,277 @@
+import gradio as gr
+import pandas as pd
+import numpy as np
+from search_fashionDB import search_images_by_text, get_similar_images, search_images_by_image
+import requests
+from io import BytesIO
+
+import requests
+from io import BytesIO
+
+
+#@st.cache_data(show_spinner="Loading FashionDB...")
+def load_data_hf():
+    # Load the Parquet file directly from Hugging Face
+    df_url = "https://huggingface.co/datasets/traopia/FashionDB/resolve/main/data_vogue_final.parquet"
+    df = pd.read_parquet(df_url)
+    df = df.explode("image_urls_sample")
+    df  = df.rename(columns={"image_urls_sample":"url", "URL":"collection"})
+
+    df_fh = pd.read_parquet("https://huggingface.co/datasets/traopia/FashionDB/resolve/main/final_info_fh.parquet")
+    df_designers = pd.read_parquet("https://huggingface.co/datasets/traopia/FashionDB/resolve/main/final_info_designers.parquet")
+
+    # Load the .npy file using requests
+    npy_url = "https://huggingface.co/datasets/traopia/FashionDB/resolve/main/fashion_clip.npy"
+    response = requests.get(npy_url)
+    response.raise_for_status()  # Raise error if download fails
+    embeddings = np.load(BytesIO(response.content))
+    image_urls = "https://huggingface.co/datasets/traopia/FashionDB/resolve/main/image_urls.npy"
+    response = requests.get(image_urls)
+    response.raise_for_status()  # Raise error if download fails
+    embeddings_urls = np.load(BytesIO(response.content), allow_pickle=True)
+
+    return df, df_fh, df_designers,  embeddings, embeddings_urls
+
+
+
+
+df, df_fh, df_designers,  embeddings, embeddings_urls = load_data_hf()
+# Suppose embeddings is a numpy array (N, D) and embeddings_urls is a list of urls/keys
+embedding_map = {url: i for i, url in enumerate(embeddings_urls)}
+
+# Filter and search
+def filter_and_search(fashion_house, designer, category, season, start_year, end_year, query):
+    filtered = df.copy()
+
+    if fashion_house:
+        filtered = filtered[filtered['fashion_house'].isin(fashion_house)]
+    
+    if designer:
+        filtered = filtered[filtered['designer_name'].isin(designer)]
+    if category:
+        filtered = filtered[filtered['category'].isin(category)]
+    if season:
+        filtered = filtered[filtered['season'].isin(season)]
+    filtered = filtered[(filtered['year'] >= start_year) & (filtered['year'] <= end_year)]
+
+    if query:
+        image_urls, metadata = search_images_by_text(query, filtered, embeddings, embeddings_urls)
+    else:
+        results = filtered.head(30) 
+        image_urls = results["url"].tolist()
+        metadata = results.to_dict(orient="records")
+    return image_urls, metadata
+
+# Display metadata and similar
+def show_metadata(idx, metadata):
+    item = metadata[idx]
+    out = ""
+    for field in ["fashion_house", "designer_name", "season", "year", "category"]:
+        if field in item and pd.notna(item[field]):
+            out += f"**{field.title()}**: {item[field]}\n"
+    if 'collection' in item and pd.notna(item['collection']):
+        out += f"\n[View Collection]({item['collection']})"
+    return out
+
+
+
+def find_similar(idx, metadata,top_k=5):
+    if not isinstance(idx, int) or idx >= len(metadata) or idx < 0:
+        return [], []
+
+    key = metadata[idx]["url"]  # assumes each row has "key" (url or id)
+
+    image_urls, metadata = get_similar_images(df, key, embeddings, embedding_map, embeddings_urls, top_k=top_k)
+    return image_urls,metadata
+    
+
+
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("# 👗 FashionDB Explorer")
+
+    with gr.Tabs():
+        # TEXT SEARCH TAB
+        with gr.Tab("Search by Text"):
+            with gr.Row():
+                fashion_house = gr.Dropdown(label="Fashion House", choices=sorted(df["fashion_house"].dropna().unique()), multiselect=True)
+                designer = gr.Dropdown(label="Fashion Designer", choices=sorted(df["designer_name"].dropna().unique()), multiselect=True)
+                category = gr.Dropdown(label="Category", choices=sorted(df["category"].dropna().unique()), multiselect=True)
+                season = gr.Dropdown(label="Season", choices=sorted(df["season"].dropna().unique()), multiselect=True)
+                min_year = int(df['year'].min())
+                max_year = int(df['year'].max())
+                start_year = gr.Slider(label="Start Year", minimum=min_year, maximum=max_year, value=2000, step=1)
+                end_year = gr.Slider(label="End Year", minimum=min_year, maximum=max_year, value=2024, step=1)
+
+            query = gr.Textbox(label="Search by text", placeholder="e.g., pink dress")
+            search_button = gr.Button("Search")
+
+            result_gallery = gr.Gallery(label="Search Results", columns=5, height="auto")
+            metadata_output = gr.Markdown()
+            reference_image = gr.Image(label="Reference Image", interactive=False)
+            similar_gallery = gr.Gallery(label="Similar Images", columns=5, height="auto")
+
+            metadata_state = gr.State([])
+            selected_idx = gr.Number(value=0, visible=False)
+
+            def handle_search(fh, dis, cat, sea, sy, ey, q):
+                imgs, meta = filter_and_search(fh, dis, cat, sea, sy, ey, q)
+                return imgs, meta, "", [], None
+
+            search_button.click(
+                handle_search,
+                inputs=[fashion_house, designer, category, season, start_year, end_year, query],
+                outputs=[result_gallery, metadata_state, metadata_output, similar_gallery, reference_image]
+            )
+
+            def handle_click(evt: gr.SelectData, metadata):
+                idx = evt.index
+                md = show_metadata(idx, metadata)
+                img_path = metadata[idx]["url"]
+                return idx, md, img_path
+
+            result_gallery.select(
+                handle_click,
+                inputs=[metadata_state],
+                outputs=[selected_idx, metadata_output, reference_image]
+            )
+
+            def show_similar(idx, metadata):
+                if idx is None or not str(idx).isdigit():
+                    return [], []
+                return find_similar(int(idx), metadata)
+
+            similar_metadata_state = gr.State()
+            similar_metadata_output = gr.Markdown()
+
+            show_similar_button = gr.Button("Show Similar Images")
+            show_similar_button.click(
+                show_similar,
+                inputs=[selected_idx, metadata_state],
+                outputs=[similar_gallery, similar_metadata_state]
+            )
+
+            def handle_similar_click(evt: gr.SelectData, metadata):
+                idx = evt.index
+                md = show_metadata(idx, metadata)
+                img_path = metadata[idx]["url"]
+                return idx, md, img_path
+
+            similar_gallery.select(
+                handle_similar_click,
+                inputs=[similar_metadata_state],
+                outputs=[selected_idx, similar_metadata_output, reference_image]
+            )
+
+        # IMAGE SEARCH TAB
+        with gr.Tab("Search by Image"):
+            with gr.Row():
+                fashion_house_img = gr.Dropdown(label="Fashion House", choices=sorted(df["fashion_house"].dropna().unique()), multiselect=True)
+                designer_img = gr.Dropdown(label="Fashion Designer", choices=sorted(df["designer_name"].dropna().unique()), multiselect=True)
+                category_img = gr.Dropdown(label="Category", choices=sorted(df["category"].dropna().unique()), multiselect=True)
+                season_img = gr.Dropdown(label="Season", choices=sorted(df["season"].dropna().unique()), multiselect=True)
+                start_year_img = gr.Slider(label="Start Year", minimum=min_year, maximum=max_year, value=2000, step=1)
+                end_year_img = gr.Slider(label="End Year", minimum=min_year, maximum=max_year, value=2024, step=1)
+
+            uploaded_image = gr.Image(label="Upload an image", type="pil")
+            search_by_image_button = gr.Button("Search by Image")
+
+            uploaded_result_gallery = gr.Gallery(label="Search Results by Image", columns=5, height="auto")
+            uploaded_metadata_state = gr.State([])
+            uploaded_metadata_output = gr.Markdown()
+            uploaded_reference_image = gr.Image(label="Reference Image", interactive=False)
+
+            def handle_search_by_image(image, fh, dis, cat, sea, sy, ey):
+                if image is None:
+                    return [], "Please upload an image first.", None
+                # Apply filters
+                filtered_df = df.copy()
+                if fh: filtered_df = filtered_df[filtered_df["fashion_house"].isin(fh)]
+                if dis: filtered_df = filtered_df[filtered_df["designer_name"].isin(fh)]
+                if cat: filtered_df = filtered_df[filtered_df["category"].isin(cat)]
+                if sea: filtered_df = filtered_df[filtered_df["season"].isin(sea)]
+                filtered_df = filtered_df[(filtered_df["year"] >= sy) & (filtered_df["year"] <= ey)]
+
+                images, metadata = search_images_by_image(image, filtered_df, embeddings, embeddings_urls)
+                return images, metadata, ""
+
+            search_by_image_button.click(
+                handle_search_by_image,
+                inputs=[uploaded_image, fashion_house_img, designer_img, category_img, season_img, start_year_img, end_year_img],
+                outputs=[uploaded_result_gallery, uploaded_metadata_state, uploaded_metadata_output]
+            )
+
+            uploaded_selected_idx = gr.Number(visible=False)
+
+            def handle_uploaded_click(evt: gr.SelectData, metadata):
+                idx = evt.index
+                md = show_metadata(idx, metadata)
+                img_path = metadata[idx]["url"]
+                return idx, md, img_path
+
+            uploaded_result_gallery.select(
+                handle_uploaded_click,
+                inputs=[uploaded_metadata_state],
+                outputs=[uploaded_selected_idx, uploaded_metadata_output, uploaded_reference_image]
+            )
+
+                        # SIMILAR IMAGE SEARCH FOR IMAGE TAB
+            uploaded_similar_gallery = gr.Gallery(label="Similar Images", columns=5, height="auto")
+            uploaded_similar_metadata_state = gr.State([])
+            uploaded_similar_metadata_output = gr.Markdown()
+
+            uploaded_show_similar_button = gr.Button("Show Similar Images")
+
+            def show_similar_uploaded(idx, metadata):
+                if idx is None or not str(idx).isdigit():
+                    return [], []
+                return find_similar(int(idx), metadata)
+
+            uploaded_show_similar_button.click(
+                show_similar_uploaded,
+                inputs=[uploaded_selected_idx, uploaded_metadata_state],
+                outputs=[uploaded_similar_gallery, uploaded_similar_metadata_state]
+            )
+
+            def handle_uploaded_similar_click(evt: gr.SelectData, metadata):
+                idx = evt.index
+                md = show_metadata(idx, metadata)
+                img_path = metadata[idx]["url"]
+                return idx, md, img_path
+
+            uploaded_similar_gallery.select(
+                handle_uploaded_similar_click,
+                inputs=[uploaded_similar_metadata_state],
+                outputs=[uploaded_selected_idx, uploaded_similar_metadata_output, uploaded_reference_image]
+            )
+
+            uploaded_back_button = gr.Button("Back to Initial Uploaded Search")
+
+            def back_to_uploaded_home():
+                return [], "", None
+
+            uploaded_back_button.click(
+                back_to_uploaded_home,
+                outputs=[uploaded_similar_gallery, uploaded_similar_metadata_output, uploaded_reference_image]
+            )
+
+        with gr.Tab("Query on FashionDB"):
+            with gr.Row():
+                gr.Markdown(
+                    "### 🔗 Query FashionDB SPARQL Endpoint\n"
+                    "[Click here to open the SPARQL endpoint](https://fashionwiki.wikibase.cloud/query/)",
+                    elem_id="sparql-link"
+                )
+
+    back_button = gr.Button("Back to Home")
+
+    def back_to_home():
+        return [], "", None  # clear similar_gallery, metadata_output, reference image
+
+    back_button.click(
+        back_to_home,
+        outputs=[similar_gallery, similar_metadata_output, reference_image]
+    )
+
+demo.launch()

search_fashionDB.py

@@ -0,0 +1,125 @@
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+
+
+from transformers import pipeline
+from PIL import Image
+import numpy as np
+import os
+import torch
+from transformers import CLIPProcessor, CLIPModel
+import pandas as pd
+
+#set device: Use GPU if availanle, otherwise mps if available otherwise CPU
+device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
+
+
+# Load Fashion-CLIP model and processor
+model_name = "patrickjohncyh/fashion-clip"
+#model_name = "openai/clip-vit-base-patch32"
+model = CLIPModel.from_pretrained(model_name).to(device)
+processor = CLIPProcessor.from_pretrained(model_name)
+
+# Initialize segmentation pipeline
+segmenter = pipeline(model="mattmdjaga/segformer_b2_clothes", device = device)
+
+
+def segment_clothing_white(img, clothes=["Background"]):
+    segments = segmenter(img)
+
+    # Create list of masks
+    mask_list = []
+    for s in segments:
+        if s['label'] in clothes:
+            mask_list.append(s['mask'])
+
+    if not mask_list:
+        print("No clothing segments found in image.")
+        return img  # Return the original image if no segments are found
+
+    # Combine all masks into a single mask
+    final_mask = np.array(mask_list[0])
+    for mask in mask_list[1:]:
+        final_mask = np.maximum(final_mask, np.array(mask))  # Combine masks using max
+
+    # Apply the mask to the image
+    img_array = np.array(img)  # Convert image to numpy array
+    final_mask = final_mask.astype(bool)  # Convert mask to boolean
+    img_array[final_mask] = [255,255,255]  # Set unmasked regions to black
+
+    # Convert back to PIL image
+    segmented_img = Image.fromarray(img_array)
+    return segmented_img
+
+def encode_image(image):
+    """Encode image into an embedding."""
+    inputs = processor(images=image, return_tensors="pt").to(device)
+
+    with torch.no_grad():
+        embedding = model.get_image_features(**inputs).cpu().numpy()  # Move to CPU for stability
+        embedding = embedding / torch.linalg.norm(torch.tensor(embedding), ord=2, dim=-1, keepdim=True)
+        embedding = embedding.numpy().astype(np.float32).flatten()
+    return embedding
+
+from PIL import Image
+import torchvision.transforms as T
+
+
+
+def search_images_by_image(uploaded_image, df, embeddings,embeddings_urls,  top_k=30):
+    # Convert to PIL
+    if isinstance(uploaded_image, str):
+        uploaded_image = Image.open(uploaded_image).convert("RGB")
+    elif isinstance(uploaded_image, np.ndarray):
+        uploaded_image = Image.fromarray(uploaded_image).convert("RGB")
+
+    # Encode with CLIP
+    image_emb = encode_image(uploaded_image)
+
+    # Similarity against ALL embeddings
+    sims = cosine_similarity([image_emb], embeddings)[0]
+    top_indices = np.argsort(sims)[::-1][:top_k]
+    top_urls = [embeddings_urls[i] for i in top_indices]
+    metadata = df[df["url"].isin(top_urls)].copy().to_dict(orient="records")
+
+
+    return top_urls, metadata
+
+
+
+
+
+
+
+
+def search_images_by_text(text, df, embeddings, embeddings_urls, top_k=30):
+    inputs = processor(text=[text], return_tensors="pt", padding=True, truncation=True, max_length=77).to(device)
+    with torch.no_grad():
+        text_emb = model.get_text_features(**inputs).cpu().numpy()
+
+    df_indices = df.index.to_numpy()
+    # slice embeddings & urls to match the filtered df
+    embeddings_filtered = embeddings[df_indices]
+    sims = cosine_similarity(text_emb, embeddings_filtered)[0]
+    sims = np.asarray(sims).flatten()  
+    top_indices = np.argsort(sims)[::-1][:top_k]
+    top_urls = [embeddings_urls[i] for i in top_indices]
+    metadata = df[df["url"].isin(top_urls)].copy().to_dict(orient="records")
+    
+    return top_urls, metadata
+
+def get_similar_images(df, image_key, embeddings, embedding_map, embeddings_urls, top_k=5):
+    if image_key not in embedding_map:
+        return pd.DataFrame()  # fallback: no match found
+    
+    index = embedding_map[image_key]
+    query_emb = embeddings[index]
+
+    sims = cosine_similarity([query_emb], embeddings)[0]
+    top_indices = np.argsort(sims)[::-1][1:top_k+1]  # skip itself
+    top_urls = [embeddings_urls[i] for i in top_indices]
+    metadata = df[df["url"].isin(top_urls)].copy().to_dict(orient="records")
+    
+    return top_urls, metadata
+
+