Set up space

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Visualized_m3.pth filter=lfs diff=lfs merge=lfs -text
+image filter=lfs diff=lfs merge=lfs -text

__init__.py

@@ -0,0 +1 @@
+from .modeling import Visualized_BGE

app.py

@@ -0,0 +1,111 @@
+import gradio as gr
+import torch
+from visual_bge.modeling import Visualized_BGE
+from qdrant_client import QdrantClient
+from qdrant_client.http.models import Filter, FieldCondition, MatchValue
+from PIL import Image
+from io import BytesIO
+import requests
+import matplotlib.pyplot as plt
+from matplotlib import font_manager
+import textwrap
+import os
+import tempfile
+import pandas as pd
+from huggingface_hub import hf_hub_download
+
+model_weight = hf_hub_download(repo_id="BAAI/bge-visualized", filename="Visualized_m3.pth")
+
+data = pd.read_csv("image_url.csv")
+
+# Load Thai font
+thai_font = font_manager.FontProperties(fname='./Sarabun-Regular.ttf')
+
+# Load model
+model = Visualized_BGE(
+    model_name_bge="BAAI/bge-m3",
+    model_weight=model_weight
+)
+model.eval()
+
+# Load Qdrant connection
+qdrant_client = QdrantClient(
+    url=os.environ.get("QDRANT_URL"),
+    api_key=os.environ.get("QDRANT_API_KEY")
+)
+
+# Visual helper function
+def visualize_results(results):
+    cols = 4
+    rows = (len(results) + cols - 1) // cols
+    fig, axs = plt.subplots(rows, cols, figsize=(cols * 3, rows * 3))
+    axs = axs.flatten() if hasattr(axs, 'flatten') else [axs]
+
+    for i, res in enumerate(results):
+        try:
+            image_url = data.iloc[int(res.payload["index"])]
+            img = Image.open(image_url) if os.path.exists(image_url) else Image.open(BytesIO(requests.get(image_url).content))
+            name = res.payload['name']
+            if len(name) > 30:
+                name = name[:27] + "..."
+            wrapped_name = textwrap.fill(name, width=15)
+            axs[i].imshow(img)
+            axs[i].set_title(f"{wrapped_name}\nScore: {res.score:.2f}", fontproperties=thai_font, fontsize=10)
+            axs[i].axis('off')
+        except Exception as e:
+            axs[i].text(0.5, 0.5, f'Error: {str(e)}', ha='center', va='center', fontsize=8)
+            axs[i].axis('off')
+
+    for j in range(len(results), len(axs)):
+        axs[j].axis('off')
+
+    plt.tight_layout(pad=3.0)
+    plt.subplots_adjust(hspace=0.5)
+    with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmpfile:
+        fig.savefig(tmpfile.name)
+        return tmpfile.name
+
+# Text Query Handler
+def search_by_text(text_input):
+    if not text_input.strip():
+        return "Please provide a text input.", None
+    query_vector = model.encode(text=text_input)[0].tolist()
+    results = qdrant_client.query_points(
+        collection_name="bge_visualized_m3",
+        query=query_vector,
+        with_payload=True,
+    ).points
+    image_path = visualize_results(results)
+    return f"Results for: {text_input}", image_path
+
+# Image Query Handler
+def search_by_image(image_input):
+    if image_input is None:
+        return "Please upload an image.", None
+    query_vector = model.encode(image=image_input)[0].tolist()
+    results = qdrant_client.query_points(
+        collection_name="bge_visualized_m3",
+        query=query_vector,
+        with_payload=True,
+    ).points
+    image_path = visualize_results(results)
+    return "Results for image query", image_path
+
+# Gradio UI
+with gr.Blocks() as demo:
+    gr.Markdown("# 🔍 Visualized BGE: Multimodal Search with Qdrant")
+    with gr.Tab("📝 Text Query"):
+        text_input = gr.Textbox(label="Enter text to search")
+        text_output = gr.Textbox(label="Query Info")
+        text_image = gr.Image(label="Results", type="filepath")
+        text_btn = gr.Button("Search")
+        text_btn.click(fn=search_by_text, inputs=text_input, outputs=[text_output, text_image])
+
+    with gr.Tab("🖼️ Image Query"):
+        image_input = gr.Image(label="Upload image to search", type="pil")
+        image_output = gr.Textbox(label="Query Info")
+        image_result = gr.Image(label="Results", type="filepath")
+        image_btn = gr.Button("Search")
+        image_btn.click(fn=search_by_image, inputs=image_input, outputs=[image_output, image_result])
+
+demo.launch()

requirements.txt

@@ -0,0 +1,14 @@
+torchvision
+timm
+einops
+ftfy
+transformers==4.52.3
+qdrant_client
+torchvision==0.22.0
+torch==2.7.0
+torchaudio==2.7.0
+gradio
+requests
+pillow
+os
+pandas

setup.py

@@ -0,0 +1,18 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="visual_bge",
+    version="0.1.0",
+    description='visual_bge',
+    long_description="./README.md",
+    long_description_content_type="text/markdown",
+    url='https://github.com/FlagOpen/FlagEmbedding/tree/master/research/visual_bge',
+    packages=find_packages(),
+    install_requires=[
+        'torchvision',
+        'timm',
+        'einops',
+        'ftfy'
+    ],
+    python_requires='>=3.6',
+)

visual_bge.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: visual_bge
+Version: 0.1.0
+Summary: visual_bge
+Home-page: https://github.com/FlagOpen/FlagEmbedding/tree/master/research/visual_bge
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: torchvision
+Requires-Dist: timm
+Requires-Dist: einops
+Requires-Dist: ftfy
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+./README.md

visual_bge.egg-info/SOURCES.txt

@@ -0,0 +1,7 @@
+README.md
+setup.py
+visual_bge.egg-info/PKG-INFO
+visual_bge.egg-info/SOURCES.txt
+visual_bge.egg-info/dependency_links.txt
+visual_bge.egg-info/requires.txt
+visual_bge.egg-info/top_level.txt

visual_bge.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

visual_bge.egg-info/requires.txt

@@ -0,0 +1,4 @@
+torchvision
+timm
+einops
+ftfy

visual_bge.egg-info/top_level.txt

@@ -0,0 +1 @@
+

visual_bge/eva_clip/__init__.py

@@ -0,0 +1,11 @@
+from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
+from .factory import create_model, create_model_and_transforms, create_model_from_pretrained, get_tokenizer, create_eva_vision_and_transforms
+from .factory import list_models, add_model_config, get_model_config, load_checkpoint
+from .loss import ClipLoss
+from .model import CLIP, CustomCLIP, CLIPTextCfg, CLIPVisionCfg,\
+    convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype
+from .openai import load_openai_model, list_openai_models
+from .pretrained import list_pretrained, list_pretrained_models_by_tag, list_pretrained_tags_by_model,\
+    get_pretrained_url, download_pretrained_from_url, is_pretrained_cfg, get_pretrained_cfg, download_pretrained
+from .tokenizer import SimpleTokenizer, tokenize
+from .transform import image_transform