spacey

src/visual_qa.py

@@ -6,7 +6,7 @@ import torch
 import os
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
-
+from datasets import load_dataset
 import chromadb
 
 from datetime import datetime
@@ -72,7 +72,18 @@ model = CLIPModel.from_pretrained(model_name).to(device)
 processor = CLIPProcessor.from_pretrained(model_name)
 
 def main_text_retrieve_images(text, result_query=None, n_retrieved=3):
-    df_emb = pd.read_json("data/fashion_show_data_all_embeddings.json", lines=True)
+
+    
+
+    # Load the dataset (no split specified, so the whole dataset)
+    dataset = load_dataset("traopia/fashion_show_data_all_embeddings.json")
+    # This returns a DatasetDict with splits as keys (usually 'train' by default).
+    # To get the whole dataset, you can access the first split like this:
+    split_name = list(dataset.keys())[0]
+    full_dataset = dataset[split_name]
+
+    # Convert to pandas DataFrame
+    df_emb = full_dataset.to_pandas()
     df_emb = df_emb.drop_duplicates(subset='image_urls')
     df_emb['fashion_clip_image'] = df_emb['fashion_clip_image'].apply(lambda x: x[0] if type(x) == list else None)
     df_emb['image_url'] = df_emb['image_urls'].apply(lambda x: x[0] if x else None)