Spaces:
Running
Running
File size: 3,021 Bytes
703c654 626e0aa 703c654 d9d27f1 703c654 d9d27f1 703c654 d9d27f1 703c654 d9d27f1 703c654 d9d27f1 703c654 d9d27f1 703c654 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 |
import gradio as gr
import torch
import io
from PIL import Image
from transformers import (
AutoImageProcessor,
AutoTokenizer,
AutoModelForCausalLM,
)
import numpy as np
model_root = "qihoo360/fg-clip-base"
model = AutoModelForCausalLM.from_pretrained(model_root,trust_remote_code=True)
device = model.device
tokenizer = AutoTokenizer.from_pretrained(model_root)
image_processor = AutoImageProcessor.from_pretrained(model_root)
import math
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
def postprocess_result(probs, labels):
pro_output = {labels[i]: probs[i] for i in range(len(labels))}
return pro_output
def Retrieval(image, candidate_labels):
"""
Takes an image and a comma-separated string of candidate labels,
and returns the classification scores.
"""
image_size=224
image = image.convert("RGB")
image = image.resize((image_size,image_size))
image_input = image_processor.preprocess(image, return_tensors='pt')['pixel_values'].to(device)
walk_short_pos = True
caption_input = torch.tensor(tokenizer(candidate_labels, max_length=77, padding="max_length", truncation=True).input_ids, dtype=torch.long, device=device)
with torch.no_grad():
image_feature = model.get_image_features(image_input)
text_feature = model.get_text_features(caption_input,walk_short_pos=walk_short_pos)
image_feature = image_feature / image_feature.norm(p=2, dim=-1, keepdim=True)
text_feature = text_feature / text_feature.norm(p=2, dim=-1, keepdim=True)
logits_per_image = image_feature @ text_feature.T
logits_per_image = model.logit_scale.exp() * logits_per_image
probs = logits_per_image.softmax(dim=1)
results = probs[0].tolist()
return results
def infer(image, candidate_labels):
candidate_labels = [label.lstrip(" ") for label in candidate_labels.split(",") if label !=""]
fg_probs = Retrieval(image, candidate_labels)
return postprocess_result(fg_probs,candidate_labels)
with gr.Blocks() as demo:
gr.Markdown("# FG-CLIP Retrieval")
gr.Markdown(
"This app uses the FG-CLIP model (qihoo360/fg-clip-base) for retrieval on CPU :"
)
with gr.Row():
with gr.Column():
image_input = gr.Image(type="pil")
text_input = gr.Textbox(label="Input a list of labels (comma seperated)")
run_button = gr.Button("Run Retrieval", visible=True)
with gr.Column():
fg_output = gr.Label(label="FG-CLIP Output", num_top_classes=11)
examples = [
["./Landscape.jpg", "red grass, yellow grass, green grass"],
["./cat.jpg", "two sleeping cats, two cats playing, three cats laying down"],
]
gr.Examples(
examples=examples,
inputs=[image_input, text_input],
outputs=fg_output,
fn=infer,
)
run_button.click(fn=infer, inputs=[image_input, text_input], outputs=fg_output)
demo.launch() |