Spaces:
Runtime error
Runtime error
File size: 2,521 Bytes
03561be |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
import json
import os
import random
from collections import defaultdict
from PIL import Image
from .vqa_dataset import VQADataset
class GQADataset(VQADataset):
"""Visual Reasoning Dataset."""
def __init__(self, tokenizer, vis_processor, vis_root, ann_paths, **kwargs):
super().__init__(tokenizer, vis_processor, vis_root, ann_paths=[], **kwargs)
self.annotation = self.load_annotations(ann_paths)
if self.sample_image:
print("randomly sample one annotation for each image")
self.annotation = self.parse_annotation(self.annotation)
self._add_instance_ids()
self.answer_prob = 1.0
@staticmethod
def load_annotations(ann_paths):
annotation = []
for ann_path in ann_paths:
ann = json.load(open(ann_path, "r"))
for k, v in ann.items():
v["question_id"] = k
annotation.append(v)
return annotation
def parse_annotation(self, annotation):
image_list = defaultdict(list)
for ann in annotation:
image_list[ann["imageId"]].append(ann)
annotation = []
for ann_list in image_list.values():
annotation.append(random.choice(ann_list))
return annotation
def process_text(self, ann):
question = ann["question"]
answer = ann["answer"]
full_answer = ann["fullAnswer"]
# TODO: check which one is better
# Random select answer or full_answer
if random.random() < self.answer_prob:
select_answer = full_answer
else:
select_answer = answer
instruction = self.prompter(question)
return dict(instruction=instruction, answer=select_answer)
def process_image(self, ann):
image_path = os.path.join(self.vis_root, ann["imageId"] + ".jpg")
image = Image.open(image_path).convert("RGB")
image = self.vis_processor(image)
return image
def build_gqa_dataset(
tokenizer,
vis_processor,
vis_root="data/gqa/images",
ann_paths=[
"data/gqa/questions/train_all_questions/train_all_questions_0.json",
"data/gqa/questions/val_all_questions.json",
],
sample_image=False,
):
return GQADataset(
tokenizer=tokenizer,
vis_processor=vis_processor,
vis_root=vis_root,
ann_paths=ann_paths,
sample_image=sample_image,
)
|