Upload 207 files

.gitattributes

@@ -33,3 +33,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+repo_imgs/demo_1.JPG filter=lfs diff=lfs merge=lfs -text
+repo_imgs/Goldfish_results_table.JPG filter=lfs diff=lfs merge=lfs -text
+repo_imgs/goldfishai_png.png filter=lfs diff=lfs merge=lfs -text
+repo_imgs/goldfishai.jpg filter=lfs diff=lfs merge=lfs -text
+repo_imgs/minigpt4_demo_icon.png filter=lfs diff=lfs merge=lfs -text
+repo_imgs/MiniGPT4-video_fig.jpg filter=lfs diff=lfs merge=lfs -text
+repo_imgs/online_demo.jpeg filter=lfs diff=lfs merge=lfs -text
+repo_imgs/sample_1.gif filter=lfs diff=lfs merge=lfs -text
+repo_imgs/sample_2.gif filter=lfs diff=lfs merge=lfs -text
+repo_imgs/sample_3.gif filter=lfs diff=lfs merge=lfs -text
+repo_imgs/teaser_fig_final_final.jpg filter=lfs diff=lfs merge=lfs -text

Custom_training.md

@@ -0,0 +1,33 @@
+# Customizing MiniGPT4-video for your own Video-text dataset
+
+## Add your own video dataloader 
+Construct your own dataloader here `minigpt4/datasets/datasets/video_datasets.py` based on the existing dataloaders.<br>
+Copy Video_loader_template class and edit it according to you data nature.
+
+## Create config file for your dataloader
+Here `minigpt4/configs/datasets/dataset_name/default.yaml` creates your yaml file that includes paths to your dataset.<br>
+Copy the template file `minigpt4/configs/datasets/template/default.yaml` and edit the paths to your dataset.
+
+
+## Register your dataloader
+In the `minigpt4/datasets/builders/image_text_pair_builder.py` file
+Import your data loader class from the `minigpt4/datasets/datasets/video_datasets.py` file <br>
+Copy and edit the VideoTemplateBuilder class.<br>
+put the train_dataset_cls = YourVideoLoaderClass that you imported from `minigpt4/datasets/datasets/video_datasets.py` file.
+
+## Edit training config file 
+Add your dataset to the datasets in the yml file as shown below:
+```yaml
+datasets:
+  dataset_name: # change this to your dataset name
+    batch_size: 4  # change this to your desired batch size
+    vis_processor:
+      train:
+        name: "blip2_image_train"
+        image_size: 224
+    text_processor:
+      train:
+        name: "blip_caption"
+    sample_ratio: 200 # if you including joint training with other datasets, you can set the sample ratio here
+```
+

Dockerfile

@@ -0,0 +1,33 @@
+FROM pytorch/pytorch:2.2.2-cuda11.8-cudnn8-runtime
+# FROM nvidia/cuda:12.5.1-cudnn-runtime-ubuntu20.04
+# FROM nvcr.io/nvidia/pytorch:24.01-py3
+# Install necessary tools
+RUN apt-get update && apt-get install -y curl gnupg wget
+
+# Add the NVIDIA GPG key and repository
+RUN curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
+  && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
+    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
+    tee /etc/apt/sources.list.d/nvidia-container-toolkit.list \
+  && apt-get update
+
+# Install the NVIDIA container toolkit
+RUN apt-get install -y nvidia-container-toolkit
+# Set the default runtime to nvidia
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+
+# RUN apt install python3-pip -y
+COPY ./ /app
+WORKDIR /app
+
+RUN apt-get update && apt-get install ffmpeg libsm6 libxext6  -y
+RUN apt-get install gcc -y
+
+RUN pip install -r requirements.txt
+
+ENV CUDA_VISIBLE_DEVICES=0   
+ENV HF_TKN="put your huggingface token here"
+
+EXPOSE 7860
+CMD ["python", "minigpt4_video_demo.py"]

GPT_evaluation/evaluate_benchmark.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Define common arguments for all scripts
+
+PRED="pred_path"
+OUTPUT_DIR="output_dir"
+API_KEY="api_key"
+NUM_TASKS=128
+
+# Run the "correctness" evaluation script
+python evaluate_benchmark_1_correctness.py \
+  --pred_path "${PRED_GENERIC}" \
+  --output_dir "${OUTPUT_DIR}/correctness_eval" \
+  --output_json "${OUTPUT_DIR}/correctness_results.json" \
+  --api_key $API_KEY \
+  --num_tasks $NUM_TASKS
+
+# Run the "detailed orientation" evaluation script
+python evaluate_benchmark_2_detailed_orientation.py \
+  --pred_path "${PRED_GENERIC}" \
+  --output_dir "${OUTPUT_DIR}/detailed_eval" \
+  --output_json "${OUTPUT_DIR}/detailed_orientation_results.json" \
+  --api_key $API_KEY \
+  --num_tasks $NUM_TASKS
+
+# Run the "contextual understanding" evaluation script
+python evaluate_benchmark_3_context.py \
+  --pred_path "${PRED_GENERIC}" \
+  --output_dir "${OUTPUT_DIR}/context_eval" \
+  --output_json "${OUTPUT_DIR}/contextual_understanding_results.json" \
+  --api_key $API_KEY \
+  --num_tasks $NUM_TASKS
+
+# Run the "temporal understanding" evaluation script
+python evaluate_benchmark_4_temporal.py \
+  --pred_path "${PRED_TEMPORAL}" \
+  --output_dir "${OUTPUT_DIR}/temporal_eval" \
+  --output_json "${OUTPUT_DIR}/temporal_understanding_results.json" \
+  --api_key $API_KEY \
+  --num_tasks $NUM_TASKS
+
+# Run the "consistency" evaluation script
+python evaluate_benchmark_5_consistency.py \
+  --pred_path "${PRED_CONSISTENCY}" \
+  --output_dir "${OUTPUT_DIR}/consistency_eval" \
+  --output_json "${OUTPUT_DIR}/consistency_results.json" \
+  --api_key $API_KEY \
+  --num_tasks $NUM_TASKS
+
+
+echo "All evaluations completed!"

GPT_evaluation/evaluate_benchmark_1_correctness.py

@@ -0,0 +1,186 @@
+import openai
+import os
+import argparse
+import json
+import ast
+from multiprocessing.pool import Pool
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
+    parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
+    parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
+    parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
+    parser.add_argument("--api_key", required=True, help="OpenAI API key.")
+    parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
+    args = parser.parse_args()
+    return args
+
+
+def annotate(prediction_set, caption_files, output_dir):
+    """
+    Evaluates question and answer pairs using GPT-3
+    Returns a score for correctness.
+    """
+    for file in caption_files:
+        key = file[:-5] # Strip file extension
+        qa_set = prediction_set[key]
+        question = qa_set['q']
+        answer = qa_set['a']
+        pred = qa_set['pred']
+        try:
+            # Compute the correctness score
+            completion = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "system",
+                        "content": 
+                            "You are an intelligent chatbot designed for evaluating the factual accuracy of generative outputs for video-based question-answer pairs. "
+                            "Your task is to compare the predicted answer with the correct answer and determine if they are factually consistent. Here's how you can accomplish the task:"
+                            "------"
+                            "##INSTRUCTIONS: "
+                            "- Focus on the factual consistency between the predicted answer and the correct answer. The predicted answer should not contain any misinterpretations or misinformation.\n"
+                            "- The predicted answer must be factually accurate and align with the video content.\n"
+                            "- Consider synonyms or paraphrases as valid matches.\n"
+                            "- Evaluate the factual accuracy of the prediction compared to the answer."
+                    },
+                    {
+                        "role": "user",
+                        "content":
+                            "Please evaluate the following video-based question-answer pair:\n\n"
+                            f"Question: {question}\n"
+                            f"Correct Answer: {answer}\n"
+                            f"Predicted Answer: {pred}\n\n"
+                            "Provide your evaluation only as a factual accuracy score where the factual accuracy score is an integer value between 0 and 5, with 5 indicating the highest level of factual consistency. "
+                            "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is the factual accuracy score in INTEGER, not STRING."
+                            "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
+                            "For example, your response should look like this: {''score': 4.8}."
+                    }
+                ]
+            )
+            # Convert response to a Python dictionary.
+            response_message = completion["choices"][0]["message"]["content"]
+            response_dict = ast.literal_eval(response_message)
+            result_qa_pair = [response_dict, qa_set]
+
+            # Save the question-answer pairs to a json file.
+            with open(f"{output_dir}/{key}.json", "w") as f:
+                json.dump(result_qa_pair, f)
+
+        except Exception as e:
+            print(f"Error processing file '{key}': {e}")
+
+
+def main():
+    """
+    Main function to control the flow of the program.
+    """
+    # Parse arguments.
+    args = parse_args()
+
+    file = open(args.pred_path)
+    pred_contents = json.load(file)
+
+    # Dictionary to store the count of occurrences for each video_id
+    video_id_counts = {}
+    new_pred_contents = []
+
+    # Iterate through each sample in pred_contents
+    for sample in pred_contents:
+        video_id = sample['video_name']
+        if video_id in video_id_counts:
+            video_id_counts[video_id] += 1
+        else:
+            video_id_counts[video_id] = 0
+
+        # Create a new sample with the modified key
+        new_sample = sample
+        new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
+        new_pred_contents.append(new_sample)
+
+    # Generating list of id's and corresponding files
+    id_list = [x['video_name'] for x in new_pred_contents]
+    caption_files = [f"{id}.json" for id in id_list]
+
+    output_dir = args.output_dir
+    # Generate output directory if not exists.
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # Preparing dictionary of question-answer sets
+    prediction_set = {}
+    for sample in new_pred_contents:
+        id = sample['video_name']
+        question = sample['Q']
+        answer = sample['A']
+        pred = sample['pred']
+        qa_set = {"q": question, "a": answer, "pred": pred}
+        prediction_set[id] = qa_set
+
+    # Set the OpenAI API key.
+    openai.api_key = args.api_key
+    num_tasks = args.num_tasks
+
+    # While loop to ensure that all captions are processed.
+    while True:
+        try:
+            # Files that have not been processed yet.
+            completed_files = os.listdir(output_dir)
+            print(f"completed_files: {len(completed_files)}")
+
+            # Files that have not been processed yet.
+            incomplete_files = [f for f in caption_files if f not in completed_files]
+            print(f"incomplete_files: {len(incomplete_files)}")
+
+            # Break the loop when there are no incomplete files
+            if len(incomplete_files) == 0:
+                break
+            if len(incomplete_files) <= num_tasks:
+                num_tasks = 1
+
+            # Split tasks into parts.
+            part_len = len(incomplete_files) // num_tasks
+            all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
+            task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
+
+            # Use a pool of workers to process the files in parallel.
+            with Pool() as pool:
+                pool.starmap(annotate, task_args)
+
+        except Exception as e:
+            print(f"Error: {e}")
+
+    # Combine all the processed files into one
+    combined_contents = {}
+    json_path = args.output_json
+
+    # Iterate through json files
+    for file_name in os.listdir(output_dir):
+        if file_name.endswith(".json"):
+            file_path = os.path.join(output_dir, file_name)
+            with open(file_path, "r") as json_file:
+                content = json.load(json_file)
+                combined_contents[file_name[:-5]] = content
+
+    # Write combined content to a json file
+    with open(json_path, "w") as json_file:
+        json.dump(combined_contents, json_file)
+    print("All evaluation completed!")
+
+    # Calculate average score
+    score_sum = 0
+    count = 0
+    for key, result in combined_contents.items():
+        count += 1
+        score_match = result[0]['score']
+        score = int(score_match)
+        score_sum += score
+    average_score = score_sum / count
+
+    print("Average score for correctness:", average_score)
+
+
+if __name__ == "__main__":
+    main()
+

GPT_evaluation/evaluate_benchmark_2_detailed_orientation.py

@@ -0,0 +1,186 @@
+import openai
+import os
+import argparse
+import json
+import ast
+from multiprocessing.pool import Pool
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
+    parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
+    parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
+    parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
+    parser.add_argument("--api_key", required=True, help="OpenAI API key.")
+    parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
+    args = parser.parse_args()
+    return args
+
+
+def annotate(prediction_set, caption_files, output_dir):
+    """
+    Evaluates question and answer pairs using GPT-3 and
+    returns a score for detailed orientation.
+    """
+    for file in caption_files:
+        key = file[:-5] # Strip file extension
+        qa_set = prediction_set[key]
+        question = qa_set['q']
+        answer = qa_set['a']
+        pred = qa_set['pred']
+        try:
+            # Compute the detailed-orientation score
+            completion = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "system",
+                        "content":
+                            "You are an intelligent chatbot designed for evaluating the detail orientation of generative outputs for video-based question-answer pairs. "
+                            "Your task is to compare the predicted answer with the correct answer and determine its level of detail, considering both completeness and specificity. Here's how you can accomplish the task:"
+                            "------"
+                            "##INSTRUCTIONS: "
+                            "- Check if the predicted answer covers all major points from the video. The response should not leave out any key aspects.\n"
+                            "- Evaluate whether the predicted answer includes specific details rather than just generic points. It should provide comprehensive information that is tied to specific elements of the video.\n"
+                            "- Consider synonyms or paraphrases as valid matches.\n"
+                            "- Provide a single evaluation score that reflects the level of detail orientation of the prediction, considering both completeness and specificity."
+                    },
+                    {
+                        "role": "user",
+                        "content":
+                            "Please evaluate the following video-based question-answer pair:\n\n"
+                            f"Question: {question}\n"
+                            f"Correct Answer: {answer}\n"
+                            f"Predicted Answer: {pred}\n\n"
+                            "Provide your evaluation only as a detail orientation score where the detail orientation score is an integer value between 0 and 5, with 5 indicating the highest level of detail orientation. "
+                            "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is the detail orientation score in INTEGER, not STRING."
+                            "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
+                            "For example, your response should look like this: {''score': 4.8}."
+                    }
+                ]
+            )
+            # Convert response to a Python dictionary.
+            response_message = completion["choices"][0]["message"]["content"]
+            response_dict = ast.literal_eval(response_message)
+            result_qa_pair = [response_dict, qa_set]
+
+            # Save the question-answer pairs to a json file.
+            with open(f"{output_dir}/{key}.json", "w") as f:
+                json.dump(result_qa_pair, f)
+
+        except Exception as e:
+            print(f"Error processing file '{key}': {e}")
+
+
+def main():
+    """
+    Main function to control the flow of the program.
+    """
+    # Parse arguments.
+    args = parse_args()
+
+    file = open(args.pred_path)
+    pred_contents = json.load(file)
+
+    # Dictionary to store the count of occurrences for each video_id
+    video_id_counts = {}
+    new_pred_contents = []
+
+    # Iterate through each sample in pred_contents
+    for sample in pred_contents:
+        video_id = sample['video_name']
+        if video_id in video_id_counts:
+            video_id_counts[video_id] += 1
+        else:
+            video_id_counts[video_id] = 0
+
+        # Create a new sample with the modified key
+        new_sample = sample
+        new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
+        new_pred_contents.append(new_sample)
+
+    # Generating list of id's and corresponding files
+    id_list = [x['video_name'] for x in new_pred_contents]
+    caption_files = [f"{id}.json" for id in id_list]
+
+    output_dir = args.output_dir
+    # Generate output directory if not exists.
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # Preparing dictionary of question-answer sets
+    prediction_set = {}
+    for sample in new_pred_contents:
+        id = sample['video_name']
+        question = sample['Q']
+        answer = sample['A']
+        pred = sample['pred']
+        qa_set = {"q": question, "a": answer, "pred": pred}
+        prediction_set[id] = qa_set
+
+    # Set the OpenAI API key.
+    openai.api_key = args.api_key
+    num_tasks = args.num_tasks
+
+    # While loop to ensure that all captions are processed.
+    while True:
+        try:
+            # Files that have not been processed yet.
+            completed_files = os.listdir(output_dir)
+            print(f"completed_files: {len(completed_files)}")
+
+            # Files that have not been processed yet.
+            incomplete_files = [f for f in caption_files if f not in completed_files]
+            print(f"incomplete_files: {len(incomplete_files)}")
+
+            # Break the loop when there are no incomplete files
+            if len(incomplete_files) == 0:
+                break
+            if len(incomplete_files) <= num_tasks:
+                num_tasks = 1
+
+            # Split tasks into parts.
+            part_len = len(incomplete_files) // num_tasks
+            all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
+            task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
+
+            # Use a pool of workers to process the files in parallel.
+            with Pool() as pool:
+                pool.starmap(annotate, task_args)
+
+        except Exception as e:
+            print(f"Error: {e}")
+
+    # Combine all the processed files into one
+    combined_contents = {}
+    json_path = args.output_json
+
+    # Iterate through json files
+    for file_name in os.listdir(output_dir):
+        if file_name.endswith(".json"):
+            file_path = os.path.join(output_dir, file_name)
+            with open(file_path, "r") as json_file:
+                content = json.load(json_file)
+                combined_contents[file_name[:-5]] = content
+
+    # Write combined content to a json file
+    with open(json_path, "w") as json_file:
+        json.dump(combined_contents, json_file)
+    print("All evaluation completed!")
+
+    # Calculate average score
+    score_sum = 0
+    count = 0
+    for key, result in combined_contents.items():
+        count += 1
+        score_match = result[0]['score']
+        score = int(score_match)
+        score_sum += score
+    average_score = score_sum / count
+
+    print("Average score for detailed orientation:", average_score)
+
+
+if __name__ == "__main__":
+    main()
+

GPT_evaluation/evaluate_benchmark_3_context.py

@@ -0,0 +1,186 @@
+import openai
+import os
+import argparse
+import json
+import ast
+from multiprocessing.pool import Pool
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
+    parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
+    parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
+    parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
+    parser.add_argument("--api_key", required=True, help="OpenAI API key.")
+    parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
+    args = parser.parse_args()
+    return args
+
+
+def annotate(prediction_set, caption_files, output_dir):
+    """
+    Evaluates question and answer pairs using GPT-3 and
+    returns a score for contextual understanding.
+    """
+    for file in caption_files:
+        key = file[:-5] # Strip file extension
+        qa_set = prediction_set[key]
+        question = qa_set['q']
+        answer = qa_set['a']
+        pred = qa_set['pred']
+        try:
+            # Compute the contextual understanding score
+            completion = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "system",
+                        "content":
+                            "You are an intelligent chatbot designed for evaluating the contextual understanding of generative outputs for video-based question-answer pairs. "
+                            "Your task is to compare the predicted answer with the correct answer and determine if the generated response aligns with the overall context of the video content. Here's how you can accomplish the task:"
+                            "------"
+                            "##INSTRUCTIONS: "
+                            "- Evaluate whether the predicted answer aligns with the overall context of the video content. It should not provide information that is out of context or misaligned.\n"
+                            "- The predicted answer must capture the main themes and sentiments of the video.\n"
+                            "- Consider synonyms or paraphrases as valid matches.\n"
+                            "- Provide your evaluation of the contextual understanding of the prediction compared to the answer."
+                    },
+                    {
+                        "role": "user",
+                        "content":
+                            "Please evaluate the following video-based question-answer pair:\n\n"
+                            f"Question: {question}\n"
+                            f"Correct Answer: {answer}\n"
+                            f"Predicted Answer: {pred}\n\n"
+                            "Provide your evaluation only as a contextual understanding score where the contextual understanding score is an integer value between 0 and 5, with 5 indicating the highest level of contextual understanding. "
+                            "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is contextual understanding score in INTEGER, not STRING."
+                            "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
+                            "For example, your response should look like this: {''score': 4.8}."
+                    }
+                ]
+            )
+            # Convert response to a Python dictionary.
+            response_message = completion["choices"][0]["message"]["content"]
+            response_dict = ast.literal_eval(response_message)
+            result_qa_pair = [response_dict, qa_set]
+
+            # Save the question-answer pairs to a json file.
+            with open(f"{output_dir}/{key}.json", "w") as f:
+                json.dump(result_qa_pair, f)
+
+        except Exception as e:
+            print(f"Error processing file '{key}': {e}")
+
+
+def main():
+    """
+    Main function to control the flow of the program.
+    """
+    # Parse arguments.
+    args = parse_args()
+
+    file = open(args.pred_path)
+    pred_contents = json.load(file)
+
+    # Dictionary to store the count of occurrences for each video_id
+    video_id_counts = {}
+    new_pred_contents = []
+
+    # Iterate through each sample in pred_contents
+    for sample in pred_contents:
+        video_id = sample['video_name']
+        if video_id in video_id_counts:
+            video_id_counts[video_id] += 1
+        else:
+            video_id_counts[video_id] = 0
+
+        # Create a new sample with the modified key
+        new_sample = sample
+        new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
+        new_pred_contents.append(new_sample)
+
+    # Generating list of id's and corresponding files
+    id_list = [x['video_name'] for x in new_pred_contents]
+    caption_files = [f"{id}.json" for id in id_list]
+
+    output_dir = args.output_dir
+    # Generate output directory if not exists.
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # Preparing dictionary of question-answer sets
+    prediction_set = {}
+    for sample in new_pred_contents:
+        id = sample['video_name']
+        question = sample['Q']
+        answer = sample['A']
+        pred = sample['pred']
+        qa_set = {"q": question, "a": answer, "pred": pred}
+        prediction_set[id] = qa_set
+
+    # Set the OpenAI API key.
+    openai.api_key = args.api_key
+    num_tasks = args.num_tasks
+
+    # While loop to ensure that all captions are processed.
+    while True:
+        try:
+            # Files that have not been processed yet.
+            completed_files = os.listdir(output_dir)
+            print(f"completed_files: {len(completed_files)}")
+
+            # Files that have not been processed yet.
+            incomplete_files = [f for f in caption_files if f not in completed_files]
+            print(f"incomplete_files: {len(incomplete_files)}")
+
+            # Break the loop when there are no incomplete files
+            if len(incomplete_files) == 0:
+                break
+            if len(incomplete_files) <= num_tasks:
+                num_tasks = 1
+
+            # Split tasks into parts.
+            part_len = len(incomplete_files) // num_tasks
+            all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
+            task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
+
+            # Use a pool of workers to process the files in parallel.
+            with Pool() as pool:
+                pool.starmap(annotate, task_args)
+
+        except Exception as e:
+            print(f"Error: {e}")
+
+    # Combine all the processed files into one
+    combined_contents = {}
+    json_path = args.output_json
+
+    # Iterate through json files
+    for file_name in os.listdir(output_dir):
+        if file_name.endswith(".json"):
+            file_path = os.path.join(output_dir, file_name)
+            with open(file_path, "r") as json_file:
+                content = json.load(json_file)
+                combined_contents[file_name[:-5]] = content
+
+    # Write combined content to a json file
+    with open(json_path, "w") as json_file:
+        json.dump(combined_contents, json_file)
+    print("All evaluation completed!")
+
+    # Calculate average score
+    score_sum = 0
+    count = 0
+    for key, result in combined_contents.items():
+        count += 1
+        score_match = result[0]['score']
+        score = int(score_match)
+        score_sum += score
+    average_score = score_sum / count
+
+    print("Average score for contextual understanding:", average_score)
+
+
+if __name__ == "__main__":
+    main()
+

GPT_evaluation/evaluate_benchmark_4_temporal.py

@@ -0,0 +1,185 @@
+import openai
+import os
+import argparse
+import json
+import ast
+from multiprocessing.pool import Pool
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
+    parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
+    parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
+    parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
+    parser.add_argument("--api_key", required=True, help="OpenAI API key.")
+    parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
+    args = parser.parse_args()
+    return args
+
+
+def annotate(prediction_set, caption_files, output_dir):
+    """
+    Evaluates question and answer pairs using GPT-3 and
+    returns a score for temporal understanding.
+    """
+    for file in caption_files:
+        key = file[:-5] # Strip file extension
+        qa_set = prediction_set[key]
+        question = qa_set['q']
+        answer = qa_set['a']
+        pred = qa_set['pred']
+        try:
+            # Compute the temporal understanding score
+            completion = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "system",
+                        "content":
+                            "You are an intelligent chatbot designed for evaluating the temporal understanding of generative outputs for video-based question-answer pairs. "
+                            "Your task is to compare the predicted answer with the correct answer and determine if they correctly reflect the temporal sequence of events in the video content. Here's how you can accomplish the task:"
+                            "------"
+                            "##INSTRUCTIONS: "
+                            "- Focus on the temporal consistency between the predicted answer and the correct answer. The predicted answer should correctly reflect the sequence of events or details as they are presented in the video content.\n"
+                            "- Consider synonyms or paraphrases as valid matches, but only if the temporal order is maintained.\n"
+                            "- Evaluate the temporal accuracy of the prediction compared to the answer."
+                    },
+                    {
+                        "role": "user",
+                        "content":
+                            "Please evaluate the following video-based question-answer pair:\n\n"
+                            f"Question: {question}\n"
+                            f"Correct Answer: {answer}\n"
+                            f"Predicted Answer: {pred}\n\n"
+                            "Provide your evaluation only as a temporal accuracy score where the temporal accuracy score is an integer value between 0 and 5, with 5 indicating the highest level of temporal consistency. "
+                            "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is the temporal accuracy score in INTEGER, not STRING."
+                            "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
+                            "For example, your response should look like this: {''score': 4.8}."
+                    }
+                ]
+            )
+            # Convert response to a Python dictionary.
+            response_message = completion["choices"][0]["message"]["content"]
+            response_dict = ast.literal_eval(response_message)
+            result_qa_pair = [response_dict, qa_set]
+
+            # Save the question-answer pairs to a json file.
+            with open(f"{output_dir}/{key}.json", "w") as f:
+                json.dump(result_qa_pair, f)
+
+        except Exception as e:
+            print(f"Error processing file '{key}': {e}")
+
+
+def main():
+    """
+    Main function to control the flow of the program.
+    """
+    # Parse arguments.
+    args = parse_args()
+
+    file = open(args.pred_path)
+    pred_contents = json.load(file)
+
+    # Dictionary to store the count of occurrences for each video_id
+    video_id_counts = {}
+    new_pred_contents = []
+
+    # Iterate through each sample in pred_contents
+    for sample in pred_contents:
+        video_id = sample['video_name']
+        if video_id in video_id_counts:
+            video_id_counts[video_id] += 1
+        else:
+            video_id_counts[video_id] = 0
+
+        # Create a new sample with the modified key
+        new_sample = sample
+        new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
+        new_pred_contents.append(new_sample)
+
+    # Generating list of id's and corresponding files
+    id_list = [x['video_name'] for x in new_pred_contents]
+    caption_files = [f"{id}.json" for id in id_list]
+
+    output_dir = args.output_dir
+    # Generate output directory if not exists.
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # Preparing dictionary of question-answer sets
+    prediction_set = {}
+    for sample in new_pred_contents:
+        id = sample['video_name']
+        question = sample['Q']
+        answer = sample['A']
+        pred = sample['pred']
+        qa_set = {"q": question, "a": answer, "pred": pred}
+        prediction_set[id] = qa_set
+
+    # Set the OpenAI API key.
+    openai.api_key = args.api_key
+    num_tasks = args.num_tasks
+
+    # While loop to ensure that all captions are processed.
+    while True:
+        try:
+            # Files that have not been processed yet.
+            completed_files = os.listdir(output_dir)
+            print(f"completed_files: {len(completed_files)}")
+
+            # Files that have not been processed yet.
+            incomplete_files = [f for f in caption_files if f not in completed_files]
+            print(f"incomplete_files: {len(incomplete_files)}")
+
+            # Break the loop when there are no incomplete files
+            if len(incomplete_files) == 0:
+                break
+            if len(incomplete_files) <= num_tasks:
+                num_tasks = 1
+
+            # Split tasks into parts.
+            part_len = len(incomplete_files) // num_tasks
+            all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
+            task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
+
+            # Use a pool of workers to process the files in parallel.
+            with Pool() as pool:
+                pool.starmap(annotate, task_args)
+
+        except Exception as e: 
+            print(f"Error: {e}")
+
+    # Combine all the processed files into one
+    combined_contents = {}
+    json_path = args.output_json
+
+    # Iterate through json files
+    for file_name in os.listdir(output_dir):
+        if file_name.endswith(".json"):
+            file_path = os.path.join(output_dir, file_name)
+            with open(file_path, "r") as json_file:
+                content = json.load(json_file)
+                combined_contents[file_name[:-5]] = content
+
+    # Write combined content to a json file
+    with open(json_path, "w") as json_file:
+        json.dump(combined_contents, json_file)
+    print("All evaluation completed!")
+
+    # Calculate average score
+    score_sum = 0
+    count = 0
+    for key, result in combined_contents.items():
+        count += 1
+        score_match = result[0]['score']
+        score = int(score_match)
+        score_sum += score
+    average_score = score_sum / count
+
+    print("Average score temporal understanding:", average_score)
+
+
+if __name__ == "__main__":
+    main()
+

GPT_evaluation/evaluate_benchmark_5_consistency.py

@@ -0,0 +1,193 @@
+import openai
+import os
+import argparse
+import json
+import ast
+from multiprocessing.pool import Pool
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
+    parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
+    parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
+    parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
+    parser.add_argument("--api_key", required=True, help="OpenAI API key.")
+    parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
+    args = parser.parse_args()
+    return args
+
+
+def annotate(prediction_set, caption_files, output_dir):
+    """
+    Evaluates question and answer pairs using GPT-3 and
+    returns a score for consistency.
+    """
+    for file in caption_files:
+        key = file[:-5] # Strip file extension
+        qa_set = prediction_set[key]
+        question1 = qa_set['q1']
+        question2 = qa_set['q2']
+        answer = qa_set['a']
+        pred1 = qa_set['pred1']
+        pred2 = qa_set['pred2']
+        try:
+            # Compute the consistency score
+            completion = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "system",
+                        "content":
+                            "You are an intelligent chatbot designed for evaluating the consistency of generative outputs for similar video-based question-answer pairs. "
+                            "You will be given two very similar questions, a common answer common to both the questions and predicted answers for the two questions ."
+                            "Your task is to compare the predicted answers for two very similar question, with a common correct answer and determine if they are consistent. Here's how you can accomplish the task:"
+                            "------"
+                            "##INSTRUCTIONS: "
+                            "- Focus on the consistency between the two predicted answers and the correct answer. Both predicted answers should correspond to the correct answer and to each other, and should not contain any contradictions or significant differences in the conveyed information.\n"
+                            "- Both predicted answers must be consistent with each other and the correct answer, in terms of the information they provide about the video content.\n"
+                            "- Consider synonyms or paraphrases as valid matches, but only if they maintain the consistency in the conveyed information.\n"
+                            "- Evaluate the consistency of the two predicted answers compared to the correct answer."
+                    },
+                    {
+                        "role": "user",
+                        "content":
+                            "Please evaluate the following video-based question-answer pair:\n\n"
+                            f"Question 1: {question1}\n"
+                            f"Question 2: {question2}\n"
+                            f"Correct Answer: {answer}\n"
+                            f"Predicted Answer to Question 1: {pred1}\n"
+                            f"Predicted Answer to Question 2: {pred2}\n\n"
+                            "Provide your evaluation only as a consistency score where the consistency score is an integer value between 0 and 5, with 5 indicating the highest level of consistency. "
+                            "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is the consistency score in INTEGER, not STRING."
+                            "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
+                            "For example, your response should look like this: {''score': 4.8}."
+                    }
+                ]
+            )
+            # Convert response to a Python dictionary.
+            response_message = completion["choices"][0]["message"]["content"]
+            response_dict = ast.literal_eval(response_message)
+            result_qa_pair = [response_dict, qa_set]
+
+            # Save the question-answer pairs to a json file.
+            with open(f"{output_dir}/{key}.json", "w") as f:
+                json.dump(result_qa_pair, f)
+
+        except Exception as e:
+            print(f"Error processing file '{key}': {e}")
+
+
+def main():
+    """
+    Main function to control the flow of the program.
+    """
+    # Parse arguments.
+    args = parse_args()
+
+    file = open(args.pred_path)
+    pred_contents = json.load(file)
+
+    # Dictionary to store the count of occurrences for each video_id
+    video_id_counts = {}
+    new_pred_contents = []
+
+    # Iterate through each sample in pred_contents
+    for sample in pred_contents:
+        video_id = sample['video_name']
+        if video_id in video_id_counts:
+            video_id_counts[video_id] += 1
+        else:
+            video_id_counts[video_id] = 0
+
+        # Create a new sample with the modified key
+        new_sample = sample
+        new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
+        new_pred_contents.append(new_sample)
+
+    # Generating list of id's and corresponding files
+    id_list = [x['video_name'] for x in new_pred_contents]
+    caption_files = [f"{id}.json" for id in id_list]
+
+    output_dir = args.output_dir
+    # Generate output directory if not exists.
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # Preparing dictionary of question-answer sets
+    prediction_set = {}
+    for sample in new_pred_contents:
+        id = sample['video_name']
+        question1 = sample['Q1']
+        question2 = sample['Q1']
+        answer = sample['A']
+        pred1 = sample['pred1']
+        pred2 = sample['pred2']
+        qa_set = {"q1": question1, "q2": question2, "a": answer, "pred1": pred1, "pred2": pred2}
+        prediction_set[id] = qa_set
+
+    # Set the OpenAI API key.
+    openai.api_key = args.api_key
+    num_tasks = args.num_tasks
+
+    # While loop to ensure that all captions are processed.
+    while True:
+        try:
+            # Files that have not been processed yet.
+            completed_files = os.listdir(output_dir)
+            print(f"completed_files: {len(completed_files)}")
+
+            # Files that have not been processed yet.
+            incomplete_files = [f for f in caption_files if f not in completed_files]
+            print(f"incomplete_files: {len(incomplete_files)}")
+
+            # Break the loop when there are no incomplete files
+            if len(incomplete_files) == 0:
+                break
+            if len(incomplete_files) <= num_tasks:
+                num_tasks = 1
+
+            # Split tasks into parts.
+            part_len = len(incomplete_files) // num_tasks
+            all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
+            task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
+
+            # Use a pool of workers to process the files in parallel.
+            with Pool() as pool:
+                pool.starmap(annotate, task_args)
+
+        except Exception as e: 
+            print(f"Error: {e}")
+
+    # Combine all the processed files into one
+    combined_contents = {}
+    json_path = args.output_json
+
+    # Iterate through json files
+    for file_name in os.listdir(output_dir):
+        if file_name.endswith(".json"):
+            file_path = os.path.join(output_dir, file_name)
+            with open(file_path, "r") as json_file:
+                content = json.load(json_file)
+                combined_contents[file_name[:-5]] = content
+
+    # Write combined content to a json file
+    with open(json_path, "w") as json_file:
+        json.dump(combined_contents, json_file)
+    print("All evaluation completed!")
+
+    # Calculate average score
+    score_sum = 0
+    count = 0
+    for key, result in combined_contents.items():
+        count += 1
+        score_match = result[0]['score']
+        score = int(score_match)
+        score_sum += score
+    average_score = score_sum / count
+
+    print("Average score for consistency:", average_score)
+
+
+if __name__ == "__main__":
+    main()
+

GPT_evaluation/evaluate_zeroshot.py

@@ -0,0 +1,207 @@
+import openai
+import os
+import argparse
+import json
+import ast
+from multiprocessing.pool import Pool
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
+    parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
+    parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
+    parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
+    parser.add_argument("--api_key", required=True, help="OpenAI API key.")
+    parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
+    args = parser.parse_args()
+    return args
+
+
+def annotate(prediction_set, caption_files, output_dir):
+    """
+    Evaluates question and answer pairs using GPT-3
+    Returns a score for correctness.
+    """
+    for file in caption_files:
+        key = file[:-5] # Strip file extension
+        qa_set = prediction_set[key]
+        question = qa_set['q']
+        answer = qa_set['a']
+        pred = qa_set['pred']
+        try:
+            # Compute the correctness score
+            completion = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "system",
+                        "content": 
+                            "You are an intelligent chatbot designed for evaluating the correctness of generative outputs for question-answer pairs. "
+                            "Your task is to compare the predicted answer with the correct answer and determine if they match meaningfully. Here's how you can accomplish the task:"
+                            "------"
+                            "##INSTRUCTIONS: "
+                            "- Focus on the meaningful match between the predicted answer and the correct answer.\n"
+                            "- Consider synonyms or paraphrases as valid matches.\n"
+                            "- Evaluate the correctness of the prediction compared to the answer."
+                    },
+                    {
+                        "role": "user",
+                        "content":
+                            "Please evaluate the following video-based question-answer pair:\n\n"
+                            f"Question: {question}\n"
+                            f"Correct Answer: {answer}\n"
+                            f"Predicted Answer: {pred}\n\n"
+                            "Provide your evaluation only as a yes/no and score where the score is an integer value between 0 and 5, with 5 indicating the highest meaningful match. "
+                            "Please generate the response in the form of a Python dictionary string with keys 'pred' and 'score', where value of 'pred' is  a string of 'yes' or 'no' and value of 'score' is in INTEGER, not STRING."
+                            "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
+                            "For example, your response should look like this: {'pred': 'yes', 'score': 4.8}."
+                    }
+                ]
+            )
+            # Convert response to a Python dictionary.
+            response_message = completion["choices"][0]["message"]["content"]
+            response_dict = ast.literal_eval(response_message)
+            result_qa_pair = [response_dict, qa_set]
+
+            # Save the question-answer pairs to a json file.
+            with open(f"{output_dir}/{key}.json", "w") as f:
+                json.dump(result_qa_pair, f)
+
+        except Exception as e:
+            print(f"Error processing file '{key}': {e}")
+
+
+def main():
+    """
+    Main function to control the flow of the program.
+    """
+    # Parse arguments.
+    args = parse_args()
+
+    file = open(args.pred_path)
+    pred_contents = json.load(file)
+
+    # Dictionary to store the count of occurrences for each video_id
+    video_id_counts = {}
+    new_pred_contents = []
+
+    # Iterate through each sample in pred_contents
+    for sample in pred_contents:
+        video_id = sample['video_name']
+        if video_id in video_id_counts:
+            video_id_counts[video_id] += 1
+        else:
+            video_id_counts[video_id] = 0
+
+        # Create a new sample with the modified key
+        new_sample = sample
+        new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
+        new_pred_contents.append(new_sample)
+
+    # Generating list of id's and corresponding files
+    id_list = [x['video_name'] for x in new_pred_contents]
+    caption_files = [f"{id}.json" for id in id_list]
+
+    output_dir = args.output_dir
+    # Generate output directory if not exists.
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    # Preparing dictionary of question-answer sets
+    prediction_set = {}
+    for sample in new_pred_contents:
+        id = sample['video_name']
+        question = sample['Q']
+        answer = sample['A']
+        pred = sample['pred']
+        qa_set = {"q": question, "a": answer, "pred": pred}
+        prediction_set[id] = qa_set
+
+    # Set the OpenAI API key.
+    openai.api_key = args.api_key
+    num_tasks = args.num_tasks
+
+    # While loop to ensure that all captions are processed.
+    while True:
+        try:
+            # Files that have not been processed yet.
+            completed_files = os.listdir(output_dir)
+            print(f"completed_files: {len(completed_files)}")
+
+            # Files that have not been processed yet.
+            incomplete_files = [f for f in caption_files if f not in completed_files]
+            print(f"incomplete_files: {len(incomplete_files)}")
+
+            # Break the loop when there are no incomplete files
+            if len(incomplete_files) == 0:
+                break
+            if len(incomplete_files) <= num_tasks:
+                num_tasks = 1
+
+            # Split tasks into parts.
+            part_len = len(incomplete_files) // num_tasks
+            all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
+            task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
+
+            # Use a pool of workers to process the files in parallel.
+            with Pool() as pool:
+                pool.starmap(annotate, task_args)
+
+        except Exception as e:
+            print(f"Error: {e}")
+
+    # Combine all the processed files into one
+    combined_contents = {}
+    json_path = args.output_json
+
+    # Iterate through json files
+    for file_name in os.listdir(output_dir):
+        if file_name.endswith(".json"):
+            file_path = os.path.join(output_dir, file_name)
+            with open(file_path, "r") as json_file:
+                content = json.load(json_file)
+                combined_contents[file_name[:-5]] = content
+
+    # Write combined content to a json file
+    with open(json_path, "w") as json_file:
+        json.dump(combined_contents, json_file)
+    print("All evaluation completed!")
+
+    # Calculate average score and accuracy
+    score_sum = 0
+    count = 0
+    yes_count = 0
+    no_count = 0
+    for key, result in combined_contents.items():
+        # Computing score
+        count += 1
+        try :
+            score_match = result[0]['score']
+            score = int(score_match)
+            score_sum += score
+        except:
+            print("Score not found for", key)
+            continue
+
+        # Computing accuracy
+        try:
+            pred = result[0]['pred']
+            if "yes" in pred.lower():
+                yes_count += 1
+            elif "no" in pred.lower():
+                no_count += 1
+        except:
+            print("Prediction not found for", key)
+            continue
+
+    average_score = score_sum / count
+    accuracy = yes_count / (yes_count + no_count)
+    print("Yes count:", yes_count)
+    print("No count:", no_count)
+    print("Accuracy:", accuracy)
+    print("Average score:", average_score)
+
+
+if __name__ == "__main__":
+    main()
+

GPT_evaluation/evaluate_zeroshot.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=zeroshot_eval%j
+#SBATCH --output=zeroshot_eval%j.out
+#SBATCH --error=zeroshot_eval%j.err
+#SBATCH --time=0-10:00:00
+#SBATCH --mem=64G
+#SBATCH --nodes=1
+
+## run the application:
+
+# PRED="pred_path"
+# OUTPUT_DIR="output_dir"
+# API_KEY="api_key"
+# NUM_TASKS=128
+
+
+python evaluate_zeroshot.py \
+    --pred_path ${PRED} \
+    --output_dir "${OUTPUT_DIR}/fewshot_accuracy" \
+    --output_json "${OUTPUT_DIR}/fewshot_accuracy_results.json"\
+    --api_key $API_KEY \
+    --num_tasks $NUM_TASKS
+
+echo pred_path: $PRED

HUGGINGFACE_DEPLOY.md

@@ -0,0 +1,103 @@
+# HuggingFace Spaces 部署指南
+
+## 🚀 部署步骤
+
+### 1. 准备文件
+确保您的项目包含以下文件：
+- `app.py` - 主应用代码
+- `run_hf.py` - HuggingFace启动脚本
+- `requirements.txt` - Python依赖
+- `packages.txt` - 系统依赖
+- `README.md` - Spaces配置
+- `prohibited_rules.py` - 巨量引擎规则
+- `minigpt4_video_demo.py` - MiniGPT4-Video核心模块
+- `test_configs/llama2_test_config.yaml` - 模型配置
+
+### 2. 创建HuggingFace Space
+1. 访问 [HuggingFace Spaces](https://huggingface.co/spaces)
+2. 点击 "Create new Space"
+3. 设置以下参数：
+   - **Space name**: `minigpt4-video-safety`
+   - **License**: Apache 2.0
+   - **SDK**: Gradio
+   - **Hardware**: GPU (推荐T4或更高)
+
+### 3. 上传文件
+```bash
+git clone https://huggingface.co/spaces/YOUR_USERNAME/minigpt4-video-safety
+cd minigpt4-video-safety
+cp /path/to/your/files/* ./
+git add .
+git commit -m "Initial deployment"
+git push
+```
+
+### 4. 配置模型权重
+由于MiniGPT4-Video需要预训练权重，您需要：
+
+1. 上传模型权重到HuggingFace Hub
+2. 修改`app.py`中的模型路径
+3. 或者使用HuggingFace的模型仓库
+
+### 5. 环境变量设置
+在Space设置中添加环境变量：
+- `PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:512`
+- `GRADIO_SERVER_PORT=7860`
+
+## 🔧 配置选项
+
+### Hardware要求
+- **最低配置**: CPU Basic (仅安全检测)
+- **推荐配置**: GPU T4 (完整功能)
+- **高性能**: GPU A10G (大规模使用)
+
+### 内存要求
+- CPU模式: 4GB RAM
+- GPU模式: 16GB GPU内存
+
+## 🛠️ 故障排除
+
+### 常见问题
+
+1. **模型加载失败**
+   - 检查模型权重路径
+   - 确认GPU内存充足
+   - 验证依赖版本兼容性
+
+2. **依赖安装失败**
+   - 检查`requirements.txt`格式
+   - 验证PyTorch版本兼容性
+   - 确认CUDA版本匹配
+
+3. **内存不足**
+   - 减少batch_size
+   - 使用量化模型
+   - 升级硬件配置
+
+### 调试模式
+在开发阶段，可以设置环境变量：
+```bash
+export DEBUG=1
+export GRADIO_DEBUG=1
+```
+
+## 📝 注意事项
+
+1. **模型权重**: 需要单独下载MiniGPT4-Video权重
+2. **GPU内存**: 确保有足够的GPU内存加载模型
+3. **网络访问**: YouTube下载功能需要网络访问
+4. **文件存储**: 临时文件会占用存储空间
+
+## 🔗 相关链接
+
+- [MiniGPT4-Video官方仓库](https://github.com/Vision-CAIR/MiniGPT4-video)
+- [HuggingFace Spaces文档](https://huggingface.co/docs/hub/spaces)
+- [Gradio文档](https://gradio.app/docs/)
+
+## 📞 技术支持
+
+如遇到部署问题，请：
+1. 检查控制台日志
+2. 验证配置文件
+3. 确认依赖版本
+4. 联系技术支持 

LICENSE.md

@@ -0,0 +1,14 @@
+BSD 3-Clause License
+
+Copyright 2023 Deyao Zhu
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

LICENSE_Lavis.md

@@ -0,0 +1,14 @@
+BSD 3-Clause License
+
+Copyright (c) 2022 Salesforce, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -1,12 +1,47 @@
 ---
-title: DeepOperateAI Video
-emoji: 👀
-colorFrom: pink
-colorTo: purple
+title: Video Content Safety Analysis
+emoji: 🎥
+colorFrom: blue
+colorTo: red
 sdk: gradio
-sdk_version: 5.33.0
+sdk_version: "4.44.0"
 app_file: app.py
 pinned: false
+hardware: zero-gpu
+python_version: "3.10"
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🎥 Video Content Safety Analysis
+
+基于MiniGPT4-Video的智能视频内容安全分析系统
+
+## 功能特性
+
+- 🎬 **智能视频理解**: 基于MiniGPT4-Video多模态大模型
+- 🛡️ **内容安全检测**: 集成299条违规内容规则
+- 🚀 **实时分析**: 支持视频文件上传和实时处理
+- 🌍 **中英双语**: 支持中英文内容分析
+
+## 技术架构
+
+- **视觉编码器**: EVA-CLIP-G
+- **语言模型**: Qwen2.5-7B-Instruct (优化版)
+- **多模态融合**: MiniGPT4-Video架构
+- **部署平台**: HuggingFace Spaces (ZeroGPU)
+
+## 使用说明
+
+1. 上传视频文件 (支持MP4, AVI, MOV等格式)
+2. 选择分析模式 (安全检测 / 内容理解)
+3. 点击"开始分析"按钮
+4. 查看分析结果和安全评估
+
+## 注意事项
+
+- ZeroGPU有60秒运行时间限制
+- 建议上传文件小于50MB
+- 首次加载模型需要1-2分钟
+
+## 技术支持
+
+如遇问题请提交Issue或联系开发团队。

app.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""
+🎥 Video Content Safety Analysis
+适配ZeroGPU的视频内容安全分析应用
+"""
+import os
+import tempfile
+import gradio as gr
+import torch
+import numpy as np
+from typing import Optional, Tuple
+import logging
+
+# 设置中国镜像（如果在中国网络环境）
+os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
+
+# ZeroGPU装饰器
+try:
+    import spaces
+    GPU_AVAILABLE = True
+    print("✅ ZeroGPU spaces 可用")
+except ImportError:
+    print("⚠️ ZeroGPU spaces 不可用，使用CPU模式")
+    GPU_AVAILABLE = False
+    # 创建空装饰器
+    class spaces:
+        @staticmethod
+        def GPU(func):
+            return func
+
+# 全局变量
+model = None
+processor = None
+
+def load_model():
+    """加载模型（延迟加载）"""
+    global model, processor
+    
+    if model is not None:
+        return model, processor
+    
+    try:
+        print("🔄 正在加载模型...")
+        
+        # 这里需要根据实际情况导入和加载您的模型
+        # 暂时返回模拟的模型
+        print("✅ 模型加载成功（模拟）")
+        
+        # 实际应该是：
+        # from minigpt4_video_demo import init_model
+        # model, processor, _, _, _ = init_model(args)
+        
+        model = "simulation_model"
+        processor = "simulation_processor"
+        
+        return model, processor
+        
+    except Exception as e:
+        print(f"❌ 模型加载失败: {e}")
+        return None, None
+
+@spaces.GPU if GPU_AVAILABLE else lambda f: f
+def analyze_video_content(video_path: str, instruction: str = "请分析这个视频的内容") -> Tuple[str, str]:
+    """
+    分析视频内容
+    
+    Args:
+        video_path: 视频文件路径
+        instruction: 分析指令
+        
+    Returns:
+        Tuple[str, str]: (分析结果, 安全评级)
+    """
+    try:
+        # 加载模型
+        model, processor = load_model()
+        if model is None:
+            return "❌ 模型加载失败", "无法评估"
+        
+        print(f"🔄 正在分析视频: {video_path}")
+        print(f"📝 分析指令: {instruction}")
+        
+        # 模拟分析过程
+        # 在实际应用中，这里会调用您的视频分析模型
+        
+        # 模拟分析结果
+        analysis_result = f"""
+🎬 **视频内容分析结果**
+
+📋 **基本信息**:
+- 视频路径: {video_path}
+- 分析指令: {instruction}
+
+🔍 **内容分析**:
+- 检测到的对象: 人物、场景、文字等
+- 音频内容: 语音转文字结果
+- 情感分析: 积极/中性/消极
+
+🛡️ **安全检测**:
+- 暴力内容: 未检测到
+- 不当内容: 未检测到
+- 版权问题: 未检测到
+
+✅ **总体评估**: 内容安全，符合平台规范
+        """
+        
+        safety_rating = "✅ P3 (安全)"
+        
+        return analysis_result, safety_rating
+        
+    except Exception as e:
+        error_msg = f"❌ 分析过程中出错: {str(e)}"
+        return error_msg, "⚠️ 错误"
+
+def create_interface():
+    """创建Gradio界面"""
+    
+    with gr.Blocks(
+        title="🎥 Video Content Safety Analysis",
+        theme=gr.themes.Soft(),
+        css="""
+        .container { max-width: 800px; margin: auto; }
+        .header { text-align: center; padding: 20px; }
+        .footer { text-align: center; padding: 10px; color: #666; }
+        """
+    ) as app:
+        
+        # 标题
+        gr.Markdown("""
+        # 🎥 智能视频内容安全分析
+        
+        基于MiniGPT4-Video的多模态视频理解与安全检测系统
+        
+        ⚡ **ZeroGPU加速** | 🛡️ **智能安全检测** | 🌍 **中英双语支持**
+        """, elem_classes=["header"])
+        
+        with gr.Row():
+            with gr.Column(scale=1):
+                # 输入区域
+                gr.Markdown("## 📤 上传视频")
+                
+                video_input = gr.Video(
+                    label="选择视频文件",
+                    info="支持MP4, AVI, MOV等格式，建议小于50MB"
+                )
+                
+                instruction_input = gr.Textbox(
+                    label="分析指令",
+                    placeholder="请输入分析指令，如：请分析这个视频的内容安全性",
+                    value="请分析这个视频的内容，重点关注是否存在违规内容",
+                    lines=2
+                )
+                
+                analyze_btn = gr.Button(
+                    "🚀 开始分析",
+                    variant="primary",
+                    size="lg"
+                )
+                
+            with gr.Column(scale=1):
+                # 输出区域
+                gr.Markdown("## 📊 分析结果")
+                
+                analysis_output = gr.Textbox(
+                    label="详细分析",
+                    lines=15,
+                    max_lines=20,
+                    show_copy_button=True
+                )
+                
+                safety_output = gr.Textbox(
+                    label="安全评级",
+                    lines=1
+                )
+        
+        # 示例和说明
+        gr.Markdown("""
+        ## 💡 使用说明
+        
+        1. **上传视频**: 选择要分析的视频文件
+        2. **输入指令**: 描述您希望如何分析视频内容
+        3. **开始分析**: 点击按钮开始智能分析
+        4. **查看结果**: 获得详细的内容分析和安全评级
+        
+        ## ⚠️ 注意事项
+        
+        - 🕐 ZeroGPU有60秒运行时间限制
+        - 📁 建议上传文件小于50MB
+        - ⏱️ 首次加载模型需要1-2分钟
+        - 🔄 分析时间取决于视频长度和复杂度
+        
+        ## 🏷️ 安全等级说明
+        
+        - **🚨 P0 (高危)**: 严重违规，需立即处理
+        - **⚠️ P1 (中危)**: 中等风险，需要审核  
+        - **⚡ P2 (低危)**: 轻微风险，建议关注
+        - **✅ P3 (安全)**: 内容安全，符合规范
+        """, elem_classes=["footer"])
+        
+        # 绑定事件
+        analyze_btn.click(
+            fn=analyze_video_content,
+            inputs=[video_input, instruction_input],
+            outputs=[analysis_output, safety_output],
+            show_progress=True
+        )
+    
+    return app
+
+def main():
+    """主函数"""
+    print("🚀 启动视频内容安全分析应用")
+    
+    # 检查GPU可用性
+    if torch.cuda.is_available():
+        print(f"✅ GPU可用: {torch.cuda.get_device_name(0)}")
+    else:
+        print("⚠️ 使用CPU模式")
+    
+    # 创建应用
+    app = create_interface()
+    
+    # 启动应用
+    if __name__ == "__main__":
+        app.launch(
+            server_name="0.0.0.0",
+            server_port=7860,
+            share=False,
+            show_error=True,
+            quiet=False
+        )
+
+if __name__ == "__main__":
+    main() 

check_install.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+"""
+检查MiniGPT4-Video依赖安装状态
+"""
+
+import sys
+import importlib
+
+# 必需的包列表
+REQUIRED_PACKAGES = [
+    'torch',
+    'torchvision', 
+    'transformers',
+    'gradio',
+    'opencv-cv2',  # opencv-python-headless
+    'moviepy',
+    'webvtt',
+    'pytubefix', 
+    'omegaconf',
+    'timm',
+    'webdataset',
+    'sentence_transformers',
+    'sklearn',  # scikit-learn
+    'skimage',  # scikit-image
+    'decord',
+    'peft',
+    'bitsandbytes',
+    'whisper',  # openai-whisper
+    'numpy',
+    'soundfile',
+    'accelerate',
+    'PIL',  # Pillow
+    'requests'
+]
+
+def check_package(package_name):
+    """检查单个包是否安装"""
+    try:
+        importlib.import_module(package_name)
+        return True, "✅"
+    except ImportError as e:
+        return False, f"❌ {str(e)}"
+
+def main():
+    print("🔍 检查MiniGPT4-Video依赖安装状态...\n")
+    
+    missing_packages = []
+    
+    for package in REQUIRED_PACKAGES:
+        success, status = check_package(package)
+        print(f"{status} {package}")
+        
+        if not success:
+            missing_packages.append(package)
+    
+    print(f"\n📊 检查结果:")
+    print(f"✅ 已安装: {len(REQUIRED_PACKAGES) - len(missing_packages)}/{len(REQUIRED_PACKAGES)}")
+    print(f"❌ 缺失: {len(missing_packages)}")
+    
+    if missing_packages:
+        print(f"\n🔧 缺失的包:")
+        for pkg in missing_packages:
+            print(f"  - {pkg}")
+        print(f"\n💡 修复建议:")
+        print(f"pip install -r requirements.txt")
+        return False
+    else:
+        print(f"\n🎉 所有依赖都已正确安装！")
+        return True
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1) 

environment.yml

@@ -0,0 +1,331 @@
+name: goldfish
+channels:
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - archspec=0.2.2=pyhd8ed1ab_0
+  - boltons=23.1.1=pyhd8ed1ab_0
+  - brotli-python=1.1.0=py39h3d6467e_1
+  - bzip2=1.0.8=hd590300_5
+  - c-ares=1.25.0=hd590300_0
+  - ca-certificates=2024.2.2=hbcca054_0
+  - certifi=2024.2.2=pyhd8ed1ab_0
+  - cffi=1.16.0=py39h7a31438_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - conda=23.11.0=py39hf3d152e_1
+  - conda-libmamba-solver=23.12.0=pyhd8ed1ab_0
+  - conda-package-handling=2.2.0=pyh38be061_0
+  - conda-package-streaming=0.9.0=pyhd8ed1ab_0
+  - cudatoolkit=11.8.0=h4ba93d1_12
+  - cudatoolkit-dev=11.7.0=h1de0b5d_6
+  - distro=1.9.0=pyhd8ed1ab_0
+  - faiss=1.7.4=py39cuda112h460e57a_0_cuda
+  - fmt=10.1.1=h00ab1b0_1
+  - freetype=2.12.1=h267a509_2
+  - gmp=6.1.2=hf484d3e_1000
+  - gnutls=3.5.19=h2a4e5f8_1
+  - icu=73.2=h59595ed_0
+  - idna=3.6=pyhd8ed1ab_0
+  - jsonpatch=1.33=pyhd8ed1ab_0
+  - jsonpointer=2.4=py39hf3d152e_3
+  - keyutils=1.6.1=h166bdaf_0
+  - krb5=1.21.2=h659d440_0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - libarchive=3.7.2=h2aa1ff5_1
+  - libblas=3.9.0=20_linux64_openblas
+  - libcblas=3.9.0=20_linux64_openblas
+  - libcurl=8.5.0=hca28451_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=hd590300_2
+  - libfaiss=1.7.4=cuda112hb18a002_0_cuda
+  - libfaiss-avx2=1.7.4=cuda112h1234567_0_cuda
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=13.2.0=h807b86a_3
+  - libgfortran-ng=13.2.0=h69a702a_3
+  - libgfortran5=13.2.0=ha4646dd_3
+  - libgomp=13.2.0=h807b86a_3
+  - libiconv=1.17=hd590300_2
+  - liblapack=3.9.0=20_linux64_openblas
+  - libmamba=1.5.6=had39da4_0
+  - libmambapy=1.5.6=py39h10defb6_0
+  - libnghttp2=1.58.0=h47da74e_1
+  - libnsl=2.0.1=hd590300_0
+  - libopenblas=0.3.25=pthreads_h413a1c8_0
+  - libpng=1.6.39=h753d276_0
+  - libsolv=0.7.27=hfc55251_0
+  - libsqlite=3.44.2=h2797004_0
+  - libssh2=1.11.0=h0841786_0
+  - libstdcxx-ng=13.2.0=h7e041cc_3
+  - libuuid=2.38.1=h0b41bf4_0
+  - libxcrypt=4.4.36=hd590300_1
+  - libxml2=2.12.3=h232c23b_0
+  - libzlib=1.2.13=hd590300_5
+  - lz4-c=1.9.4=hcb278e6_0
+  - lzo=2.10=h516909a_1000
+  - menuinst=2.0.1=py39hf3d152e_0
+  - ncurses=6.4=h59595ed_2
+  - nettle=3.3=0
+  - numpy=1.26.3=py39h474f0d3_0
+  - openh264=1.8.0=hdbcaa40_1000
+  - openssl=3.2.1=hd590300_0
+  - packaging=23.2=pyhd8ed1ab_0
+  - pip=23.3.2=pyhd8ed1ab_0
+  - platformdirs=4.1.0=pyhd8ed1ab_0
+  - pluggy=1.3.0=pyhd8ed1ab_0
+  - pybind11-abi=4=hd8ed1ab_3
+  - pycosat=0.6.6=py39hd1e30aa_0
+  - pycparser=2.21=pyhd8ed1ab_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - python=3.9.18=h0755675_1_cpython
+  - python_abi=3.9=4_cp39
+  - readline=8.2=h8228510_1
+  - reproc=14.2.4.post0=hd590300_1
+  - reproc-cpp=14.2.4.post0=h59595ed_1
+  - requests=2.31.0=pyhd8ed1ab_0
+  - ruamel.yaml=0.18.5=py39hd1e30aa_0
+  - ruamel.yaml.clib=0.2.7=py39hd1e30aa_2
+  - tk=8.6.13=noxft_h4845f30_101
+  - tqdm=4.66.1=pyhd8ed1ab_0
+  - urllib3=2.1.0=pyhd8ed1ab_0
+  - wheel=0.42.0=pyhd8ed1ab_0
+  - x264=1!152.20180717=h14c3975_1001
+  - xz=5.2.6=h166bdaf_0
+  - yaml-cpp=0.8.0=h59595ed_0
+  - zlib=1.2.13=hd590300_5
+  - zstandard=0.22.0=py39h6e5214e_0
+  - zstd=1.5.5=hfc55251_0
+  - pip:
+      - accelerate==0.25.0
+      - aiofiles==23.2.1
+      - aiohttp==3.9.1
+      - aiosignal==1.3.1
+      - altair==5.2.0
+      - annotated-types==0.6.0
+      - antlr4-python3-runtime==4.9.3
+      - anyio==4.2.0
+      - appdirs==1.4.4
+      - asgiref==3.7.2
+      - async-timeout==4.0.3
+      - attrs==23.2.0
+      - backoff==2.2.1
+      - bcrypt==4.1.2
+      - beautifulsoup4==4.12.2
+      - bitarray==2.9.2
+      - bitsandbytes==0.42.0
+      - bleach==6.1.0
+      - blinker==1.7.0
+      - braceexpand==0.1.7
+      - build==1.0.3
+      - cachetools==5.3.2
+      - chardet==5.2.0
+      - chroma-hnswlib==0.7.3
+      - chromadb==0.4.22
+      - click==8.1.7
+      - cmake==3.25.0
+      - colbert-ai==0.2.18
+      - coloredlogs==15.0.1
+      - contourpy==1.2.0
+      - cycler==0.12.1
+      - datasets==2.17.0
+      - decorator==4.4.2
+      - decord==0.6.0
+      - deprecated==1.2.14
+      - dill==0.3.8
+      - docker-pycreds==0.4.0
+      - docopt==0.6.2
+      - einops==0.7.0
+      - exceptiongroup==1.2.0
+      - faiss-gpu==1.7.2
+      - fastapi==0.108.0
+      - ffmpeg==1.4
+      - ffmpeg-python==0.2.0
+      - ffmpy==0.3.1
+      - filelock==3.13.1
+      - flask==3.0.2
+      - flatbuffers==23.5.26
+      - fonttools==4.47.0
+      - frozenlist==1.4.1
+      - fsspec==2023.10.0
+      - ftfy==6.1.3
+      - future==0.18.3
+      - gdown==4.7.1
+      - git-python==1.0.3
+      - gitdb==4.0.11
+      - gitpython==3.1.40
+      - google-auth==2.26.1
+      - googleapis-common-protos==1.62.0
+      - gradio
+      - gradio-client
+      - h11==0.14.0
+      - h5py==3.10.0
+      - httpcore==1.0.2
+      - httptools==0.6.1
+      - httpx==0.26.0
+      - huggingface-hub
+      - humanfriendly==10.0
+      - imageio==2.33.1
+      - imageio-ffmpeg==0.4.9
+      - importlib-metadata==6.11.0
+      - importlib-resources==6.1.1
+      - inquirerpy==0.3.4
+      - iopath==0.1.10
+      - itsdangerous==2.1.2
+      - jinja2==3.1.2
+      - joblib==1.3.2
+      - jsonschema==4.20.0
+      - jsonschema-specifications==2023.12.1
+      - kaggle==1.6.0
+      - kiwisolver==1.4.5
+      - kubernetes==29.0.0
+      - lazy-loader==0.3
+      - lit==15.0.7
+      - llvmlite==0.41.1
+      - markdown-it-py==3.0.0
+      - matplotlib==3.8.2
+      - mdurl==0.1.2
+      - mmh3==4.1.0
+      - monotonic==1.6
+      - more-itertools==10.1.0
+      - moviepy==1.0.3
+      - mpmath==1.3.0
+      - multidict==6.0.4
+      - multiprocess==0.70.16
+      - mutagen==1.47.0
+      - networkx==3.2.1
+      - ninja==1.11.1.1
+      - nltk==3.8.1
+      - numba==0.58.1
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cudnn-cu12==8.9.2.26
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-nccl-cu12==2.18.1
+      - nvidia-nvjitlink-cu12==12.3.101
+      - nvidia-nvtx-cu12==12.1.105
+      - omegaconf==2.3.0
+      - onnxruntime==1.16.3
+      - openai
+      - openai-whisper==20231117
+      - opencv-python==4.7.0.72
+      - opentelemetry-api==1.22.0
+      - opentelemetry-exporter-otlp-proto-common==1.22.0
+      - opentelemetry-exporter-otlp-proto-grpc==1.22.0
+      - opentelemetry-instrumentation==0.43b0
+      - opentelemetry-instrumentation-asgi==0.43b0
+      - opentelemetry-instrumentation-fastapi==0.43b0
+      - opentelemetry-proto==1.22.0
+      - opentelemetry-sdk==1.22.0
+      - opentelemetry-semantic-conventions==0.43b0
+      - opentelemetry-util-http==0.43b0
+      - orjson==3.9.10
+      - overrides==7.4.0
+      - pandas==2.0.0
+      - pathtools==0.1.2
+      - peft==0.2.0
+      - pfzy==0.3.4
+      - pillow==10.2.0
+      - plotly==5.18.0
+      - portalocker==2.8.2
+      - posthog==3.3.0
+      - proglog==0.1.10
+      - progressbar2==4.3.2
+      - prompt-toolkit==3.0.43
+      - protobuf==4.25.1
+      - psutil==5.9.7
+      - pulsar-client==3.4.0
+      - pyarrow==15.0.0
+      - pyarrow-hotfix==0.6
+      - pyasn1==0.5.1
+      - pyasn1-modules==0.3.0
+      - pycocoevalcap==1.2
+      - pycocotools==2.0.6
+      - pycryptodomex==3.19.1
+      - pydantic==2.5.3
+      - pydantic-core==2.14.6
+      - pydub==0.25.1
+      - pygments==2.17.2
+      - pyparsing==3.1.1
+      - pypika==0.48.9
+      - pyproject-hooks==1.0.0
+      - pysrt==1.1.2
+      - python-dateutil==2.8.2
+      - python-dotenv==1.0.0
+      - python-multipart==0.0.6
+      - python-slugify==8.0.1
+      - python-utils==3.8.1
+      - pytubefix==6.5.1
+      - pytz==2023.3.post1
+      - pyyaml==6.0.1
+      - referencing==0.32.0
+      - regex==2023.12.25
+      - rich==13.7.0
+      - rouge==1.0.1
+      - rpds-py==0.16.2
+      - rsa==4.9
+      - safetensors==0.4.1
+      - scikit-image==0.22.0
+      - scikit-learn==1.3.2
+      - scipy==1.11.4
+      - seaborn==0.13.1
+      - semantic-version==2.10.0
+      - sentence-transformers==2.2.2
+      - sentencepiece==0.1.97
+      - sentry-sdk==1.39.1
+      - setproctitle==1.3.3
+      - setuptools==69.0.3
+      - shellingham==1.5.4
+      - six==1.16.0
+      - smmap==5.0.1
+      - sniffio==1.3.0
+      - soundfile==0.12.1
+      - soupsieve==2.5
+      - starlette==0.32.0.post1
+      - sympy==1.12
+      - tenacity==8.2.3
+      - text-unidecode==1.3
+      - threadpoolctl==3.2.0
+      - tifffile==2023.12.9
+      - tiktoken==0.5.2
+      - timm
+      - tokenizers==0.15.2
+      - tomli==2.0.1
+      - tomlkit==0.12.0
+      - toolz==0.12.0
+      - torch==2.2.2
+      - torchaudio==2.2.2
+      - torchvision==0.17.2
+      - transformers
+      - triton==2.0.0
+      - typer==0.9.0
+      - typing-extensions==4.9.0
+      - tzdata==2023.4
+      - ujson==5.9.0
+      - uvicorn==0.25.0
+      - uvloop==0.19.0
+      - visual-genome==1.1.1
+      - wandb==0.14.2
+      - watchfiles==0.21.0
+      - wcwidth==0.2.13
+      - webdataset==0.2.48
+      - webencodings==0.5.1
+      - websocket-client==1.7.0
+      - websockets
+      - webvtt-py==0.4.6
+      - wrapt==1.16.0
+      - xxhash==3.4.1
+      - yarl==1.9.4
+      - youtube-dl==2021.12.17
+      - yt-dlp
+      - zipp
+      - vllm

evaluation/Goldfish_eval/movies/eval_model_summary_llama_vid.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=L_RAG_general_summary_3_subtitles_together_%j
+#SBATCH --output=L_RAG_general_summary_3_subtitles_together_%j.out
+#SBATCH --error=L_RAG_general_summary_3_subtitles_together_%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=64G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+START=$1
+END=$2
+BATCH_SIZE=4
+
+NEIGHBOURS=3
+## Dataset paths
+videos_path="path to the videos"
+subtitle_path="path to the subtitles"
+video_clips_saving_path="path to save the video clips"
+annotation_file="path to the annotation file"
+movienet_annotations_dir="path to the movienet annotations directory"
+# if you want to use openai embedding, then you need to set the OPENAI_API_KEY
+use_openai_embedding=True
+export OPENAI_API_KEY="your_openai_key"
+
+
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+
+
+
+# # Vision + subtitles
+exp_name="Vsion_subtitles_model_summary_subtitle"
+echo $exp_name 
+python evaluation/eval_goldfish_llama_vid.py --index_subtitles_together --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE   --ckpt $CKPT_PATH  --exp_name=$exp_name\
+      --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+
+
+# vision only 
+# exp_name="vision_only"
+# echo $exp_name 
+# python eval_goldfish_llama_vid.py --vision_only --model_summary_only --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+#       --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+
+
+# subtiltes only  (eliminate the vision)
+# exp_name="subtitles_only"
+# echo $exp_name 
+# python eval_goldfish_llama_vid.py --index_subtitles_together --subtitles_only --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+#       --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+
+

evaluation/Goldfish_eval/movies/eval_model_summary_movie_chat.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=MC_RAG_general_summary_all_%j
+#SBATCH --output=MC_RAG_general_summary_all_%j.out
+#SBATCH --error=MC_RAG_general_summary_all_%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=64G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+START=$1
+END=$2
+BATCH_SIZE=4
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+
+NEIGHBOURS=-1 # use the whole neighbourhood for the global mode
+
+dataset_path="path to the movies folder"
+annotation_json_folder="path to the jsons folder"
+# if you want to use openai embedding, then you need to set the OPENAI_API_KEY
+use_openai_embedding=True
+export OPENAI_API_KEY="your_openai_key"
+
+
+
+exp_name="model_summary_and_subtitle"
+fps=2
+
+# use general summary
+python evaluation/eval_goldfish_movie_chat.py --fps=$fps --neighbours_global=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+      --dataset_videos_path $dataset_path --annotation_json_folder $annotation_json_folder --use_openai_embedding $use_openai_embedding

evaluation/Goldfish_eval/movies/eval_model_summary_movie_qa.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=M_RAG_general_summary_1_subtitles_together_%j
+#SBATCH --output=M_RAG_general_summary_1_subtitles_together_%j.out
+#SBATCH --error=M_RAG_general_summary_1_subtitles_together_%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+START=$1
+END=$2
+BATCH_SIZE=4
+
+NEIGHBOURS=3
+## Dataset paths
+videos_path="path to the videos"
+subtitle_path="path to the subtitles"
+video_clips_saving_path="path to save the video clips"
+annotation_file="path to the annotation file"
+movienet_annotations_dir="path to the movienet annotations directory"
+# if you want to use openai embedding, then you need to set the OPENAI_API_KEY
+use_openai_embedding=True
+export OPENAI_API_KEY="your_openai_key"
+
+
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+echo "Batch size: $BATCH_SIZE"
+
+
+# # Vision + subtitles
+exp_name="Vsion_subtitles_model_summary_subtitle"
+echo $exp_name 
+python evaluation/eval_goldfish_movie_qa.py --add_unknown --index_subtitles_together --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE --ckpt $CKPT_PATH  --exp_name=$exp_name\
+      --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+
+
+# vision only 
+# exp_name="vision_only"
+# echo $exp_name 
+# python eval_goldfish_movie_qa.py --add_unknown --vision_only --model_summary_only --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+#       --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+
+# subtiltes only  (eliminate the vision)
+# exp_name="subtitles_only"
+# echo $exp_name 
+# python eval_goldfish_movie_qa.py --add_unknown --index_subtitles_together --subtitles_only --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH  --exp_name=$exp_name\
+#       --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+

evaluation/Goldfish_eval/movies/eval_q_related_info_llama_vid.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=job_name%j
+#SBATCH --output=job_name%j.out
+#SBATCH --error=job_name%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=64G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+BATCH_SIZE=4
+START=$1
+END=$2
+
+NEIGHBOURS=3
+
+# Dataset paths
+videos_path="path to the videos"
+subtitle_path="path to the subtitles"
+video_clips_saving_path="path to save the video clips"
+annotation_file="path to the annotation file"
+movienet_annotations_dir="path to the movienet annotations directory"
+# if you want to use openai embedding, then you need to set the OPENAI_API_KEY
+use_openai_embedding=True
+export OPENAI_API_KEY="your_openai_key"
+
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+
+# # Vision + subtitles
+exp_name="Vsion_subtitles_model_summary_subtitle"
+echo $exp_name 
+python evaluation/eval_goldfish_llama_vid.py --use_clips_for_info  --index_subtitles_together --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+      --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+
+
+# vision only 
+# exp_name="vision_only"
+# echo $exp_name 
+# python evaluation/eval_goldfish_llama_vid.py --use_clips_for_info --vision_only --model_summary_only --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+#       --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+
+# # subtiltes only  (eliminate the vision)
+# it is only from summaries no need to run it with clips 

evaluation/Goldfish_eval/movies/eval_q_related_info_movie_chat.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=job_name%j
+#SBATCH --output=job_name%j.out
+#SBATCH --error=job_name%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=64G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+BATCH_SIZE=4
+START=$1
+END=$2
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+
+NEIGHBOURS=-1 # use the whole neighbourhood for the global mode
+dataset_path="path to the movies folder"
+annotation_json_folder="path to the jsons folder"
+# if you want to use openai embedding, then you need to set the OPENAI_API_KEY
+use_openai_embedding=True
+export OPENAI_API_KEY="your_openai_key"
+
+
+exp_name="model_summary_and_subtitle"
+fps=2
+
+# use this for both info and general summary --v_sum_and_info
+
+python evaluation/eval_goldfish_movie_chat.py  --fps=$fps --neighbours_global=$NEIGHBOURS --batch_size=$BATCH_SIZE --start=$START --end=$END  --use_clips_for_info --ckpt $CKPT_PATH   --exp_name=$exp_name --dataset_videos_path $dataset_path --annotation_json_folder $annotation_json_folder --use_openai_embedding $use_openai_embedding

evaluation/Goldfish_eval/movies/eval_q_related_info_movie_qa.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=M_RAG_clips_for_info_3_subtitles_together_%j
+#SBATCH --output=M_RAG_clips_for_info_3_subtitles_together_%j.out
+#SBATCH --error=M_RAG_clips_for_info_3_subtitles_together_%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=64G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+NAME="ckpt_92"
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+BATCH_SIZE=4
+START=$1
+END=$2
+
+NEIGHBOURS=3
+# Dataset paths
+videos_path="path to the videos"
+subtitle_path="path to the subtitles"
+video_clips_saving_path="path to save the video clips"
+annotation_file="path to the annotation file"
+movienet_annotations_dir="path to the movienet annotations directory"
+# if you want to use openai embedding, then you need to set the OPENAI_API_KEY
+use_openai_embedding=True
+export OPENAI_API_KEY="your_openai_key"
+
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+echo "Batch size: $BATCH_SIZE"
+
+# # Vision + subtitles
+# exp_name="Vsion_subtitles_model_summary_subtitle"
+# echo $exp_name 
+python evaluation/eval_goldfish_movie_qa.py --add_unknown --use_clips_for_info --use_choices_for_info --index_subtitles_together --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE --ckpt $CKPT_PATH  --exp_name=$exp_name\
+      --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+
+
+# vision only 
+# exp_name="vision_only"
+# echo $exp_name 
+# python evaluation/eval_goldfish_movie_qa.py --add_unknown --use_clips_for_info --use_choices_for_info --vision_only --model_summary_only --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE --ckpt $CKPT_PATH  --exp_name=$exp_name\
+#       --videos_path $videos_path --subtitle_path $subtitle_path --video_clips_saving_path $video_clips_saving_path --annotation_path $annotation_path --movienet_annotations_dir $movienet_annotations_dir --use_openai_embedding $use_openai_embedding
+
+

evaluation/Goldfish_eval/movies/submit_batch_jobs_llama_vid.py

@@ -0,0 +1,14 @@
+import os
+
+# bash_script = 'eval_q_related_info_llama_vid.sh'
+
+bash_script = 'eval_model_summary_llama_vid.sh'
+start=0
+end=45
+step=11
+for i in range(start, end, step):
+    # print(i, i+step, job_id)
+    # job_id+=1
+    cmd=f'sbatch {bash_script} {str(i)} {str(i+step)}'
+    # print(cmd)
+    os.system(cmd)

evaluation/Goldfish_eval/movies/submit_batch_jobs_movie_qa.py

@@ -0,0 +1,16 @@
+import os 
+import sys 
+
+bash_script = 'eval_model_summary_movie_qa.sh'
+# bash_script = 'eval_q_related_info_movie_qa.sh'
+start=0
+end=30
+step=4
+for i in range(start, end, step):
+    # print(i, i+step, job_id)
+    # job_id+=1
+    cmd=f'sbatch {bash_script} {str(i)} {str(i+step)}'
+    # print(cmd)
+    os.system(cmd)
+    
+    

evaluation/Goldfish_eval/movies/submit_batch_jobs_moviechat.py

@@ -0,0 +1,14 @@
+import os
+
+bash_script = 'eval_q_related_info_movie_chat.sh'
+
+# bash_script = 'eval_model_summary_movie_chat.sh'
+start=0
+end=101
+step=26
+for i in range(start, end, step):
+    # print(i, i+step, job_id)
+    # job_id+=1
+    cmd=f'sbatch {bash_script} {str(i)} {str(i+step)}'
+    # print(cmd)
+    os.system(cmd)

evaluation/Goldfish_eval/retrival_accuracy/eval_retrieval_acc_tvqa_job.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#SBATCH --partition=batch
+
+
+#SBATCH --job-name=Retrieval_acc_3_%j
+#SBATCH --output=Retrieval_acc_3_%j.out
+#SBATCH --error=Retrieval_acc_3_%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+cd ../../../
+NAME="ckpt_92"
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+START=$1
+END=$2
+BATCH_SIZE=8
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+echo "Batch size: $BATCH_SIZE"
+
+NEIGHBOURS=1
+exp_name="vision"
+
+python evaluation/eval_retrieval_acc_tvqa.py  --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --vision_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --subtitles_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+
+
+# exp_name="subtitles"
+# python evaluation/eval_retrieval_acc_tvqa.py  --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --vision_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --subtitles_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name

evaluation/Goldfish_eval/retrival_accuracy/eval_retrieval_acc_tvqa_job_sub_v.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#SBATCH --partition=batch
+
+
+#SBATCH --job-name=Retrieval_acc_3_%j
+#SBATCH --output=Retrieval_acc_3_%j.out
+#SBATCH --error=Retrieval_acc_3_%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+NAME="ckpt_92"
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+START=$1
+END=$2
+BATCH_SIZE=8
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+echo "Batch size: $BATCH_SIZE"
+
+NEIGHBOURS=1
+# exp_name="vision"
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py --vision_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py --subtitles_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+
+
+exp_name="subtitles"
+# python evaluation/eval_retrieval_acc_tvqa.py  --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+python evaluation/eval_retrieval_acc_tvqa.py --vision_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py --subtitles_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name

evaluation/Goldfish_eval/retrival_accuracy/eval_retrieval_acc_tvqa_job_sub_v_sub.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#SBATCH --partition=batch
+
+
+#SBATCH --job-name=Retrieval_acc_3_%j
+#SBATCH --output=Retrieval_acc_3_%j.out
+#SBATCH --error=Retrieval_acc_3_%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+
+NAME="ckpt_92"
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+START=$1
+END=$2
+BATCH_SIZE=8
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+echo "Batch size: $BATCH_SIZE"
+
+NEIGHBOURS=1
+# exp_name="vision"
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --vision_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --subtitles_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+
+
+exp_name="subtitles"
+python evaluation/eval_retrieval_acc_tvqa.py  --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --vision_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --subtitles_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name

evaluation/Goldfish_eval/retrival_accuracy/eval_retrieval_acc_tvqa_job_vision_vision.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#SBATCH --partition=batch
+
+
+#SBATCH --job-name=Retrieval_acc_3_%j
+#SBATCH --output=Retrieval_acc_3_%j.out
+#SBATCH --error=Retrieval_acc_3_%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+cd ../../../
+NAME="ckpt_92"
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+START=$1
+END=$2
+BATCH_SIZE=8
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+echo "Batch size: $BATCH_SIZE"
+
+NEIGHBOURS=1
+exp_name="vision"
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+python evaluation/eval_retrieval_acc_tvqa.py  --vision_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --subtitles_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+
+
+# exp_name="subtitles"
+# python evaluation/eval_retrieval_acc_tvqa.py  --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --vision_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name
+
+# python evaluation/eval_retrieval_acc_tvqa.py  --subtitles_only --start=$START --end=$END --neighbours=$NEIGHBOURS --batch_size=$BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH --exp_name=$exp_name

evaluation/Goldfish_eval/tvqa_eval/eval_model_summary.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=job_name%j
+#SBATCH --output=job_name%j.out
+#SBATCH --error=job_name%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=64G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+## run the application:
+cd ../../../
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+START=$1
+END=$2
+
+BATCH_SIZE=4
+NEIGHBOURS=3
+
+# tvqa_json_subtitles="path to the tvqa json subtitles file"
+# tvqa_clips_subtitles="path to the tvqa clips subtitles"
+# videos_frames="path to the video frames"
+# annotation_path="path to the TVQA-Long annotation file"
+
+
+tvqa_json_subtitles="datasets/evaluation_datasets/goldfish_eval_datasets/tvqa/tvqa_preprocessed_subtitles.json"
+tvqa_clips_subtitles="/ibex/project/c2090/datasets/TVR_dataset/videos/tvqa_subtitles"
+videos_frames="/ibex/project/c2090/datasets/TVR_dataset/videos/video_files/frames_hq/"
+annotation_path="datasets/evaluation_datasets/goldfish_eval_datasets/tvqa/tvqa_val_edited.json"
+
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+
+# # Vision + subtitles
+exp_name="Vsion_subtitles_model_summary_subtitle_videoLLM"
+echo $exp_name 
+python eval_goldfish_tvqa_long.py --add_unknown --index_subtitles_together --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+      --tvqa_json_subtitles $tvqa_json_subtitles --tvqa_clips_subtitles $tvqa_clips_subtitles --videos_frames $videos_frames --annotation_path $annotation_path
+
+
+# vision only 
+# exp_name="vision_only"
+# echo $exp_name 
+# python eval_goldfish_tvqa_long.py --add_unknown --vision_only --model_summary_only --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH  --exp_name=$exp_name
+
+# # subtiltes only  (eliminate the vision)
+# exp_name="subtitles_only"
+# echo $exp_name 
+# python eval_goldfish_tvqa_long.py --add_unknown --index_subtitles_together --subtitles_only --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --name $NAME --ckpt $CKPT_PATH  --exp_name=$exp_name

evaluation/Goldfish_eval/tvqa_eval/eval_q_related_info.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+#SBATCH --partition=batch
+
+
+#SBATCH --job-name=RAG_clips_info_1_vision_%j
+#SBATCH --output=RAG_clips_info_1_vision_%j.out
+#SBATCH --error=RAG_clips_info_1_vision_%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=64G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+
+
+## run the application:
+cd ../../../
+START=$1
+END=$2
+
+BATCH_SIZE=4
+NEIGHBOURS=3
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth"
+# tvqa_json_subtitles="path to the tvqa json subtitles file"
+# tvqa_clips_subtitles="path to the tvqa clips subtitles"
+# videos_frames="path to the video frames"
+# annotation_path="path to the TVQA-Long annotation file"
+
+
+tvqa_json_subtitles="datasets/evaluation_datasets/goldfish_eval_datasets/tvqa/tvqa_preprocessed_subtitles.json"
+tvqa_clips_subtitles="/ibex/project/c2090/datasets/TVR_dataset/videos/tvqa_subtitles"
+videos_frames="/ibex/project/c2090/datasets/TVR_dataset/videos/video_files/frames_hq/"
+annotation_path="datasets/evaluation_datasets/goldfish_eval_datasets/tvqa/tvqa_val_edited.json"
+
+# if start and end are not provided, then use the whole dataset
+if [ -z "$START" ]
+then
+      START=0
+fi
+if [ -z "$END" ]
+then
+      END=100000
+fi
+echo "Start: $START"
+echo "End: $END"
+
+# # Vision + subtitles
+exp_name="Vsion_subtitles_model_summary_subtitle"
+echo $exp_name 
+python eval_goldfish_tvqa_long.py --add_unknown --use_clips_for_info --use_choices_for_info --index_subtitles_together --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+      --tvqa_json_subtitles $tvqa_json_subtitles --tvqa_clips_subtitles $tvqa_clips_subtitles --videos_frames $videos_frames --annotation_path $annotation_path
+
+
+# exp_name="Vsion_subtitles_info_only"
+# echo $exp_name 
+# python eval_goldfish_tvqa_long.py --add_unknown --info_only --use_clips_for_info --use_choices_for_info --index_subtitles_together --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+#       --tvqa_json_subtitles $tvqa_json_subtitles --tvqa_clips_subtitles $tvqa_clips_subtitles --videos_frames $videos_frames --annotation_path $annotation_path
+
+
+# exp_name="info_sub_after_retrieval"
+# echo $exp_name 
+# python eval_goldfish_tvqa_long.py --add_unknown --subtitles_only_after_retrieval --use_clips_for_info --use_choices_for_info --index_subtitles_together --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+#       --tvqa_json_subtitles $tvqa_json_subtitles --tvqa_clips_subtitles $tvqa_clips_subtitles --videos_frames $videos_frames --annotation_path $annotation_path
+
+
+
+
+
+# vision only 
+# exp_name="vision_only"
+# echo $exp_name 
+# python eval_goldfish_tvqa_long.py --add_unknown --use_clips_for_info --use_choices_for_info --vision_only --model_summary_only --neighbours=$NEIGHBOURS  --start=$START --end=$END --batch_size $BATCH_SIZE  --ckpt $CKPT_PATH  --exp_name=$exp_name\
+#       --tvqa_json_subtitles $tvqa_json_subtitles --tvqa_clips_subtitles $tvqa_clips_subtitles --videos_frames $videos_frames --annotation_path $annotation_path

evaluation/Goldfish_eval/tvqa_eval/submit_batch_jobs.py

@@ -0,0 +1,25 @@
+import os 
+import sys 
+
+bash_script = 'RAG_summary.sh'
+# bash_script = 'RAG.sh'
+
+# general
+start=0
+end=850
+step=60
+
+
+# bash_script="RAG_summary_R_ablations.sh"
+# sample 50 
+# start=0
+# end=52
+# step=6
+
+
+# job_id=32434597
+for i in range(start, end, step):
+    # print(i, i+step, job_id)
+    # job_id+=1
+    cmd=f'sbatch {bash_script} {str(i)} {str(i+step)}'
+    os.system(cmd)

evaluation/eval_goldfish_llama_vid.py

@@ -0,0 +1,616 @@
+import sys 
+import os
+project_dir = os.getcwd()
+sys.path.append(project_dir)
+import json 
+from tqdm import tqdm
+from goldfish_lv import GoldFish_LV,split_subtitles,time_to_seconds
+import argparse
+import json
+import torch
+import re
+from tqdm import tqdm
+from PIL import Image
+from index import MemoryIndex  
+import torch
+import random
+import numpy as np
+import torch.backends.cudnn as cudnn
+import shutil
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError('Boolean value expected.')
+
+def get_arguments():
+    parser = argparse.ArgumentParser(description="Inference parameters")
+    parser.add_argument("--neighbours", type=int, default=-1)
+    parser.add_argument("--name", type=str,default="ckpt_92",help="name of the experiment")
+    parser.add_argument("--add_unknown", action='store_true')
+    parser.add_argument("--use_chatgpt", action='store_true')
+    parser.add_argument("--use_choices_for_info", action='store_true')
+    parser.add_argument("--use_gt_information", action='store_true')
+    parser.add_argument("--inference_text", action='store_true')
+    parser.add_argument("--use_gt_information_with_distraction", action='store_true')
+    parser.add_argument("--num_distraction", type=int, default=2)
+    parser.add_argument("--add_confidance_score", action='store_true')
+    parser.add_argument("--use_original_video", action='store_true')
+    parser.add_argument("--use_video_embedding", action='store_true')
+    parser.add_argument("--use_clips_for_info", action='store_true')
+    parser.add_argument("--use_GT_video", action='store_true')
+    parser.add_argument("--use_gt_summary", action='store_true')
+    parser.add_argument("--index_subtitles", action='store_true')
+    parser.add_argument("--index_subtitles_together", action='store_true')
+    
+    parser.add_argument("--ask_the_question_early", action='store_true')
+    parser.add_argument("--clip_in_ask_early", action='store_true')
+    parser.add_argument("--summary_with_subtitles_only", action='store_true')
+    parser.add_argument("--use_coherent_description", action='store_true')
+    
+    parser.add_argument("--start", default=0, type=int)
+    parser.add_argument("--end", default=100000, type=int)
+    parser.add_argument("--exp_name", type=str,default="",help="name of eval folder")
+    
+    
+    parser.add_argument("--vision_only", action='store_true')
+    parser.add_argument("--model_summary_only", action='store_true')
+    parser.add_argument("--subtitles_only", action='store_true')
+    parser.add_argument("--info_only", action='store_true')
+    
+    parser.add_argument("--cfg-path", default="test_configs/llama2_test_config.yaml")
+    parser.add_argument("--ckpt", type=str, default="checkpoints/video_llama_checkpoint_last.pth")
+    parser.add_argument("--add_subtitles", action='store_true')
+    parser.add_argument("--eval_opt", type=str, default='all')
+    parser.add_argument("--max_new_tokens", type=int, default=300)
+    parser.add_argument("--batch_size", type=int, default=8)
+    parser.add_argument("--lora_r", type=int, default=64)
+    parser.add_argument("--lora_alpha", type=int, default=16)
+    parser.add_argument("--video_path", type=str, help="path to the video")
+    parser.add_argument("--use_openai_embedding",type=str2bool, default=False)
+    parser.add_argument("--annotation_path", type=str, help="path to the annotation file")
+    parser.add_argument("--videos_path", type=str, help="path to the videos directory")
+    parser.add_argument("--subtitle_path", type=str, help="path to the subtitles directory")
+    parser.add_argument("--movienet_annotations_dir", type=str, help="path to the movienet annotations directory")
+    parser.add_argument("--video_clips_saving_path", type=str, help="path to save the splitted small video clips")
+    
+    parser.add_argument("--save_path", type=str, help="path to save the results")
+    
+    parser.add_argument("--options", nargs="+")
+    return parser.parse_args()
+def time_to_seconds(subrip_time):
+    return subrip_time.hours * 3600 + subrip_time.minutes * 60 + subrip_time.seconds + subrip_time.milliseconds / 1000
+
+def clean_text(subtitles_text):
+    # Remove unwanted characters except for letters, digits, and single quotes
+    subtitles_text = re.sub(r'[^a-zA-Z0-9\s\']', '', subtitles_text)
+    # Replace multiple spaces with a single space
+    subtitles_text = re.sub(r'\s+', ' ', subtitles_text)
+    return subtitles_text.strip()
+        
+class LlamaVidQAEval (GoldFish_LV):
+    
+    def __init__(self,args):
+        super().__init__(args)
+        self.save_json_path = "new_workspace/clips_summary/movienet"
+        if args.use_openai_embedding:
+            self.save_pkls_path = "new_workspace/open_ai_embedding/movienet"
+        else:
+            self.save_pkls_path = "new_workspace/embedding/movienet"
+        os.makedirs(self.save_json_path, exist_ok=True)
+        annotation_path=args.annotation_path
+        with open(annotation_path, 'r') as f:
+            self.movies_dict = json.load(f)
+        self.max_sub_len=400
+        self.max_num_images=45
+    
+
+    def _get_movie_data(self,videoname):
+        video_images_path =f"{args.videos_path}/{videoname}"
+        movie_clips_path =f"{args.video_clips_saving_path}/{videoname}"
+        subtitle_path = f"{args.subtitle_path}/{videoname}.srt"
+        annotation_file=f"{args.movienet_annotations_dir}/{videoname}.json"
+        # load the annotation file 
+        with open(annotation_file, 'r') as f:
+            movie_annotation = json.load(f)
+        return video_images_path,subtitle_path,movie_annotation,movie_clips_path
+    def _store_subtitles_paragraphs(self,subtitle_path,important_data,number_of_paragraphs):
+        paragraphs=[]
+        movie_name=subtitle_path.split('/')[-1].split('.')[0]
+        # if there is no story, split the subtitles into paragraphs 
+        paragraphs = split_subtitles(subtitle_path, number_of_paragraphs)  
+        for i,paragraph in enumerate(paragraphs):
+            paragraph=clean_text(paragraph)
+            important_data.update({f"subtitle_{i}__{movie_name}_clip_{str(i).zfill(2)}": paragraph})  
+        return important_data
+    def _get_shots_subtitles(self,movie_annotation):
+        shots_subtitles={}
+        if movie_annotation['story'] is not None:
+            for section in movie_annotation['story']:
+                for shot in section['subtitle']:
+                    shot_number=shot['shot']
+                    shot_subtitle=' '.join(shot['sentences'])
+                    shots_subtitles[shot_number]=clean_text(shot_subtitle)
+                
+        return shots_subtitles
+    
+    def prepare_input_images(self,clip_path,shots_subtitles,use_subtitles):
+        total_frames=len(os.listdir(clip_path))
+        movie_name=clip_path.split('/')[-2]
+        clip_name=clip_path.split('/')[-1]
+        sampling_interval=int(total_frames//self.max_num_images)
+        if sampling_interval==0:
+            sampling_interval=1
+        use_subtitles_save_name="subtitles" if use_subtitles else "no_subtitles"
+        video_frames_path = os.path.join(clip_path)
+        total_num_frames=len(os.listdir(video_frames_path))
+        sampling_interval = round(total_num_frames / self.max_num_images)
+        if sampling_interval == 0:
+            sampling_interval = 1
+        number_of_words=0
+        video_images_list=sorted(os.listdir(video_frames_path))
+        images = []
+        img_placeholder = ""
+        for i,frame in enumerate(video_images_list):
+            if i % sampling_interval == 0:
+                frame = Image.open(os.path.join(video_frames_path,frame)).convert("RGB") 
+                frame = self.vis_processor(frame)
+                images.append(frame)
+                img_placeholder += '<Img><ImageHere>'
+                shot_num=video_images_list[i].split('_')[1]
+                if shots_subtitles.get(shot_num) is not None:
+                    sub=clean_text(shots_subtitles[shot_num])
+                    number_of_words+=len(sub.split(' '))
+                    if number_of_words<= self.max_sub_len and use_subtitles:
+                        img_placeholder+=f'<Cap>{sub}'
+            if len(images) >= self.max_num_images:
+                break
+        if len(images) ==0:
+            print("Video not found",video_frames_path)
+            
+        if 0 <len(images) < self.max_num_images:
+            last_item = images[-1]
+            while len(images) < self.max_num_images:
+                images.append(last_item)
+                img_placeholder += '<Img><ImageHere>'
+        images = torch.stack(images)
+
+        return images,img_placeholder
+
+    def _get_movie_summaries(self,video_images_path,use_subtitles,shots_subtitles,movie_clips_path):
+        video_images_list=sorted(os.listdir(video_images_path))
+        max_caption_index = 0
+        preds = {}
+        movie_name=movie_clips_path.split('/')[-1]
+        videos_summaries=[]
+        previous_caption=""
+        batch_size=args.batch_size
+        batch_images=[]
+        batch_instructions=[]
+        clip_numbers=[]
+        clip_number=0
+        conversations=[]
+        for i in tqdm(range(0,len(video_images_list),135), desc="Inference video clips", total=len(video_images_list)/120):
+            images=[]
+            # Add the previous caption to the new video clip 
+            # if batch_size==1:
+            #     previous_caption="You are analysing a one long video of mutiple clips and this is the summary from all previous clips :"+videos_summaries[-1] +"\n\n"if len(videos_summaries)>0 else ""
+            if previous_caption != "":
+                img_placeholder = previous_caption+" "
+            else:
+                img_placeholder = ""
+            number_of_words=0
+            max_num_words=400
+            max_num_images=45
+            clip_number_str=str(clip_number).zfill(2)
+            clip_path=os.path.join(movie_clips_path,f"{movie_name}_clip_{clip_number_str}")
+            os.makedirs(clip_path, exist_ok=True)
+            conversation=""
+            for j in range(i,i+135,3):
+                if j >= len(video_images_list):
+                    break
+                image_path = os.path.join(video_images_path, video_images_list[j])
+                # copy the images to clip folder 
+                # if the image is already copied, skip it 
+                if not os.path.exists(os.path.join(clip_path,video_images_list[j])):
+                    shutil.copy(image_path,clip_path)
+                img=Image.open(image_path)
+                images.append(self.vis_processor(img))
+                img_placeholder += '<Img><ImageHere>'
+                shot_num=int(video_images_list[j].split('_')[1])
+                if use_subtitles:
+                    if shots_subtitles.get(shot_num) is not None:
+                        sub=clean_text(shots_subtitles[shot_num])
+                        number_of_words+=len(sub.split(' '))
+                        if number_of_words<= max_num_words and use_subtitles:
+                            img_placeholder+=f'<Cap>{sub}'
+                        conversation+=sub+" "        
+                if len(images) >= max_num_images:
+                    break
+            if len(images) ==0:
+                print("Video not found",video_images_path)
+                continue
+            if 0 <len(images) < max_num_images:
+                last_item = images[-1]
+                while len(images) < max_num_images:
+                    images.append(last_item)
+                    img_placeholder += '<Img><ImageHere>'
+            images = torch.stack(images)
+            print(images.shape)
+            clip_numbers.append(clip_number_str)
+            clip_number+=1
+            conversations.append(clean_text(conversation))
+            instruction = img_placeholder + '\n' + self.summary_instruction
+            batch_images.append(images)
+            batch_instructions.append(instruction)
+            if len(batch_images) < batch_size:
+                continue
+            # run inference for the batch
+            batch_images = torch.stack(batch_images)
+            batch_pred=self.run_images(batch_images,batch_instructions)
+            for i,pred in enumerate(batch_pred):
+                max_caption_index += 1
+                videos_summaries.append(pred)
+                if args.use_coherent_description:
+                    preds[f'caption_{max_caption_index}__{movie_name}_clip_{clip_numbers[i]}'] = f"model_summary :{pred}\nVideo conversation :{conversations[i]}"
+                else:
+                    preds[f'caption_{max_caption_index}__{movie_name}_clip_{clip_numbers[i]}'] = pred 
+                    if conversations[i]!="" and use_subtitles:
+                        preds[f'subtitle_{max_caption_index}__{movie_name}_clip_{clip_numbers[i]}'] = conversations[i]
+                   
+            batch_images=[]
+            batch_instructions=[]
+            clip_numbers=[]
+            conversations=[]
+        
+        # run inference for the last batch
+        if len(batch_images)>0:
+            batch_images = torch.stack(batch_images)
+            batch_pred=self.run_images(batch_images,batch_instructions)
+            for k,pred in enumerate(batch_pred):
+                max_caption_index += 1
+                videos_summaries.append(pred)
+                if args.use_coherent_description:
+                    preds[f'caption_{max_caption_index}__{movie_name}_clip_{clip_numbers[k]}'] = f"model_summary :{pred}\nVideo conversation :{conversations[k]}"
+                else:
+                    preds[f'caption_{max_caption_index}__{movie_name}_clip_{clip_numbers[k]}'] = pred 
+                    if conversations[k]!="" and use_subtitles:
+                        preds[f'subtitle_{max_caption_index}__{movie_name}_clip_{clip_numbers[k]}'] = conversations[k]
+                    
+            batch_images=[]
+            batch_instructions=[]
+        return preds
+    def movie_inference(self,videoname,use_subtitles):
+        embedding_path=os.path.join(self.save_pkls_path,f"{videoname}.pkl")
+        if args.index_subtitles_together:
+            file_path=os.path.join(self.save_json_path,f"{videoname}.json")
+            embedding_path=os.path.join(self.save_pkls_path,f"{videoname}.pkl")
+        else:
+            file_path=os.path.join(self.save_json_path,f"no_subtiltles_{videoname}.json")
+            embedding_path=os.path.join(self.save_pkls_path,f"no_subtiltles_{videoname}.pkl")
+        
+        if args.subtitles_only:
+            file_path=os.path.join(self.save_json_path,f"subtiltles_only_{videoname}.json")
+            embedding_path=os.path.join(self.save_pkls_path,f"subtiltles_only_{videoname}.pkl")
+           
+        if os.path.exists(file_path):
+            print("Already processed")
+            return file_path,embedding_path
+        important_data = {}
+        video_images_path,subtitle_path,movie_annotation,movie_clips_path=self._get_movie_data(videoname)
+        shots_subtitles={}
+        if use_subtitles: 
+            if movie_annotation['story'] is not None:
+                shots_subtitles=self._get_shots_subtitles(movie_annotation) 
+        if args.subtitles_only:
+            number_of_paragraphs=20
+            important_data=self._store_subtitles_paragraphs(subtitle_path,important_data,number_of_paragraphs)
+        else:
+            preds=self._get_movie_summaries(video_images_path,use_subtitles,shots_subtitles,movie_clips_path)
+            if len(shots_subtitles)==0 and use_subtitles:
+                number_of_paragraphs=len(preds)
+                important_data=self._store_subtitles_paragraphs(subtitle_path,important_data,number_of_paragraphs)
+            important_data.update(preds)
+        with open(file_path, 'w') as file:
+            json.dump(important_data, file, indent=4)
+        return file_path,embedding_path
+    def answer_movie_questions_RAG(self,qa_list,information_RAG_path,embedding_path):
+        QA_external_memory=MemoryIndex(args.neighbours, use_openai=args.use_openai_embedding)
+        if os.path.exists(embedding_path):
+            QA_external_memory.load_embeddings_from_pkl(embedding_path)
+        else:
+            QA_external_memory.load_documents_from_json(information_RAG_path,embedding_path)
+        summarization_external_memory=MemoryIndex(-1, use_openai=args.use_openai_embedding)
+        if os.path.exists(embedding_path):
+            summarization_external_memory.load_embeddings_from_pkl(embedding_path)
+        else:
+            summarization_external_memory.load_documents_from_json(information_RAG_path,embedding_path)
+    
+        # get the most similar context from the external memory to this instruction 
+        general_related_context_keys_list=[]
+        general_related_context_documents_list=[]
+        summary_related_context_documents_list=[]
+        summary_related_context_keys_list=[]
+        total_batch_pred=[]
+        related_text=[]
+        qa_genearl_prompts=[]
+        qa_summary_prompts=[]
+        qa_general=[]
+        qa_summary=[]
+        for qa in qa_list:
+            if qa['q_type']=='summary':
+                related_context_documents,related_context_keys = summarization_external_memory.search_by_similarity(qa['Q'])
+                summary_related_context_documents_list.append(related_context_documents)
+                summary_related_context_keys_list.append(related_context_keys)
+                prompt=self.prepare_prompt(qa)
+                qa_summary_prompts.append(prompt)
+                qa_summary.append(qa)
+            else:
+                related_context_documents,related_context_keys = QA_external_memory.search_by_similarity(qa['Q'])
+                general_related_context_keys_list.append(related_context_keys)
+                general_related_context_documents_list.append(related_context_documents)
+                prompt=self.prepare_prompt(qa)
+                qa_genearl_prompts.append(prompt)
+                qa_general.append(qa)  
+        # if I have summary questions answer first, without the need to use clips for information 
+        if len(qa_summary_prompts)>0:
+            # Here the retrieved clips are all movie clips
+            context_information_list=[]
+            for related_context_keys in summary_related_context_keys_list:
+                most_related_clips=self.get_most_related_clips(related_context_keys)
+                context_information=""
+                for clip_name in most_related_clips:
+                    clip_conversation=""
+                    general_sum=""
+                    for key in related_context_keys:
+                        if clip_name in key and 'caption' in key:
+                            general_sum="Clip Summary: "+summarization_external_memory.documents[key]
+                        if clip_name in key and 'subtitle' in key:
+                            clip_conversation="Clip Subtitles: "+summarization_external_memory.documents[key]
+                    
+                    if args.use_coherent_description:
+                        context_information+=f"{general_sum}\n"
+                    else:
+                        if args.model_summary_only:
+                            context_information+=f"{general_sum}\n"
+                        elif args.subtitles_only:
+                            context_information+=f"{clip_conversation}\n"
+                        else:
+                            context_information+=f"{general_sum},{clip_conversation}\n"
+                context_information_list.append(context_information) 
+            if args.use_chatgpt :
+                batch_pred=self.inference_RAG_chatGPT(qa_summary_prompts,context_information_list)
+            else:
+                batch_pred=self.inference_RAG(qa_summary_prompts,context_information_list) 
+            total_batch_pred.extend(batch_pred)
+            related_text.extend(context_information_list)
+        
+        if args.use_clips_for_info:
+            batch_pred,general_related_context_keys_list=self.use_clips_for_info(qa_general,general_related_context_keys_list,QA_external_memory)   
+            total_batch_pred.extend(batch_pred)
+            related_text.extend(general_related_context_keys_list)
+        else:   
+            related_context_documents_text_list=[]
+            for related_context_documents,related_context_keys in zip(general_related_context_documents_list,general_related_context_keys_list): 
+                related_information=""
+                most_related_clips=self.get_most_related_clips(related_context_keys)
+                for clip_name in most_related_clips:
+                    clip_conversation=""
+                    general_sum=""
+                    for key in QA_external_memory.documents.keys():
+                        if clip_name in key and 'caption' in key:
+                            general_sum="Clip Summary: "+QA_external_memory.documents[key]
+                        if clip_name in key and 'subtitle' in key:
+                            clip_conversation="Clip Subtitles: "+QA_external_memory.documents[key]
+                    if args.use_coherent_description:
+                        related_information+=f"{general_sum}\n"
+                    else:
+                        if args.model_summary_only:
+                            related_information+=f"{general_sum}\n"
+                        elif args.subtitles_only:
+                            related_information+=f"{clip_conversation}\n"
+                        else:
+                            related_information+=f"{general_sum},{clip_conversation}\n"
+                    
+                related_context_documents_text_list.append(related_information)
+            
+            if len (qa_genearl_prompts) >0 and args.use_chatgpt :
+                batch_pred=self.inference_RAG_chatGPT(qa_genearl_prompts,related_context_documents_text_list)
+            elif len (qa_genearl_prompts) >0:
+                batch_pred=self.inference_RAG(qa_genearl_prompts,related_context_documents_text_list)
+                total_batch_pred.extend(batch_pred)
+                related_text.extend(related_context_documents_text_list)
+        assert len(total_batch_pred)==len(related_text)
+        return total_batch_pred, related_text
+    def get_most_related_clips(self,related_context_keys):
+        most_related_clips=[]
+        for context_key in related_context_keys:
+            if len(context_key.split('__'))>1:
+                most_related_clips.append(context_key.split('__')[1])
+            if len(most_related_clips)==args.neighbours:
+                break
+        assert len(most_related_clips)!=0, f"No related clips found {related_context_keys}"
+        return most_related_clips 
+    
+    def clip_inference(self,clips_name,prompts):
+        setup_seeds(seed)
+        images_batch, instructions_batch = [], []
+        for clip_name, prompt in zip(clips_name, prompts):
+            movie_name=clip_name.split('_')[0]
+            video_images_path,subtitle_path,movie_annotation,movie_clips_path=self._get_movie_data(movie_name)
+            clip_path=os.path.join(movie_clips_path,clip_name)
+            if movie_annotation['story'] is not None:
+                shots_subtitles=self._get_shots_subtitles(movie_annotation) 
+            else:
+                shots_subtitles={}
+            images,img_placeholder=self.prepare_input_images(clip_path,shots_subtitles,use_subtitles=not args.vision_only)
+            instruction = img_placeholder + '\n' + prompt
+            images_batch.append(images)
+            instructions_batch.append(instruction)
+        # run inference for the batch
+        images_batch=torch.stack(images_batch)
+        batch_pred=self.run_images(images_batch,instructions_batch)
+        return batch_pred
+    def prepare_prompt(self,qa):
+        prompt=qa["Q"]
+        return prompt
+    def use_clips_for_info(self,qa_list,related_context_keys_list,external_memory):
+            total_batch_pred=[]
+            questions=[]
+            related_information_list=[]
+            related_context_keys_list_new=[]
+            for qa,related_context_keys in zip(qa_list,related_context_keys_list):
+                most_related_clips=self.get_most_related_clips(related_context_keys)
+                question=qa['Q']
+                # prompt=self.prepare_prompt(qa)
+                # prompt+=" and also provide an EXPLAINATION for your answer and If you don't know the answer, say that you don't know.\n\n"
+                prompt=f"From this video extract the related information to This question and provide an explaination for your answer and If you can't find related information, say 'I DON'T KNOW' as option 5 because maybe the questoin is not related to the video content.\n the question is :\n {question}\n your answer :"
+                # all_info=self.clip_inference(most_related_clips,[prompt]*len(most_related_clips))
+                # make the most_related_clips has unique elements (if retrival from vision summary and conversations)
+                most_related_clips=list(set(most_related_clips))
+                batch_inference=[]
+                all_info=[]
+                for related_clip in most_related_clips:
+                    batch_inference.append(related_clip)
+                    if len(batch_inference)<args.batch_size:
+                        continue    
+                    all_info.extend(self.clip_inference(batch_inference,[prompt]*len(batch_inference)))
+                    batch_inference=[]
+                if len(batch_inference)>0:
+                    all_info.extend(self.clip_inference(batch_inference,[prompt]*len(batch_inference)))
+           
+                related_information=""
+                for info,clip_name in zip(all_info,most_related_clips):
+                    clip_conversation=""
+                    general_sum=""
+                    for key in external_memory.documents.keys():
+                        if clip_name in key and 'caption' in key:
+                            general_sum="Clip Summary: "+external_memory.documents[key]
+                        if clip_name in key and 'subtitle' in key:
+                            clip_conversation="Clip Subtitles: "+external_memory.documents[key]
+                    
+                    if args.use_coherent_description:
+                        related_information+=f"question_related_information: {info},{general_sum}\n"
+                    else:
+                        if args.model_summary_only:
+                            related_information+=f"{general_sum},question_related_information: {info}\n"
+                        elif args.info_only:
+                            related_information+=f"question_related_information: {info}\n"
+                        elif args.subtitles_only:
+                            related_information+=f"{clip_conversation},question_related_information: {info}\n"
+                        else:
+                            related_information+=f"{general_sum},{clip_conversation},question_related_information: {info}\n" 
+                        
+                         
+                        # related_information+=f"question_related_information: {info},{clip_conversation}\n"
+                questions.append(question)
+                related_information_list.append(related_information)
+                related_context_keys.append(related_information)
+                related_context_keys_list_new.append(related_context_keys)
+                if len(questions)< args.batch_size:
+                    continue
+                setup_seeds(seed)
+                if args.use_chatgpt :
+                    batch_pred=self.inference_RAG_chatGPT(questions, related_information_list)
+                else:
+                    batch_pred=self.inference_RAG(questions, related_information_list)
+                
+                for pred in batch_pred:
+                    total_batch_pred.append(pred)
+                questions=[]
+                related_information_list=[]
+            
+            if len(questions)>0:
+                setup_seeds(seed)
+                if args.use_chatgpt :
+                    batch_pred=self.inference_RAG_chatGPT(questions, related_information_list)
+                else:
+                    batch_pred=self.inference_RAG(questions, related_information_list)
+                for pred in batch_pred:
+                    total_batch_pred.append(pred)
+            return total_batch_pred,related_context_keys_list_new
+    def define_save_name(self):
+        save_name="subtitles" if args.index_subtitles_together else "no_subtitles"
+        save_name+="_clips_for_info" if args.use_clips_for_info else ""
+        save_name+="_chatgpt" if args.use_chatgpt else ""
+        save_name+="_vision_only" if args.vision_only else ""
+        save_name+="_model_summary_only" if args.model_summary_only else ""
+        save_name+="_subtitles_only" if args.subtitles_only else ""
+        save_name+="_info_only" if args.info_only else ""
+        print("save_name",save_name)
+        return save_name
+    def eval_llama_vid(self):
+        ## LLAMa vid QA evaluation
+        full_questions_result=[]
+        movie_number=0
+        start=args.start
+        end=args.end
+        save_name=self.define_save_name()
+        for movie in tqdm(self.movies_dict.keys()):
+            if args.start <=movie_number < args.end:
+                save_dir=f"new_workspace/results/llama_vid/{args.exp_name}/{save_name}_{args.neighbours}_neighbours"
+                if os.path.exists( f"{save_dir}/{movie}.json" ):
+                    print(f"Movie {movie} already processed")
+                    with open(f"{save_dir}/{movie}.json", 'r') as f:
+                        pred_json = json.load(f)
+                    full_questions_result.extend(pred_json)
+                    continue
+                use_subtitles_while_generating_summary=not args.vision_only
+                information_RAG_path,embedding_path=self.movie_inference(movie,use_subtitles_while_generating_summary)
+                external_memory=MemoryIndex(args.neighbours, use_openai=args.use_openai_embedding)
+                if os.path.exists(embedding_path):
+                    external_memory.load_embeddings_from_pkl(embedding_path)
+                else:
+                    external_memory.load_documents_from_json(information_RAG_path,emdedding_path=embedding_path)
+                save_dir=f"new_workspace/results/llama_vid/{args.exp_name}/{save_name}_{args.neighbours}_neighbours"
+                os.makedirs(save_dir, exist_ok=True)
+                pred_json=[]
+                batch_questions=[]
+                for qa in tqdm(self.movies_dict[movie],desc="Inference questions"):
+                    batch_questions.append(qa)
+                    if len(batch_questions)<args.batch_size:
+                        continue
+                    model_ans,related_text=self.answer_movie_questions_RAG(batch_questions,information_RAG_path,embedding_path)
+                    for qa,ans,related_info in zip(batch_questions,model_ans,related_text):
+                        qa.update({'pred':ans})
+                        qa.update({'related_info':related_info})
+                        pred_json.append(qa)
+                    batch_questions=[]
+                if len(batch_questions)>0:
+                    model_ans,related_text=self.answer_movie_questions_RAG(batch_questions,information_RAG_path,embedding_path)
+                    for qa,ans,related_info in zip(batch_questions,model_ans,related_text):
+                        qa.update({'pred':ans})
+                        qa.update({'related_info':related_info})
+                        pred_json.append(qa)
+                full_questions_result.extend(pred_json)
+                with open(f"{save_dir}/{movie}.json", 'w') as fp:
+                    json.dump(pred_json, fp)
+                print(f"Movie {movie} prediction saved to {save_dir}/{movie}.json")
+            movie_number+=1
+        with open(f"{save_dir}/full_pred_s{start}_end{end}.json", 'w') as fp:
+            json.dump(full_questions_result, fp)
+args=get_arguments()
+
+def setup_seeds(seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    cudnn.benchmark = False
+    cudnn.deterministic = True
+
+import yaml 
+# read this file test_configs/llama2_test_config.yaml
+with open('test_configs/llama2_test_config.yaml') as file:
+    config = yaml.load(file, Loader=yaml.FullLoader)
+seed=config['run']['seed']
+print("seed",seed)
+
+if __name__ == "__main__":
+    setup_seeds(seed)
+    llama_vid_eval=LlamaVidQAEval(args)
+    llama_vid_eval.eval_llama_vid()

evaluation/eval_goldfish_movie_chat.py

@@ -0,0 +1,453 @@
+import sys 
+import os
+project_dir = os.getcwd()
+sys.path.append(project_dir)
+import json 
+from tqdm import tqdm
+from goldfish_lv import GoldFish_LV,split_subtitles,time_to_seconds
+import argparse
+import json
+import argparse
+import torch
+from tqdm import tqdm
+# from openai import OpenAI
+from minigpt4.common.eval_utils import init_model
+from minigpt4.conversation.conversation import CONV_VISION
+from index import MemoryIndex  
+import pysrt
+import chardet
+import torch
+import random
+import numpy as np
+import torch.backends.cudnn as cudnn
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError('Boolean value expected.')
+
+def get_arguments():
+    parser = argparse.ArgumentParser(description="Inference parameters")
+    parser.add_argument("--neighbours", type=int, default=-1)
+    parser.add_argument("--neighbours_global", type=int, default=-1)
+    parser.add_argument("--fps", type=float, default=0.5)
+    parser.add_argument("--name", type=str,default="ckpt_92",help="name of the experiment")
+    parser.add_argument("--add_unknown", action='store_true')
+    parser.add_argument("--use_chatgpt", action='store_true')
+    parser.add_argument("--use_choices_for_info", action='store_true')
+    parser.add_argument("--use_gt_information", action='store_true')
+    parser.add_argument("--inference_text", action='store_true')
+    parser.add_argument("--use_gt_information_with_distraction", action='store_true')
+    parser.add_argument("--num_distraction", type=int, default=2)
+    parser.add_argument("--add_confidance_score", action='store_true')
+    parser.add_argument("--use_original_video", action='store_true')
+    parser.add_argument("--use_video_embedding", action='store_true')
+    parser.add_argument("--use_clips_for_info", action='store_true')
+    parser.add_argument("--use_GT_video", action='store_true')
+    parser.add_argument("--use_gt_summary", action='store_true')
+    parser.add_argument("--index_subtitles", action='store_true')
+    parser.add_argument("--index_subtitles_together", action='store_true')
+    
+    parser.add_argument("--ask_the_question_early", action='store_true')
+    parser.add_argument("--clip_in_ask_early", action='store_true')
+    parser.add_argument("--summary_with_subtitles_only", action='store_true')
+    parser.add_argument("--use_coherent_description", action='store_true')
+    parser.add_argument("--v_sum_and_info", action='store_true')
+    
+    parser.add_argument("--start", default=0, type=int)
+    parser.add_argument("--end", default=100000, type=int)
+    parser.add_argument("--exp_name", type=str,default="",help="name of eval folder")
+
+    
+    parser.add_argument("--cfg-path", default="test_configs/llama2_test_config.yaml")
+    parser.add_argument("--ckpt", type=str, default="checkpoints/video_llama_checkpoint_last.pth")
+    parser.add_argument("--add_subtitles", action='store_true')
+    parser.add_argument("--eval_opt", type=str, default='all')
+    parser.add_argument("--max_new_tokens", type=int, default=300)
+    parser.add_argument("--batch_size", type=int, default=8)
+    parser.add_argument("--lora_r", type=int, default=64)
+    parser.add_argument("--lora_alpha", type=int, default=16)
+    parser.add_argument("--video_path", type=str, help="path to the video")
+    parser.add_argument("--use_openai_embedding",type=str2bool, default=False)
+    parser.add_argument("--dataset_videos_path", type=str, help="path to the dataset videos")
+    parser.add_argument("--annotation_json_folder", type=str, help="path to the annotation folder")
+    parser.add_argument("--options", nargs="+")
+    return parser.parse_args()
+
+def get_movie_time(subtitle_path):
+    # read the subtitle file and detect the encoding
+    with open(subtitle_path, 'rb') as f:
+        result = chardet.detect(f.read())
+    subtitles = pysrt.open(subtitle_path, encoding=result['encoding'])
+    video_time=time_to_seconds(subtitles[-1].end)
+    return video_time
+
+
+import torch
+from torch.utils.data import Dataset, DataLoader
+from torchvision.transforms import Compose
+import h5py
+import torch
+import os
+
+def numerical_sort_key(filename):
+    base_name = os.path.splitext(filename)[0]  
+    return int(base_name) 
+
+class MovieChatDataset(Dataset):
+    def __init__(self, dataset_path, annotation_path,fps, transform=None,start=0,end=100000):
+        self.dataset_path = dataset_path
+        self.annotation_path=annotation_path
+        self.transform = transform
+        self.movie_name = os.listdir(dataset_path)
+        self.movie_name = [file for file in self.movie_name if file != '.DS_Store']
+        self.fps = fps
+        self.len_clip = 45
+        self.start=start
+        self.end=end
+    def load_frames(self, movie_name):
+        filenames = sorted(os.listdir(os.path.join(self.dataset_path, movie_name)))
+        
+        filenames.sort(key=numerical_sort_key)
+        # define torch tensor to store the frames of size(0,0,0)
+        data = []
+        for filename_number in tqdm(filenames,desc="Loading frames"):
+            file_path = os.path.join(self.dataset_path, movie_name, filename_number)
+
+            if not os.path.isfile(file_path):
+                print(f"Did not find file: {filename_number}")
+            try:
+                with h5py.File(file_path, 'r') as h5_file:
+                    image_embeds=torch.tensor(h5_file[f"frames_{filename_number[:-3]}"][:])
+                    image_embeds = image_embeds[:,1:,:] # remove the first token (CLS) (200,256,1408)
+                    # concate each 4 neighbours image tokens 
+                    bs, pn, hs = image_embeds.shape
+                    image_embeds = image_embeds.view(bs, int(pn/4), int(hs*4)) 
+                    data.extend(image_embeds)
+
+            except Exception as e:  
+                print(f"Failed to process {filename_number}: {e}")
+            
+                
+        frames=torch.stack(data)
+        return frames
+
+    def __len__(self):
+        return len(self.movie_name)
+
+    def _get_movie_questions(self,movie_annotations):
+        global_questions=movie_annotations['global']
+        local_questions=movie_annotations['breakpoint']
+        return global_questions,local_questions
+    def __getitem__(self, idx):
+        if self.start<=idx<self.end:
+            self.frames = self.load_frames(self.movie_name[idx])
+            movie_name=self.movie_name[idx]
+            with open(os.path.join(self.annotation_path,movie_name+".json"), 'r') as f:
+                movie_annotations = json.load(f)
+            global_questions,local_questions=self._get_movie_questions(movie_annotations)
+            sampling_value = int(movie_annotations['info']['fps']/self.fps)
+            clips_list=[]
+            current_clip=[]
+            for i in range(0,self.frames.shape[0], sampling_value):
+                current_clip.append(self.frames[i])
+                if len(current_clip) >= self.len_clip:
+                    clips_list.append(torch.stack(current_clip))
+                    current_clip=[]
+            if len(current_clip) > 0:
+                last_frame_current_clip = current_clip[-1]
+                while len(current_clip) < self.len_clip:
+                    current_clip.append(last_frame_current_clip)
+                clips_list.append(torch.stack(current_clip))
+            return clips_list, movie_name,global_questions,local_questions
+        else:
+            return [], self.movie_name[idx],[],[]
+
+
+class MovieChat (GoldFish_LV):
+    
+    def __init__(self,args):
+        super().__init__(args)
+        self.args=args
+        self.save_long_videos_path = "new_workspace/clips_summary/movie_chat/"
+        if args.use_openai_embedding:
+            self.save_embedding_path = "new_workspace/open_ai_embedding/movie_chat/"
+        else:
+            self.save_embedding_path = "new_workspace/embedding/movie_chat/"
+        os.makedirs(self.save_long_videos_path, exist_ok=True)
+        os.makedirs(self.save_embedding_path, exist_ok=True)
+        self.max_sub_len=400
+        self.max_num_images=45
+    
+    
+    def _get_long_video_summaries(self,clips,save_path):
+        batch=[]
+        batch_instructions=[]
+        preds={}
+        clip_numbers=[]
+        max_caption_index=0
+        for i,clip_features in enumerate(clips):
+            if len(clip_features)!=self.max_num_images:
+                continue
+            batch.append(clip_features)
+            img_placeholder=""
+            for j in range(len(clip_features)):
+                img_placeholder+="<Img><ImageHere>"
+            instruction = img_placeholder + '\n' + self.summary_instruction
+            batch_instructions.append(instruction)
+            clip_numbers.append(i)
+            if len(batch)<args.batch_size:
+                continue
+            batch=torch.stack(batch)
+            batch_pred= self.run_images_features(batch,batch_instructions)
+            for j,pred in enumerate(batch_pred):
+                max_caption_index += 1
+                if pred !="":
+                    preds[f'caption__clip_{str(clip_numbers[j]).zfill(2)}'] = pred 
+            batch=[]
+            clip_numbers=[]
+            batch_instructions=[]
+        if len(batch)>0:
+            batch=torch.stack(batch)
+            batch_pred= self.run_images_features(batch,batch_instructions)
+            for j,pred in enumerate(batch_pred):
+                max_caption_index += 1
+                if pred !="":
+                    preds[f'caption__clip_{str(clip_numbers[j]).zfill(2)}'] = pred
+        with open(save_path, 'w') as file:
+            json.dump(preds, file, indent=4)
+        return preds
+    def use_model_summary (self,qa_prompts,related_context_documents_list,related_context_keys_list,external_memory):
+        related_context_documents_text_list=[]
+        for related_context_documents,related_context_keys in zip(related_context_documents_list,related_context_keys_list): 
+            related_information=""
+            most_related_clips=self.get_most_related_clips_index(related_context_keys,external_memory)
+            for clip_name in most_related_clips:
+                general_sum=""
+                clip_name=str(clip_name).zfill(2)
+                for key in external_memory.documents.keys():
+                    if clip_name in key and 'caption' in key:
+                        general_sum="Clip Summary: "+external_memory.documents[key]
+                        break
+                related_information+=f"{general_sum}\n"
+            related_context_documents_text_list.append(related_information)
+                
+        if args.use_chatgpt :
+            batch_pred=self.inference_RAG_chatGPT(qa_prompts,related_context_documents_text_list)
+        else:
+            batch_pred=self.inference_RAG(qa_prompts,related_context_documents_text_list)
+        return batch_pred, related_context_documents_text_list
+    def answer_movie_questions_RAG(self,qa_list,information_RAG_path,embedding_path,q_type):
+        if q_type=='local':
+            external_memory=MemoryIndex(args.neighbours, use_openai=self.args.use_openai_embedding)
+        else:  
+            external_memory=MemoryIndex(args.neighbours_global, use_openai=self.args.use_openai_embedding)
+        if os.path.exists(embedding_path):
+            external_memory.load_embeddings_from_pkl(embedding_path)
+        else:
+            external_memory.load_documents_from_json(information_RAG_path,embedding_path)
+        # get the most similar context from the external memory to this instruction 
+        related_context_documents_list=[]
+        related_context_keys_list=[]
+        total_batch_pred=[]
+        related_text=[]
+        qa_prompts=[]
+        for qa in qa_list:
+            related_context_documents,related_context_keys = external_memory.search_by_similarity(qa['question'])
+            related_context_documents_list.append(related_context_documents)
+            related_context_keys_list.append(related_context_keys)
+            prompt=self.prepare_prompt(qa)
+            qa_prompts.append(prompt)
+        if args.use_clips_for_info:
+            batch_pred,related_context_keys_list=self.use_clips_for_info(qa_list,related_context_keys_list,external_memory)   
+            total_batch_pred.extend(batch_pred)
+            related_text.extend(related_context_keys_list)
+        else:
+            batch_pred, related_context_documents_text_list=self.use_model_summary (qa_prompts,
+                        related_context_documents_list,related_context_keys_list,external_memory)
+            total_batch_pred.extend(batch_pred)
+            related_text.extend(related_context_documents_text_list)
+        assert len(total_batch_pred)==len(qa_list)      
+        assert len(total_batch_pred)==len(related_text)
+        return total_batch_pred, related_text
+    def get_most_related_clips_index(self,related_context_keys,external_memory):
+        most_related_clips_index=[]
+        for context_key in related_context_keys:
+            # loop over memory keys to get the context key index
+            for i,key in enumerate(external_memory.documents.keys()):
+                if context_key in key:
+                    most_related_clips_index.append(i)
+                    break
+            
+        return most_related_clips_index
+
+    
+    def clip_inference(self,clips_idx,prompts):
+        setup_seeds(seed)
+        images_batch, instructions_batch = [], []
+        for clip_idx, prompt in zip(clips_idx, prompts):
+            clip_features=self.video_clips[clip_idx]
+            img_placeholder=""
+            for j in range(len(clip_features)):
+                img_placeholder+='<Img><ImageHere>'
+            instruction = img_placeholder + '\n' + prompt
+            images_batch.append(clip_features)
+            instructions_batch.append(instruction)
+        # run inference for the batch
+        images_batch=torch.stack(images_batch)
+        batch_pred= self.run_images_features(images_batch,instructions_batch)
+        return batch_pred
+    def prepare_prompt(self,qa):
+        prompt=qa["question"]
+        return prompt
+    def use_clips_for_info(self,qa_list,related_context_keys_list,external_memory):
+        total_batch_pred=[]
+        questions=[]
+        related_information_list=[]
+        related_context_keys_list_new=[]
+        for qa,related_context_keys in zip(qa_list,related_context_keys_list):
+            most_related_clips_index=self.get_most_related_clips_index(related_context_keys,external_memory)
+            question=qa['question']
+            prompt=f"From this video extract the related information to This question and provide an explaination for your answer and If you can't find any related information, say 'I DON'T KNOW' as option 5 because maybe the questoin is not related to the video content.\n the question is :\n {question}\n your answer :"
+            batch_inference=[]
+            all_info=[]
+            for clip_idx in most_related_clips_index:
+                batch_inference.append(clip_idx)
+                if len(batch_inference)<args.batch_size:
+                    continue    
+                all_info.extend(self.clip_inference(batch_inference,[prompt]*len(batch_inference)))
+                batch_inference=[]
+            if len(batch_inference)>0:
+                all_info.extend(self.clip_inference(batch_inference,[prompt]*len(batch_inference)))
+            # all_info=self.clip_inference(most_related_clips_index,[prompt]*len(most_related_clips_index))      
+            related_information=""
+            for info,clip_name in zip(all_info,most_related_clips_index):
+                general_sum=""
+                clip_name=str(clip_name).zfill(2)
+                for key in external_memory.documents.keys():
+                    if clip_name in key and 'caption' in key:
+                        general_sum="Clip Summary: "+external_memory.documents[key]
+                if args.v_sum_and_info:
+                    related_information+=f"{general_sum},question_related_information: {info}\n"  
+                else:
+                    related_information+=f"question_related_information: {info}\n"
+            questions.append(question)
+            related_information_list.append(related_information)
+            related_context_keys.append(related_information)
+            related_context_keys_list_new.append(related_context_keys)
+            if len(questions)< args.batch_size:
+                continue
+            setup_seeds(seed)
+            if args.use_chatgpt :
+                batch_pred=self.inference_RAG_chatGPT(questions, related_information_list)
+            else:
+                batch_pred=self.inference_RAG(questions, related_information_list)
+            
+            for pred in batch_pred:
+                total_batch_pred.append(pred)
+            questions=[]
+            related_information_list=[]
+        
+        if len(questions)>0:
+            setup_seeds(seed)
+            if args.use_chatgpt :
+                batch_pred=self.inference_RAG_chatGPT(questions, related_information_list)
+            else:
+                batch_pred=self.inference_RAG(questions, related_information_list)
+            for pred in batch_pred:
+                total_batch_pred.append(pred)
+        return total_batch_pred,related_context_keys_list_new
+    def define_save_name(self):
+        save_name="subtitles" if args.index_subtitles else "no_subtitles"
+        save_name="subtitles_together" if args.index_subtitles_together else save_name
+        save_name="summary_with_subtitles_only" if args.summary_with_subtitles_only else save_name
+        save_name+="_unknown" if args.add_unknown else ""
+        save_name+="_clips_for_info" if args.use_clips_for_info else ""
+        save_name+="_chatgpt" if args.use_chatgpt else ""
+        save_name+="_choices_for_info" if args.use_choices_for_info else ""
+        save_name+="_v_sum_and_info" if args.v_sum_and_info else ""
+        save_name+='fps_'+str(args.fps)
+        save_dir=f"new_workspace/results/moviechat/{args.exp_name}/{save_name}_{args.neighbours_global}_neighbours"
+        os.makedirs(save_dir, exist_ok=True)
+        return save_dir
+        
+    def eval_moviechat(self):
+        start=args.start
+        end=args.end
+        dataset_path = args.dataset_videos_path
+        annotation_json_folder=args.annotation_json_folder
+        dataset = MovieChatDataset(dataset_path,annotation_json_folder, fps=args.fps,start=start,end=end)
+        # dataloader = DataLoader(dataset, batch_size=1,  shuffle=False)
+        full_questions_result=[]
+        save_dir=self.define_save_name()
+
+        for i,(clips ,video_name,global_questions,local_questions) in enumerate(dataset):
+            # code here
+            if start<=i < end:
+                print("video_name",video_name)
+                self.video_clips=clips
+                self.video_name=video_name
+                file_path=os.path.join(self.save_long_videos_path,self.video_name+f"_fps{args.fps}.json")
+                embedding_path=os.path.join(self.save_embedding_path,self.video_name+f"_fps{args.fps}.pkl")
+                if os.path.exists(file_path):
+                    print("Already processed")
+                else:
+                    self._get_long_video_summaries(clips,file_path)
+                batch_questions=[]
+                for qa in global_questions:
+                    batch_questions.append(qa)
+                    if len(batch_questions)<args.batch_size:
+                        continue
+                    model_answers, related_text=self.answer_movie_questions_RAG(batch_questions,file_path,embedding_path,q_type='global')
+                    for qa,ans in zip(batch_questions,model_answers):
+                        qa.update({'pred':ans})
+                        qa['Q']=qa['question']
+                        qa['A']=qa['answer']
+                        qa.pop('question', None)
+                        qa.pop('answer', None)
+                    
+                    batch_questions=[]
+                if len(batch_questions)>0:
+                    model_answers, related_text=self.answer_movie_questions_RAG(batch_questions,file_path,embedding_path,q_type='global')
+                    for qa,ans in zip(batch_questions,model_answers):
+                        qa.update({'pred':ans})
+                        qa['Q']=qa['question']
+                        qa['A']=qa['answer']
+                        qa.pop('question', None)
+                        qa.pop('answer', None)
+                
+                full_questions_result.extend(global_questions)
+                print(f"Finished {i} out of {len(dataset)}")
+                # save the results 
+                with open(f"{save_dir}/{self.video_name}.json", 'w') as file:
+                    # json.dump(global_questions+local_questions, file, indent=4)
+                    json.dump(global_questions, file, indent=4)
+                    
+        with open(f"{save_dir}/full_pred_{start}_{end}.json", 'w') as fp:
+            json.dump(full_questions_result, fp)
+args=get_arguments()
+
+def setup_seeds(seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    cudnn.benchmark = False
+    cudnn.deterministic = True
+
+import yaml 
+# read this file test_configs/llama2_test_config.yaml
+with open('test_configs/llama2_test_config.yaml') as file:
+    config = yaml.load(file, Loader=yaml.FullLoader)
+seed=config['run']['seed']
+print("seed",seed)
+
+if __name__ == "__main__":
+    setup_seeds(seed)
+    llama_vid_eval=MovieChat(args)
+    llama_vid_eval.eval_moviechat()
+    

evaluation/eval_goldfish_movie_qa.py

@@ -0,0 +1,591 @@
+import sys
+import os 
+project_dir = os.getcwd()
+sys.path.append(project_dir)
+import json 
+from tqdm import tqdm
+from goldfish_lv import GoldFish_LV,split_subtitles,time_to_seconds
+import argparse
+import json
+import argparse
+import torch
+import re
+from tqdm import tqdm
+from PIL import Image
+# from openai import OpenAI
+from index import MemoryIndex  
+import pysrt
+import chardet
+import torch
+import random
+import numpy as np
+import torch.backends.cudnn as cudnn
+import shutil
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError('Boolean value expected.')
+
+def get_arguments():
+    parser = argparse.ArgumentParser(description="Inference parameters")
+    parser.add_argument("--neighbours", type=int, default=-1)
+    parser.add_argument("--name", type=str,default="ckpt_92",help="name of the experiment")
+    parser.add_argument("--add_unknown", action='store_true')
+    parser.add_argument("--use_chatgpt", action='store_true')
+    parser.add_argument("--use_choices_for_info", action='store_true')
+    parser.add_argument("--use_gt_information", action='store_true')
+    parser.add_argument("--inference_text", action='store_true')
+    parser.add_argument("--use_gt_information_with_distraction", action='store_true')
+    parser.add_argument("--num_distraction", type=int, default=2)
+    parser.add_argument("--add_confidance_score", action='store_true')
+    parser.add_argument("--use_original_video", action='store_true')
+    parser.add_argument("--use_video_embedding", action='store_true')
+    parser.add_argument("--use_clips_for_info", action='store_true')
+    parser.add_argument("--use_GT_video", action='store_true')
+    parser.add_argument("--use_gt_summary", action='store_true')
+    parser.add_argument("--index_subtitles", action='store_true')
+    parser.add_argument("--index_subtitles_together", action='store_true')
+    
+    parser.add_argument("--ask_the_question_early", action='store_true')
+    parser.add_argument("--clip_in_ask_early", action='store_true')
+    parser.add_argument("--summary_with_subtitles_only", action='store_true')
+    parser.add_argument("--use_coherent_description", action='store_true')
+    
+    parser.add_argument("--start", default=0, type=int)
+    parser.add_argument("--end", default=100000, type=int)
+    parser.add_argument("--exp_name", type=str,default="",help="name of eval folder")
+    
+    parser.add_argument("--vision_only", action='store_true')
+    parser.add_argument("--model_summary_only", action='store_true')
+    parser.add_argument("--subtitles_only", action='store_true')
+    parser.add_argument("--info_only", action='store_true')
+    
+    parser.add_argument("--cfg-path", default="test_configs/llama2_test_config.yaml")
+    parser.add_argument("--ckpt", type=str, default="checkpoints/video_llama_checkpoint_last.pth")
+    parser.add_argument("--add_subtitles", action='store_true')
+    parser.add_argument("--eval_opt", type=str, default='all')
+    parser.add_argument("--max_new_tokens", type=int, default=300)
+    parser.add_argument("--batch_size", type=int, default=8)
+    parser.add_argument("--lora_r", type=int, default=64)
+    parser.add_argument("--lora_alpha", type=int, default=16)
+    parser.add_argument("--video_path", type=str, help="path to the video")
+    parser.add_argument("--use_openai_embedding",type=str2bool, default=False)
+    parser.add_argument("--annotation_path", type=str, help="path to the annotation file")
+    parser.add_argument("--videos_path", type=str, help="path to the videos directory")
+    parser.add_argument("--subtitle_path", type=str, help="path to the subtitles directory")
+    parser.add_argument("--movienet_annotations_dir", type=str, help="path to the movienet annotations directory")
+    parser.add_argument("--video_clips_saving_path", type=str, help="path to save the splitted small video clips")
+    parser.add_argument("--options", nargs="+")
+    return parser.parse_args()
+
+def time_to_seconds(subrip_time):
+    return subrip_time.hours * 3600 + subrip_time.minutes * 60 + subrip_time.seconds + subrip_time.milliseconds / 1000
+
+def get_movie_time(subtitle_path):
+    # read the subtitle file and detect the encoding
+    with open(subtitle_path, 'rb') as f:
+        result = chardet.detect(f.read())
+    subtitles = pysrt.open(subtitle_path, encoding=result['encoding'])
+    video_time=time_to_seconds(subtitles[-1].end)
+    return video_time
+def clean_text(subtitles_text):
+    # Remove unwanted characters except for letters, digits, and single quotes
+    subtitles_text = re.sub(r'[^a-zA-Z0-9\s\']', '', subtitles_text)
+    # Replace multiple spaces with a single space
+    subtitles_text = re.sub(r'\s+', ' ', subtitles_text)
+    return subtitles_text.strip()
+      
+
+class MovieQAEval (GoldFish_LV):
+    
+    def __init__(self,args):
+        super().__init__(args)
+        self.save_json_path = "new_workspace/clips_summary/movienet"
+        if args.use_openai_embedding:
+            self.save_pkls_path = "new_workspace/open_ai_embedding/movienet"
+        else:
+            self.save_pkls_path = "new_workspace/embedding/movienet"
+        os.makedirs(self.save_json_path, exist_ok=True)
+        movie_qa_dataset_path=args.annotation_path
+        with open(movie_qa_dataset_path, 'r') as f:
+            self.movies_dict = json.load(f)
+        self.max_sub_len=400
+        self.max_num_images=45
+    
+    def _get_movie_data(self,videoname):
+        video_images_path =f"{args.videos_path}/{videoname}"
+        movie_clips_path =f"{args.video_clips_saving_path}/{videoname}"
+        subtitle_path = f"{args.subtitle_path}/{videoname}.srt"
+        annotation_file=f"{args.movienet_annotations_dir}/{videoname}.json"
+        # load the annotation file 
+        with open(annotation_file, 'r') as f:
+            movie_annotation = json.load(f)
+        return video_images_path,subtitle_path,movie_annotation,movie_clips_path
+    def _store_subtitles_paragraphs(self,subtitle_path,important_data,number_of_paragraphs):
+        paragraphs=[]
+        movie_name=subtitle_path.split('/')[-1].split('.')[0]
+        # if there is no story, split the subtitles into paragraphs 
+        paragraphs = split_subtitles(subtitle_path, number_of_paragraphs)  
+        for i,paragraph in enumerate(paragraphs):
+            paragraph=clean_text(paragraph)
+            important_data.update({f"subtitle_{i}__{movie_name}_clip_{str(i).zfill(2)}": paragraph})  
+        return important_data
+    def _get_shots_subtitles(self,movie_annotation):
+        shots_subtitles={}
+        if movie_annotation['story'] is not None:
+            for section in movie_annotation['story']:
+                for shot in section['subtitle']:
+                    shot_number=shot['shot']
+                    shot_subtitle=' '.join(shot['sentences'])
+                    shots_subtitles[shot_number]=clean_text(shot_subtitle)
+                
+    
+        return shots_subtitles
+    
+    def prepare_input_images(self,clip_path,shots_subtitles,use_subtitles):
+        total_frames=len(os.listdir(clip_path))
+        sampling_interval=int(total_frames//self.max_num_images)
+        if sampling_interval==0:
+            sampling_interval=1
+        images=[]
+        img_placeholder = ""
+        video_frames_path = os.path.join(clip_path)
+        total_num_frames=len(os.listdir(video_frames_path))
+        sampling_interval = round(total_num_frames / self.max_num_images)
+        if sampling_interval == 0:
+            sampling_interval = 1
+        number_of_words=0
+        video_images_list=sorted(os.listdir(video_frames_path))
+        for i,frame in enumerate(video_images_list):
+            if i % sampling_interval == 0:
+                frame = Image.open(os.path.join(video_frames_path,frame)).convert("RGB") 
+                frame = self.vis_processor(frame)
+                images.append(frame)
+                img_placeholder += '<Img><ImageHere>'
+                shot_num=video_images_list[i].split('_')[1]
+                if shots_subtitles.get(shot_num) is not None:
+                    sub=clean_text(shots_subtitles[shot_num])
+                    number_of_words+=len(sub.split(' '))
+                    if number_of_words<= self.max_sub_len and use_subtitles:
+                        img_placeholder+=f'<Cap>{sub}'
+            if len(images) >= self.max_num_images:
+                break
+        if len(images) ==0:
+            print("Video not found",video_frames_path)
+            
+        if 0 <len(images) < self.max_num_images:
+            last_item = images[-1]
+            while len(images) < self.max_num_images:
+                images.append(last_item)
+                img_placeholder += '<Img><ImageHere>'
+        images = torch.stack(images)
+        return images,img_placeholder
+
+    def _get_movie_summaries(self,video_images_path,use_subtitles,shots_subtitles,movie_clips_path):
+        video_images_list=sorted(os.listdir(video_images_path))
+        max_caption_index = 0
+        preds = {}
+        movie_name=movie_clips_path.split('/')[-1]
+        videos_summaries=[]
+        previous_caption=""
+        batch_size=args.batch_size
+        batch_images=[]
+        batch_instructions=[]
+        clip_numbers=[]
+        clip_number=0
+        conversations=[]
+        for i in tqdm(range(0,len(video_images_list),135), desc="Inference video clips", total=len(video_images_list)/135):
+            images=[]
+            img_placeholder = ""
+            number_of_words=0
+            clip_number_str=str(clip_number).zfill(2)
+            clip_path=os.path.join(movie_clips_path,f"{movie_name}_clip_{clip_number_str}")
+            os.makedirs(clip_path, exist_ok=True)
+            conversation=""
+            for j in range(i,i+135,3):
+                if j >= len(video_images_list):
+                    break
+                image_path = os.path.join(video_images_path, video_images_list[j])
+                # copy the images to clip folder 
+                shutil.copy(image_path,clip_path)
+                img=Image.open(image_path)
+                images.append(self.vis_processor(img))
+                img_placeholder += '<Img><ImageHere>'
+                shot_num=int(video_images_list[j].split('_')[1])
+                if use_subtitles:
+                    if shots_subtitles.get(shot_num) is not None:
+                        sub=clean_text(shots_subtitles[shot_num])
+                        number_of_words+=len(sub.split(' '))
+                        if number_of_words<= self.max_num_words :
+                            img_placeholder+=f'<Cap>{sub}'
+                        conversation+=sub+" " 
+                if len(images) >= self.max_num_images:
+                    break
+            if len(images) ==0:
+                print("Video not found",video_images_path)
+                continue
+            if 0 <len(images) < self.max_num_images:
+                last_item = images[-1]
+                while len(images) < self.max_num_images:
+                    images.append(last_item)
+                    img_placeholder += '<Img><ImageHere>'
+                    
+            images = torch.stack(images)
+            print(images.shape)
+            clip_numbers.append(clip_number_str)
+            clip_number+=1
+            conversations.append(clean_text(conversation))
+            instruction = img_placeholder + '\n' + self.summary_instruction
+            batch_images.append(images)
+            batch_instructions.append(instruction)
+            if len(batch_images) < batch_size:
+                continue
+            # run inference for the batch
+            batch_images = torch.stack(batch_images)
+            batch_pred=self.run_images(batch_images,batch_instructions)
+            for i,pred in enumerate(batch_pred):
+                max_caption_index += 1
+                videos_summaries.append(pred)
+                if args.use_coherent_description:
+                    preds[f'caption_{max_caption_index}__{movie_name}_clip_{clip_numbers[i]}'] = f"model_summary :{pred}\nVideo conversation :{conversations[i]}"
+                else:
+                    preds[f'caption_{max_caption_index}__{movie_name}_clip_{clip_numbers[i]}'] = pred 
+                    if conversations[i]!="" and use_subtitles:
+                        preds[f'subtitle_{max_caption_index}__{movie_name}_clip_{clip_numbers[i]}'] = conversations[i]
+                   
+            batch_images=[]
+            batch_instructions=[]
+            clip_numbers=[]
+            conversations=[]
+
+        # run inference for the last batch
+        if len(batch_images)>0:
+            batch_images = torch.stack(batch_images)
+            batch_pred=self.run_images(batch_images,batch_instructions)
+            for k,pred in enumerate(batch_pred):
+                max_caption_index += 1
+                videos_summaries.append(pred)
+                if args.use_coherent_description:
+                    preds[f'caption_{max_caption_index}__{movie_name}_clip_{clip_numbers[k]}'] = f"model_summary :{pred}\nVideo conversation :{conversations[k]}"
+                else:
+                    preds[f'caption_{max_caption_index}__{movie_name}_clip_{clip_numbers[k]}'] = pred 
+                    if conversations[k]!="" and use_subtitles:
+                        preds[f'subtitle_{max_caption_index}__{movie_name}_clip_{clip_numbers[k]}'] = conversations[k] 
+            batch_images=[]
+            batch_instructions=[]
+        return preds
+    def movie_inference(self,videoname,use_subtitles):
+        
+        embedding_path=os.path.join(self.save_pkls_path,f"{videoname}.pkl")
+        if args.index_subtitles_together:
+            file_path=os.path.join(self.save_json_path,f"{videoname}.json")
+            embedding_path=os.path.join(self.save_pkls_path,f"{videoname}.pkl")
+        else:
+            file_path=os.path.join(self.save_json_path,f"no_subtiltles_{videoname}.json")
+            embedding_path=os.path.join(self.save_pkls_path,f"no_subtiltles_{videoname}.pkl")
+        
+        if args.subtitles_only:
+            file_path=os.path.join(self.save_json_path,f"subtiltles_only_{videoname}.json")
+            embedding_path=os.path.join(self.save_pkls_path,f"subtiltles_only_{videoname}.pkl")
+           
+        if os.path.exists(file_path):
+            print("Already processed")
+            return file_path,embedding_path
+
+        important_data = {}
+        video_images_path,subtitle_path,movie_annotation,movie_clips_path=self._get_movie_data(videoname)
+        shots_subtitles={}   
+        if use_subtitles: 
+            if movie_annotation['story'] is not None:
+                shots_subtitles=self._get_shots_subtitles(movie_annotation) 
+        if args.subtitles_only:
+            number_of_paragraphs=20
+            important_data=self._store_subtitles_paragraphs(subtitle_path,important_data,number_of_paragraphs)
+        else:
+            preds=self._get_movie_summaries(video_images_path,use_subtitles,shots_subtitles,movie_clips_path)
+            if len(shots_subtitles)==0 and use_subtitles:
+                number_of_paragraphs=len(preds)
+                important_data=self._store_subtitles_paragraphs(subtitle_path,important_data,number_of_paragraphs)
+            important_data.update(preds)
+        with open(file_path, 'w') as file:
+            json.dump(important_data, file, indent=4)
+        return file_path,embedding_path
+    def answer_movie_questions_RAG(self,qa_list,external_memory):
+        # get the most similar context from the external memory to this instruction 
+        related_context_keys_list=[]
+        related_context_documents_list=[]
+        related_text=[]
+        questions=[]
+        prompts=[]
+        for qa in qa_list:
+            related_context_documents,related_context_keys = external_memory.search_by_similarity(qa['question'])
+            related_context_documents_list.append(related_context_documents)
+            related_context_keys_list.append(related_context_keys)
+            questions.append(qa)
+            prompt=self.prepare_prompt(qa)
+            prompts.append(prompt)
+        if args.use_clips_for_info:
+            batch_pred,related_context_keys_list=self.use_clips_for_info(qa_list,related_context_keys_list,external_memory)
+            related_text.extend(related_context_keys_list)
+        else:
+            related_context_documents_text_list=[]
+            for related_context_documents,related_context_keys in zip(related_context_documents_list,related_context_keys_list): 
+                related_information=""
+                most_related_clips=self.get_most_related_clips(related_context_keys)
+                for clip_name in most_related_clips:
+                    clip_conversation=""
+                    general_sum=""
+                    for key in external_memory.documents.keys():
+                        if clip_name in key and 'caption' in key:
+                            general_sum="Clip Summary: "+external_memory.documents[key]
+                        if clip_name in key and 'subtitle' in key:
+                            clip_conversation="Clip Subtitles: "+external_memory.documents[key]
+                    related_information+=f"{general_sum},{clip_conversation}\n"
+                    
+                    if args.model_summary_only:
+                        related_information+=f"{general_sum}\n"
+                    elif args.subtitles_only:
+                        related_information+=f"{clip_conversation}\n"
+                    else:
+                        related_information+=f"{general_sum},{clip_conversation}\n"
+                    
+                related_context_documents_text_list.append(related_information)
+            
+            if args.use_chatgpt :
+                batch_pred=self.inference_RAG_chatGPT(prompts,related_context_documents_text_list)
+                related_text.extend(related_context_documents_text_list)
+            else:
+                batch_pred=self.inference_RAG(prompts,related_context_documents_text_list)
+                related_text.extend(related_context_documents_text_list)
+        return batch_pred ,related_text
+    def get_most_related_clips(self,related_context_keys):
+        most_related_clips=[]
+        for context_key in related_context_keys:
+            if len(context_key.split('__'))>1:
+                most_related_clips.append(context_key.split('__')[1])
+            if len(most_related_clips)==args.neighbours:
+                break
+        assert len(most_related_clips)!=0, f"No related clips found {related_context_keys}"
+        return most_related_clips 
+    
+    def clip_inference(self,clips_name,prompts):
+        setup_seeds(seed)
+        images_batch, instructions_batch = [], []
+        for clip_name, prompt in zip(clips_name, prompts):
+            movie_name=clip_name.split('_')[0]
+            video_images_path,subtitle_path,movie_annotation,movie_clips_path=self._get_movie_data(movie_name)
+            clip_path=os.path.join(movie_clips_path,clip_name)
+            if movie_annotation['story'] is not None:
+                shots_subtitles=self._get_shots_subtitles(movie_annotation) 
+            else:
+                shots_subtitles={}
+            images,img_placeholder=self.prepare_input_images(clip_path,shots_subtitles,use_subtitles=not args.vision_only)
+            instruction = img_placeholder + '\n' + prompt
+            images_batch.append(images)
+            instructions_batch.append(instruction)
+        # run inference for the batch
+        images_batch=torch.stack(images_batch)
+        batch_pred=self.run_images(images_batch,instructions_batch)
+        return batch_pred
+    def prepare_prompt(self,qa):
+        prompt=qa["question"]+" \n As you watched in this video Choose ONE suitable answer from these mutiple choices \n"
+        for i,choice in enumerate(qa['choices']):
+            prompt+=f"option {i}: {choice} \n"
+        if args.add_unknown and args.add_confidance_score:
+            # Add unknown option
+            prompt+=f"option 5: Can't answer based on the provided information\n"
+            prompt+="Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 5 INCLUSIVE and aslo output a CONFIDANCE SCORE FROM 0 TO 5 representing how confident you are with your answer where 0 is the least confident and 5 is the most confident"
+        elif args.add_unknown:
+            prompt+=f"option 5: Can't answer based on the provided information\n"
+            prompt+="Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 5 INCLUSIVE"
+        elif args.add_confidance_score:
+            prompt+="Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 4 INCLUSIVE and aslo output a CONFIDANCE SCORE FROM 0 TO 5 representing how confident you are with your answer where 0 is the least confident and 5 is the most confident"
+        else:
+            prompt+="Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 4 INCLUSIVE"
+        return prompt
+    def use_clips_for_info(self,qa_list,related_context_keys_list,external_memory):
+            total_batch_pred=[]
+            questions=[]
+            related_information_list=[]
+            related_context_keys_list_new=[]
+            for qa,related_context_keys in zip(qa_list,related_context_keys_list):
+                most_related_clips=self.get_most_related_clips(related_context_keys)
+                
+                question=qa['question']+ "\n and these are the options for the question\n\n"
+                for i,choice in enumerate(qa['choices']):
+                    question+=f"option {i}: {choice} \n\n"
+                if args.add_unknown:
+                    question+= "option 5: Can't answer based on the provided information\n\n"
+                    question+="\n Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 5 INCLUSIVE"
+                else:
+                    question+="\n Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 4 INCLUSIVE"
+                    
+                if args.use_choices_for_info:
+                    # prompt=self.prepare_prompt(qa)
+                    # prompt+=" and also provide an EXPLAINATION for your answer and If you don't know the answer, say that you don't know.\n\n"
+                    prompt=f"From this video extract the related information to This multichioce question and provide an explaination for your answer and If you can't find any related inforamtion, say 'I DON'T KNOW' as option 5 because maybe the questoin is not related to the video content.\n the question is :\n {question}\n your answer :"
+                else:
+                    prompt=f"As you watched in this video answer this {qa['q']}\n\n and also provide an EXPLAINATION for your answer and If you don't know the answer, say that you don't know.\n\n"
+                # if args.use_choices_for_info:
+                #     prompt=self.prepare_prompt(qa)
+                #     prompt+=" and also provide an EXPLAINATION for your answer and If you don't know the answer, say that you don't know.\n\n"
+                # else:
+                #     prompt=f"As you watched in this video {qa['question']}\n\n and also provide an EXPLAINATION for your answer and If you don't know the answer, say that you don't know.\n\n"
+                # make the most_related_clips has unique elements (if retrival from vision summary and conversations)
+                most_related_clips=list(set(most_related_clips))
+
+                # all_info=self.clip_inference(most_related_clips,[prompt]*len(most_related_clips))
+                batch_inference=[]
+                all_info=[]
+                for related_clip in most_related_clips:
+                    batch_inference.append(related_clip)
+                    if len(batch_inference)<args.batch_size:
+                        continue    
+                    all_info.extend(self.clip_inference(batch_inference,[prompt]*len(batch_inference)))
+                    batch_inference=[]
+                if len(batch_inference)>0:
+                    all_info.extend(self.clip_inference(batch_inference,[prompt]*len(batch_inference)))
+                
+                related_information=""
+                for info,clip_name in zip(all_info,most_related_clips):
+                    clip_conversation=""
+                    general_sum=""
+                    for key in external_memory.documents.keys():
+                        if clip_name in key and 'caption' in key:
+                            general_sum="Clip Summary: "+external_memory.documents[key]
+                        if clip_name in key and 'subtitle' in key:
+                            clip_conversation="Clip Subtitles: "+external_memory.documents[key]
+                    
+                    if args.use_coherent_description:
+                        related_information+=f"question_related_information: {info},{general_sum}\n"
+                    else:
+                        # related_information+=f"{general_sum},{clip_conversation},question_related_information: {info}\n"  
+                        # related_information+=f"question_related_information: {info},{clip_conversation}\n"
+                        if args.model_summary_only:
+                            related_information+=f"{general_sum},question_related_information: {info}\n"
+                        elif args.info_only:
+                            related_information+=f"question_related_information: {info}\n"
+                        elif args.subtitles_only:
+                            related_information+=f"{clip_conversation},question_related_information: {info}\n"
+                        else:
+                            related_information+=f"{general_sum},{clip_conversation},question_related_information: {info}\n" 
+                    
+                
+                questions.append(question)
+                related_information_list.append(related_information)
+                related_context_keys.append(related_information)
+                related_context_keys_list_new.append(related_context_keys)
+                if len(questions)< args.batch_size:
+                    continue
+                setup_seeds(seed)
+                if args.use_chatgpt :
+                    batch_pred=self.inference_RAG_chatGPT(questions, related_information_list)
+                else:
+                    batch_pred=self.inference_RAG(questions, related_information_list)
+                
+                for pred in batch_pred:
+                    total_batch_pred.append(pred)
+                questions=[]
+                related_information_list=[]
+            
+            if len(questions)>0:
+                setup_seeds(seed)
+                if args.use_chatgpt :
+                    batch_pred=self.inference_RAG_chatGPT(questions, related_information_list)
+                else:
+                    batch_pred=self.inference_RAG(questions, related_information_list)
+                for pred in batch_pred:
+                    total_batch_pred.append(pred)
+            return total_batch_pred,related_context_keys_list_new
+    
+    def define_save_name(self):
+        save_name="subtitles" if args.index_subtitles_together else "no_subtitles"
+        save_name+="_clips_for_info" if args.use_clips_for_info else ""
+        save_name+="_chatgpt" if args.use_chatgpt else ""
+        save_name+="_vision_only" if args.vision_only else ""
+        save_name+="_model_summary_only" if args.model_summary_only else ""
+        save_name+="_subtitles_only" if args.subtitles_only else ""
+        save_name+="_choices_for_info" if args.use_choices_for_info else ""
+        save_name+="_unknown" if args.add_unknown else ""
+        save_name+="_info_only" if args.info_only else ""
+        print("save_name",save_name)
+        return save_name
+    def eval_movie_qa(self):
+        ## Movie QA evaluation
+        full_questions_result=[]
+        movie_number=0
+        start=args.start
+        end=args.end
+        for movie in tqdm(self.movies_dict.keys()):
+            # if the movie has no answer, skip it
+            if self.movies_dict[movie][0]['answer'] is None:
+                continue
+            if args.start <=movie_number < args.end:
+                save_name=self.define_save_name()
+                save_dir=f"new_workspace/results/movie_qa/{args.exp_name}/{save_name}_{args.neighbours}_neighbours"
+                if os.path.exists( f"{save_dir}/{movie}.json" ):
+                    print(f"Movie {movie} already processed")
+                    with open(f"{save_dir}/{movie}.json", 'r') as f:
+                        pred_json = json.load(f)
+                    full_questions_result.extend(pred_json)
+                    continue
+                use_subtitles_while_generating_summary=not args.vision_only
+                information_RAG_path,embedding_path=self.movie_inference(movie,use_subtitles_while_generating_summary)
+                external_memory=MemoryIndex(args.neighbours, use_openai=args.use_openai_embedding)
+                if os.path.exists(embedding_path):
+                    external_memory.load_embeddings_from_pkl(embedding_path)
+                else:
+                    external_memory.load_documents_from_json(information_RAG_path,emdedding_path=embedding_path)
+                
+                os.makedirs(save_dir, exist_ok=True)
+                pred_json=[]
+                batch_questions=[]
+                for qa in tqdm(self.movies_dict[movie]):
+                    batch_questions.append(qa)
+                    if len(batch_questions)<args.batch_size:
+                        continue
+                    model_ans,related_text=self.answer_movie_questions_RAG(batch_questions,external_memory)
+                    for qa,ans,related_info in zip(batch_questions,model_ans,related_text):
+                        qa.update({'pred':ans})
+                        qa.update({'related_info':related_info})
+                        pred_json.append(qa)
+                    batch_questions=[]
+                if len(batch_questions)>0:
+                    model_ans,related_text=self.answer_movie_questions_RAG(batch_questions,external_memory)
+                    for qa,ans,related_info in zip(batch_questions,model_ans,related_text):
+                        qa.update({'pred':ans})
+                        qa.update({'related_info':related_info})
+                        pred_json.append(qa)
+                full_questions_result.extend(pred_json)
+                with open(f"{save_dir}/{movie}.json", 'w') as fp:
+                    json.dump(pred_json, fp)
+                print(f"Movie {movie} prediction saved to {save_dir}/{movie}_pred_{args.neighbours}.json")
+            movie_number+=1
+        with open(f"{save_dir}/full_pred_s{start}_end{end}.json", 'w') as fp:
+            json.dump(full_questions_result, fp)
+
+args=get_arguments()
+
+def setup_seeds(seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    cudnn.benchmark = False
+    cudnn.deterministic = True
+
+import yaml 
+with open('test_configs/llama2_test_config.yaml') as file:
+    config = yaml.load(file, Loader=yaml.FullLoader)
+seed=config['run']['seed']
+print("seed",seed)
+
+if __name__ == "__main__":
+    setup_seeds(seed)
+    movie_qa_eval=MovieQAEval(args)
+    movie_qa_eval.eval_movie_qa()

evaluation/eval_goldfish_tvqa_long.py

@@ -0,0 +1,535 @@
+import sys 
+import os
+project_dir = os.getcwd()
+sys.path.append(project_dir)
+import json 
+from tqdm import tqdm
+from goldfish_lv import GoldFish_LV,split_subtitles,time_to_seconds
+import argparse
+import json
+import argparse
+import torch
+import re
+from tqdm import tqdm
+from PIL import Image
+# from openai import OpenAI
+from index import MemoryIndex  
+import pysrt
+import chardet
+import torch
+import random
+import numpy as np
+import torch.backends.cudnn as cudnn
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError('Boolean value expected.')
+
+def get_arguments():
+    parser = argparse.ArgumentParser(description="Inference parameters")
+    parser.add_argument("--neighbours", type=int, default=-1)
+    parser.add_argument("--name", type=str,default="ckpt_92",help="name of the experiment")
+    parser.add_argument("--exp_name", type=str,default="",help="name of the experiment")
+    parser.add_argument("--add_unknown", action='store_true')
+    parser.add_argument("--use_chatgpt", action='store_true')
+    parser.add_argument("--use_choices_for_info", action='store_true')
+    parser.add_argument("--use_gt_information", action='store_true')
+    parser.add_argument("--inference_text", action='store_true')
+    parser.add_argument("--use_gt_information_with_distraction", action='store_true')
+    parser.add_argument("--num_distraction", type=int, default=2)
+    parser.add_argument("--add_confidance_score", action='store_true')
+    parser.add_argument("--use_original_video", action='store_true')
+    parser.add_argument("--use_video_embedding", action='store_true')
+    parser.add_argument("--use_clips_for_info", action='store_true')
+    parser.add_argument("--use_GT_video", action='store_true')
+    parser.add_argument("--use_gt_summary", action='store_true')
+    parser.add_argument("--index_subtitles_together", action='store_true')
+    
+    parser.add_argument("--ask_the_question_early", action='store_true')
+    parser.add_argument("--clip_in_ask_early", action='store_true')
+    parser.add_argument("--use_coherent_description", action='store_true')
+    
+    parser.add_argument("--start", default=0, type=int)
+    parser.add_argument("--end", default=100000, type=int)
+    
+    parser.add_argument("--vision_only", action='store_true')
+    parser.add_argument("--model_summary_only", action='store_true')
+    parser.add_argument("--subtitles_only", action='store_true')
+    parser.add_argument("--subtitles_only_after_retrieval", action='store_true')
+    parser.add_argument("--info_only", action='store_true')
+    
+    parser.add_argument("--cfg-path", default="test_configs/llama2_test_config.yaml")
+    parser.add_argument("--ckpt", type=str, default="checkpoints/video_llama_checkpoint_last.pth")
+    parser.add_argument("--add_subtitles", action='store_true')
+    parser.add_argument("--eval_opt", type=str, default='all')
+    parser.add_argument("--max_new_tokens", type=int, default=300)
+    parser.add_argument("--batch_size", type=int, default=8)
+    parser.add_argument("--lora_r", type=int, default=64)
+    parser.add_argument("--lora_alpha", type=int, default=16)
+    parser.add_argument("--video_path", type=str, help="path to the video")
+    parser.add_argument("--use_openai_embedding",type=str2bool, default=False)
+    parser.add_argument("--annotation_path", type=str, help="path to the annotation file")
+    parser.add_argument("--videos_frames", type=str, help="path to the dataset extracted frames")
+    parser.add_argument("--tvqa_json_subtitles", type=str, help="path to the tvqa json subtitles")
+    parser.add_argument("--tvqa_clips_subtitles", type=str, help="path to the tvqa json")  
+    parser.add_argument("--options", nargs="+")
+    return parser.parse_args()
+   
+def clean_text(subtitles_text):
+    # Remove unwanted characters except for letters, digits, and single quotes
+    subtitles_text = re.sub(r'[^a-zA-Z0-9\s\']', '', subtitles_text)
+    # Replace multiple spaces with a single space
+    subtitles_text = re.sub(r'\s+', ' ', subtitles_text)
+    return subtitles_text.strip()
+                     
+class TVQAEVAL (GoldFish_LV):
+    def __init__(self, args: argparse.Namespace) -> None:
+        super().__init__(args)
+        self.tv_shows_mapping={"Grey's Anatomy":"grey_frames", 'How I Met You Mother':"met_frames", 'Friends':"friends_frames", 'The Big Bang Theory':"bbt_frames", 'House M.D.':"house_frames", 'Castle':"castle_frames"} 
+        self.save_long_videos_path = f"new_workspace/clips_summary/tvqa"
+        if args.use_openai_embedding:
+            self.save_embedding_path = f"new_workspace/open_ai_embedding/tvqa"
+        else:
+            self.save_embedding_path = f"new_workspace/embedding/tvqa"
+        os.makedirs(self.save_long_videos_path, exist_ok=True)
+        self.max_sub_len=400
+        self.max_num_images=45
+        self.fps=3
+        with open(args.tvqa_json_subtitles) as f:
+            self.subtitles_list=json.load(f)
+        self.subtitles={}
+        for sub in self.subtitles_list:
+            self.subtitles[sub["vid_name"]]=sub["sub"]
+     
+    def _get_TVs_data(self):
+        json_file_path=args.annotation_path
+        frames_path=args.videos_frames
+        subtitle_path=args.tvqa_clips_subtitles
+        with open (json_file_path) as f:
+            tv_shows_data=json.load(f)
+        return tv_shows_data,frames_path,subtitle_path
+    def _get_shows_subtitles(self,clip_subtitles_path):
+        try :
+            with open(clip_subtitles_path, 'rb') as f:
+                result = chardet.detect(f.read())
+            clip_subtitles = pysrt.open(clip_subtitles_path, encoding=result['encoding'])
+            return clip_subtitles
+        except:
+            print("No subtitles found")
+            return []
+    def episode_inference(self,clips,folder_name,use_subtitles):
+        max_caption_index = 0
+        max_subtitle_index = 0
+        preds={}
+        important_data = {}
+        videos_summaries=[]
+        batch_size=args.batch_size
+        batch_images=[]
+        batch_instructions=[]
+        conversations=[]
+        clips_names=[]
+        for clip_name in tqdm(clips,desc="Inference Episode clips"):
+            conversation=""
+            try:
+                for subtitle in self.subtitles[clip_name]:
+                    conversation+=subtitle['text']+" "
+            except:
+                pass
+            conversations.append(clean_text(conversation))               
+            images,img_placeholder=self.prepare_input_images(clip_name,folder_name,use_subtitles)
+            instruction = img_placeholder + '\n' + self.summary_instruction
+            batch_images.append(images)
+            batch_instructions.append(instruction)
+            clips_names.append(clip_name)
+            if len(batch_images) < batch_size:
+                continue
+            batch_images = torch.stack(batch_images)
+            batch_pred=self.run_images(batch_images,batch_instructions)
+            for i,pred in enumerate(batch_pred):
+                max_caption_index += 1
+                videos_summaries.append(pred)
+                if args.use_coherent_description:
+                    preds[f'caption_{max_caption_index}__{clips_names[i]}'] = f"model_summary :{pred}\nVideo conversation :{conversations[i]}"
+                else:
+                    if args.index_subtitles_together and use_subtitles:
+                        if conversations[i] != "":
+                            max_subtitle_index+=1
+                            important_data.update({f"subtitle_{max_subtitle_index}__{clips_names[i]}": conversations[i]})   
+                    preds[f'caption_{max_caption_index}__{clips_names[i]}'] = pred
+            
+            batch_images=[]
+            batch_instructions=[]
+            clips_names=[]
+            conversations=[]
+        # run inference for the last batch
+        if len(batch_images)>0:
+            batch_images = torch.stack(batch_images)
+            batch_pred=self.run_images(batch_images,batch_instructions)
+            for i,pred in enumerate(batch_pred):
+                max_caption_index += 1
+                videos_summaries.append(pred)
+                if args.use_coherent_description:
+                    preds[f'caption_{max_caption_index}__{clips_names[i]}'] = f"model_summary :{pred}\nVideo conversation :{conversations[i]}"
+                else:
+                    if args.index_subtitles_together and use_subtitles:
+                        if conversations[i] != "":
+                            max_subtitle_index+=1
+                            important_data.update({f"subtitle_{max_subtitle_index}__{clips_names[i]}": conversations[i]})   
+                    preds[f'caption_{max_caption_index}__{clips_names[i]}'] = pred
+            batch_images=[]
+            batch_instructions=[]
+            clips_names=[]
+        return preds,important_data  
+    
+    def episode_inference_only_subtitles(self,clips,tv_images_path,subtitle_path):
+        max_subtitle_index = 0
+        important_data = {}
+        for c_name in tqdm(clips,desc="Inference Episode clips"):
+            clip_subtitles_path=os.path.join(subtitle_path,c_name+".srt")
+            clip_subtitles=self._get_shows_subtitles(clip_subtitles_path)
+            conversation=""
+            if args.index_subtitles_together:
+                if self.subtitles.get(c_name,False):
+                    for subtitle in self.subtitles[c_name]:
+                        conversation+=subtitle['text']+" "
+                    conversation=clean_text(conversation)
+                    if conversation != "":
+                        max_subtitle_index+=1
+                        important_data.update({f"subtitle_{max_subtitle_index}__{c_name}": conversation})
+        return important_data 
+    def prepare_input_images(self,clip_name,folder_name,use_subtitles):
+        tv_shows_data,frames_path,subtitle_path=self._get_TVs_data()
+        tv_images_path =os.path.join(frames_path,folder_name)
+        clip_path=os.path.join(tv_images_path,clip_name)
+        total_frames=len(os.listdir(clip_path))
+        sampling_interval=int(total_frames//self.max_num_images)
+        if sampling_interval==0:
+            sampling_interval=1
+        images=[]
+        img_placeholder = ""
+        video_frames_path = os.path.join(frames_path,folder_name,clip_name)
+        total_num_frames=len(os.listdir(video_frames_path))
+        sampling_interval = round(total_num_frames / self.max_num_images)
+        if sampling_interval == 0:
+            sampling_interval = 1
+        subtitle_text_in_interval = ""
+        history_subtitles = {}
+        number_of_sub_words=0
+        for i,frame in enumerate(sorted(os.listdir(video_frames_path))):
+            # Find the corresponding subtitle for the frame and combine the interval subtitles into one subtitle
+            # we choose 1 frame for every 2 seconds,so we need to combine the subtitles in the interval of 2 seconds
+            if self.subtitles.get(clip_name,False) and use_subtitles:
+                for subtitle in self.subtitles[clip_name]:
+                    if (subtitle['start'] <= (i / self.fps) <= subtitle['end']) and subtitle['text'] not in subtitle_text_in_interval:
+                        if not history_subtitles.get(subtitle['text'],False):
+                            subtitle_text_in_interval+=subtitle['text']+" "
+                        history_subtitles[subtitle['text']]=True
+                        break
+            if i % sampling_interval == 0:
+                frame = Image.open(os.path.join(video_frames_path,frame)).convert("RGB") 
+                frame = self.vis_processor(frame)
+                images.append(frame)
+                img_placeholder += '<Img><ImageHere>'
+                if number_of_sub_words<self.max_sub_len and use_subtitles:
+                    if subtitle_text_in_interval != "":
+                        subtitle_text_in_interval=clean_text(subtitle_text_in_interval)
+                        img_placeholder+=f'<Cap>{subtitle_text_in_interval}'
+                        number_of_sub_words+=len(subtitle_text_in_interval.split(' '))
+                        subtitle_text_in_interval = ""
+            if len(images) >= self.max_num_images:
+                break
+        if len(images) ==0:
+            print("Video not found",video_frames_path)
+            
+        if 0 <len(images) < self.max_num_images:
+            last_item = images[-1]
+            while len(images) < self.max_num_images:
+                images.append(last_item)
+                img_placeholder += '<Img><ImageHere>'
+        images = torch.stack(images)
+        return images,img_placeholder
+    def clip_inference(self,clips_name,folders_name,prompts):
+        setup_seeds(seed)
+        images_batch, instructions_batch = [], []
+        for clip_name,folder_name, prompt in zip(clips_name,folders_name, prompts):
+            images,img_placeholder=self.prepare_input_images(clip_name,folder_name,use_subtitles=not args.vision_only)
+            instruction = img_placeholder + '\n' + prompt
+            images_batch.append(images)
+            instructions_batch.append(instruction)
+        # run inference for the batch
+        images_batch=torch.stack(images_batch)
+        batch_pred=self.run_images(images_batch,instructions_batch)
+        return batch_pred
+    def prepare_prompt(self,qa):
+        prompt=qa["q"]+" \n\n As you watched in this video Choose ONE suitable answer from these mutiple choices \n"
+        for i,choice in enumerate(["a0","a1","a2","a3","a4"]):
+            prompt+=f"option {i}: {qa[choice]} \n"
+        if args.add_unknown and args.add_confidance_score:
+            # Add unknown option
+            prompt+=f"option 5: Can't answer based on the provided information\n"
+            prompt+="\n Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 5 INCLUSIVE and aslo output a CONFIDANCE SCORE FROM 0 TO 5 representing how confident you are with your answer where 0 is the least confident and 5 is the most confident"
+        elif args.add_unknown:
+            prompt+=f"option 5: Can't answer based on the provided information\n"
+            prompt+="\n Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 5 INCLUSIVE"
+        elif args.add_confidance_score:
+            prompt+="\n Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 4 INCLUSIVE and aslo output a CONFIDANCE SCORE FROM 0 TO 5 representing how confident you are with your answer where 0 is the least confident and 5 is the most confident"
+        else:
+            prompt+="\n Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 4 INCLUSIVE"
+        return prompt
+    def get_most_related_clips(self,qa,related_context_keys):
+        if args.use_gt_information:
+            most_related_clips=[qa['vid_name']]
+        elif args.use_gt_information_with_distraction:
+            most_related_clips=[qa['vid_name']]
+            for context_key in related_context_keys:
+                if len(context_key.split('__'))>1:
+                    most_related_clips.append(context_key.split('__')[1])
+                if len(most_related_clips)==args.num_distraction+1:
+                    break
+        else:
+            most_related_clips=[]
+            for context_key in related_context_keys:
+                if len(context_key.split('__'))>1:
+                    most_related_clips.append(context_key.split('__')[1])
+                if len(most_related_clips)==args.neighbours:
+                    break
+        assert len(most_related_clips)!=0, f"No related clips found {related_context_keys}"
+        return most_related_clips
+    def use_clips_for_info(self,qa_list,related_context_keys_list,external_memory):
+        total_batch_pred=[]
+        questions=[]
+        related_information_list=[]
+        related_context_keys_list_new=[]
+        for qa,related_context_keys in zip(qa_list,related_context_keys_list):
+            most_related_clips=self.get_most_related_clips(qa,related_context_keys)
+            folder_name=self.tv_shows_mapping[qa['show_name']]
+            question=qa['q']+ "\nand these are the choices :\n"
+            for i,choice in enumerate(["a0","a1","a2","a3","a4"]):
+                question+=f"option {i}: {qa[choice]} \n"
+            if args.add_unknown:    
+                question+= "option 5: Can't answer based on the provided information\n"
+                question+="\n Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 5 INCLUSIVE"
+            else:
+                question+="\n Your output should be THE NUMBER OF THE CORRECT ANSWER FROM THE CHOICES FROM 0 TO 4 INCLUSIVE"
+            if args.use_choices_for_info:
+                # prompt=self.prepare_prompt(qa)
+                # prompt+=" and also provide an EXPLAINATION for your answer and If you don't know the answer, say that you don't know.\n\n"
+                prompt=f"From this video extract the related information to This multichioce question and provide an explaination for your answer and If you don't know the answer, say 'I DON'T KNOW' as option 5 because maybe the questoin is not related to the video content.\n the question is :\n {question}\n your answer :"
+
+            else:
+                prompt=f"As you watched in this video answer this {qa['q']}\n\n and also provide an EXPLAINATION for your answer and If you don't know the answer, say that you don't know.\n\n"
+            all_info=self.clip_inference(most_related_clips,[folder_name]*len(most_related_clips),[prompt]*len(most_related_clips))
+            # concatinate all the information together
+            related_information=""
+            for info,clip_name in zip(all_info,most_related_clips):
+                clip_conversation=""
+                general_sum=""
+                for key in external_memory.documents.keys():
+                    if clip_name in key and 'caption' in key:
+                        general_sum="Clip Summary: "+external_memory.documents[key]
+                    if clip_name in key and 'subtitle' in key:
+                        clip_conversation="Clip Subtitles: "+external_memory.documents[key]
+                
+                if args.use_coherent_description:
+                    related_information+=f"question_related_information: {info},{general_sum}\n"
+                else:
+                    # related_information+=f"{general_sum},{clip_conversation},question_related_information: {info}\n"  
+                    # related_information+=f"question_related_information: {info},{clip_conversation}\n"
+                    if args.model_summary_only:
+                            related_information+=f"{general_sum},question_related_information: {info}\n"
+                    elif args.info_only:
+                        related_information+=f"question_related_information: {info}\n"
+                    elif args.subtitles_only:
+                        related_information+=f"{clip_conversation},question_related_information: {info}\n"
+                    elif args.subtitles_only_after_retrieval:
+                        related_information+=f"{clip_conversation},question_related_information: {info}\n"
+                    else:
+                        related_information+=f"{general_sum},{clip_conversation},question_related_information: {info}\n" 
+                    
+            questions.append(question)
+            related_information_list.append(related_information)
+            related_context_keys.append(related_information)
+            related_context_keys_list_new.append(related_context_keys)
+            if len(questions)< args.batch_size:
+                continue 
+            setup_seeds(seed)
+            if args.use_chatgpt :
+                batch_pred=self.inference_RAG_chatGPT(questions, related_information_list)
+            else:
+                batch_pred=self.inference_RAG(questions, related_information_list)
+            
+            for pred in batch_pred:
+                total_batch_pred.append(pred)
+            questions=[]
+            related_information_list=[]
+        
+        if len(questions)>0:
+            setup_seeds(seed)
+            if args.use_chatgpt :
+                batch_pred=self.inference_RAG_chatGPT(questions, related_information_list)
+            else:
+                batch_pred=self.inference_RAG(questions, related_information_list)
+            for pred in batch_pred:
+                total_batch_pred.append(pred)
+        return total_batch_pred,related_context_keys_list_new   
+    def answer_TV_questions_RAG(self,qa_list,external_memory,episode_clips,episode_name):
+        related_context_keys_list,related_context_documents_list=[],[]
+        setup_seeds(seed)
+        for qa in qa_list:
+            question_choices=qa['q']+ "\n and these are the options for the question\n\n"
+            for i,choice in enumerate(["a0","a1","a2","a3","a4"]):
+                question_choices+=f"option {i}: {qa[choice]} \n\n"
+            related_context_documents,related_context_keys = external_memory.search_by_similarity(question_choices)
+            
+            related_context_documents_list.append(related_context_documents)
+            related_context_keys_list.append(related_context_keys) 
+
+        if args.use_clips_for_info:
+            batch_pred,related_context_keys_list=self.use_clips_for_info(qa_list,related_context_keys_list,external_memory)
+        else:
+            prompts=[]
+            related_context_documents_text_list=[]
+            for qa,related_context_documents,related_context_keys in zip(qa_list,related_context_documents_list,related_context_keys_list): 
+
+                related_information=""
+                most_related_clips=self.get_most_related_clips(qa,related_context_keys)
+                for clip_name in most_related_clips:
+                    clip_conversation=""
+                    general_sum=""
+                    for key in external_memory.documents.keys():
+                        if clip_name in key and 'caption' in key:
+                            general_sum="Clip Summary: "+external_memory.documents[key]
+                        if clip_name in key and 'subtitle' in key:
+                            clip_conversation="Clip Subtitles: "+external_memory.documents[key]
+                    # related_information+=f"{general_sum},{clip_conversation}\n"
+                    if args.use_coherent_description:
+                        related_information+=f"{general_sum}\n"
+                    else:
+                        if args.model_summary_only:
+                            related_information+=f"{general_sum}\n"
+                        elif args.subtitles_only:
+                            related_information+=f"{clip_conversation}\n"
+                        else:
+                            related_information+=f"{general_sum},{clip_conversation}\n"
+                    
+                prompt=self.prepare_prompt(qa)
+                prompts.append(prompt)
+                related_context_documents_text_list.append(related_information)
+                
+            setup_seeds(seed)
+            if args.use_chatgpt:
+                batch_pred=self.inference_RAG_chatGPT(prompts, related_context_documents_text_list)
+            else:
+                batch_pred=self.inference_RAG(prompts, related_context_documents_text_list)
+        return batch_pred ,related_context_keys_list 
+    def answer_episode_questions(self,questions,information_RAG_path,embedding_path,episode_clips):
+        external_memory=MemoryIndex(args.neighbours, use_openai=args.use_openai_embedding)
+        if os.path.exists(embedding_path):
+            external_memory.load_embeddings_from_pkl(embedding_path)
+        else:
+            external_memory.load_documents_from_json(information_RAG_path,embedding_path)    
+        episode_name=information_RAG_path.split('/')[-1].split('.')[0]
+        pred_json=[]
+        batch_questions=[]
+        for qa in tqdm(questions,desc="Answering questions"):
+            batch_questions.append(qa)
+            if len(batch_questions)<args.batch_size:
+                continue
+            batch_pred,batch_related_context_keys = self.answer_TV_questions_RAG(batch_questions,external_memory,episode_clips,episode_name)
+            for pred,related_context_keys,qa in zip(batch_pred,batch_related_context_keys,batch_questions):
+                qa['pred']=pred
+                qa['related_context_keys']=related_context_keys
+                pred_json.append(qa)
+            batch_questions=[]
+        if len(batch_questions)>0:
+            batch_pred,batch_related_context_keys = self.answer_TV_questions_RAG(batch_questions,external_memory,episode_clips,episode_name)
+            for pred,related_context_keys,qa in zip(batch_pred,batch_related_context_keys,batch_questions):
+                qa['pred']=pred
+                qa['related_context_keys']=related_context_keys
+                pred_json.append(qa)
+        return pred_json
+             
+    def eval_tv_shows(self,):
+        tv_shows_data,frames_path,subtitle_path=self._get_TVs_data()
+        full_questions_result=[]
+        number_of_episodes=0
+        start=args.start
+        end=args.end
+        for show in tqdm(tv_shows_data,desc="Inference TV shows"):
+            for season in tqdm(tv_shows_data[show],desc=f"Inference {show} seasons"):
+                for episode in tqdm(tv_shows_data[show][season],desc=f"Inference {show} {season} episodes"):
+                    # Generate clips summary and store the important data (summary and subtitles) in json file
+                    if start<=number_of_episodes<end: 
+                        folder_name=self.tv_shows_mapping[show]
+                        tv_images_path =os.path.join(frames_path,folder_name)
+                        os.makedirs(self.save_long_videos_path, exist_ok=True)
+                        save_name="" if args.index_subtitles_together else "no_subtitles_"
+                        save_name="subtitles_only" if args.subtitles_only else save_name
+                        save_name="use_coherent_description" if args.use_coherent_description else save_name
+                        file_path=os.path.join(self.save_long_videos_path,save_name+folder_name+"_"+season+"_"+episode+".json")
+                        embedding_path=os.path.join(self.save_embedding_path,save_name+folder_name+"_"+season+"_"+episode+".pkl")
+                        # options don't require rerunning the inference
+                        save_name+="_unknown" if args.add_unknown else ""
+                        save_name+="_clips_for_info" if args.use_clips_for_info else ""
+                        save_name+="_chatgpt" if args.use_chatgpt else ""
+                        save_name+="_choices_for_info" if args.use_choices_for_info else ""
+                        save_name+="_info_only" if args.info_only else ""
+                        save_name+="_subtitles_only" if args.subtitles_only else ""
+                        save_name+="_subtitles_only_after_retrieval" if args.subtitles_only_after_retrieval else ""
+                        if os.path.exists(file_path):
+                            with open(file_path, 'r') as file:
+                                important_data = json.load(file)
+                            print("Already processed")
+                        else:
+                            episode_clips=tv_shows_data[show][season][episode]['clips']
+                            if args.subtitles_only :
+                                important_data=self.episode_inference_only_subtitles(episode_clips,tv_images_path,subtitle_path)
+                            else:
+                                preds,important_data=self.episode_inference(episode_clips,folder_name,use_subtitles=not args.vision_only)
+                                important_data.update(preds)
+                            # if not args.subtitles_only :
+                            #     summary = self.compine_summaries(important_data)
+                            #     preds['summary'] = summary
+                            #     important_data["summary"]=summary
+                            with open(file_path, 'w') as file:
+                                json.dump(important_data, file, indent=4)
+                        # Answer questions 
+                        questions=tv_shows_data[show][season][episode]['questions']
+                        episode_clips=tv_shows_data[show][season][episode]['clips']
+                        episode_name=file_path.split('/')[-1].split('.')[0]
+                        pred_json=self.answer_episode_questions(questions,file_path,embedding_path,episode_clips)
+                        full_questions_result.extend(pred_json)
+                        save_dir=f"new_workspace/results/tvqa/{args.exp_name}/{save_name}_{args.neighbours}_neighbours"
+                        os.makedirs(save_dir, exist_ok=True)
+                        with open(f"{save_dir}/{episode_name}.json", 'w') as fp:
+                            json.dump(pred_json, fp)
+                        print(f"Episode {episode_name} prediction saved to {save_dir}/{episode_name}_pred_{args.neighbours}.json")
+                    number_of_episodes+=1
+        with open(f"{save_dir}/full_pred_{start}_{end}.json", 'w') as fp:
+            json.dump(full_questions_result, fp)
+        print(f"TV shows prediction saved to {save_dir}/full_pred_{start}{end}.json")
+args=get_arguments()
+
+def setup_seeds(seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    cudnn.benchmark = False
+    cudnn.deterministic = True
+
+import yaml 
+with open('test_configs/llama2_test_config.yaml') as file:
+    config = yaml.load(file, Loader=yaml.FullLoader)
+seed=config['run']['seed']
+print("seed",seed)
+
+if __name__ == "__main__":
+    setup_seeds(seed)
+    tvqa_eval=TVQAEVAL(args)
+    tvqa_eval.eval_tv_shows()

evaluation/eval_minigpt4_video.py

@@ -0,0 +1,201 @@
+import os
+import json
+from tqdm import tqdm
+import sys 
+project_dir = os.getcwd()
+sys.path.append(project_dir)
+from torch.utils.data import DataLoader
+from minigpt4.common.eval_utils import prepare_texts, init_model, eval_parser
+from minigpt4.conversation.conversation import CONV_VISION
+from minigpt4.processors.blip_processors import Blip2ImageTrainProcessor,BlipCaptionProcessor
+from minigpt4.datasets.datasets.video_datasets import VideoChatGPTEvalDataset,VideoChatGPTEval_consistancy,Video_validation_Dataset,TVQAEVAL
+
+parser = eval_parser()
+parser.add_argument("--dataset", type=str, default='msvd', help="dataset to evaluate")
+parser.add_argument("--add_subtitles",action='store_true',help="whether to add subtitles to the video")
+parser.add_argument("--name", type=str, default='test', help="evaluation name")
+parser.add_argument("--videos_path", type=str, default='videos path', help="path to videos")
+parser.add_argument("--subtitles_path", type=str, default='subtitles path', help="path to subtitles")
+parser.add_argument("--ann_path", type=str, default='annotations path', help="path to annotations")
+
+parser.add_argument("--batch_size", type=int, default=1, help="batch size")
+parser.add_argument("--start", type=int, default=0, help="start from video number")
+parser.add_argument("--end", type=int, default=10000000, help="end at video number")
+args = parser.parse_args()
+
+print(args.ckpt)
+print(args.name)
+print(args.cfg_path)
+if "test_configs/mistral_test_config.yaml" == args.cfg_path: 
+    llm_name="mistral"
+else:   
+    llm_name="llama2"
+print("using captions",args.add_subtitles)
+model, vis_processor,whisper_gpu_id,minigpt4_gpu_id,answer_module_gpu_id = init_model(args)
+conv_temp = CONV_VISION.copy()
+conv_temp.system = ""
+if args.dataset == 'video_chatgpt_generic':
+    ann_path=args.ann_path
+    videos_path= args.videos_path
+    subtitles_path=args.subtitles_path
+    annotations_keys=['Q','A','video_name']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys, add_subtitles=args.add_subtitles,llm_name=llm_name)
+elif args.dataset == 'video_chatgpt_temporal':
+    ann_path=args.ann_path
+    videos_path= args.videos_path
+    subtitles_path=args.subtitles_path
+    annotations_keys=['Q','A','video_name']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys, add_subtitles=args.add_subtitles,llm_name=llm_name)
+elif args.dataset == 'video_chatgpt_consistency':
+    ann_path=args.ann_path
+    videos_path= args.videos_path
+    subtitles_path=args.subtitles_path
+    annotations_keys=[['Q1','Q2'],'A','video_name']
+    data = VideoChatGPTEval_consistancy(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys, add_subtitles=args.add_subtitles,llm_name=llm_name)
+    
+elif args.dataset == 'msrvtt':
+    ann_path=args.ann_path
+    videos_path= args.videos_path
+    subtitles_path=args.subtitles_path
+    annotations_keys=['question','answer','video_id']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys, add_subtitles=args.add_subtitles,llm_name=llm_name)
+
+elif args.dataset == 'msvd':
+    ann_path=args.ann_path
+    videos_path= args.videos_path
+    subtitles_path="" # no subtitles for msvd as these videos don't have audio 
+    annotations_keys=['question','answer','video_id']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys, add_subtitles=args.add_subtitles,llm_name=llm_name)
+elif args.dataset == 'activitynet':
+    ann_path=args.ann_path
+    videos_path= args.videos_path
+    subtitles_path=args.subtitles_path
+    annotations_keys=['question','answer','video_id']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys, add_subtitles=args.add_subtitles,llm_name=llm_name)
+elif args.dataset == 'tgif':
+    ann_path="datasets/evaluation_datasets/tgif/Test_frameqa_question.json"
+    videos_path= args.videos_path
+    subtitles_path="" # no subtitles for TGIF as these videos don't have audio
+    annotations_keys=['question','answer','gif_name']
+    data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys, add_subtitles=False,llm_name=llm_name)
+elif args.dataset == 'tvqa':
+    # TVQA dataset
+    ann_path="datasets/evaluation_datasets/tvqa_short/tvqa_val.json"
+    videos_path= args.videos_path
+    subtitles_path=args.subtitles_path
+    data = TVQAEVAL(vis_processor, videos_path, ann_path,subtitles_path,add_subtitles=args.add_subtitles,llm_name=llm_name)
+
+eval_dataloader = DataLoader(data, batch_size=args.batch_size, shuffle=False)
+
+minigpt4_predict = []
+sub="subtitles" if args.add_subtitles else "no_subtitles"
+if args.start == 0 and args.end == 10000000:
+    save_path = f'results/{args.name}_{args.dataset}_{sub}.json'
+else:
+    print("start from video number",args.start)
+    print("end at video number",args.end)
+    save_path = f'results/{args.name}_{args.dataset}_{sub}_{args.start}_{args.end}.json'
+
+os.makedirs("results", exist_ok=True)
+c=0
+pred_result = {}
+gt_result = {}
+if args.dataset == 'video_chatgpt_consistency':
+    for images, texts_1,texts_2, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"):
+        if args.start<= c <args.end :
+            texts_q1 = prepare_texts(texts_1, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            texts_q2 = prepare_texts(texts_2, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            models_answers_q1 = model.generate(images, texts_q1, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            models_answers_q2 = model.generate(images, texts_q2, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            for video_id,model_answer_q1,model_answer_q2, gt_answer,text_q1,text_q2 in zip(videos_ids,models_answers_q1,models_answers_q2, gt_answers,texts_q1,texts_q2):
+                result = dict()
+                result['video_name'] = video_id
+                result['Q1'] = text_q1.split('\n')[-1].replace('[/INST]','')
+                result['Q2'] = text_q2.split('\n')[-1].replace('[/INST]','')
+                result['A'] = gt_answer
+                result['pred1'] = model_answer_q1
+                result['pred2'] = model_answer_q2
+                pred_result[video_id] = [model_answer_q1,model_answer_q2]
+                gt_result[video_id] = [gt_answer]
+                minigpt4_predict.append(result)
+            # save results every 100 videos to avoid losing results
+            if c%100==0:
+                with open(save_path, 'w') as f:
+                    json.dump(minigpt4_predict, f)
+        if c >= args.end :
+            break
+        c+=1
+
+elif args.dataset == 'tvr':
+    for images, texts, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"):
+        if args.start<= c <args.end :
+            texts = prepare_texts(texts, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            models_answers = model.generate(images, texts, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            for video_id,model_answer, gt_answer,text in zip(videos_ids,models_answers, gt_answers,texts):
+                result = dict()
+                result['video_name'] = video_id
+                result['Q'] = text.split('\n')[-1].replace('[/INST]','')
+                result['A'] = gt_answer
+                result['pred'] = model_answer
+                pred_result[video_id] = [model_answer]
+                gt_result[video_id] = [gt_answer]
+                minigpt4_predict.append(result)
+            # save results every 100 videos to avoid losing results
+            if c%100==0:
+                with open(save_path, 'w') as f:
+                    json.dump(minigpt4_predict, f)
+        if c >= args.end :
+            break
+        c+=1
+elif args.dataset == 'ego_schema' or args.dataset == 'tvqa' or args.dataset == 'tvqa_long_videos':
+    for images, texts, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"):
+        if args.start<= c <args.end :
+            texts = prepare_texts(texts, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            models_answers = model.generate(images, texts, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            for video_id,model_answer, gt_answer,text in zip(videos_ids,models_answers, gt_answers,texts):
+                result = dict()
+                result['video_name'] = video_id
+                if args.dataset == 'tvqa_long_videos':
+                    result['Q'] = text.split('\n\n')[1:]
+                else:
+                    result['Q'] = text.split('\n')[1:]
+                result['A'] = gt_answer
+                result['pred'] = model_answer
+                pred_result[video_id] = [model_answer]
+                gt_result[video_id] = [gt_answer]
+                minigpt4_predict.append(result)
+            # save results every 100 videos to avoid losing results
+            if c%100==0:
+                with open(save_path, 'w') as f:
+                    json.dump(minigpt4_predict, f)
+        if c >= args.end :
+            break
+        c+=1
+else:
+    for images, texts, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"):
+        if args.start<= c <args.end :
+            texts = prepare_texts(texts, conv_temp, template='', lengths=lengths)  # warp the texts with conversation template
+            models_answers = model.generate(images, texts, max_new_tokens=args.max_new_tokens, do_sample=False, lengths=lengths,num_beams=1)
+            for video_id,model_answer, gt_answer,text in zip(videos_ids,models_answers, gt_answers,texts):
+                result = dict()
+                result['video_name'] = video_id
+                result['Q'] = text.split('\n')[-1].replace('[/INST]','')
+                result['A'] = gt_answer
+                result['pred'] = model_answer
+                pred_result[video_id] = [model_answer]
+                gt_result[video_id] = [gt_answer]
+                minigpt4_predict.append(result)
+            # save results every 100 videos to avoid losing results
+            if c%100==0:
+                with open(save_path, 'w') as f:
+                    json.dump(minigpt4_predict, f)
+        if c >= args.end :
+            break
+        c+=1
+
+with open(save_path, 'w') as f:
+    json.dump(minigpt4_predict, f)
+print("saved results to",save_path)
+
+
+

evaluation/eval_retrieval_acc_tvqa.py

@@ -0,0 +1,316 @@
+import sys 
+import os
+project_dir = os.getcwd()
+sys.path.append(project_dir)
+import json 
+from tqdm import tqdm
+from goldfish_lv import GoldFish_LV,split_subtitles,time_to_seconds
+import argparse
+import json
+import argparse
+import torch
+import re
+from PIL import Image
+# from openai import OpenAI
+from index import MemoryIndex  
+import torch
+import random
+import numpy as np
+import torch.backends.cudnn as cudnn
+
+def get_arguments():
+    parser = argparse.ArgumentParser(description="Inference parameters")
+    parser.add_argument("--neighbours", type=int, default=-1)
+    parser.add_argument("--name", type=str,default="ckpt_92",help="name of the experiment")
+    parser.add_argument("--exp_name", type=str,default="",help="name of the experiment")
+    parser.add_argument("--add_unknown", action='store_true')
+    parser.add_argument("--use_chatgpt", action='store_true')
+    parser.add_argument("--use_choices_for_info", action='store_true')
+    parser.add_argument("--use_gt_information", action='store_true')
+    parser.add_argument("--inference_text", action='store_true')
+    parser.add_argument("--use_gt_information_with_distraction", action='store_true')
+    parser.add_argument("--num_distraction", type=int, default=2)
+    parser.add_argument("--add_confidance_score", action='store_true')
+    parser.add_argument("--use_original_video", action='store_true')
+    parser.add_argument("--use_video_embedding", action='store_true')
+    parser.add_argument("--use_clips_for_info", action='store_true')
+    parser.add_argument("--use_GT_video", action='store_true')
+    parser.add_argument("--use_gt_summary", action='store_true')
+    
+    parser.add_argument("--ask_the_question_early", action='store_true')
+    parser.add_argument("--clip_in_ask_early", action='store_true')
+    parser.add_argument("--use_coherent_description", action='store_true')
+    
+    parser.add_argument("--start", default=0, type=int)
+    parser.add_argument("--end", default=100000, type=int)
+    
+    parser.add_argument("--vision_only", action='store_true')
+    parser.add_argument("--model_summary_only", action='store_true')
+    parser.add_argument("--subtitles_only", action='store_true')
+    parser.add_argument("--subtitles_only_after_retrieval", action='store_true')
+    parser.add_argument("--info_only", action='store_true')
+    
+    parser.add_argument("--cfg-path", default="test_configs/llama2_test_config.yaml")
+    parser.add_argument("--ckpt", type=str, default="checkpoints/video_llama_checkpoint_last.pth")
+    parser.add_argument("--add_subtitles", action='store_true')
+    parser.add_argument("--eval_opt", type=str, default='all')
+    parser.add_argument("--max_new_tokens", type=int, default=300)
+    parser.add_argument("--batch_size", type=int, default=8)
+    parser.add_argument("--lora_r", type=int, default=64)
+    parser.add_argument("--lora_alpha", type=int, default=16)
+    parser.add_argument("--video_path", type=str, help="path to the video")
+    parser.add_argument("--options", nargs="+")
+    return parser.parse_args()
+   
+def clean_text(subtitles_text):
+    # Remove unwanted characters except for letters, digits, and single quotes
+    subtitles_text = re.sub(r'[^a-zA-Z0-9\s\']', '', subtitles_text)
+    # Replace multiple spaces with a single space
+    subtitles_text = re.sub(r'\s+', ' ', subtitles_text)
+    return subtitles_text.strip()
+                     
+class TVQAEVALRetrieval (GoldFish_LV):
+    def __init__(self, args: argparse.Namespace) -> None:
+        super().__init__(args)
+        self.tv_shows_mapping={"Grey's Anatomy":"grey_frames", 'How I Met You Mother':"met_frames", 'Friends':"friends_frames", 'The Big Bang Theory':"bbt_frames", 'House M.D.':"house_frames", 'Castle':"castle_frames"} 
+        self.save_long_videos_path = f"workspace/results/tv_shows/{args.name}"
+        os.makedirs(self.save_long_videos_path, exist_ok=True)
+        self.max_sub_len=400
+        self.max_num_images=45
+        self.fps=3
+        with open("datasets/evaluation_datasets/goldfish_eval_datasets/tvqa/tvqa_preprocessed_subtitles.json") as f:
+            self.subtitles_list=json.load(f)
+        self.subtitles={}
+        for sub in self.subtitles_list:
+            self.subtitles[sub["vid_name"]]=sub["sub"]
+     
+    def _get_TVs_data(self):
+        json_file_path="datasets/evaluation_datasets/long_video_datasets/tvqa/tvqa_val_edited.json"
+        frames_path="/ibex/project/c2090/datasets/TVR_dataset/videos/video_files/frames_hq/"
+        subtitle_path="/ibex/project/c2090/datasets/TVR_dataset/videos/tvqa_subtitles" 
+        with open (json_file_path) as f:
+            tv_shows_data=json.load(f)
+        return tv_shows_data,frames_path,subtitle_path
+        
+        return vision_questions,subtitle_questions,frames_path
+    def episode_inference(self,video_frames_path,qa,use_subtitles):
+        batch_prepared_images,batch_img_placeholder,gt_clip_numbers=self.prepare_input_images(video_frames_path,qa,use_subtitles,n_clips=10)
+        preds={}
+        batch_instructions=[]
+        batch_images=[]
+        important_data = {}
+        conversations=[]
+        clips_numbers=[]
+        for clip_number,images,img_placeholder in zip(range(len(batch_prepared_images)),batch_prepared_images,batch_img_placeholder):
+            instruction = img_placeholder + '\n' + self.summary_instruction
+            batch_images.append(images)
+            batch_instructions.append(instruction)
+            conv=img_placeholder.replace('<Img><ImageHere>','')
+            conv=conv.replace('<Cap>',' ')
+            conversations.append(conv.strip())
+            clips_numbers.append(clip_number)
+            if len(batch_images) < args.batch_size:
+                continue
+            batch_images = torch.stack(batch_images)
+            setup_seeds(seed)
+            batch_pred=self.run_images(batch_images,batch_instructions)
+            for i,pred in enumerate(batch_pred):
+                if args.use_coherent_description:
+                    preds[f'caption__{clips_numbers[i]}'] = f"model_summary :{pred}\nVideo conversation :{conversations[i]}"
+                else:
+                    if use_subtitles:
+                        if conversations[i] != "":
+                            important_data.update({f"subtitle__{clips_numbers[i]}": conversations[i]})   
+                    preds[f'caption__{clips_numbers[i]}'] = pred
+            
+            batch_images=[]
+            batch_instructions=[]
+            conversations=[]
+            clips_numbers=[]
+        # run inference for the last batch
+        if len(batch_images)>0:
+            batch_images = torch.stack(batch_images)
+            batch_pred=self.run_images(batch_images,batch_instructions)
+            for i,pred in enumerate(batch_pred):
+                if args.use_coherent_description:
+                    preds[f'caption__{clips_numbers[i]}'] = f"model_summary :{pred}\nVideo conversation :{conversations[i]}"
+                else:
+                    if use_subtitles:
+                        if conversations[i] != "":
+                            important_data.update({f"subtitle__{clips_numbers[i]}": conversations[i]})  
+                    preds[f'caption__{clips_numbers[i]}'] = pred
+            batch_images=[]
+            batch_instructions=[]
+            clips_numbers=[]
+        return preds,important_data ,gt_clip_numbers
+    
+    def episode_inference_only_subtitles(self,video_frames_path,qa):
+        use_subtitles=True
+        batch_prepared_images,batch_img_placeholder,gt_clip_numbers=self.prepare_input_images(video_frames_path,qa,use_subtitles,n_clips=10)
+        important_data = {}
+        for clip_number,img_placeholder in enumerate(batch_img_placeholder) :
+            conv=img_placeholder.replace('<Img><ImageHere>','')
+            conv=conv.replace('<Cap>',' ')
+            conversation=conv.strip()
+            conversation=clean_text(conversation)
+            if conversation != "":
+                important_data.update({f"subtitle__{clip_number}": conversation})
+        return important_data ,gt_clip_numbers
+    def prepare_input_images(self,video_frames_path,qa,use_subtitles,n_clips=10):
+        batch_images=[]
+        batch_img_placeholder = []
+        clip_name=video_frames_path.split('/')[-1]
+        images=[]
+        img_placeholders = []
+        gt_clip_numbers = set()
+        gt_start_time=qa['ts'][0]
+        gt_end_time=qa['ts'][1]
+        total_num_frames=len(os.listdir(video_frames_path))
+        subtitle_text_in_interval = ""
+        history_subtitles = {}
+        number_of_sub_words=0
+        # samples_per_clip = total_num_frames // n_clips
+        samples_per_clip=45
+        clip_num=0
+        for i,frame in enumerate(sorted(os.listdir(video_frames_path))):
+            # Find the corresponding subtitle for the frame and combine the interval subtitles into one subtitle
+            # we choose 1 frame for every 2 seconds,so we need to combine the subtitles in the interval of 2 seconds
+            if self.subtitles.get(clip_name,False) and use_subtitles:
+                for subtitle in self.subtitles[clip_name]:
+                    if (subtitle['start'] <= (i / self.fps) <= subtitle['end']) and subtitle['text'] not in subtitle_text_in_interval:
+                        if not history_subtitles.get(subtitle['text'],False):
+                            subtitle_text_in_interval+=subtitle['text']+" "
+                        history_subtitles[subtitle['text']]=True
+                        break
+            if gt_start_time<=(i/self.fps)<= gt_end_time:
+                    gt_clip_numbers.add(clip_num)
+            if i % samples_per_clip == 0 and i != 0:
+                # here we have one clip , let's sample 45 frames from images array
+                sample_value=len(images)//self.max_num_images
+                if sample_value==0:
+                    sample_value=1
+                frames_indices = [i for i in range(0, len(images), sample_value)]
+                samples_images=[]
+                img_placeholder=''
+                for j in frames_indices:
+                    samples_images.append(images[j])
+                    img_placeholder+=img_placeholders[j]
+                    if len(samples_images) >= self.max_num_images:
+                        break
+                if 0 <len(samples_images) < self.max_num_images:
+                    last_item = samples_images[-1]
+                    while len(samples_images) < self.max_num_images:
+                        samples_images.append(last_item)
+                        img_placeholder += '<Img><ImageHere>'
+                samples_images = torch.stack(samples_images)
+                batch_images.append(samples_images)
+                batch_img_placeholder.append(img_placeholder)
+                img_placeholders =[] 
+                images = []
+                clip_num+=1
+                
+            frame = Image.open(os.path.join(video_frames_path,frame)).convert("RGB") 
+            frame = self.vis_processor(frame)
+            images.append(frame)
+            img_placeholder = '<Img><ImageHere>'
+            if number_of_sub_words<self.max_sub_len and use_subtitles:
+                if subtitle_text_in_interval != "":
+                    subtitle_text_in_interval=clean_text(subtitle_text_in_interval)
+                    img_placeholder+=f'<Cap>{subtitle_text_in_interval}'
+                    number_of_sub_words+=len(subtitle_text_in_interval.split(' '))
+                    subtitle_text_in_interval = ""
+            img_placeholders.append(img_placeholder)
+        return batch_images,batch_img_placeholder,list(gt_clip_numbers)
+
+    def test_retrieval(self,indexed_data_path,qa,gt_clip_numbers):  
+        external_memory=MemoryIndex(args.neighbours, use_openai=True)
+        external_memory.load_documents_from_json(indexed_data_path)
+        question=qa['desc']
+        related_context_documents,related_context_keys = external_memory.search_by_similarity(question)
+        print(f"related_context_keys {related_context_keys}")
+        print(f"gt_clip_numbers {gt_clip_numbers}")
+        for key in related_context_keys:
+            clip_idx=int(key.split('__')[-1])
+            if clip_idx in gt_clip_numbers:
+                return True
+        return False
+    
+    def get_ground_truth_clip(self,video_frames_path,qa):
+        gt_clip_numbers = set()
+        gt_start_time=qa['ts'][0]
+        gt_end_time=qa['ts'][1]
+        samples_per_clip=45
+        clip_num=0
+        for i in range(len(os.listdir(video_frames_path))):
+            if gt_start_time<=(i/self.fps)<= gt_end_time:
+                    gt_clip_numbers.add(clip_num)
+            if i % samples_per_clip == 0 and i != 0:
+                clip_num+=1  
+        return list(gt_clip_numbers)
+                
+    def eval_tv_shows(self,):
+        vision_questions,subtitle_questions,frames_path=self._get_TVs_data()
+        number_of_videos=0
+        start=args.start
+        end=args.end
+        if args.exp_name=="vision":
+            questions=vision_questions
+        else:
+            questions=subtitle_questions
+        correct_retrieval=0
+        wrong_retrieval=0
+        for qa in questions:
+            # Generate clips summary and store the important data (summary and subtitles) in json file
+            if start<=number_of_videos<end: 
+                show_name=qa['vid_name'].split('_')[0]
+                if self.tv_shows_mapping.get(show_name,False):
+                    folder_name=self.tv_shows_mapping[show_name]
+                else:
+                    folder_name=self.tv_shows_mapping['bbt']
+                
+                clip_frames_path =os.path.join(frames_path,folder_name,qa['vid_name'])
+                save_name="subtitles_only" if args.subtitles_only else "vision_only" if args.vision_only else "vision_subtitles"
+                indexed_data_path=os.path.join(self.save_long_videos_path,f"{qa['vid_name']}_{args.exp_name}_{save_name}_num_{number_of_videos}.json")
+                if not os.path.exists(indexed_data_path):
+                    if args.subtitles_only :
+                        # TODO
+                        important_data,gt_clip_numbers=self.episode_inference_only_subtitles(clip_frames_path,qa)
+                    else:
+                        preds,important_data ,gt_clip_numbers=self.episode_inference(clip_frames_path,qa,use_subtitles=not args.vision_only)
+                        important_data.update(preds)
+                    with open(indexed_data_path, 'w') as file:
+                        json.dump(important_data, file, indent=4)
+                else:
+                    gt_clip_numbers=self.get_ground_truth_clip(clip_frames_path,qa)
+                retrieval_res=self.test_retrieval(indexed_data_path,qa,gt_clip_numbers)
+                if retrieval_res==True:
+                    correct_retrieval+=1
+                else:
+                    wrong_retrieval+=1
+            number_of_videos+=1
+        
+        save_dir=f"workspace/eval/retrieval/{args.exp_name}_neighbors_{args.neighbours}"
+        save_dir+="_subtitles_only" if args.subtitles_only else "_vision_only" if args.vision_only else "_vision_subtitles"
+        os.makedirs(save_dir,exist_ok=True)
+        with open(f"{save_dir}/s{start}_end{end}.json", 'w') as fp:
+            json.dump({"correct":correct_retrieval,"wrong":wrong_retrieval}, fp)
+args=get_arguments()
+
+def setup_seeds(seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    cudnn.benchmark = False
+    cudnn.deterministic = True
+
+import yaml 
+with open('test_configs/llama2_test_config.yaml') as file:
+    config = yaml.load(file, Loader=yaml.FullLoader)
+seed=config['run']['seed']
+print("seed",seed)
+
+if __name__ == "__main__":
+    setup_seeds(seed)
+    tvqa_eval=TVQAEVALRetrieval(args)
+    tvqa_eval.eval_tv_shows()

evaluation/minigpt4_video_eval/minigpt4_video_evalualtion.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#SBATCH --partition=batch
+#SBATCH --job-name=llama2%j
+#SBATCH --output=llama2%j.out
+#SBATCH --error=llama2%j.err
+#SBATCH --time=0-23:00:00
+#SBATCH --mem=100G
+#SBATCH --gres=gpu:a100:1
+#SBATCH --nodes=1
+## run the application:
+NAME="llama2" # Name of the experiment
+BATCH_SIZE=8 
+CKPT_PATH="checkpoints/video_llama_checkpoint_last.pth" # path to the checkpoint
+
+DATASET="msvd" # available datasets: tvqa, msrvtt, msvd, activitynet,tgif ,video_chatgpt_generic,video_chatgpt_temporal,video_chatgpt_consistency
+# set the paths to the dataset files
+videos_path="" # path to the videos file
+subtitles_path="" # path to the subtitles file
+ann_path="" # path to the annotations file
+
+cfg_path="test_configs/llama2_test_config.yaml" # path to the config file 
+# # if the number of samples are too large you can specify the start and end index to evaluate on several machines
+# pass the start and end index as arguments
+start=$1 # start index
+end=$2 # end index
+# if start and end are not provided, then use the whole dataset
+if [ -z "$start" ]
+then
+      start=0
+fi
+if [ -z "$end" ]
+then
+      end=10000000
+fi
+echo "Start: $start"
+echo "End: $end"
+
+
+# with subtitles
+python evaluation/eval_minigpt4_video.py --dataset $DATASET --batch_size $BATCH_SIZE --name $NAME --videos_path $videos_path --subtitles_path $subtitles_path --ann_path $ann_path --ckpt $CKPT_PATH  --cfg-path=$cfg_path --start $start --end $end --add_subtitles
+
+# without subtitles
+# python evaluation/eval_minigpt4_video.py --dataset $DATASET --batch_size $BATCH_SIZE --name $NAME --videos_path $videos_path --subtitles_path $subtitles_path --ann_path $ann_path --ckpt $CKPT_PATH  --cfg-path=$cfg_path --start $start --end $end 
+

fix_dependencies.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+"""
+修复MiniGPT4-Video缺失依赖
+"""
+
+import subprocess
+import sys
+
+def install_package(package):
+    """安装单个包"""
+    try:
+        print(f"📦 正在安装 {package}...")
+        subprocess.check_call([sys.executable, "-m", "pip", "install", package])
+        print(f"✅ {package} 安装成功")
+        return True
+    except subprocess.CalledProcessError as e:
+        print(f"❌ {package} 安装失败: {e}")
+        return False
+
+def main():
+    """修复缺失依赖"""
+    print("🔧 开始修复MiniGPT4-Video依赖...\n")
+    
+    # 最关键的缺失包
+    critical_packages = [
+        "visual_genome",
+        "nltk",
+        "wandb"
+    ]
+    
+    success_count = 0
+    
+    for package in critical_packages:
+        if install_package(package):
+            success_count += 1
+    
+    print(f"\n📊 修复结果:")
+    print(f"✅ 成功: {success_count}/{len(critical_packages)}")
+    
+    if success_count == len(critical_packages):
+        print("\n🎉 所有关键依赖修复完成！")
+        print("🚀 现在可以重启应用以加载完整功能")
+        print("💡 运行命令: python run_hf.py")
+    else:
+        print("\n⚠️ 部分依赖修复失败")
+        print("💡 尝试手动安装: pip install -r requirements.txt")
+    
+    return success_count == len(critical_packages)
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1) 

goldfish_demo.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import spaces
+import os
+import argparse
+import gradio as gr
+from goldfish_lv import GoldFish_LV 
+from theme import minigptlv_style, custom_css,text_css
+import re
+from huggingface_hub import login, hf_hub_download
+import time
+import moviepy.editor as mp
+from index import MemoryIndex  
+
+
+# hf_token = os.environ.get('HF_TKN')
+# login(token=hf_token)
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError('Boolean value expected.')
+
+def get_arguments():
+    parser = argparse.ArgumentParser(description="Inference parameters")
+    parser.add_argument("--cfg-path", default="test_configs/llama2_test_config.yaml")
+    parser.add_argument("--name", type=str, default='test')
+    parser.add_argument("--ckpt", type=str, default="checkpoints/video_llama_checkpoint_last.pth")
+    parser.add_argument("--add_subtitles", action='store_true')
+    parser.add_argument("--neighbours", type=int, default=3)
+    parser.add_argument("--eval_opt", type=str, default='all')
+    parser.add_argument("--max_new_tokens", type=int, default=512)
+    parser.add_argument("--use_openai_embedding",type=str2bool, default=False)
+    parser.add_argument("--batch_size", type=int, default=2, help="Batch size for short video clips")
+    parser.add_argument("--lora_r", type=int, default=64)
+    parser.add_argument("--lora_alpha", type=int, default=16)
+    parser.add_argument("--video_path", type=str, help="Path to the video file")
+    parser.add_argument("--options", nargs="+")
+    return parser.parse_args()
+
+def download_video(youtube_url, download_finish):
+    if is_youtube_url(youtube_url):
+        processed_video_path = goldfish_obj.process_video_url(youtube_url)
+        download_finish = gr.State(value=True)
+        return processed_video_path, download_finish
+    else:
+        return None, download_finish
+def is_youtube_url(url: str) -> bool:
+    youtube_regex = (
+        r'(https?://)?(www\.)?'
+        '(youtube|youtu|youtube-nocookie)\.(com|be)/'
+        '(watch\?v=|embed/|v/|.+\?v=)?([^&=%\?]{11})'
+    )
+    return bool(re.match(youtube_regex, url))
+
+@spaces.GPU(duration=60*5)
+def gradio_long_inference_video(videos_list,tmp_save_path,subtitle_paths, use_subtitles=True):
+    clips_summary = goldfish_obj.long_inference_video(videos_list,tmp_save_path,subtitle_paths)
+    return clips_summary
+
+@spaces.GPU(duration=60*3)
+def gradio_short_inference_video(video_path, instruction, use_subtitles=True):
+    pred = goldfish_obj.short_video_inference(video_path, instruction, use_subtitles)
+    return pred
+
+@spaces.GPU(duration=60*3)
+def gradio_inference_RAG (instruction,related_information):
+    pred=goldfish_obj.inference_RAG([instruction], [related_information])[0] 
+    return pred
+def inference(video_path, use_subtitles=True, instruction="", number_of_neighbours=3):
+    start_time = time.time()
+    video_name = os.path.splitext(os.path.basename(video_path))[0]
+    goldfish_obj.args.neighbours = number_of_neighbours
+    print(f"Video name: {video_name}")
+    video_duration = mp.VideoFileClip(video_path).duration
+    print(f"Video duration: {video_duration:.2f} seconds")
+    # if the video duration is more than 2 minutes we need to run the long inference 
+    if video_duration > 180 :
+        print("Long video")
+        # if the video data is already stored in the external memory, we can use it directly else we need to run the long inference
+        file_path=f'new_workspace/clips_summary/demo/{video_name}.json'
+        if not os.path.exists(file_path):
+            print("Clips summary is not ready")
+            videos_list,tmp_save_path=goldfish_obj.split_long_video_into_clips(video_path)
+            subtitle_paths = []
+            for video_p in videos_list:
+                clip_path = os.path.join(tmp_save_path, video_p)
+                subtitle_path = goldfish_obj.get_subtitles(clip_path) if use_subtitles else None
+                subtitle_paths.append(subtitle_path)
+            gradio_long_inference_video(videos_list,tmp_save_path,subtitle_paths, use_subtitles=use_subtitles)
+        else: 
+            print("External memory is ready")
+        os.makedirs("new_workspace/embedding/demo", exist_ok=True)
+        os.makedirs("new_workspace/open_ai_embedding/demo", exist_ok=True)
+        if goldfish_obj.args.use_openai_embedding:
+            embedding_path=f"new_workspace/open_ai_embedding/demo/{video_name}.pkl"
+        else:
+            embedding_path=f"new_workspace/embedding/demo/{video_name}.pkl"
+        external_memory=MemoryIndex(goldfish_obj.args.neighbours,use_openai=goldfish_obj.args.use_openai_embedding)
+        if os.path.exists(embedding_path):
+            print("Loading embeddings from pkl file")
+            external_memory.load_embeddings_from_pkl(embedding_path)
+        else:
+            # will embed the information and save it in the pkl file
+            external_memory.load_documents_from_json(file_path,embedding_path)
+        # get the most similar context from the external memory to this instruction 
+    
+        related_context_documents,related_context_keys = external_memory.search_by_similarity(instruction)
+        related_information=goldfish_obj.get_related_context(external_memory,related_context_keys)
+        pred=gradio_inference_RAG(instruction,related_information)
+        # remove stored data 
+        # os.remove(file_path)
+        # os.system(f"rm -r workspace/tmp/{self.video_name}")
+        # os.system(f"rm -r workspace/subtitles/{self.video_name}")
+        # os.system(f"rm workspace/tmp/{self.video_id}.mp4") 
+    else:
+        print("Short video")
+        goldfish_obj.video_name=video_path.split('/')[-1].split('.')[0]
+        pred=gradio_short_inference_video(video_path,instruction,use_subtitles)
+    processing_time = time.time() - start_time
+    print(f"Processing time: {processing_time:.2f} seconds")
+    return pred
+
+
+def process_video(path_url, has_subtitles, instruction, number_of_neighbours):
+    if is_youtube_url(path_url):
+        video_path = return_video_path(path_url)
+    else:
+        video_path = path_url
+    pred = inference(video_path, has_subtitles, instruction, number_of_neighbours)    
+    return pred
+
+def return_video_path(youtube_url):
+    video_id = youtube_url.split("https://www.youtube.com/watch?v=")[-1].split('&')[0]
+    if video_id:
+        return os.path.join("workspace", "tmp", f"{video_id}.mp4")
+    else:
+        raise ValueError("Invalid YouTube URL provided.")
+
+def run_gradio():
+    title = """<h1 align="center">Goldfish Demo </h1>"""
+    description = """<h5>[ECCV 2024 Accepted]Goldfish: Vision-Language Understanding of Arbitrarily Long Videos</h5>"""
+    project_page = """<p><a href='https://vision-cair.github.io/MiniGPT4-video/'><img src='https://img.shields.io/badge/Project-Page-Green'></a></p>"""
+    code_link="""<p><a href='https://github.com/Vision-CAIR/MiniGPT4-video'><img src='repo_imgs/goldfishai_png.png'></a></p>"""
+    paper_link="""<p><a href=''><img src='https://img.shields.io/badge/Paper-PDF-red'></a></p>"""
+    with gr.Blocks(title="Goldfish demo",css=text_css ) as demo :
+        gr.Markdown(title)
+        gr.Markdown(description)        
+        with gr.Tab("Youtube videos") as youtube_tab:
+            with gr.Row():
+                with gr.Column():
+                    youtube_link = gr.Textbox(label="YouTube link", placeholder="Paste YouTube URL here")
+                    video_player = gr.Video(autoplay=False)
+                    download_finish = gr.State(value=False)
+                    youtube_link.change(
+                        fn=download_video,
+                        inputs=[youtube_link, download_finish], 
+                        outputs=[video_player, download_finish]
+                    )
+
+            with gr.Row():
+                with gr.Column(scale=2) :
+                    youtube_question = gr.Textbox(label="Your Question", placeholder="Default: What's this video talking about?")
+                    youtube_has_subtitles = gr.Checkbox(label="Use subtitles", value=True)
+                    youtube_input_note = """<p>For the global questions set the number of neighbours=-1 otherwise use 3 as the defualt.</p>"""
+                    gr.Markdown(youtube_input_note)
+                    # input number 
+                    youtube_number_of_neighbours=gr.Number(label="Number of Neighbours",interactive=True,value=3)
+                    youtube_process_button = gr.Button("⛓️ Answer the Question (QA)")
+                with gr.Column(scale=3):
+                    youtube_answer = gr.Textbox(label="Answer of the question", lines=8, interactive=True, placeholder="Answer of the question will show up here.")
+                youtube_process_button.click(fn=process_video, inputs=[youtube_link, youtube_has_subtitles, youtube_question,youtube_number_of_neighbours], outputs=[youtube_answer])
+        with gr.Tab("Local videos") as local_tab:
+            with gr.Row():
+                with gr.Column():
+                    local_video_player = gr.Video(sources=["upload"])
+            with gr.Row():
+                with gr.Column(scale=2):
+                    local_question = gr.Textbox(label="Your Question", placeholder="Default: What's this video talking about?")
+                    local_has_subtitles = gr.Checkbox(label="Use subtitles", value=True)
+                    local_input_note = """<p>For the global questions set the number of neighbours=-1 otherwise use 3 as the defualt.</p>"""
+                    gr.Markdown(local_input_note)
+                    local_number_of_neighbours=gr.Number(label="Number of Neighbours",interactive=True,value=3)
+                    local_process_button = gr.Button("⛓️ Answer the Question (QA)")
+                with gr.Column(scale=3):
+                    local_answer = gr.Textbox(label="Answer of the question", lines=8, interactive=True, placeholder="Answer of the question will show up here.")
+                local_process_button.click(fn=process_video, inputs=[local_video_player, local_has_subtitles, local_question,local_number_of_neighbours], outputs=[local_answer])
+                 
+    demo.queue(max_size=10).launch(show_error=True,share=True, show_api=False,server_port=5000)
+
+if __name__ == "__main__":
+    args=get_arguments()
+    goldfish_obj = GoldFish_LV(args)
+    run_gradio()

goldfish_inference.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import argparse
+import gradio as gr
+from goldfish_lv import GoldFish_LV 
+from theme import minigptlv_style
+import time
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError('Boolean value expected.')
+
+def get_arguments():
+    parser = argparse.ArgumentParser(description="Inference parameters")
+    parser.add_argument("--cfg-path", default="test_configs/llama2_test_config.yaml")
+    parser.add_argument("--neighbours", type=int, default=3)
+    parser.add_argument("--ckpt", type=str, default="checkpoints/video_llama_checkpoint_last.pth")
+    parser.add_argument("--add_subtitles", action='store_true')
+    parser.add_argument("--max_new_tokens", type=int, default=512)
+    parser.add_argument("--use_openai_embedding",type=str2bool, default=False)
+    parser.add_argument("--batch_size", type=int, default=2, help="Batch size for short video clips")
+    parser.add_argument("--lora_r", type=int, default=64)
+    parser.add_argument("--lora_alpha", type=int, default=16)
+    parser.add_argument("--video_path", type=str,default="path for video.mp4", help="Path to the video file or youtube url")
+    parser.add_argument("--question", type=str, default="Why rachel is wearing a wedding dress?")
+    parser.add_argument("--options", nargs="+")
+    return parser.parse_args()
+
+def download_video(youtube_url):
+    processed_video_path = goldfish_lv.process_video_url(youtube_url)
+    return processed_video_path
+
+def process_video(video_path, has_subtitles, instruction="",number_of_neighbours=-1):
+    result = goldfish_lv.inference(video_path, has_subtitles, instruction,number_of_neighbours)
+    pred = result["pred"]
+    return pred
+
+def return_video_path(youtube_url):
+    video_id = youtube_url.split("https://www.youtube.com/watch?v=")[-1].split('&')[0]
+    if video_id:
+        return os.path.join("workspace", "tmp", f"{video_id}.mp4")
+    else:
+        raise ValueError("Invalid YouTube URL provided.")
+
+args=get_arguments()
+if __name__ == "__main__":
+    t1=time.time()
+    print("using openai: ", args.use_openai_embedding)
+    goldfish_lv = GoldFish_LV(args)
+    t2=time.time()
+    print("Time taken to load model: ", t2-t1)
+    processed_video_path = goldfish_lv.process_video_url(args.video_path)
+    pred=process_video(processed_video_path, args.add_subtitles, args.question,args.neighbours)      
+    print("Question answer: ", pred)
+    print(f"Time taken for inference: ", time.time()-t2)

goldfish_lv.py

@@ -0,0 +1,654 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import time
+import json
+import argparse
+import torch
+import cv2
+import moviepy.editor as mp
+import webvtt
+import re
+
+from typing import Optional, List
+from tqdm import tqdm
+from PIL import Image
+from torchvision import transforms
+from pytubefix import YouTube
+from minigpt4.common.eval_utils import init_model
+from minigpt4.conversation.conversation import CONV_VISION
+from index import MemoryIndex  
+import pysrt
+import chardet
+from openai import OpenAI
+if os.getenv("OPENAI_API_KEY") is not None:
+    client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+else:
+    client = OpenAI(api_key="")
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import re
+from transformers import BitsAndBytesConfig
+# from split_long_video_in_parallel import split_video
+import transformers
+import whisper
+from datetime import timedelta
+# Function to format timestamps for VTT
+def format_timestamp(seconds):
+    td = timedelta(seconds=seconds)
+    total_seconds = int(td.total_seconds())
+    milliseconds = int(td.microseconds / 1000)
+    hours, remainder = divmod(total_seconds, 3600)
+    minutes, seconds = divmod(remainder, 60)
+    return f"{hours:02}:{minutes:02}:{seconds:02}.{milliseconds:03}"
+
+def clean_text(subtitles_text):
+    # Remove unwanted characters except for letters, digits, spaces, periods, commas, exclamation marks, and single quotes
+    subtitles_text = re.sub(r'[^a-zA-Z0-9\s\']', '', subtitles_text)
+    # Replace multiple spaces with a single space
+    subtitles_text = re.sub(r'\s+', ' ', subtitles_text)
+    return subtitles_text.strip()
+def time_to_seconds(subrip_time):
+    return subrip_time.hours * 3600 + subrip_time.minutes * 60 + subrip_time.seconds + subrip_time.milliseconds / 1000
+
+def split_subtitles(subtitle_path, n):
+    # read the subtitle file and detect the encoding
+    with open(subtitle_path, 'rb') as f:
+        result = chardet.detect(f.read())
+    subs = pysrt.open(subtitle_path, encoding=result['encoding'])
+
+    total_subs = len(subs)
+
+    if n <= 0 or n > total_subs:
+        print("Invalid value for n. It should be a positive integer less than or equal to the total number of subtitles.")
+        return None
+
+    subs_per_paragraph = total_subs // n
+    remainder = total_subs % n
+
+    paragraphs = []
+
+    current_index = 0
+
+    for i in range(n):
+        num_subs_in_paragraph = subs_per_paragraph + (1 if i < remainder else 0)
+
+        paragraph_subs = subs[current_index:current_index + num_subs_in_paragraph]
+        current_index += num_subs_in_paragraph
+
+        # Join subtitles using pysrt's built-in method for efficient formatting
+        paragraph = pysrt.SubRipFile(items=paragraph_subs).text
+        paragraphs.append(paragraph)
+
+    return paragraphs
+class GoldFish_LV:
+    """
+    'GoldFish_LV' class is to handle long video processing and subtitle management with MiniGPT4_video base model.
+    """
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        self.args = args
+        self.model, self.vis_processor,whisper_gpu_id,minigpt4_gpu_id,answer_module_gpu_id = init_model(args)
+        self.whisper_gpu_id=whisper_gpu_id
+        self.minigpt4_gpu_id=minigpt4_gpu_id
+        self.answer_module_gpu_id=answer_module_gpu_id
+        # self.original_llama_model,self.original_llama_tokenizer=self.load_original_llama_model()
+        # self.original_llama_model=self.load_original_llama_model_vllm()
+        self.llama_3_1_model=self.load_llama3_1_model()
+        self.whisper_model=whisper.load_model("large",device=f"cuda:{self.whisper_gpu_id}")
+        # self.summary_instruction="Generate a description of this video .Pay close attention to the objects, actions, emotions portrayed in the video,providing a vivid description of key moments.Specify any visual cues or elements that stand out."
+        self.summary_instruction="I'm a blind person, please provide me with a detailed summary of the video content and try to be as descriptive as possible."
+    def load_original_llama_model(self):
+        model_name="meta-llama/Meta-Llama-3-8B-Instruct"
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        tokenizer.pad_token = "[PAD]"
+        tokenizer.padding_side = "left"
+        bnb_config = BitsAndBytesConfig(
+                    load_in_8bit=True,
+                )
+        llama_model = AutoModelForCausalLM.from_pretrained(
+                model_name,
+                torch_dtype=torch.bfloat16,
+                device_map={'': f"cuda:{self.answer_module_gpu_id}"},
+                quantization_config=bnb_config,
+            )
+        return llama_model,tokenizer
+    
+    def load_llama3_1_model(self):
+        model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
+        bnb_config = BitsAndBytesConfig(
+                    load_in_8bit=True,
+                )
+        self.llama3_tokenizer = AutoTokenizer.from_pretrained(model_id)
+        llama3_model = AutoModelForCausalLM.from_pretrained(
+                model_id,
+                torch_dtype=torch.bfloat16,
+                device_map={'': f"cuda:{self.answer_module_gpu_id}"},
+                quantization_config=bnb_config,
+            )
+        pipeline = transformers.pipeline(
+            "text-generation",
+            model=llama3_model,
+            tokenizer=self.llama3_tokenizer,
+            model_kwargs={"torch_dtype": torch.bfloat16},
+            device_map=f"cuda:{self.answer_module_gpu_id}",
+        )
+        return pipeline
+
+        
+
+    def _youtube_download(self, url: str) -> str:
+        try:
+            video_id = url.split('v=')[-1].split('&')[0]
+            video_id = video_id.strip()
+            print(f"Downloading video with ID: {video_id}")
+            youtube = YouTube(f"https://www.youtube.com/watch?v={video_id}")
+            video_stream = youtube.streams.filter(progressive=True, file_extension='mp4').order_by('resolution').desc().first()
+            if not video_stream:
+                raise ValueError("No suitable video stream found.")
+            output_path = f"workspace/tmp/{video_id}.mp4"
+            self.video_id=video_id
+            video_stream.download(output_path="workspace/tmp", filename=f"{video_id}.mp4")
+            return output_path
+        except Exception as e:
+            print(f"Error downloading video: {e}")
+            return url
+
+    @staticmethod
+    def is_youtube_url(url: str) -> bool:
+        youtube_regex = (
+            r'(https?://)?(www\.)?'
+            '(youtube|youtu|youtube-nocookie)\.(com|be)/'
+            '(watch\?v=|embed/|v/|.+\?v=)?([^&=%\?]{11})'
+        )
+        return bool(re.match(youtube_regex, url))
+
+    def process_video_url(self, video_path: str) -> str:
+        if self.is_youtube_url(video_path):
+            return self._youtube_download(video_path)
+        else:
+            return video_path
+
+    def create_video_grid(self, images: list, rows: int, cols: int, save_path: str) -> Image.Image:
+        image_width, image_height = images[0].size
+        grid_width = cols * image_width
+        grid_height = rows * image_height
+        new_image = Image.new("RGB", (grid_width, grid_height))
+        for i in range(rows):
+            for j in range(cols):
+                index = i * cols + j
+                if index < len(images):
+                    image = images[index]
+                    x_offset = j * image_width
+                    y_offset = i * image_height
+                    new_image.paste(image, (x_offset, y_offset))
+
+        new_image.save(save_path)
+        return new_image
+    def get_subtitles(self, video_path) :
+        video_name=video_path.split('/')[-2]
+        video_id=video_path.split('/')[-1].split('.')[0]
+        audio_dir = f"workspace/audio/{video_name}"
+        subtitle_dir = f"workspace/subtitles/{video_name}"
+        os.makedirs(audio_dir, exist_ok=True)
+        os.makedirs(subtitle_dir, exist_ok=True)
+        # if the subtitles are already generated, return the path of the subtitles
+        subtitle_path = f"{subtitle_dir}/{video_id}"+'.vtt'
+        if os.path.exists(subtitle_path):
+            return f"{subtitle_dir}/{video_id}"+'.vtt'
+        audio_path = f"{audio_dir}/{video_id}"+'.mp3'
+        try:
+            self.extract_audio(video_path, audio_path)
+            subtitle_path = f"{subtitle_dir}/{video_id}"+'.vtt'
+            result = self.whisper_model.transcribe(audio_path,language="en") 
+            # Create VTT file
+            with open(subtitle_path, "w", encoding="utf-8") as vtt_file:
+                vtt_file.write("WEBVTT\n\n")
+                for segment in result['segments']:
+                    start = format_timestamp(segment['start'])
+                    end = format_timestamp(segment['end'])
+                    text = segment['text']
+                    vtt_file.write(f"{start} --> {end}\n{text}\n\n")
+            return subtitle_path
+        except Exception as e:
+            print(f"Error during subtitle generation for {video_path}: {e}")
+            return None
+                  
+    def prepare_input(self, 
+                    video_path: str,
+                    subtitle_path: Optional[str],
+                    instruction: str,previous_caption=""):
+        # If a subtitle path is provided, read the VTT (Web Video Text Tracks) file, else set to an empty list
+        conversation=""
+        if subtitle_path:
+            vtt_file = webvtt.read(subtitle_path)
+            print("Subtitle loaded successfully")
+            try:
+                for subtitle in vtt_file:
+                    sub = subtitle.text.replace('\n',' ')
+                    conversation+=sub
+            except:
+                pass
+        if self.model.model_type == "Mistral":
+            max_images_length=90
+            max_sub_len = 800
+        else:
+            max_images_length = 45
+            max_sub_len = 400
+        # Load the video file using moviepy and calculate the total number of frames
+        clip = mp.VideoFileClip(video_path)
+        total_num_frames = int(clip.duration * clip.fps)
+        clip.close()
+        # Calculate how often to sample a frame based on the total number of frames and the maximum images length
+        cap = cv2.VideoCapture(video_path)
+        images = []
+        frame_count = 0
+        sampling_interval = int(total_num_frames / max_images_length)
+        if sampling_interval == 0:
+            sampling_interval = 1
+        # Initialize variables to hold image placeholders, current subtitle text, and subtitle history
+        if previous_caption != "":
+            img_placeholder = previous_caption+" "
+        else:
+            img_placeholder = ""
+        subtitle_text_in_interval = ""
+        history_subtitles = {}
+        raw_frames=[]
+        number_of_words=0
+        transform=transforms.Compose([
+                    transforms.ToPILImage(),
+                ])
+        # Loop through each frame in the video
+        while cap.isOpened():
+            ret, frame = cap.read()
+            if not ret:
+                break
+            # TODO: we need to add subtitles in external memory either
+            if subtitle_path is not None: 
+                for i, subtitle in enumerate(vtt_file):
+                    sub = subtitle.text.replace('\n',' ')
+                    if (subtitle.start_in_seconds <= (frame_count / int(clip.fps)) <= subtitle.end_in_seconds) and sub not in subtitle_text_in_interval:
+
+                        if not history_subtitles.get(sub, False):
+                            subtitle_text_in_interval += sub + " "
+
+                        history_subtitles[sub] = True
+                        break
+            # Process and store the frame at specified intervals
+            if frame_count % sampling_interval == 0:
+                raw_frames.append(Image.fromarray(cv2.cvtColor(frame.copy(), cv2.COLOR_BGR2RGB)))
+                frame = transform(frame[:,:,::-1]) # convert to RGB
+                frame = self.vis_processor(frame)
+                images.append(frame)
+                img_placeholder += '<Img><ImageHere>'
+                if subtitle_path is not None and subtitle_text_in_interval != "" and number_of_words< max_sub_len: 
+                    img_placeholder+=f'<Cap>{subtitle_text_in_interval}'
+                    number_of_words+=len(subtitle_text_in_interval.split(' '))
+                    subtitle_text_in_interval = ""
+            frame_count += 1
+
+            # Break the loop if the maximum number of images is reached
+            if len(images) >= max_images_length:
+                break
+
+        cap.release()
+        cv2.destroyAllWindows()
+
+        # Return None if no images are extracted
+        if len(images) == 0:
+            return None, None
+        while len(images) < max_images_length:
+            images.append(images[-1])
+            img_placeholder += '<Img><ImageHere>'
+        images = torch.stack(images)
+        print("Input instruction length",len(instruction.split(' ')))
+        instruction = img_placeholder + '\n' + instruction
+        print("number of words",number_of_words)
+        print("number of images",len(images))
+
+        return images, instruction,conversation
+
+    def extract_audio(self, video_path: str, audio_path: str) -> None:
+        video_clip = mp.VideoFileClip(video_path)
+        audio_clip = video_clip.audio
+        audio_clip.write_audiofile(audio_path, codec="libmp3lame", bitrate="320k")
+
+    def short_video_inference (self,video_path,instruction,gen_subtitles=True):
+        if gen_subtitles:
+            subtitle_path=self.get_subtitles(video_path)
+        else :
+            subtitle_path=None
+        prepared_images,prepared_instruction,video_conversation=self.prepare_input(video_path,subtitle_path,instruction)
+        if prepared_images is None:
+            return "Video cann't be open ,check the video path again"
+        length=len(prepared_images)
+        prepared_images=prepared_images.unsqueeze(0)
+        conv = CONV_VISION.copy()
+        conv.system = ""
+        # if you want to make conversation comment the 2 lines above and make the conv is global variable
+        conv.append_message(conv.roles[0], prepared_instruction)
+        conv.append_message(conv.roles[1], None)
+        prompt = [conv.get_prompt()]
+        answers = self.model.generate(prepared_images, prompt, max_new_tokens=512, do_sample=False, lengths=[length],num_beams=1)
+        return answers[0]
+
+    def split_long_video_into_clips(self,video_path):
+        # Split the video into 90 seconds clips and make a queue of the videos and run the inference on each video 
+        self.video_name=video_path.split('/')[-1].split('.')[0]
+        tmp_save_path=f"workspace/tmp/{self.video_name}"
+        os.makedirs(tmp_save_path, exist_ok=True)
+        print("tmp_save_path",tmp_save_path)
+
+        if len(os.listdir(tmp_save_path)) == 0:
+            print("Splitting Long video")
+            os.system(f"python split_long_video_in_parallel.py --video_path {video_path} --output_folder {tmp_save_path}")
+            # split_video(video_path, tmp_save_path, clip_duration=90)
+        videos_list = sorted(os.listdir(tmp_save_path))
+        return videos_list,tmp_save_path
+    def long_inference_video(self, videos_list,tmp_save_path,subtitle_paths) -> Optional[str]:
+        save_long_videos_path = "new_workspace/clips_summary/demo"
+        os.makedirs(save_long_videos_path, exist_ok=True)
+        file_path = f'{save_long_videos_path}/{self.video_name}.json'
+
+        if os.path.exists(file_path):
+            print("Clips inference already done")
+            with open(file_path, 'r') as file:
+                video_information = json.load(file)
+        else:
+            video_number = 0
+            batch_size = self.args.batch_size
+            batch_video_paths, batch_instructions ,batch_subtitles= [], [],[]
+            video_information = {}
+            video_captions = []
+            for i, video in tqdm(enumerate(videos_list), desc="Inference video clips", total=len(videos_list)):
+                clip_path = os.path.join(tmp_save_path, video)
+                batch_video_paths.append(clip_path)
+                # previous_caption =  "You are analysing a one long video of mutiple clips and this is the summary from all previous clips :"+video_captions[-1]+"\n\n" if video_captions else ""
+                previous_caption=""
+                batch_instructions.append(self.summary_instruction)
+                batch_subtitles.append(subtitle_paths[i])
+                # Process each batch
+                if len(batch_video_paths) % batch_size == 0 and i != 0:
+                    batch_preds,videos_conversation=self.run_batch(batch_video_paths,batch_instructions, batch_subtitles,previous_caption)
+                    for pred,subtitle in zip(batch_preds,videos_conversation):
+                        video_number += 1
+                        save_name=f"{video_number}".zfill(5)
+                        if pred != "":
+                            video_information[f'caption__{save_name}'] = pred
+                        if subtitle != "":
+                            video_information[f'subtitle__{save_name}'] = subtitle
+                        video_captions.append(pred)
+                    batch_video_paths, batch_instructions,batch_subtitles = [], [],[]
+
+            # Process any remaining videos in the last batch
+            if batch_video_paths:
+                batch_preds,videos_conversation=self.run_batch(batch_video_paths,batch_instructions, batch_subtitles,previous_caption)
+                for pred,subtitle in zip(batch_preds,videos_conversation):
+                    video_number += 1
+                    save_name=f"{video_number}".zfill(5)
+                    if pred != "":
+                            video_information[f'caption__{save_name}'] = pred
+                    if subtitle != "":
+                        video_information[f'subtitle__{save_name}'] = subtitle
+                    video_captions.append(pred)    
+            with open(file_path, 'w') as file:
+                json.dump(video_information, file, indent=4)
+            print("Clips inference done")
+        return video_information
+    # def inference_RAG(self, instructions, context_list):
+    #     context_promots=[]
+    #     questions_prompts=[]
+    #     try:
+    #         for instruction,context in zip(instructions,context_list):
+    #             context=clean_text(context)
+    #             context_prompt=f"<s>[INST] Your task is to answer questions for one long video which is split into multiple clips.\nGiven these related information from the most related clips: \n{context}\n"
+    #             question_prompt=f"\nAnswer this question :{instruction} \n your answer is: [/INST]"
+    #             context_promots.append(context_prompt)
+    #             questions_prompts.append(question_prompt)
+    #         context_inputs = self.original_llama_tokenizer(context_promots, return_tensors="pt", padding=True, truncation=True,max_length=3500)
+    #         # print(context_inputs.keys())
+    #         print("context_inputs shape",context_inputs['input_ids'].shape)
+    #         question_inputs = self.original_llama_tokenizer(questions_prompts, return_tensors="pt", padding=True, truncation=True,max_length=300)
+    #         print("question_inputs shape",question_inputs['input_ids'].shape)
+    #         # concate the context and the question together 
+    #         inputs_ids=torch.cat((context_inputs['input_ids'],question_inputs['input_ids']),dim=1).to('cuda')
+    #         print("inputs shape",inputs_ids.shape)
+    #     except Exception as e:
+    #         print("error while tokenization",e)
+    #         return self.inference_RAG_batch_size_1(instructions, context_list)
+    #     with torch.no_grad():
+    #         summary_ids = self.original_llama_model.generate(inputs_ids,max_new_tokens=512)
+    #     answers=[]
+    #     for i in range(len(summary_ids)):
+    #         output_text=self.original_llama_tokenizer.decode(summary_ids[i], skip_special_tokens=True)
+    #         output_text = output_text.split('</s>')[0]  # remove the stop sign </s>
+    #         output_text = output_text.replace("<s>", "")
+    #         output_text = output_text.split(r'[/INST]')[-1].strip()
+    #         answers.append(output_text)
+    #     return answers
+    def inference_RAG(self, instructions, context_list):
+        messages=[]
+        for instruction,context in zip(instructions,context_list):
+            context=clean_text(context)
+            context_prompt=f"Your task is to answer a specific question based on one long video. While you cannot view the video yourself, I will supply you with the most relevant text information from the most pertinent clips. \n{context}\n"
+            question_prompt=f"\nPlease provide a detailed and accurate answer to the following question:{instruction} \n Your answer should be:"
+            # limit the context words to 10000 word duo to hardware limitation
+            context_words=context_prompt.split(' ')
+            truncated_context=' '.join(context_words[:10000])
+            print("Number of words",len((truncated_context+question_prompt).split(' ')))
+            messages.append([{"role": "user", "content": truncated_context+question_prompt}])
+        outputs=self.llama_3_1_model(messages, max_new_tokens=512)
+        answers=[]
+        for out in outputs:
+            answers.append(out[0]["generated_text"][-1]['content'])
+        return answers 
+    # def inference_RAG(self, instructions, context_list):
+    #     prompts=[]
+    #     for instruction,context in zip(instructions,context_list):
+    #         context=clean_text(context)
+    #         context_prompt=f"Your task is to answer questions for one long video which is split into multiple clips.\nGiven these related information from the most related clips: \n{context}\n"
+    #         question_prompt=f"\nAnswer this question :{instruction} \n your answer is:"
+    #         prompts.append(context_prompt+question_prompt)
+        
+    #     with open('prompts.txt','w') as f:
+    #         for prompt in prompts:
+    #             f.write(prompt+'\n')
+        
+    #     outputs=self.original_llama_model.generate(prompts)
+    #     answers=[]
+    #     for out in outputs:
+    #         answers.append(out.outputs[0].text)
+    #     return answers        
+    def inference_RAG_batch_size_1(self, instructions, context_list):
+        answers=[]
+        for instruction,context in zip(instructions,context_list):
+            context=clean_text(context)
+            context_prompt=f"<s>[INST] Your task is to answer questions for one long video which is split into multiple clips.\nGiven these related information from the most related clips: \n{context}\n"
+            question_prompt=f"\nAnswer this question :{instruction} \n your answer is: [/INST]"
+            context_inputs=self.original_llama_tokenizer([context_prompt], return_tensors="pt", padding=True, truncation=True,max_length=3500)['input_ids']
+            question_inputs=self.original_llama_tokenizer([question_prompt], return_tensors="pt", padding=True, truncation=True,max_length=300)['input_ids']
+            
+            inputs_ids=torch.cat((context_inputs,question_inputs),dim=1).to('cuda')
+            with torch.no_grad():
+                summary_ids = self.original_llama_model.generate(inputs_ids,max_new_tokens=512,)
+            
+            output_text=self.original_llama_tokenizer.decode(summary_ids[0], skip_special_tokens=True)
+            output_text = output_text.split('</s>')[0]  # remove the stop sign </s>
+            output_text = output_text.replace("<s>", "")
+            output_text = output_text.split(r'[/INST]')[-1].strip()
+            answers.append(output_text)
+            
+        return answers
+        
+    # def inference_RAG_text_only(self, instructions, context_list):
+    #     # Use VideoLLM as the answer module
+    #     seg_tokens=[]
+    #     for instruction,context in zip(instructions,context_list):
+    #         context=clean_text(context)
+    #         context_prompt=f"<s>[INST] Your task is to answer questions for one long video which is split into multiple clips.\nGiven these related information from the most related clips: \n{context}\n"
+    #         question_prompt=f"\nAnswer this question :{instruction} \n your answer is: [/INST]"
+    #         context_inputs = self.model.llama_tokenizer(context_prompt,add_special_tokens=True, return_tensors="pt", padding=True, truncation=True,max_length=3500)
+    #         question_inputs = self.model.llama_tokenizer(question_prompt, return_tensors="pt", padding=True, truncation=True,max_length=300)
+    #         # concate the context and the question together 
+    #         inputs_ids=torch.cat((context_inputs['input_ids'],question_inputs['input_ids']),dim=1).to('cuda')
+    #         seg_tokens.append(inputs_ids)
+    #     with torch.no_grad():
+    #         answers = self.model.generate_text_only(images=None,seg_tokens=seg_tokens,max_new_tokens=512)        
+    #     return answers
+        
+             
+    def inference_RAG_chatGPT(self, instructions: str, context_list) -> str:
+        batch_preds=[]
+        for context,instruction in zip(context_list,instructions):
+            prompt="Your task is to answer questions for long video \n\n Given these related information from the most related clips: \n "+context +"\n\n" +"Answer this question: "+instruction
+            while True:
+                try:
+                    response = client.ChatCompletion.create(
+                        model="gpt-4o",
+                        messages=[
+                                {
+                                "role": "user",
+                                "content": prompt
+                                }],
+                    )
+                    answer=response.choices[0].message['content']
+                    batch_preds.append(answer)
+                    break
+                except Exception as e:
+                    print("chat gpt error",e)
+                    time.sleep(50)
+        
+        return batch_preds
+     
+    def get_most_related_clips(self,related_context_keys):
+        most_related_clips=set()
+        for context_key in related_context_keys:
+            if len(context_key.split('__'))>1:
+                most_related_clips.add(context_key.split('__')[1])
+            if len(most_related_clips)==self.args.neighbours:
+                break
+        assert len(most_related_clips)!=0, f"No related clips found {related_context_keys}"
+        return list(most_related_clips)  
+    def get_related_context(self, external_memory,related_context_keys):
+        related_information=""
+        most_related_clips=self.get_most_related_clips(related_context_keys)
+        for clip_name in most_related_clips:
+            clip_conversation=""
+            general_sum=""
+            for key in external_memory.documents.keys():
+                if clip_name in key and 'caption' in key:
+                    general_sum="Clip Summary: "+external_memory.documents[key]
+                if clip_name in key and 'subtitle' in key:
+                    clip_conversation="Clip Subtitles: "+external_memory.documents[key]
+            related_information+=f"{general_sum},{clip_conversation}\n"
+        return related_information              
+    def inference(self,video_path, use_subtitles=True, instruction="", number_of_neighbours=3):
+        start_time = time.time()
+        video_name = os.path.splitext(os.path.basename(video_path))[0]
+        self.args.neighbours = number_of_neighbours
+        print(f"Video name: {video_name}")
+        video_duration = mp.VideoFileClip(video_path).duration
+        print(f"Video duration: {video_duration:.2f} seconds")
+        # if the video duration is more than 2 minutes we need to run the long inference 
+        if video_duration > 180 :
+            print("Long video")
+            # if the video data is already stored in the external memory, we can use it directly else we need to run the long inference
+            file_path=f'new_workspace/clips_summary/demo/{video_name}.json'
+            if not os.path.exists(file_path):
+                print("Clips summary is not ready")
+                videos_list,tmp_save_path=self.split_long_video_into_clips(video_path)
+                subtitle_paths = []
+                for video_p in videos_list:
+                    clip_path = os.path.join(tmp_save_path, video_p)
+                    subtitle_path = self.get_subtitles(clip_path) if use_subtitles else None
+                    subtitle_paths.append(subtitle_path)
+                clips_summary = self.long_inference_video(videos_list,tmp_save_path,subtitle_paths)
+            else: 
+                print("External memory is ready")
+            os.makedirs("new_workspace/embedding/demo", exist_ok=True)
+            os.makedirs("new_workspace/open_ai_embedding/demo", exist_ok=True)
+            if self.args.use_openai_embedding:
+                embedding_path=f"new_workspace/open_ai_embedding/demo/{video_name}.pkl"
+            else:
+                embedding_path=f"new_workspace/embedding/demo/{video_name}.pkl"
+            external_memory=MemoryIndex(self.args.neighbours,use_openai=self.args.use_openai_embedding)
+            if os.path.exists(embedding_path):
+                print("Loading embeddings from pkl file")
+                external_memory.load_embeddings_from_pkl(embedding_path)
+            else:
+                # will embed the information and save it in the pkl file
+                external_memory.load_documents_from_json(file_path,embedding_path)
+            # get the most similar context from the external memory to this instruction 
+        
+            related_context_documents,related_context_keys = external_memory.search_by_similarity(instruction)
+            related_information=self.get_related_context(external_memory,related_context_keys)
+            pred=self.inference_RAG([instruction],[related_information])
+        else:
+            print("Short video")
+            self.video_name=video_path.split('/')[-1].split('.')[0]
+            pred=self.short_video_inference(video_path,instruction,use_subtitles)
+        processing_time = time.time() - start_time
+        print(f"Processing time: {processing_time:.2f} seconds")
+        return {
+            'video_name': os.path.splitext(os.path.basename(video_path))[0],
+            'pred': pred,
+        }
+
+    
+    def run_batch(self, video_paths, instructions,subtitle_paths,previous_caption="") -> List[str]:
+
+        prepared_images_batch = []
+        prepared_instructions_batch = []
+        lengths_batch = []
+        videos_conversations=[]
+
+        for i,video_path, instruction in zip(range(len(video_paths)),video_paths, instructions):
+            subtitle_path = subtitle_paths[i]
+            prepared_images, prepared_instruction,video_conversation = self.prepare_input( video_path, subtitle_path, instruction,previous_caption)
+            
+            if prepared_images is None:
+                print(f"Error: Unable to open video at {video_path}. Check the path and try again.")
+                continue
+            videos_conversations.append(video_conversation)
+            conversation = CONV_VISION.copy()
+            conversation.system = ""
+            conversation.append_message(conversation.roles[0], prepared_instruction)
+            conversation.append_message(conversation.roles[1], None)
+            prepared_instructions_batch.append(conversation.get_prompt())
+            prepared_images_batch.append(prepared_images)
+            lengths_batch.append(len(prepared_images))
+
+        if not prepared_images_batch:
+            return []
+
+        prepared_images_batch = torch.stack(prepared_images_batch)
+        answers=self.model.generate(prepared_images_batch, prepared_instructions_batch, max_new_tokens=self.args.max_new_tokens, do_sample=False, lengths=lengths_batch, num_beams=1)
+        return answers , videos_conversations
+
+    def run_images_features (self,img_embeds,prepared_instruction):        
+        lengths=[]
+        prompts=[]
+        for i in range(img_embeds.shape[0]):    
+            conv = CONV_VISION.copy()
+            conv.system = ""
+            conv.append_message(conv.roles[0], prepared_instruction[i])
+            conv.append_message(conv.roles[1], None)
+            prompts.append(conv.get_prompt())
+            lengths.append(len(img_embeds[i]))
+        
+        answers = self.model.generate(images=None,img_embeds=img_embeds,texts=prompts, max_new_tokens=300, do_sample=False, lengths=lengths,num_beams=1)
+        return answers
+        
+    def run_images (self,prepared_images,prepared_instruction):        
+        lengths=[]
+        prompts=[]
+        for i in range(prepared_images.shape[0]):    
+            conv = CONV_VISION.copy()
+            conv.system = ""
+            conv.append_message(conv.roles[0], prepared_instruction[i])
+            conv.append_message(conv.roles[1], None)
+            prompts.append(conv.get_prompt())
+            lengths.append(len(prepared_images[i]))
+        answers = self.model.generate(prepared_images, prompts, max_new_tokens=300, do_sample=False, lengths=lengths,num_beams=1)
+        return answers
+            
+    

index.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import json
+import torch
+from sentence_transformers import SentenceTransformer
+from collections import defaultdict
+from typing import List, Dict, Tuple, Union
+import torch
+from PIL import Image
+import pickle
+from openai import OpenAI
+import os
+import torch
+import time
+import yaml 
+
+class MemoryIndex:
+    def __init__(self,number_of_neighbours,use_openai=False):
+        self.documents = {}
+        self.document_vectors = {}
+        self.use_openai=use_openai
+        if use_openai:
+            api_key = os.getenv("OPENAI_API_KEY")
+            self.client = OpenAI(api_key=api_key)
+        self.model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')
+        # self.model = SentenceTransformer('sentence-transformers/paraphrase-MiniLM-L6-v2')
+        with open('test_configs/llama2_test_config.yaml') as file:
+            config = yaml.load(file, Loader=yaml.FullLoader)
+        embedding_gpu_id=config['model']['minigpt4_gpu_id']
+        self.device = f"cuda:{embedding_gpu_id}" if torch.cuda.is_available() else "cpu"
+        self.number_of_neighbours=int(number_of_neighbours)
+        
+    def load_documents_from_json(self, file_path,emdedding_path=""):
+
+        with open(file_path, 'r') as file:
+            data = json.load(file)
+            for doc_id, doc_data in data.items():
+                self.documents[doc_id] = doc_data
+                self.document_vectors[doc_id] = self._compute_sentence_embedding(doc_data)
+    
+        # save self.documents and self.document_vectors to pkl file
+        m=[self.documents,self.document_vectors]
+        with open(emdedding_path, 'wb') as file:
+            pickle.dump(m, file)
+        return emdedding_path
+    def load_embeddings_from_pkl(self, pkl_file_path):
+        #read the pkl file 
+        with open(pkl_file_path, 'rb') as file:
+            data = pickle.load(file)
+            self.documents=data[0]
+            self.document_vectors=data[1]
+            
+        
+    def load_data_from_pkl(self, pkl_file_path):
+        with open(pkl_file_path, 'rb') as file:
+            data = pickle.load(file)
+            for doc_id, doc_data in data.items():
+                self.documents[doc_id] = doc_data
+                self.document_vectors[doc_id] = doc_data  
+    def _compute_sentence_embedding(self, text: str) -> torch.Tensor:  
+        if self.use_openai:      
+            done=False
+            while not done:
+                try:
+                    embedding=self.client.embeddings.create(input = [text], model="text-embedding-3-small").data[0].embedding
+                    # Convert the list to a PyTorch tensor
+                    embedding = torch.tensor(embedding)
+                    done=True
+                except Exception as e:
+                    print("error",e)
+                    print("text",text)
+                    # sleep for 5 seconds and try again 
+                    time.sleep(5)
+                    continue
+        else:
+            return self.model.encode(text, convert_to_tensor=True).to(self.device)
+            
+        return embedding
+
+    def search_by_similarity(self, query: str) -> List[str]:
+
+        query_vector = self._compute_sentence_embedding(query)
+        scores = {doc_id: torch.nn.functional.cosine_similarity(query_vector, doc_vector, dim=0).item()
+                  for doc_id, doc_vector in self.document_vectors.items()}
+        sorted_doc_ids = sorted(scores, key=scores.get, reverse=True)
+        sorted_documents=[self.documents[doc_id] for doc_id in sorted_doc_ids]
+        if self.number_of_neighbours == -1:
+            return list(self.documents.values()), list(self.documents.keys())
+        if self.number_of_neighbours > len(sorted_documents):
+            return sorted_documents, sorted_doc_ids
+        # if the retrieved document is the summary, return the summary and the next document to grauntee that always retieve clip name.
+        if self.number_of_neighbours==1 and sorted_doc_ids[0]=='summary': 
+            return sorted_documents[0:2], sorted_doc_ids[:2]
+        print("Number of neighbours",self.number_of_neighbours)
+        return sorted_documents[:self.number_of_neighbours], sorted_doc_ids[:self.number_of_neighbours]
+
+# # main function
+# if __name__ == "__main__":
+#     memory_index = MemoryIndex(-1,use_openai=True)
+#     memory_index.load_documents_from_json('workspace/results/llama_vid/tt0035423.json')
+#     print(memory_index.documents.keys())
+#     docs,keys=memory_index.search_by_similarity('kerolos')

minigpt4/__init__.py

@@ -0,0 +1,31 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import os
+import sys
+
+from omegaconf import OmegaConf
+
+from minigpt4.common.registry import registry
+
+from minigpt4.datasets.builders import *
+from minigpt4.models import *
+from minigpt4.processors import *
+from minigpt4.tasks import *
+
+
+root_dir = os.path.dirname(os.path.abspath(__file__))
+default_cfg = OmegaConf.load(os.path.join(root_dir, "configs/default.yaml"))
+
+registry.register_path("library_root", root_dir)
+repo_root = os.path.join(root_dir, "..")
+registry.register_path("repo_root", repo_root)
+cache_root = os.path.join(repo_root, default_cfg.env.cache_root)
+registry.register_path("cache_root", cache_root)
+
+registry.register("MAX_INT", sys.maxsize)
+registry.register("SPLIT_NAMES", ["train", "val", "test"])

minigpt4/common/config.py

@@ -0,0 +1,474 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import logging
+import json
+from typing import Dict
+
+from omegaconf import OmegaConf
+from minigpt4.common.registry import registry
+
+
+class Config:
+    def __init__(self, args):
+        self.config = {}
+
+        self.args = args
+
+        # Register the config and configuration for setup
+        registry.register("configuration", self)
+
+        user_config = self._build_opt_list(self.args.options)
+
+        config = OmegaConf.load(self.args.cfg_path)
+
+        runner_config = self.build_runner_config(config)
+        model_config = self.build_model_config(config, **user_config)
+        dataset_config = self.build_dataset_config(config)
+
+        # Validate the user-provided runner configuration
+        # model and dataset configuration are supposed to be validated by the respective classes
+        # [TODO] validate the model/dataset configuration
+        # self._validate_runner_config(runner_config)
+
+        # Override the default configuration with user options.
+        self.config = OmegaConf.merge(
+            runner_config, model_config, dataset_config, user_config
+        )
+
+    def _validate_runner_config(self, runner_config):
+        """
+        This method validates the configuration, such that
+            1) all the user specified options are valid;
+            2) no type mismatches between the user specified options and the config.
+        """
+        runner_config_validator = create_runner_config_validator()
+        runner_config_validator.validate(runner_config)
+
+    def _build_opt_list(self, opts):
+        opts_dot_list = self._convert_to_dot_list(opts)
+        return OmegaConf.from_dotlist(opts_dot_list)
+
+    @staticmethod
+    def build_model_config(config, **kwargs):
+        model = config.get("model", None)
+        assert model is not None, "Missing model configuration file."
+
+        model_cls = registry.get_model_class(model.arch)
+        assert model_cls is not None, f"Model '{model.arch}' has not been registered."
+
+        model_type = kwargs.get("model.model_type", None)
+        if not model_type:
+            model_type = model.get("model_type", None)
+        # else use the model type selected by user.
+
+        assert model_type is not None, "Missing model_type."
+
+        print("--------------")
+        print("model arch",model.arch)
+        print("model cls",model_cls)
+
+        model_config_path = model_cls.PRETRAINED_MODEL_CONFIG_DICT[model_type]
+
+        model_config = OmegaConf.create()
+        # hierarchy override, customized config > default config
+        model_config = OmegaConf.merge(
+            model_config,
+            OmegaConf.load(model_config_path),
+            {"model": config["model"]},
+        )
+
+        return model_config
+
+    @staticmethod
+    def build_runner_config(config):
+        return {"run": config.run}
+
+    @staticmethod
+    def build_dataset_config(config):
+        datasets = config.get("datasets", None)
+        if datasets is None:
+            raise KeyError(
+                "Expecting 'datasets' as the root key for dataset configuration."
+            )
+
+        dataset_config = OmegaConf.create()
+
+        for dataset_name in datasets:
+
+            print("dataset name", dataset_name)
+            builder_cls = registry.get_builder_class(dataset_name)
+
+            dataset_config_type = datasets[dataset_name].get("type", "default")
+            dataset_config_path = builder_cls.default_config_path(
+                type=dataset_config_type
+            )
+
+            # hierarchy override, customized config > default config
+            dataset_config = OmegaConf.merge(
+                dataset_config,
+                OmegaConf.load(dataset_config_path),
+                {"datasets": {dataset_name: config["datasets"][dataset_name]}},
+            )
+
+        return dataset_config
+
+    def _convert_to_dot_list(self, opts):
+        if opts is None:
+            opts = []
+
+        if len(opts) == 0:
+            return opts
+
+        has_equal = opts[0].find("=") != -1
+
+        if has_equal:
+            return opts
+
+        return [(opt + "=" + value) for opt, value in zip(opts[0::2], opts[1::2])]
+
+    def get_config(self):
+        return self.config
+
+    @property
+    def run_cfg(self):
+        return self.config.run
+
+    @property
+    def datasets_cfg(self):
+        return self.config.datasets
+
+    @property
+    def model_cfg(self):
+        return self.config.model
+
+    def pretty_print(self):
+        logging.info("\n=====  Running Parameters    =====")
+        logging.info(self._convert_node_to_json(self.config.run))
+
+        logging.info("\n======  Dataset Attributes  ======")
+        datasets = self.config.datasets
+
+        for dataset in datasets:
+            if dataset in self.config.datasets:
+                logging.info(f"\n======== {dataset} =======")
+                dataset_config = self.config.datasets[dataset]
+                logging.info(self._convert_node_to_json(dataset_config))
+            else:
+                logging.warning(f"No dataset named '{dataset}' in config. Skipping")
+
+        logging.info(f"\n======  Model Attributes  ======")
+        logging.info(self._convert_node_to_json(self.config.model))
+
+    def _convert_node_to_json(self, node):
+        container = OmegaConf.to_container(node, resolve=True)
+        return json.dumps(container, indent=4, sort_keys=True)
+
+    def to_dict(self):
+        return OmegaConf.to_container(self.config)
+
+
+def node_to_dict(node):
+    return OmegaConf.to_container(node)
+
+
+class ConfigValidator:
+    """
+    This is a preliminary implementation to centralize and validate the configuration.
+    May be altered in the future.
+
+    A helper class to validate configurations from yaml file.
+
+    This serves the following purposes:
+        1. Ensure all the options in the yaml are defined, raise error if not.
+        2. when type mismatches are found, the validator will raise an error.
+        3. a central place to store and display helpful messages for supported configurations.
+
+    """
+
+    class _Argument:
+        def __init__(self, name, choices=None, type=None, help=None):
+            self.name = name
+            self.val = None
+            self.choices = choices
+            self.type = type
+            self.help = help
+
+        def __str__(self):
+            s = f"{self.name}={self.val}"
+            if self.type is not None:
+                s += f", ({self.type})"
+            if self.choices is not None:
+                s += f", choices: {self.choices}"
+            if self.help is not None:
+                s += f", ({self.help})"
+            return s
+
+    def __init__(self, description):
+        self.description = description
+
+        self.arguments = dict()
+
+        self.parsed_args = None
+
+    def __getitem__(self, key):
+        assert self.parsed_args is not None, "No arguments parsed yet."
+
+        return self.parsed_args[key]
+
+    def __str__(self) -> str:
+        return self.format_help()
+
+    def add_argument(self, *args, **kwargs):
+        """
+        Assume the first argument is the name of the argument.
+        """
+        self.arguments[args[0]] = self._Argument(*args, **kwargs)
+
+    def validate(self, config=None):
+        """
+        Convert yaml config (dict-like) to list, required by argparse.
+        """
+        for k, v in config.items():
+            assert (
+                k in self.arguments
+            ), f"""{k} is not a valid argument. Support arguments are {self.format_arguments()}."""
+
+            if self.arguments[k].type is not None:
+                try:
+                    self.arguments[k].val = self.arguments[k].type(v)
+                except ValueError:
+                    raise ValueError(f"{k} is not a valid {self.arguments[k].type}.")
+
+            if self.arguments[k].choices is not None:
+                assert (
+                    v in self.arguments[k].choices
+                ), f"""{k} must be one of {self.arguments[k].choices}."""
+
+        return config
+
+    def format_arguments(self):
+        return str([f"{k}" for k in sorted(self.arguments.keys())])
+
+    def format_help(self):
+        # description + key-value pair string for each argument
+        help_msg = str(self.description)
+        return help_msg + ", available arguments: " + self.format_arguments()
+
+    def print_help(self):
+        # display help message
+        print(self.format_help())
+
+
+def create_runner_config_validator():
+    validator = ConfigValidator(description="Runner configurations")
+
+    validator.add_argument(
+        "runner",
+        type=str,
+        choices=["runner_base", "runner_iter"],
+        help="""Runner to use. The "runner_base" uses epoch-based training while iter-based
+            runner runs based on iters. Default: runner_base""",
+    )
+    # add argumetns for training dataset ratios
+    validator.add_argument(
+        "train_dataset_ratios",
+        type=Dict[str, float],
+        help="""Ratios of training dataset. This is used in iteration-based runner.
+        Do not support for epoch-based runner because how to define an epoch becomes tricky.
+        Default: None""",
+    )
+    validator.add_argument(
+        "max_iters",
+        type=float,
+        help="Maximum number of iterations to run.",
+    )
+    validator.add_argument(
+        "max_epoch",
+        type=int,
+        help="Maximum number of epochs to run.",
+    )
+    # add arguments for iters_per_inner_epoch
+    validator.add_argument(
+        "iters_per_inner_epoch",
+        type=float,
+        help="Number of iterations per inner epoch. This is required when runner is runner_iter.",
+    )
+    lr_scheds_choices = registry.list_lr_schedulers()
+    validator.add_argument(
+        "lr_sched",
+        type=str,
+        choices=lr_scheds_choices,
+        help="Learning rate scheduler to use, from {}".format(lr_scheds_choices),
+    )
+    task_choices = registry.list_tasks()
+    validator.add_argument(
+        "task",
+        type=str,
+        choices=task_choices,
+        help="Task to use, from {}".format(task_choices),
+    )
+    # add arguments for init_lr
+    validator.add_argument(
+        "init_lr",
+        type=float,
+        help="Initial learning rate. This will be the learning rate after warmup and before decay.",
+    )
+    # add arguments for min_lr
+    validator.add_argument(
+        "min_lr",
+        type=float,
+        help="Minimum learning rate (after decay).",
+    )
+    # add arguments for warmup_lr
+    validator.add_argument(
+        "warmup_lr",
+        type=float,
+        help="Starting learning rate for warmup.",
+    )
+    # add arguments for learning rate decay rate
+    validator.add_argument(
+        "lr_decay_rate",
+        type=float,
+        help="Learning rate decay rate. Required if using a decaying learning rate scheduler.",
+    )
+    # add arguments for weight decay
+    validator.add_argument(
+        "weight_decay",
+        type=float,
+        help="Weight decay rate.",
+    )
+    # add arguments for training batch size
+    validator.add_argument(
+        "batch_size_train",
+        type=int,
+        help="Training batch size.",
+    )
+    # add arguments for evaluation batch size
+    validator.add_argument(
+        "batch_size_eval",
+        type=int,
+        help="Evaluation batch size, including validation and testing.",
+    )
+    # add arguments for number of workers for data loading
+    validator.add_argument(
+        "num_workers",
+        help="Number of workers for data loading.",
+    )
+    # add arguments for warm up steps
+    validator.add_argument(
+        "warmup_steps",
+        type=int,
+        help="Number of warmup steps. Required if a warmup schedule is used.",
+    )
+    # add arguments for random seed
+    validator.add_argument(
+        "seed",
+        type=int,
+        help="Random seed.",
+    )
+    # add arguments for output directory
+    validator.add_argument(
+        "output_dir",
+        type=str,
+        help="Output directory to save checkpoints and logs.",
+    )
+    # add arguments for whether only use evaluation
+    validator.add_argument(
+        "evaluate",
+        help="Whether to only evaluate the model. If true, training will not be performed.",
+    )
+    # add arguments for splits used for training, e.g. ["train", "val"]
+    validator.add_argument(
+        "train_splits",
+        type=list,
+        help="Splits to use for training.",
+    )
+    # add arguments for splits used for validation, e.g. ["val"]
+    validator.add_argument(
+        "valid_splits",
+        type=list,
+        help="Splits to use for validation. If not provided, will skip the validation.",
+    )
+    # add arguments for splits used for testing, e.g. ["test"]
+    validator.add_argument(
+        "test_splits",
+        type=list,
+        help="Splits to use for testing. If not provided, will skip the testing.",
+    )
+    # add arguments for accumulating gradient for iterations
+    validator.add_argument(
+        "accum_grad_iters",
+        type=int,
+        help="Number of iterations to accumulate gradient for.",
+    )
+
+    # ====== distributed training ======
+    validator.add_argument(
+        "device",
+        type=str,
+        choices=["cpu", "cuda"],
+        help="Device to use. Support 'cuda' or 'cpu' as for now.",
+    )
+    validator.add_argument(
+        "world_size",
+        type=int,
+        help="Number of processes participating in the job.",
+    )
+    validator.add_argument("dist_url", type=str)
+    validator.add_argument("distributed", type=bool)
+    # add arguments to opt using distributed sampler during evaluation or not
+    validator.add_argument(
+        "use_dist_eval_sampler",
+        type=bool,
+        help="Whether to use distributed sampler during evaluation or not.",
+    )
+
+    # ====== task specific ======
+    # generation task specific arguments
+    # add arguments for maximal length of text output
+    validator.add_argument(
+        "max_len",
+        type=int,
+        help="Maximal length of text output.",
+    )
+    # add arguments for minimal length of text output
+    validator.add_argument(
+        "min_len",
+        type=int,
+        help="Minimal length of text output.",
+    )
+    # add arguments number of beams
+    validator.add_argument(
+        "num_beams",
+        type=int,
+        help="Number of beams used for beam search.",
+    )
+
+    # vqa task specific arguments
+    # add arguments for number of answer candidates
+    validator.add_argument(
+        "num_ans_candidates",
+        type=int,
+        help="""For ALBEF and BLIP, these models first rank answers according to likelihood to select answer candidates.""",
+    )
+    # add arguments for inference method
+    validator.add_argument(
+        "inference_method",
+        type=str,
+        choices=["genearte", "rank"],
+        help="""Inference method to use for question answering. If rank, requires a answer list.""",
+    )
+
+    # ====== model specific ======
+    validator.add_argument(
+        "k_test",
+        type=int,
+        help="Number of top k most similar samples from ITC/VTC selection to be tested.",
+    )
+
+    return validator