Upload 4 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+delay_tyre.mp4 filter=lfs diff=lfs merge=lfs -text

app (1).py

@@ -0,0 +1,31 @@
+import gradio as gr
+
+from process import inference
+
+def clickit(video, prompt):
+    return inference(
+        video,
+        prompt
+    )
+
+with gr.Blocks() as blok:
+    with gr.Row():
+        with gr.Column():
+            video = gr.Video(
+                label="video input",
+            )
+            prompt = gr.Text(
+                label="Prompt",
+                value="Please describe this video in detail."
+            )
+        with gr.Column():
+            button = gr.Button("Caption it", variant="primary")
+            text = gr.Text(label="Output")
+        
+        button.click(
+            fn=clickit,
+            inputs=[video, prompt],
+            outputs=[text]
+        )
+
+blok.launch()

delay_tyre.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63a245902a9509f492fda6537c84ab53c3582f868503982b53419c01fee6e592
+size 7352910

process.py

@@ -0,0 +1,111 @@
+import io
+import spaces
+import argparse
+import numpy as np
+import torch
+from decord import cpu, VideoReader, bridge
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import BitsAndBytesConfig
+
+MODEL_PATH = "THUDM/cogvlm2-llama3-caption"
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+TORCH_TYPE = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.get_device_capability()[
+    0] >= 8 else torch.float16
+
+parser = argparse.ArgumentParser(description="CogVLM2-Video CLI Demo")
+parser.add_argument('--quant', type=int, choices=[4, 8], help='Enable 4-bit or 8-bit precision loading', default=4)
+args = parser.parse_args([])
+
+def load_video(video_data, strategy='chat'):
+    bridge.set_bridge('torch')
+    mp4_stream = video_data
+    num_frames = 24
+    decord_vr = VideoReader(io.BytesIO(mp4_stream), ctx=cpu(0))
+    frame_id_list = None
+    total_frames = len(decord_vr)
+    
+    if strategy == 'base':
+        clip_end_sec = 60
+        clip_start_sec = 0
+        start_frame = int(clip_start_sec * decord_vr.get_avg_fps())
+        end_frame = min(total_frames,
+                        int(clip_end_sec * decord_vr.get_avg_fps())) if clip_end_sec is not None else total_frames
+        frame_id_list = np.linspace(start_frame, end_frame - 1, num_frames, dtype=int)
+    elif strategy == 'chat':
+        timestamps = decord_vr.get_frame_timestamp(np.arange(total_frames))
+        timestamps = [i[0] for i in timestamps]
+        max_second = round(max(timestamps)) + 1
+        frame_id_list = []
+        for second in range(max_second):
+            closest_num = min(timestamps, key=lambda x: abs(x - second))
+            index = timestamps.index(closest_num)
+            frame_id_list.append(index)
+            if len(frame_id_list) >= num_frames:
+                break
+                
+    video_data = decord_vr.get_batch(frame_id_list)
+    video_data = video_data.permute(3, 0, 1, 2)
+    return video_data
+
+# Configure quantization
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=TORCH_TYPE,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4"
+)
+
+tokenizer = AutoTokenizer.from_pretrained(
+    MODEL_PATH,
+    trust_remote_code=True,
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_PATH,
+    torch_dtype=TORCH_TYPE,
+    trust_remote_code=True,
+    quantization_config=quantization_config,
+    device_map="auto"
+).eval()
+
+@spaces.GPU
+def predict(prompt, video_data, temperature):
+    strategy = 'chat'
+    video = load_video(video_data, strategy=strategy)
+    history = []
+    query = prompt
+    inputs = model.build_conversation_input_ids(
+        tokenizer=tokenizer,
+        query=query,
+        images=[video],
+        history=history,
+        template_version=strategy
+    )
+    
+    inputs = {
+        'input_ids': inputs['input_ids'].unsqueeze(0).to(DEVICE),
+        'token_type_ids': inputs['token_type_ids'].unsqueeze(0).to(DEVICE),
+        'attention_mask': inputs['attention_mask'].unsqueeze(0).to(DEVICE),
+        'images': [[inputs['images'][0].to(DEVICE).to(TORCH_TYPE)]],
+    }
+    
+    gen_kwargs = {
+        "max_new_tokens": 2048,
+        "pad_token_id": 128002,
+        "top_k": 1,
+        "do_sample": False,
+        "top_p": 0.1,
+        "temperature": temperature,
+    }
+    
+    with torch.no_grad():
+        outputs = model.generate(**inputs, **gen_kwargs)
+        outputs = outputs[:, inputs['input_ids'].shape[1]:]
+        response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return response
+
+def inference(video, prompt):
+    temperature = 0.1
+    video_data = open(video, 'rb').read()
+    response = predict(prompt, video_data, temperature)
+    return response

requirements (1).txt

@@ -0,0 +1,26 @@
+decord>=0.6.0
+#根据https://download.pytorch.org/whl/torch/，python版本为[3.8,3.11]
+torch==2.1.0
+torchvision== 0.16.0
+pytorchvideo==0.1.5
+xformers
+transformers==4.42.4
+#git+https://github.com/huggingface/transformers.git
+huggingface-hub>=0.23.0
+pillow
+chainlit>=1.0
+pydantic>=2.7.1
+timm>=0.9.16
+openai>=1.30.1
+loguru>=0.7.2
+pydantic>=2.7.1
+einops
+sse-starlette>=2.1.0
+flask
+gunicorn
+gevent
+requests
+gradio
+accelerate
+bitsandbytes>=0.39.0
+spaces