Create app.py

app.py

@@ -0,0 +1,54 @@
+import gradio as gr
+from tqdm import tqdm
+import requests, os, ctypes, json, argparse, os, array, sys
+
+def ensure_file(filename, src):
+    if not os.path.exists(filename):
+        response = requests.get(src, stream=True)
+        total_size = int(response.headers.get('content-length', 0))
+
+        with open(filename, 'wb') as file:
+            with tqdm(total=total_size, unit='8', unit_scale=True, desc=filename, ncols=80) as progress_bar:
+                for data in response.iter_content(chunk_size=1024):
+                    if data:
+                        file.write(data)
+                        progress_bar.update(len(data))
+
+        print(f'Download Completed.')
+
+ensure_file("mmproj-model-f16.gguf", "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf")
+ensure_file("ggml-model-q4_k.gguf", "https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-q4_k.gguf")
+
+from llama_cpp import Llama, clip_model_load, llava_image_embed_make_with_filename, llava_image_embed_make_with_bytes, llava_image_embed_p, llava_image_embed_free, llava_validate_embed_size, llava_eval_image_embed
+
+ctx_clip = clip_model_load("mmproj-model-f16.gguf".encode('utf-8'))
+llm = Llama(model_path="ggml-model-q4_k.gguf", n_ctx=2048)
+
+def generate(image, ins='Describe the image'):
+    if len(ins) < 1:
+        ins = 'Describe the image'
+    image_embed = llava_image_embed_make_with_filename(ctx_clip=ctx_clip, n_threads=1, filename=image.encode('utf-8'))
+
+    n_past = ctypes.c_int(llm.n_tokens)
+    n_past_p = ctypes.byref(n_past)
+    llava_eval_image_embed(llm.ctx, image_embed, llm.n_batch, n_past_p)
+    llm.n_tokens = n_past.value
+    llava_image_embed_free(image_embed)
+
+    llm.eval(llm.tokenize(ins.encode('utf-8')))
+
+    max_target_len = 256
+    res = ''
+    for i in range(max_target_len):
+        t_id = llm.sample(temp=0.3)
+        t = llm.detokenize([t_id]).decode('utf-8')
+        if t == '</s>':
+            break 
+        res += t
+        llm.eval([t_id])
+
+    return res 
+
+
+iface = gr.Interface(generate, inputs=[gr.Image(type='filepath'), gr.Textbox()], outpus='text')
+iface.launch()