Datasculptor ffahmed commited on
Commit
cba26b3
·
0 Parent(s):

Duplicate from microsoft-cognitive-service/mm-react

Browse files

Co-authored-by: fai ah <[email protected]>

.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tflite filter=lfs diff=lfs merge=lfs -text
29
+ *.tgz filter=lfs diff=lfs merge=lfs -text
30
+ *.wasm filter=lfs diff=lfs merge=lfs -text
31
+ *.xz filter=lfs diff=lfs merge=lfs -text
32
+ *.zip filter=lfs diff=lfs merge=lfs -text
33
+ *.zst filter=lfs diff=lfs merge=lfs -text
34
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
35
+ *.png filter=lfs diff=lfs merge=lfs -text
Dockerfile ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.10.9
2
+
3
+ WORKDIR /src
4
+
5
+ COPY ./MM-REACT /src/MM-REACT
6
+
7
+ COPY ./requirements.txt /src/requirements.txt
8
+
9
+ COPY ./langchain-0.0.94-py3-none-any.whl /src/langchain-0.0.94-py3-none-any.whl
10
+
11
+ RUN pip install --no-cache-dir /src/langchain-0.0.94-py3-none-any.whl
12
+
13
+ RUN pip install --no-cache-dir --upgrade -r /src/requirements.txt
14
+
15
+ WORKDIR /src/MM-REACT
16
+
17
+
18
+ CMD ["python", "app.py", "--port", "7860", "--openAIModel", "azureChatGPT", "--noIntermediateConv"]
MM-REACT/app.py ADDED
@@ -0,0 +1,507 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ import io
3
+ import os
4
+ from typing import Optional, Tuple
5
+ import datetime
6
+ import sys
7
+ import gradio as gr
8
+ import requests
9
+ import json
10
+ from threading import Lock
11
+ from langchain import ConversationChain, LLMChain
12
+ from langchain.agents import load_tools, initialize_agent, Tool
13
+ from langchain.tools.bing_search.tool import BingSearchRun, BingSearchAPIWrapper
14
+ from langchain.chains.conversation.memory import ConversationBufferMemory
15
+ from langchain.llms import OpenAI
16
+ from langchain.chains import PALChain
17
+ from langchain.llms import AzureOpenAI
18
+ from langchain.utilities import ImunAPIWrapper, ImunMultiAPIWrapper
19
+ from openai.error import AuthenticationError, InvalidRequestError, RateLimitError
20
+ import argparse
21
+ import logging
22
+ from opencensus.ext.azure.log_exporter import AzureLogHandler
23
+ import uuid
24
+
25
+ logger = None
26
+
27
+
28
+ OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
29
+ BUG_FOUND_MSG = "Some Functionalities not supported yet. Please refresh and hit 'Click to wake up MM-REACT'"
30
+ AUTH_ERR_MSG = "OpenAI key needed"
31
+ REFRESH_MSG = "Please refresh and hit 'Click to wake up MM-REACT'"
32
+ MAX_TOKENS = 512
33
+
34
+
35
+ ############## ARGS #################
36
+ AGRS = None
37
+ #####################################
38
+
39
+
40
+ def get_logger():
41
+ global logger
42
+ if logger is None:
43
+ logger = logging.getLogger(__name__)
44
+ logger.addHandler(AzureLogHandler())
45
+ return logger
46
+
47
+
48
+ # load chain
49
+ def load_chain(history, log_state):
50
+ global ARGS
51
+
52
+ if ARGS.openAIModel == 'openAIGPT35':
53
+ # openAI GPT 3.5
54
+ llm = OpenAI(temperature=0, max_tokens=MAX_TOKENS)
55
+ elif ARGS.openAIModel == 'azureChatGPT':
56
+ # for Azure OpenAI ChatGPT
57
+ llm = AzureOpenAI(deployment_name="text-chat-davinci-002", model_name="text-chat-davinci-002", temperature=0, max_tokens=MAX_TOKENS)
58
+ elif ARGS.openAIModel == 'azureGPT35turbo':
59
+ # for Azure OpenAI gpt3.5 turbo
60
+ llm = AzureOpenAI(deployment_name="gpt-35-turbo-version-0301", model_name="gpt-35-turbo (version 0301)", temperature=0, max_tokens=MAX_TOKENS)
61
+ elif ARGS.openAIModel == 'azureTextDavinci003':
62
+ # for Azure OpenAI text davinci
63
+ llm = AzureOpenAI(deployment_name="text-davinci-003", model_name="text-davinci-003", temperature=0, max_tokens=MAX_TOKENS)
64
+
65
+ memory = ConversationBufferMemory(memory_key="chat_history")
66
+
67
+
68
+ #############################
69
+ # loading all tools
70
+
71
+ imun_dense = ImunAPIWrapper(
72
+ imun_url="https://ehazarwestus.cognitiveservices.azure.com/computervision/imageanalysis:analyze",
73
+ params="api-version=2023-02-01-preview&model-version=latest&features=denseCaptions",
74
+ imun_subscription_key=os.environ.get("IMUN_SUBSCRIPTION_KEY2"))
75
+
76
+ imun = ImunAPIWrapper()
77
+ imun = ImunMultiAPIWrapper(imuns=[imun, imun_dense])
78
+
79
+ imun_celeb = ImunAPIWrapper(
80
+ imun_url="https://cvfiahmed.cognitiveservices.azure.com/vision/v3.2/models/celebrities/analyze",
81
+ params="")
82
+
83
+ imun_read = ImunAPIWrapper(
84
+ imun_url="https://vigehazar.cognitiveservices.azure.com/formrecognizer/documentModels/prebuilt-read:analyze",
85
+ params="api-version=2022-08-31",
86
+ imun_subscription_key=os.environ.get("IMUN_OCR_SUBSCRIPTION_KEY"))
87
+
88
+ imun_receipt = ImunAPIWrapper(
89
+ imun_url="https://vigehazar.cognitiveservices.azure.com/formrecognizer/documentModels/prebuilt-receipt:analyze",
90
+ params="api-version=2022-08-31",
91
+ imun_subscription_key=os.environ.get("IMUN_OCR_SUBSCRIPTION_KEY"))
92
+
93
+ imun_businesscard = ImunAPIWrapper(
94
+ imun_url="https://vigehazar.cognitiveservices.azure.com/formrecognizer/documentModels/prebuilt-businessCard:analyze",
95
+ params="api-version=2022-08-31",
96
+ imun_subscription_key=os.environ.get("IMUN_OCR_SUBSCRIPTION_KEY"))
97
+
98
+ imun_layout = ImunAPIWrapper(
99
+ imun_url="https://vigehazar.cognitiveservices.azure.com/formrecognizer/documentModels/prebuilt-layout:analyze",
100
+ params="api-version=2022-08-31",
101
+ imun_subscription_key=os.environ.get("IMUN_OCR_SUBSCRIPTION_KEY"))
102
+
103
+ bing = BingSearchAPIWrapper(k=2)
104
+
105
+ def edit_photo(query: str) -> str:
106
+ endpoint = os.environ.get("PHOTO_EDIT_ENDPOINT_URL")
107
+ query = query.strip()
108
+ url_idx = query.rfind(" ")
109
+ img_url = query[url_idx + 1:].strip()
110
+ if img_url.endswith((".", "?")):
111
+ img_url = img_url[:-1]
112
+ if not img_url.startswith(("http://", "https://")):
113
+ return "Invalid image URL"
114
+ img_url = img_url.replace("0.0.0.0", os.environ.get("PHOTO_EDIT_ENDPOINT_URL_SHORT"))
115
+ instruction = query[:url_idx]
116
+ # This should be some internal IP to wherever the server runs
117
+ job = {"image_path": img_url, "instruction": instruction}
118
+ response = requests.post(endpoint, json=job)
119
+ if response.status_code != 200:
120
+ return "Could not finish the task try again later!"
121
+ return "Here is the edited image " + endpoint + response.json()["edited_image"]
122
+
123
+ # these tools should not step on each other's toes
124
+ tools = [
125
+ Tool(
126
+ name="PAL-MATH",
127
+ func=PALChain.from_math_prompt(llm).run,
128
+ description=(
129
+ "A wrapper around calculator. "
130
+ "A language model that is really good at solving complex word math problems."
131
+ "Input should be a fully worded hard word math problem."
132
+ )
133
+ ),
134
+ Tool(
135
+ name = "Image Understanding",
136
+ func=imun.run,
137
+ description=(
138
+ "A wrapper around Image Understanding. "
139
+ "Useful for when you need to understand what is inside an image (objects, texts, people)."
140
+ "Input should be an image url, or path to an image file (e.g. .jpg, .png)."
141
+ )
142
+ ),
143
+ Tool(
144
+ name = "OCR Understanding",
145
+ func=imun_read.run,
146
+ description=(
147
+ "A wrapper around OCR Understanding (Optical Character Recognition). "
148
+ "Useful after Image Understanding tool has found text or handwriting is present in the image tags."
149
+ "This tool can find the actual text, written name, or product name in the image."
150
+ "Input should be an image url, or path to an image file (e.g. .jpg, .png)."
151
+ )
152
+ ),
153
+ Tool(
154
+ name = "Receipt Understanding",
155
+ func=imun_receipt.run,
156
+ description=(
157
+ "A wrapper receipt understanding. "
158
+ "Useful after Image Understanding tool has recognized a receipt in the image tags."
159
+ "This tool can find the actual receipt text, prices and detailed items."
160
+ "Input should be an image url, or path to an image file (e.g. .jpg, .png)."
161
+ )
162
+ ),
163
+ Tool(
164
+ name = "Business Card Understanding",
165
+ func=imun_businesscard.run,
166
+ description=(
167
+ "A wrapper around business card understanding. "
168
+ "Useful after Image Understanding tool has recognized businesscard in the image tags."
169
+ "This tool can find the actual business card text, name, address, email, website on the card."
170
+ "Input should be an image url, or path to an image file (e.g. .jpg, .png)."
171
+ )
172
+ ),
173
+ Tool(
174
+ name = "Layout Understanding",
175
+ func=imun_layout.run,
176
+ description=(
177
+ "A wrapper around layout and table understanding. "
178
+ "Useful after Image Understanding tool has recognized businesscard in the image tags."
179
+ "This tool can find the actual business card text, name, address, email, website on the card."
180
+ "Input should be an image url, or path to an image file (e.g. .jpg, .png)."
181
+ )
182
+ ),
183
+ Tool(
184
+ name = "Celebrity Understanding",
185
+ func=imun_celeb.run,
186
+ description=(
187
+ "A wrapper around celebrity understanding. "
188
+ "Useful after Image Understanding tool has recognized people in the image tags that could be celebrities."
189
+ "This tool can find the name of celebrities in the image."
190
+ "Input should be an image url, or path to an image file (e.g. .jpg, .png)."
191
+ )
192
+ ),
193
+ BingSearchRun(api_wrapper=bing),
194
+ Tool(
195
+ name = "Photo Editing",
196
+ func=edit_photo,
197
+ description=(
198
+ "A wrapper around photo editing. "
199
+ "Useful to edit an image with a given instruction."
200
+ "Input should be an image url, or path to an image file (e.g. .jpg, .png)."
201
+ )
202
+ ),
203
+ ]
204
+
205
+ chain = initialize_agent(tools, llm, agent="conversational-assistant", verbose=True, memory=memory, return_intermediate_steps=True, max_iterations=4)
206
+ log_state = log_state or ""
207
+ print ("log_state {}".format(log_state))
208
+ log_state = str(uuid.uuid1())
209
+ print("langchain reloaded")
210
+ # eproperties = {'custom_dimensions': {'key_1': 'value_1', 'key_2': 'value_2'}}
211
+ properties = {'custom_dimensions': {'session': log_state}}
212
+ get_logger().warning("langchain reloaded", extra=properties)
213
+ history = []
214
+ history.append(("Show me what you got!", "Hi Human, Please upload an image to get started!"))
215
+
216
+ return history, history, chain, log_state, \
217
+ gr.Textbox.update(visible=True), \
218
+ gr.Button.update(visible=True), \
219
+ gr.UploadButton.update(visible=True), \
220
+ gr.Row.update(visible=True), \
221
+ gr.HTML.update(visible=True), \
222
+ gr.Button.update(variant="secondary")
223
+
224
+
225
+ # executes input typed by human
226
+ def run_chain(chain, inp):
227
+ # global chain
228
+
229
+ output = ""
230
+ try:
231
+ output = chain.conversation(input=inp, keep_short=ARGS.noIntermediateConv)
232
+ # output = chain.run(input=inp)
233
+ except AuthenticationError as ae:
234
+ output = AUTH_ERR_MSG + str(datetime.datetime.now()) + ". " + str(ae)
235
+ print("output", output)
236
+ except RateLimitError as rle:
237
+ output = "\n\nRateLimitError: " + str(rle)
238
+ except ValueError as ve:
239
+ output = "\n\nValueError: " + str(ve)
240
+ except InvalidRequestError as ire:
241
+ output = "\n\nInvalidRequestError: " + str(ire)
242
+ except Exception as e:
243
+ output = "\n\n" + BUG_FOUND_MSG + ":\n\n" + str(e)
244
+
245
+ return output
246
+
247
+ # simple chat function wrapper
248
+ class ChatWrapper:
249
+
250
+ def __init__(self):
251
+ self.lock = Lock()
252
+
253
+ def __call__(
254
+ self, inp: str, history: Optional[Tuple[str, str]], chain: Optional[ConversationChain], log_state
255
+ ):
256
+
257
+ """Execute the chat functionality."""
258
+ self.lock.acquire()
259
+ try:
260
+ print("\n==== date/time: " + str(datetime.datetime.now()) + " ====")
261
+ print("inp: " + inp)
262
+
263
+ properties = {'custom_dimensions': {'session': log_state}}
264
+ get_logger().warning("inp: " + inp, extra=properties)
265
+
266
+
267
+ history = history or []
268
+ # If chain is None, that is because no API key was provided.
269
+ output = "Please paste your OpenAI key from openai.com to use this app. " + str(datetime.datetime.now())
270
+
271
+ ########################
272
+ # multi line
273
+ outputs = run_chain(chain, inp)
274
+
275
+ outputs = process_chain_output(outputs)
276
+
277
+ print (" len(outputs) {}".format(len(outputs)))
278
+ for i, output in enumerate(outputs):
279
+ if i==0:
280
+ history.append((inp, output))
281
+ else:
282
+ history.append((None, output))
283
+
284
+
285
+ except Exception as e:
286
+ raise e
287
+ finally:
288
+ self.lock.release()
289
+
290
+ print (history)
291
+ properties = {'custom_dimensions': {'session': log_state}}
292
+ if outputs is None:
293
+ outputs = ""
294
+ get_logger().warning(str(json.dumps(outputs)), extra=properties)
295
+
296
+ return history, history, ""
297
+
298
+ def add_image_with_path(state, chain, imagepath, log_state):
299
+ global ARGS
300
+ state = state or []
301
+
302
+ url_input_for_chain = "http://0.0.0.0:{}/file={}".format(ARGS.port, imagepath)
303
+
304
+ outputs = run_chain(chain, url_input_for_chain)
305
+
306
+ ########################
307
+ # multi line response handling
308
+ outputs = process_chain_output(outputs)
309
+
310
+ for i, output in enumerate(outputs):
311
+ if i==0:
312
+ # state.append((f"![](/file={imagepath})", output))
313
+ state.append(((imagepath,), output))
314
+ else:
315
+ state.append((None, output))
316
+
317
+
318
+ print (state)
319
+ properties = {'custom_dimensions': {'session': log_state}}
320
+ get_logger().warning("url_input_for_chain: " + url_input_for_chain, extra=properties)
321
+ if outputs is None:
322
+ outputs = ""
323
+ get_logger().warning(str(json.dumps(outputs)), extra=properties)
324
+ return state, state
325
+
326
+
327
+ # upload image
328
+ def add_image(state, chain, image, log_state):
329
+ global ARGS
330
+ state = state or []
331
+
332
+ # handling spaces in image path
333
+ imagepath = image.name.replace(" ", "%20")
334
+
335
+ url_input_for_chain = "http://0.0.0.0:{}/file={}".format(ARGS.port, imagepath)
336
+
337
+ outputs = run_chain(chain, url_input_for_chain)
338
+
339
+ ########################
340
+ # multi line response handling
341
+ outputs = process_chain_output(outputs)
342
+
343
+ for i, output in enumerate(outputs):
344
+ if i==0:
345
+ state.append(((imagepath,), output))
346
+ else:
347
+ state.append((None, output))
348
+
349
+
350
+ print (state)
351
+ properties = {'custom_dimensions': {'session': log_state}}
352
+ get_logger().warning("url_input_for_chain: " + url_input_for_chain, extra=properties)
353
+ if outputs is None:
354
+ outputs = ""
355
+ get_logger().warning(str(json.dumps(outputs)), extra=properties)
356
+ return state, state
357
+
358
+ # extract image url from response and process differently
359
+ def replace_with_image_markup(text):
360
+ img_url = None
361
+ text= text.strip()
362
+ url_idx = text.rfind(" ")
363
+ img_url = text[url_idx + 1:].strip()
364
+ if img_url.endswith((".", "?")):
365
+ img_url = img_url[:-1]
366
+
367
+ # if img_url is not None:
368
+ # img_url = f"![](/file={img_url})"
369
+ return img_url
370
+
371
+ # multi line response handling
372
+ def process_chain_output(outputs):
373
+ global ARGS
374
+ EMPTY_AI_REPLY = "AI:"
375
+ # print("outputs {}".format(outputs))
376
+ if isinstance(outputs, str): # single line output
377
+ if outputs.strip() == EMPTY_AI_REPLY:
378
+ outputs = REFRESH_MSG
379
+ outputs = [outputs]
380
+ elif isinstance(outputs, list): # multi line output
381
+ if ARGS.noIntermediateConv: # remove the items with assistant in it.
382
+ cleanOutputs = []
383
+ for output in outputs:
384
+ if output.strip() == EMPTY_AI_REPLY:
385
+ output = REFRESH_MSG
386
+ # found an edited image url to embed
387
+ img_url = None
388
+ # print ("type list: {}".format(output))
389
+ if "assistant: here is the edited image " in output.lower():
390
+ img_url = replace_with_image_markup(output)
391
+ cleanOutputs.append("Assistant: Here is the edited image")
392
+ if img_url is not None:
393
+ cleanOutputs.append((img_url,))
394
+ else:
395
+ cleanOutputs.append(output)
396
+ # cleanOutputs = cleanOutputs + output+ "."
397
+ outputs = cleanOutputs
398
+
399
+ return outputs
400
+
401
+
402
+ def init_and_kick_off():
403
+ global ARGS
404
+ # initalize chatWrapper
405
+ chat = ChatWrapper()
406
+
407
+ exampleTitle = """<h3>Examples to start conversation..</h3>"""
408
+ comingSoon = """<center><b><p style="color:Red;">MM-REACT: March 21th version with image understanding capabilities</p></b></center>"""
409
+
410
+ with gr.Blocks(css="#tryButton {width: 120px;}") as block:
411
+ llm_state = gr.State()
412
+ history_state = gr.State()
413
+ chain_state = gr.State()
414
+ log_state = gr.State()
415
+
416
+ reset_btn = gr.Button(value="!!!CLICK to wake up MM-REACT!!!", variant="primary", elem_id="resetbtn").style(full_width=True)
417
+ gr.HTML(comingSoon)
418
+
419
+ example_image_size = 90
420
+ col_min_width = 80
421
+ button_variant = "primary"
422
+ with gr.Row():
423
+ with gr.Column(scale=1.0, min_width=100):
424
+ chatbot = gr.Chatbot(elem_id="chatbot", label="MM-REACT Bot").style(height=620)
425
+ with gr.Column(scale=0.20, min_width=200, visible=False) as exampleCol:
426
+ with gr.Row():
427
+ grExampleTitle = gr.HTML(exampleTitle, visible=False)
428
+ with gr.Row():
429
+ with gr.Column(scale=0.50, min_width=col_min_width):
430
+ example3Image = gr.Image("images/receipt.png", interactive=False).style(height=example_image_size, width=example_image_size)
431
+ with gr.Column(scale=0.50, min_width=col_min_width):
432
+ example3ImageButton = gr.Button(elem_id="tryButton", value="Try it!", variant=button_variant).style(full_width=True)
433
+ # dummy text field to hold the path
434
+ example3ImagePath = gr.Text("images/receipt.png", interactive=False, visible=False)
435
+ with gr.Row():
436
+ with gr.Column(scale=0.50, min_width=col_min_width):
437
+ example1Image = gr.Image("images/money.png", interactive=False).style(height=example_image_size, width=example_image_size)
438
+ with gr.Column(scale=0.50, min_width=col_min_width):
439
+ example1ImageButton = gr.Button(elem_id="tryButton", value="Try it!", variant=button_variant).style(full_width=True)
440
+ # dummy text field to hold the path
441
+ example1ImagePath = gr.Text("images/money.png", interactive=False, visible=False)
442
+ with gr.Row():
443
+ with gr.Column(scale=0.50, min_width=col_min_width):
444
+ example2Image = gr.Image("images/cartoon.png", interactive=False).style(height=example_image_size, width=example_image_size)
445
+ with gr.Column(scale=0.50, min_width=col_min_width):
446
+ example2ImageButton = gr.Button(elem_id="tryButton", value="Try it!", variant=button_variant).style(full_width=True)
447
+ # dummy text field to hold the path
448
+ example2ImagePath = gr.Text("images/cartoon.png", interactive=False, visible=False)
449
+ with gr.Row():
450
+ with gr.Column(scale=0.50, min_width=col_min_width):
451
+ example4Image = gr.Image("images/product.png", interactive=False).style(height=example_image_size, width=example_image_size)
452
+ with gr.Column(scale=0.50, min_width=col_min_width):
453
+ example4ImageButton = gr.Button(elem_id="tryButton", value="Try it!", variant=button_variant).style(full_width=True)
454
+ # dummy text field to hold the path
455
+ example4ImagePath = gr.Text("images/product.png", interactive=False, visible=False)
456
+ with gr.Row():
457
+ with gr.Column(scale=0.50, min_width=col_min_width):
458
+ example5Image = gr.Image("images/celebrity.png", interactive=False).style(height=example_image_size, width=example_image_size)
459
+ with gr.Column(scale=0.50, min_width=col_min_width):
460
+ example5ImageButton = gr.Button(elem_id="tryButton", value="Try it!", variant=button_variant).style(full_width=True)
461
+ # dummy text field to hold the path
462
+ example5ImagePath = gr.Text("images/celebrity.png", interactive=False, visible=False)
463
+
464
+
465
+
466
+ with gr.Row():
467
+ with gr.Column(scale=0.75):
468
+ message = gr.Textbox(label="Upload a pic and ask!",
469
+ placeholder="Type your question about the uploaded image",
470
+ lines=1, visible=False)
471
+ with gr.Column(scale=0.15):
472
+ submit = gr.Button(value="Send", variant="secondary", visible=False).style(full_width=True)
473
+ with gr.Column(scale=0.10, min_width=0):
474
+ btn = gr.UploadButton("🖼️", file_types=["image"], visible=False).style(full_width=True)
475
+
476
+
477
+ message.submit(chat, inputs=[message, history_state, chain_state, log_state], outputs=[chatbot, history_state, message])
478
+
479
+ submit.click(chat, inputs=[message, history_state, chain_state, log_state], outputs=[chatbot, history_state, message])
480
+
481
+ btn.upload(add_image, inputs=[history_state, chain_state, btn, log_state], outputs=[history_state, chatbot])
482
+
483
+ # load the chain
484
+ reset_btn.click(load_chain, inputs=[history_state, log_state], outputs=[chatbot, history_state, chain_state, log_state, message, submit, btn, exampleCol, grExampleTitle, reset_btn])
485
+
486
+ # setup listener click for the examples
487
+ example1ImageButton.click(add_image_with_path, inputs=[history_state, chain_state, example1ImagePath, log_state], outputs=[history_state, chatbot])
488
+ example2ImageButton.click(add_image_with_path, inputs=[history_state, chain_state, example2ImagePath, log_state], outputs=[history_state, chatbot])
489
+ example3ImageButton.click(add_image_with_path, inputs=[history_state, chain_state, example3ImagePath, log_state], outputs=[history_state, chatbot])
490
+ example4ImageButton.click(add_image_with_path, inputs=[history_state, chain_state, example4ImagePath, log_state], outputs=[history_state, chatbot])
491
+ example5ImageButton.click(add_image_with_path, inputs=[history_state, chain_state, example5ImagePath, log_state], outputs=[history_state, chatbot])
492
+
493
+
494
+ # launch the app
495
+ block.launch(server_name="0.0.0.0", server_port = ARGS.port)
496
+
497
+ if __name__ == '__main__':
498
+ parser = argparse.ArgumentParser()
499
+
500
+ parser.add_argument('--port', type=int, required=False, default=7860)
501
+ parser.add_argument('--openAIModel', type=str, required=False, default='azureChatGPT')
502
+ parser.add_argument('--noIntermediateConv', default=False, action='store_true', help='if this flag is turned on no intermediate conversation should be shown')
503
+
504
+ global ARGS
505
+ ARGS = parser.parse_args()
506
+
507
+ init_and_kick_off()
MM-REACT/images/cartoon.png ADDED

Git LFS Details

  • SHA256: 21802525f0fa269131e78c5ac55d150bdd6cb728a473b09a871e28a02c5dd123
  • Pointer size: 132 Bytes
  • Size of remote file: 1.06 MB
MM-REACT/images/celebrity.png ADDED

Git LFS Details

  • SHA256: 9c60f49627655d70c76ff0106f61412db1735cad1820ed1b87f9ca7e287d9763
  • Pointer size: 132 Bytes
  • Size of remote file: 2.24 MB
MM-REACT/images/money.png ADDED

Git LFS Details

  • SHA256: 0c7f6155049624573c2852538c86ffaab41c258f04462d0f9dc1b73b86949030
  • Pointer size: 131 Bytes
  • Size of remote file: 763 kB
MM-REACT/images/product.png ADDED

Git LFS Details

  • SHA256: b6bd4efae45dadf27cd9b960a56aded711d0c8ec3e41ae0012eb8dd0c8c92ceb
  • Pointer size: 131 Bytes
  • Size of remote file: 655 kB
MM-REACT/images/receipt.png ADDED

Git LFS Details

  • SHA256: 86a1c65f0e82c855cbd766178fe9080fe9c084678da79f2e695680c0ee97b9eb
  • Pointer size: 131 Bytes
  • Size of remote file: 656 kB
README.md ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: mm-react
3
+ emoji: 💻
4
+ colorFrom: indigo
5
+ colorTo: pink
6
+ sdk: docker
7
+ pinned: false
8
+ license: other
9
+ duplicated_from: microsoft-cognitive-service/mm-react
10
+ ---
11
+
12
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
langchain-0.0.94-py3-none-any.whl ADDED
Binary file (319 kB). View file
 
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ opencensus==0.11.0
2
+ opencensus-context==0.1.3
3
+ opencensus-ext-azure==1.1.6
4
+ opencensus-ext-logging==0.1.1
5
+ imagesize==1.4.1
6
+ gradio==3.21.0
7
+ openai==0.26.4
8
+ requests==2.28.2