mszel commited on
Commit
2577d27
·
1 Parent(s): ed1dc36

Correcting the Image RAG pipeline

Browse files
examples/LynxScribe Image RAG CHANGED
@@ -1,24 +1,10 @@
1
  {
2
  "edges": [
3
  {
4
- "id": "Input chat 1 LynxScribe Image RAG Query 1",
5
- "source": "Input chat 1",
6
- "sourceHandle": "output",
7
- "target": "LynxScribe Image RAG Query 1",
8
- "targetHandle": "text"
9
- },
10
- {
11
- "id": "LynxScribe Image RAG Query 1 View image 1",
12
- "source": "LynxScribe Image RAG Query 1",
13
  "sourceHandle": "output",
14
- "target": "View image 1",
15
- "targetHandle": "embedding_similarities"
16
- },
17
- {
18
- "id": "Cloud-sourced File Loader 1 LynxScribe Image RAG Builder 1",
19
- "source": "Cloud-sourced File Loader 1",
20
- "sourceHandle": "output",
21
- "target": "LynxScribe Image RAG Builder 1",
22
  "targetHandle": "file_urls"
23
  },
24
  {
@@ -26,21 +12,28 @@
26
  "source": "LynxScribe Image Describer 1",
27
  "sourceHandle": "output",
28
  "target": "LynxScribe Image RAG Builder 1",
29
- "targetHandle": "image_describer"
30
  },
31
  {
32
- "id": "LynxScribe RAG Graph Vector Store 1 LynxScribe Image RAG Builder 1",
33
- "source": "LynxScribe RAG Graph Vector Store 1",
34
  "sourceHandle": "output",
35
- "target": "LynxScribe Image RAG Builder 1",
36
  "targetHandle": "rag_graph"
37
  },
38
  {
39
- "id": "LynxScribe Image RAG Builder 1 LynxScribe Image RAG Query 1",
40
- "source": "LynxScribe Image RAG Builder 1",
41
  "sourceHandle": "output",
42
  "target": "LynxScribe Image RAG Query 1",
43
- "targetHandle": "rag_graph"
 
 
 
 
 
 
 
44
  }
45
  ],
46
  "env": "LynxScribe",
@@ -75,7 +68,7 @@
75
  "type": "basic"
76
  },
77
  "params": {
78
- "chat": "Show me a picture about pills!"
79
  },
80
  "status": "done",
81
  "title": "Input chat"
@@ -84,8 +77,8 @@
84
  "height": 214.0,
85
  "id": "Input chat 1",
86
  "position": {
87
- "x": -302.70349900314835,
88
- "y": -140.67386384008685
89
  },
90
  "type": "basic",
91
  "width": 387.0
@@ -97,23 +90,8 @@
97
  "display": null,
98
  "error": null,
99
  "meta": {
100
- "inputs": {
101
- "rag_graph": {
102
- "name": "rag_graph",
103
- "position": "bottom",
104
- "type": {
105
- "type": "<class 'inspect._empty'>"
106
- }
107
- },
108
- "text": {
109
- "name": "text",
110
- "position": "left",
111
- "type": {
112
- "type": "<class 'inspect._empty'>"
113
- }
114
- }
115
- },
116
- "name": "LynxScribe Image RAG Query",
117
  "outputs": {
118
  "output": {
119
  "name": "output",
@@ -124,78 +102,77 @@
124
  }
125
  },
126
  "params": {
127
- "top_k": {
128
- "default": 3.0,
129
- "name": "top_k",
130
  "type": {
131
- "type": "<class 'int'>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
  }
133
  }
134
  },
 
 
 
 
135
  "type": "basic"
136
  },
137
  "params": {
138
- "top_k": "3"
 
 
139
  },
140
  "status": "done",
141
- "title": "LynxScribe Image RAG Query"
142
  },
143
  "dragHandle": ".bg-primary",
144
- "height": 200.0,
145
- "id": "LynxScribe Image RAG Query 1",
146
  "position": {
147
- "x": 908.9211080204011,
148
- "y": -132.3031800030364
149
  },
150
  "type": "basic",
151
- "width": 200.0
152
  },
153
  {
154
  "data": {
155
- "__execution_delay": null,
156
- "collapsed": false,
157
- "display": "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test/capsules-1079838_1280.jpg",
158
  "error": null,
159
  "meta": {
160
  "inputs": {
161
- "embedding_similarities": {
162
- "name": "embedding_similarities",
163
  "position": "left",
164
  "type": {
165
  "type": "<class 'inspect._empty'>"
166
  }
167
  }
168
  },
169
- "name": "View image",
170
- "outputs": {},
171
- "params": {},
172
- "type": "image"
173
- },
174
- "params": {},
175
- "status": "done",
176
- "title": "View image"
177
- },
178
- "dragHandle": ".bg-primary",
179
- "height": 1170.0,
180
- "id": "View image 1",
181
- "position": {
182
- "x": 1426.7020124006506,
183
- "y": -293.16229409169125
184
- },
185
- "type": "image",
186
- "width": 750.0
187
- },
188
- {
189
- "data": {
190
- "display": null,
191
- "error": null,
192
- "meta": {
193
- "inputs": {},
194
  "name": "LynxScribe Image Describer",
195
  "outputs": {
196
  "output": {
197
  "name": "output",
198
- "position": "top",
199
  "type": {
200
  "type": "None"
201
  }
@@ -217,7 +194,7 @@
217
  }
218
  },
219
  "llm_prompt_path": {
220
- "default": "../lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
221
  "name": "llm_prompt_path",
222
  "type": {
223
  "type": "<class 'str'>"
@@ -232,108 +209,118 @@
232
  }
233
  },
234
  "position": {
235
- "x": 1066.0,
236
- "y": 713.0
237
  },
238
  "type": "basic"
239
  },
240
  "params": {
241
  "llm_interface": "openai",
242
  "llm_prompt_name": "cot_picture_descriptor",
243
- "llm_prompt_path": "../lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
244
  "llm_visual_model": "gpt-4o"
245
  },
246
  "status": "done",
247
  "title": "LynxScribe Image Describer"
248
  },
249
  "dragHandle": ".bg-primary",
250
- "height": 363.0,
251
  "id": "LynxScribe Image Describer 1",
252
  "position": {
253
- "x": 127.85361236096924,
254
- "y": 687.0518781863441
255
  },
256
  "type": "basic",
257
- "width": 401.0
258
  },
259
  {
260
  "data": {
 
 
261
  "display": null,
262
  "error": null,
263
  "meta": {
264
- "inputs": {},
265
- "name": "LynxScribe RAG Graph Vector Store",
 
 
 
 
 
 
 
 
266
  "outputs": {
267
  "output": {
268
  "name": "output",
269
- "position": "top",
270
  "type": {
271
  "type": "None"
272
  }
273
  }
274
  },
275
  "params": {
276
- "collection_name": {
277
- "default": "lynx",
278
- "name": "collection_name",
279
  "type": {
280
  "type": "<class 'str'>"
281
  }
282
  },
283
- "name": {
284
- "default": "faiss",
285
- "name": "name",
286
  "type": {
287
  "type": "<class 'str'>"
288
  }
289
  },
290
- "num_dimensions": {
291
- "default": 3072.0,
292
- "name": "num_dimensions",
293
  "type": {
294
- "type": "<class 'int'>"
295
  }
296
  },
297
- "text_embedder_interface": {
298
- "default": "openai",
299
- "name": "text_embedder_interface",
300
  "type": {
301
- "type": "<class 'str'>"
302
  }
303
  },
304
- "text_embedder_model_name_or_path": {
305
- "default": "text-embedding-3-large",
306
- "name": "text_embedder_model_name_or_path",
307
  "type": {
308
  "type": "<class 'str'>"
309
  }
310
  }
311
  },
312
  "position": {
313
- "x": 790.0,
314
- "y": 633.0
315
  },
316
  "type": "basic"
317
  },
318
  "params": {
319
- "collection_name": "lynx",
320
- "name": "faiss",
321
- "num_dimensions": 3072.0,
322
  "text_embedder_interface": "openai",
323
- "text_embedder_model_name_or_path": "text-embedding-3-large"
 
 
 
324
  },
325
  "status": "done",
326
- "title": "LynxScribe RAG Graph Vector Store"
327
  },
328
  "dragHandle": ".bg-primary",
329
- "height": 436.0,
330
- "id": "LynxScribe RAG Graph Vector Store 1",
331
  "position": {
332
- "x": 595.4558693958389,
333
- "y": 688.0989202130276
334
  },
335
  "type": "basic",
336
- "width": 339.0
337
  },
338
  {
339
  "data": {
@@ -342,8 +329,23 @@
342
  "display": null,
343
  "error": null,
344
  "meta": {
345
- "inputs": {},
346
- "name": "Cloud-sourced File Loader",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
347
  "outputs": {
348
  "output": {
349
  "name": "output",
@@ -354,110 +356,72 @@
354
  }
355
  },
356
  "params": {
357
- "accepted_file_types": {
358
- "default": ".jpg, .jpeg, .png",
359
- "name": "accepted_file_types",
360
- "type": {
361
- "type": "<class 'str'>"
362
- }
363
- },
364
- "cloud_provider": {
365
- "default": "gcp",
366
- "name": "cloud_provider",
367
- "type": {
368
- "type": "<class 'str'>"
369
- }
370
- },
371
- "folder_URL": {
372
- "default": "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test",
373
- "name": "folder_URL",
374
  "type": {
375
- "type": "<class 'str'>"
376
  }
377
  }
378
  },
379
  "position": {
380
- "x": 248.0,
381
- "y": 419.0
382
  },
383
  "type": "basic"
384
  },
385
  "params": {
386
- "accepted_file_types": ".jpg, .jpeg, .png",
387
- "cloud_provider": "gcp",
388
- "folder_URL": "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test"
389
  },
390
  "status": "done",
391
- "title": "Cloud-sourced File Loader"
392
  },
393
  "dragHandle": ".bg-primary",
394
- "height": 291.0,
395
- "id": "Cloud-sourced File Loader 1",
396
  "position": {
397
- "x": -479.7367372966062,
398
- "y": 213.81581567584843
399
  },
400
  "type": "basic",
401
- "width": 512.0
402
  },
403
  {
404
  "data": {
405
- "display": null,
406
  "error": null,
407
  "meta": {
408
  "inputs": {
409
- "file_urls": {
410
- "name": "file_urls",
411
  "position": "left",
412
  "type": {
413
  "type": "<class 'inspect._empty'>"
414
  }
415
- },
416
- "image_describer": {
417
- "name": "image_describer",
418
- "position": "bottom",
419
- "type": {
420
- "type": "<class 'inspect._empty'>"
421
- }
422
- },
423
- "rag_graph": {
424
- "name": "rag_graph",
425
- "position": "bottom",
426
- "type": {
427
- "type": "<class 'inspect._empty'>"
428
- }
429
- }
430
- },
431
- "name": "LynxScribe Image RAG Builder",
432
- "outputs": {
433
- "output": {
434
- "name": "output",
435
- "position": "right",
436
- "type": {
437
- "type": "None"
438
- }
439
  }
440
  },
 
 
441
  "params": {},
442
  "position": {
443
- "x": 480.0,
444
- "y": 388.0
445
  },
446
- "type": "basic"
447
  },
448
  "params": {},
449
  "status": "done",
450
- "title": "LynxScribe Image RAG Builder"
451
  },
452
  "dragHandle": ".bg-primary",
453
- "height": 313.0,
454
- "id": "LynxScribe Image RAG Builder 1",
455
  "position": {
456
- "x": 243.62049392420903,
457
- "y": 215.6136303371116
458
  },
459
- "type": "basic",
460
- "width": 526.0
461
  }
462
  ]
463
  }
 
1
  {
2
  "edges": [
3
  {
4
+ "id": "Cloud-sourced File Listing 1 LynxScribe Image Describer 1",
5
+ "source": "Cloud-sourced File Listing 1",
 
 
 
 
 
 
 
6
  "sourceHandle": "output",
7
+ "target": "LynxScribe Image Describer 1",
 
 
 
 
 
 
 
8
  "targetHandle": "file_urls"
9
  },
10
  {
 
12
  "source": "LynxScribe Image Describer 1",
13
  "sourceHandle": "output",
14
  "target": "LynxScribe Image RAG Builder 1",
15
+ "targetHandle": "image_descriptions"
16
  },
17
  {
18
+ "id": "LynxScribe Image RAG Builder 1 LynxScribe Image RAG Query 1",
19
+ "source": "LynxScribe Image RAG Builder 1",
20
  "sourceHandle": "output",
21
+ "target": "LynxScribe Image RAG Query 1",
22
  "targetHandle": "rag_graph"
23
  },
24
  {
25
+ "id": "Input chat 1 LynxScribe Image RAG Query 1",
26
+ "source": "Input chat 1",
27
  "sourceHandle": "output",
28
  "target": "LynxScribe Image RAG Query 1",
29
+ "targetHandle": "text"
30
+ },
31
+ {
32
+ "id": "LynxScribe Image RAG Query 1 LynxScribe Image Result Viewer 1",
33
+ "source": "LynxScribe Image RAG Query 1",
34
+ "sourceHandle": "output",
35
+ "target": "LynxScribe Image Result Viewer 1",
36
+ "targetHandle": "embedding_similarities"
37
  }
38
  ],
39
  "env": "LynxScribe",
 
68
  "type": "basic"
69
  },
70
  "params": {
71
+ "chat": "show me a picture about 2 doctors"
72
  },
73
  "status": "done",
74
  "title": "Input chat"
 
77
  "height": 214.0,
78
  "id": "Input chat 1",
79
  "position": {
80
+ "x": 51.51211115780683,
81
+ "y": -147.75474103115954
82
  },
83
  "type": "basic",
84
  "width": 387.0
 
90
  "display": null,
91
  "error": null,
92
  "meta": {
93
+ "inputs": {},
94
+ "name": "Cloud-sourced File Listing",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  "outputs": {
96
  "output": {
97
  "name": "output",
 
102
  }
103
  },
104
  "params": {
105
+ "accepted_file_types": {
106
+ "default": ".jpg, .jpeg, .png",
107
+ "name": "accepted_file_types",
108
  "type": {
109
+ "type": "<class 'str'>"
110
+ }
111
+ },
112
+ "cloud_provider": {
113
+ "default": "gcp",
114
+ "name": "cloud_provider",
115
+ "type": {
116
+ "enum": [
117
+ "GCP",
118
+ "AWS",
119
+ "AZURE"
120
+ ]
121
+ }
122
+ },
123
+ "folder_URL": {
124
+ "default": "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test",
125
+ "name": "folder_URL",
126
+ "type": {
127
+ "type": "<class 'str'>"
128
  }
129
  }
130
  },
131
+ "position": {
132
+ "x": 1271.0,
133
+ "y": 603.0
134
+ },
135
  "type": "basic"
136
  },
137
  "params": {
138
+ "accepted_file_types": ".jpg, .jpeg, .png",
139
+ "cloud_provider": "GCP",
140
+ "folder_URL": "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test"
141
  },
142
  "status": "done",
143
+ "title": "Cloud-sourced File Listing"
144
  },
145
  "dragHandle": ".bg-primary",
146
+ "height": 308.0,
147
+ "id": "Cloud-sourced File Listing 1",
148
  "position": {
149
+ "x": -733.5815993327456,
150
+ "y": 418.3880816741662
151
  },
152
  "type": "basic",
153
+ "width": 613.0
154
  },
155
  {
156
  "data": {
157
+ "__execution_delay": 0.0,
158
+ "collapsed": null,
159
+ "display": null,
160
  "error": null,
161
  "meta": {
162
  "inputs": {
163
+ "file_urls": {
164
+ "name": "file_urls",
165
  "position": "left",
166
  "type": {
167
  "type": "<class 'inspect._empty'>"
168
  }
169
  }
170
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
171
  "name": "LynxScribe Image Describer",
172
  "outputs": {
173
  "output": {
174
  "name": "output",
175
+ "position": "right",
176
  "type": {
177
  "type": "None"
178
  }
 
194
  }
195
  },
196
  "llm_prompt_path": {
197
+ "default": "uploads/image_description_prompts.yaml",
198
  "name": "llm_prompt_path",
199
  "type": {
200
  "type": "<class 'str'>"
 
209
  }
210
  },
211
  "position": {
212
+ "x": 1331.0,
213
+ "y": 686.0
214
  },
215
  "type": "basic"
216
  },
217
  "params": {
218
  "llm_interface": "openai",
219
  "llm_prompt_name": "cot_picture_descriptor",
220
+ "llm_prompt_path": "uploads/image_description_prompts.yaml",
221
  "llm_visual_model": "gpt-4o"
222
  },
223
  "status": "done",
224
  "title": "LynxScribe Image Describer"
225
  },
226
  "dragHandle": ".bg-primary",
227
+ "height": 366.0,
228
  "id": "LynxScribe Image Describer 1",
229
  "position": {
230
+ "x": 94.4350838249984,
231
+ "y": 389.7616279503166
232
  },
233
  "type": "basic",
234
+ "width": 362.0
235
  },
236
  {
237
  "data": {
238
+ "__execution_delay": 0.0,
239
+ "collapsed": null,
240
  "display": null,
241
  "error": null,
242
  "meta": {
243
+ "inputs": {
244
+ "image_descriptions": {
245
+ "name": "image_descriptions",
246
+ "position": "left",
247
+ "type": {
248
+ "type": "<class 'inspect._empty'>"
249
+ }
250
+ }
251
+ },
252
+ "name": "LynxScribe Image RAG Builder",
253
  "outputs": {
254
  "output": {
255
  "name": "output",
256
+ "position": "right",
257
  "type": {
258
  "type": "None"
259
  }
260
  }
261
  },
262
  "params": {
263
+ "text_embedder_interface": {
264
+ "default": "openai",
265
+ "name": "text_embedder_interface",
266
  "type": {
267
  "type": "<class 'str'>"
268
  }
269
  },
270
+ "text_embedder_model_name_or_path": {
271
+ "default": "text-embedding-3-large",
272
+ "name": "text_embedder_model_name_or_path",
273
  "type": {
274
  "type": "<class 'str'>"
275
  }
276
  },
277
+ "vdb_collection_name": {
278
+ "default": "lynx",
279
+ "name": "vdb_collection_name",
280
  "type": {
281
+ "type": "<class 'str'>"
282
  }
283
  },
284
+ "vdb_num_dimensions": {
285
+ "default": 3072.0,
286
+ "name": "vdb_num_dimensions",
287
  "type": {
288
+ "type": "<class 'int'>"
289
  }
290
  },
291
+ "vdb_provider_name": {
292
+ "default": "faiss",
293
+ "name": "vdb_provider_name",
294
  "type": {
295
  "type": "<class 'str'>"
296
  }
297
  }
298
  },
299
  "position": {
300
+ "x": 1714.0,
301
+ "y": 740.0
302
  },
303
  "type": "basic"
304
  },
305
  "params": {
 
 
 
306
  "text_embedder_interface": "openai",
307
+ "text_embedder_model_name_or_path": "text-embedding-3-small",
308
+ "vdb_collection_name": "lynx",
309
+ "vdb_num_dimensions": "1536",
310
+ "vdb_provider_name": "faiss"
311
  },
312
  "status": "done",
313
+ "title": "LynxScribe Image RAG Builder"
314
  },
315
  "dragHandle": ".bg-primary",
316
+ "height": 463.0,
317
+ "id": "LynxScribe Image RAG Builder 1",
318
  "position": {
319
+ "x": 634.1082253159385,
320
+ "y": 341.7237080874875
321
  },
322
  "type": "basic",
323
+ "width": 309.0
324
  },
325
  {
326
  "data": {
 
329
  "display": null,
330
  "error": null,
331
  "meta": {
332
+ "inputs": {
333
+ "rag_graph": {
334
+ "name": "rag_graph",
335
+ "position": "bottom",
336
+ "type": {
337
+ "type": "<class 'inspect._empty'>"
338
+ }
339
+ },
340
+ "text": {
341
+ "name": "text",
342
+ "position": "left",
343
+ "type": {
344
+ "type": "<class 'inspect._empty'>"
345
+ }
346
+ }
347
+ },
348
+ "name": "LynxScribe Image RAG Query",
349
  "outputs": {
350
  "output": {
351
  "name": "output",
 
356
  }
357
  },
358
  "params": {
359
+ "top_k": {
360
+ "default": 3.0,
361
+ "name": "top_k",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
362
  "type": {
363
+ "type": "<class 'int'>"
364
  }
365
  }
366
  },
367
  "position": {
368
+ "x": 1865.0,
369
+ "y": 363.0
370
  },
371
  "type": "basic"
372
  },
373
  "params": {
374
+ "top_k": "3"
 
 
375
  },
376
  "status": "done",
377
+ "title": "LynxScribe Image RAG Query"
378
  },
379
  "dragHandle": ".bg-primary",
380
+ "height": 205.0,
381
+ "id": "LynxScribe Image RAG Query 1",
382
  "position": {
383
+ "x": 1064.0579569918539,
384
+ "y": -140.79102876607624
385
  },
386
  "type": "basic",
387
+ "width": 263.0
388
  },
389
  {
390
  "data": {
391
+ "display": "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test/surgery-1807541_1280.jpg",
392
  "error": null,
393
  "meta": {
394
  "inputs": {
395
+ "embedding_similarities": {
396
+ "name": "embedding_similarities",
397
  "position": "left",
398
  "type": {
399
  "type": "<class 'inspect._empty'>"
400
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
401
  }
402
  },
403
+ "name": "LynxScribe Image Result Viewer",
404
+ "outputs": {},
405
  "params": {},
406
  "position": {
407
+ "x": 2207.0,
408
+ "y": 327.0
409
  },
410
+ "type": "image"
411
  },
412
  "params": {},
413
  "status": "done",
414
+ "title": "LynxScribe Image Result Viewer"
415
  },
416
  "dragHandle": ".bg-primary",
417
+ "height": 622.0,
418
+ "id": "LynxScribe Image Result Viewer 1",
419
  "position": {
420
+ "x": 1550.5086064306404,
421
+ "y": -349.93521115271193
422
  },
423
+ "type": "image",
424
+ "width": 802.0
425
  }
426
  ]
427
  }
{lynxkite-lynxscribe/promptdb → examples/uploads}/image_description_prompts.yaml RENAMED
File without changes
lynxkite-lynxscribe/src/lynxkite_lynxscribe/lynxscribe_ops.py CHANGED
@@ -1,9 +1,11 @@
1
  """
2
  LynxScribe configuration and testing in LynxKite.
 
3
  """
4
 
5
  from google.cloud import storage
6
  from copy import deepcopy
 
7
  import asyncio
8
  import pandas as pd
9
  import joblib
@@ -44,10 +46,17 @@ op = ops.op_registration(ENV)
44
  output_on_top = ops.output_position(output="top")
45
 
46
 
47
- @op("Cloud-sourced File Loader")
 
 
 
 
 
 
 
48
  def cloud_file_loader(
49
  *,
50
- cloud_provider: str = "gcp",
51
  folder_URL: str = "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test",
52
  accepted_file_types: str = ".jpg, .jpeg, .png",
53
  ):
@@ -60,7 +69,7 @@ def cloud_file_loader(
60
 
61
  accepted_file_types = tuple([t.strip() for t in accepted_file_types.split(",")])
62
 
63
- if cloud_provider == "gcp":
64
  client = storage.Client()
65
  url_useful_part = folder_URL.split(".com/")[-1]
66
  bucket_name = url_useful_part.split("/")[0]
@@ -118,66 +127,41 @@ def ls_rag_graph(
118
  return {"rag_graph": rag_graph}
119
 
120
 
121
- @output_on_top
122
  @op("LynxScribe Image Describer")
123
  @mem.cache
124
- def ls_image_describer(
 
125
  *,
126
  llm_interface: str = "openai",
127
  llm_visual_model: str = "gpt-4o",
128
- llm_prompt_path: str = "../lynxkite-lynxscribe/promptdb/image_description_prompts.yaml",
129
  llm_prompt_name: str = "cot_picture_descriptor",
130
  # api_key_name: str = "OPENAI_API_KEY",
131
  ):
132
  """
133
- Returns with an image describer instance.
134
- TODO: adding a relative path to the prompt path + adding model kwargs
 
 
135
  """
136
 
 
 
 
 
137
  llm_params = {"name": llm_interface}
138
  # if api_key_name:
139
  # llm_params["api_key"] = os.getenv(api_key_name)
140
  llm = get_llm_engine(**llm_params)
141
 
 
142
  prompt_base = load_config(llm_prompt_path)[llm_prompt_name]
143
-
144
- return {
145
- "image_describer": {
146
- "llm": llm,
147
- "prompt_base": prompt_base,
148
- "model": llm_visual_model,
149
- }
150
- }
151
-
152
-
153
- @ops.input_position(image_describer="bottom", rag_graph="bottom")
154
- @op("LynxScribe Image RAG Builder")
155
- @mem.cache
156
- async def ls_image_rag_builder(
157
- file_urls,
158
- image_describer,
159
- rag_graph,
160
- ):
161
- """
162
- Based on an input image folder (currently only supports GCP storage),
163
- the function builds up an image RAG graph, where the nodes are the
164
- descriptions of the images (and of all image objects).
165
-
166
- In a later phase, synthetic questions and "named entities" will also
167
- be added to the graph.
168
- """
169
-
170
- # handling inputs
171
- image_describer = image_describer[0]["image_describer"]
172
- image_urls = file_urls["file_urls"]
173
- rag_graph = rag_graph[0]["rag_graph"]
174
-
175
- # generate prompts from inputs
176
  prompt_list = []
 
177
  for i in range(len(image_urls)):
178
  image = image_urls[i]
179
 
180
- _prompt = deepcopy(image_describer["prompt_base"])
181
  for message in _prompt:
182
  if isinstance(message["content"], list):
183
  for _message_part in message["content"]:
@@ -185,13 +169,14 @@ async def ls_image_rag_builder(
185
  _message_part["image_url"] = {"url": image}
186
 
187
  prompt_list.append(_prompt)
 
 
188
  ch_prompt_list = [
189
- ChatCompletionPrompt(model=image_describer["model"], messages=prompt)
190
  for prompt in prompt_list
191
  ]
192
 
193
  # get the image descriptions
194
- llm = image_describer["llm"]
195
  tasks = [
196
  llm.acreate_completion(completion_prompt=_prompt) for _prompt in ch_prompt_list
197
  ]
@@ -201,27 +186,86 @@ async def ls_image_rag_builder(
201
  for result in out_completions
202
  ]
203
 
204
- # generate combination of descriptions and embed them
205
- text_embedder = rag_graph.kg_base.text_embedder
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
206
 
207
  dict_list_df = []
208
- for _i, _result in enumerate(results):
209
- url_res = image_urls[_i]
 
210
 
211
- if "overall description" in _result:
212
  dict_list_df.append(
213
  {
214
- "image_url": url_res,
215
- "description": _result["overall description"],
216
  "source": "overall description",
217
  }
218
  )
219
 
220
- if "details" in _result:
221
- for dkey in _result["details"].keys():
222
- text = f"The picture's description is: {_result['overall description']}\n\nThe description of the {dkey} is: {_result['details'][dkey]}"
223
  dict_list_df.append(
224
- {"image_url": url_res, "description": text, "source": "details"}
225
  )
226
 
227
  pdf_descriptions = pd.DataFrame(dict_list_df)
@@ -257,7 +301,7 @@ async def ls_image_rag_builder(
257
 
258
  @op("LynxScribe RAG Graph Saver")
259
  def ls_save_rag_graph(
260
- knowledge_base,
261
  *,
262
  image_rag_out_path: str = "image_test_rag_graph.pickle",
263
  ):
@@ -265,7 +309,10 @@ def ls_save_rag_graph(
265
  Saves the RAG graph to a pickle file.
266
  """
267
 
268
- knowledge_base.kg_base.save(image_rag_out_path)
 
 
 
269
  return None
270
 
271
 
@@ -294,10 +341,12 @@ async def search_context(rag_graph, text, *, top_k=3):
294
  return {"embedding_similarities": result_list}
295
 
296
 
297
- @op("View image", view="image")
298
  def view_image(embedding_similarities):
299
  """
300
- Plotting the selected image.
 
 
301
  """
302
  embedding_similarities = embedding_similarities["embedding_similarities"]
303
  return embedding_similarities[0]["image_url"]
 
1
  """
2
  LynxScribe configuration and testing in LynxKite.
3
+ TODO: all these outputs should contain metadata. So the next task can check the input type, etc.
4
  """
5
 
6
  from google.cloud import storage
7
  from copy import deepcopy
8
+ from enum import Enum
9
  import asyncio
10
  import pandas as pd
11
  import joblib
 
46
  output_on_top = ops.output_position(output="top")
47
 
48
 
49
+ # defining the cloud provider enum
50
+ class CloudProvider(Enum):
51
+ GCP = "gcp"
52
+ AWS = "aws"
53
+ AZURE = "azure"
54
+
55
+
56
+ @op("Cloud-sourced File Listing")
57
  def cloud_file_loader(
58
  *,
59
+ cloud_provider: CloudProvider = CloudProvider.GCP,
60
  folder_URL: str = "https://storage.googleapis.com/lynxkite_public_data/lynxscribe-images/image-rag-test",
61
  accepted_file_types: str = ".jpg, .jpeg, .png",
62
  ):
 
69
 
70
  accepted_file_types = tuple([t.strip() for t in accepted_file_types.split(",")])
71
 
72
+ if cloud_provider == CloudProvider.GCP:
73
  client = storage.Client()
74
  url_useful_part = folder_URL.split(".com/")[-1]
75
  bucket_name = url_useful_part.split("/")[0]
 
127
  return {"rag_graph": rag_graph}
128
 
129
 
 
130
  @op("LynxScribe Image Describer")
131
  @mem.cache
132
+ async def ls_image_describer(
133
+ file_urls,
134
  *,
135
  llm_interface: str = "openai",
136
  llm_visual_model: str = "gpt-4o",
137
+ llm_prompt_path: str = "uploads/image_description_prompts.yaml",
138
  llm_prompt_name: str = "cot_picture_descriptor",
139
  # api_key_name: str = "OPENAI_API_KEY",
140
  ):
141
  """
142
+ Returns with image descriptions from a list of image URLs.
143
+
144
+ TODO: making the inputs more flexible (e.g. accepting file locations, URLs, binaries, etc.).
145
+ the input dictionary should contain some meta info: e.g., what is in the list...
146
  """
147
 
148
+ # handling inputs
149
+ image_urls = file_urls["file_urls"]
150
+
151
+ # loading the LLM
152
  llm_params = {"name": llm_interface}
153
  # if api_key_name:
154
  # llm_params["api_key"] = os.getenv(api_key_name)
155
  llm = get_llm_engine(**llm_params)
156
 
157
+ # preparing the prompts
158
  prompt_base = load_config(llm_prompt_path)[llm_prompt_name]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  prompt_list = []
160
+
161
  for i in range(len(image_urls)):
162
  image = image_urls[i]
163
 
164
+ _prompt = deepcopy(prompt_base)
165
  for message in _prompt:
166
  if isinstance(message["content"], list):
167
  for _message_part in message["content"]:
 
169
  _message_part["image_url"] = {"url": image}
170
 
171
  prompt_list.append(_prompt)
172
+
173
+ # creating the prompt objects
174
  ch_prompt_list = [
175
+ ChatCompletionPrompt(model=llm_visual_model, messages=prompt)
176
  for prompt in prompt_list
177
  ]
178
 
179
  # get the image descriptions
 
180
  tasks = [
181
  llm.acreate_completion(completion_prompt=_prompt) for _prompt in ch_prompt_list
182
  ]
 
186
  for result in out_completions
187
  ]
188
 
189
+ # getting the image descriptions (list of dictionaries {image_url: URL, description: description})
190
+ # TODO: some result class could be a better idea (will be developed in LynxScribe)
191
+ image_descriptions = [
192
+ {"image_url": image_urls[i], "description": results[i]}
193
+ for i in range(len(image_urls))
194
+ ]
195
+
196
+ return {"image_descriptions": image_descriptions}
197
+
198
+
199
+ @op("LynxScribe Image RAG Builder")
200
+ @mem.cache
201
+ async def ls_image_rag_builder(
202
+ image_descriptions,
203
+ *,
204
+ vdb_provider_name: str = "faiss",
205
+ vdb_num_dimensions: int = 3072,
206
+ vdb_collection_name: str = "lynx",
207
+ text_embedder_interface: str = "openai",
208
+ text_embedder_model_name_or_path: str = "text-embedding-3-large",
209
+ # api_key_name: str = "OPENAI_API_KEY",
210
+ ):
211
+ """
212
+ Based on image descriptions, and embedding/VDB parameters,
213
+ the function builds up an image RAG graph, where the nodes are the
214
+ descriptions of the images (and of all image objects).
215
+
216
+ In a later phase, synthetic questions and "named entities" will also
217
+ be added to the graph.
218
+ """
219
+
220
+ # handling inputs
221
+ image_descriptions = image_descriptions["image_descriptions"]
222
+
223
+ # Building up the empty RAG graph
224
+
225
+ # a) Define LLM interface and get a text embedder
226
+ llm_params = {"name": text_embedder_interface}
227
+ # if api_key_name:
228
+ # llm_params["api_key"] = os.getenv(api_key_name)
229
+ llm = get_llm_engine(**llm_params)
230
+ text_embedder = TextEmbedder(llm=llm, model=text_embedder_model_name_or_path)
231
+
232
+ # b) getting the vector store
233
+ # TODO: vdb_provider_name should be ENUM, and other parameters should appear accordingly
234
+ if vdb_provider_name == "chromadb":
235
+ vector_store = get_vector_store(
236
+ name=vdb_provider_name, collection_name=vdb_collection_name
237
+ )
238
+ elif vdb_provider_name == "faiss":
239
+ vector_store = get_vector_store(
240
+ name=vdb_provider_name, num_dimensions=vdb_num_dimensions
241
+ )
242
+ else:
243
+ raise ValueError(f"Vector store name '{vdb_provider_name}' is not supported.")
244
+
245
+ # c) building up the RAG graph
246
+ rag_graph = RAGGraph(
247
+ PandasKnowledgeBaseGraph(vector_store=vector_store, text_embedder=text_embedder)
248
+ )
249
 
250
  dict_list_df = []
251
+ for image_description_tuple in image_descriptions:
252
+ image_url = image_description_tuple["image_url"]
253
+ image_description = image_description_tuple["description"]
254
 
255
+ if "overall description" in image_description:
256
  dict_list_df.append(
257
  {
258
+ "image_url": image_url,
259
+ "description": image_description["overall description"],
260
  "source": "overall description",
261
  }
262
  )
263
 
264
+ if "details" in image_description:
265
+ for dkey in image_description["details"].keys():
266
+ text = f"The picture's description is: {image_description['overall description']}\n\nThe description of the {dkey} is: {image_description['details'][dkey]}"
267
  dict_list_df.append(
268
+ {"image_url": image_url, "description": text, "source": "details"}
269
  )
270
 
271
  pdf_descriptions = pd.DataFrame(dict_list_df)
 
301
 
302
  @op("LynxScribe RAG Graph Saver")
303
  def ls_save_rag_graph(
304
+ rag_graph,
305
  *,
306
  image_rag_out_path: str = "image_test_rag_graph.pickle",
307
  ):
 
309
  Saves the RAG graph to a pickle file.
310
  """
311
 
312
+ # reading inputs
313
+ rag_graph = rag_graph[0]["rag_graph"]
314
+
315
+ rag_graph.kg_base.save(image_rag_out_path)
316
  return None
317
 
318
 
 
341
  return {"embedding_similarities": result_list}
342
 
343
 
344
+ @op("LynxScribe Image Result Viewer", view="image")
345
  def view_image(embedding_similarities):
346
  """
347
+ Plotting the TOP images (from embedding similarities).
348
+
349
+ TODO: later on, the user can scroll the images and send feedbacks
350
  """
351
  embedding_similarities = embedding_similarities["embedding_similarities"]
352
  return embedding_similarities[0]["image_url"]