Charles Kabui
commited on
Commit
·
0da14c5
1
Parent(s):
c33e07b
replaced pdf2image with PyMuPDF
Browse files- analysis.ipynb +0 -0
- app.py +0 -12
- main.py +92 -65
- requirements.txt +10 -0
- utils/get_RGB_image.py +2 -1
- utils/visualize_bboxes_on_image.py +129 -100
analysis.ipynb
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
app.py
CHANGED
|
@@ -1,15 +1,3 @@
|
|
| 1 |
-
import os
|
| 2 |
-
os.system("apt install -y poppler-utils")
|
| 3 |
-
os.system("python -m pip install --upgrade pip")
|
| 4 |
-
os.system("python -m pip install pdf2image==1.16.3")
|
| 5 |
-
os.system("python -m pip install torch==2.1.0")
|
| 6 |
-
os.system("python -m pip install 'git+https://github.com/facebookresearch/detectron2.git@898507047cf441a1e4be7a729270961c401c4354'")
|
| 7 |
-
os.system("python -m pip install layoutparser==0.3.4 layoutparser[layoutmodels] layoutparser[ocr]")
|
| 8 |
-
os.system("python -m pip install Pillow==9.5.0")
|
| 9 |
-
os.system("python -m pip install imagehash==4.3.1")
|
| 10 |
-
os.system("python -m pip install tensorflow==2.15.0 tensorflow-estimator==2.15.0")
|
| 11 |
-
os.system("python -m pip install scikit-learn==1.3.2")
|
| 12 |
-
|
| 13 |
from main import app
|
| 14 |
|
| 15 |
model_path = './model/trained_model/model_final.pth'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
from main import app
|
| 2 |
|
| 3 |
model_path = './model/trained_model/model_final.pth'
|
main.py
CHANGED
|
@@ -1,17 +1,19 @@
|
|
| 1 |
import traceback
|
| 2 |
import gradio as gr
|
| 3 |
from utils.get_RGB_image import get_RGB_image, is_online_file, steam_online_file
|
| 4 |
-
from pdf2image import convert_from_path, convert_from_bytes
|
| 5 |
import layoutparser as lp
|
| 6 |
from PIL import Image
|
| 7 |
from utils.get_features import get_features
|
| 8 |
from imagehash import average_hash
|
| 9 |
from sklearn.metrics.pairwise import cosine_similarity
|
| 10 |
from utils.visualize_bboxes_on_image import visualize_bboxes_on_image
|
|
|
|
| 11 |
|
| 12 |
-
label_map = {0: 'Caption', 1: 'Footnote', 2: 'Formula', 3: 'List-item', 4: 'Page-footer',
|
|
|
|
| 13 |
label_names = list(label_map.values())
|
| 14 |
-
color_map = {'Caption': '#
|
|
|
|
| 15 |
cache = {
|
| 16 |
'output_document_image_1_hash': None,
|
| 17 |
'output_document_image_2_hash': None,
|
|
@@ -29,14 +31,16 @@ visualize_bboxes_on_image_kwargs = {
|
|
| 29 |
'label_rectangle_left_margin': 0,
|
| 30 |
'label_rectangle_top_margin': 0
|
| 31 |
}
|
| 32 |
-
vectors_types = ['vectors', 'weighted_vectors',
|
|
|
|
|
|
|
| 33 |
|
| 34 |
def similarity_fn(model: lp.Detectron2LayoutModel, document_image_1: Image.Image, document_image_2: Image.Image, vectors_type: str):
|
| 35 |
message = None
|
| 36 |
annotations = {
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
}
|
| 41 |
show_vectors_type = False
|
| 42 |
try:
|
|
@@ -50,7 +54,8 @@ def similarity_fn(model: lp.Detectron2LayoutModel, document_image_1: Image.Image
|
|
| 50 |
document_image_1_features = cache['document_image_1_features']
|
| 51 |
document_image_1 = cache['original_document_image_1']
|
| 52 |
else:
|
| 53 |
-
document_image_1_features = get_features(
|
|
|
|
| 54 |
cache['document_image_1_features'] = document_image_1_features
|
| 55 |
cache['original_document_image_1'] = document_image_1
|
| 56 |
|
|
@@ -58,105 +63,126 @@ def similarity_fn(model: lp.Detectron2LayoutModel, document_image_1: Image.Image
|
|
| 58 |
document_image_2_features = cache['document_image_2_features']
|
| 59 |
document_image_2 = cache['original_document_image_2']
|
| 60 |
else:
|
| 61 |
-
document_image_2_features = get_features(
|
|
|
|
| 62 |
cache['document_image_2_features'] = document_image_2_features
|
| 63 |
cache['original_document_image_2'] = document_image_2
|
| 64 |
|
| 65 |
[[similarity]] = cosine_similarity(
|
| 66 |
[
|
| 67 |
cache['document_image_1_features'][vectors_type]
|
| 68 |
-
],
|
| 69 |
[
|
| 70 |
cache['document_image_2_features'][vectors_type]
|
| 71 |
])
|
| 72 |
message = f'<pre style="{pre_message_style}">Similarity between the two documents is: {round(similarity, 4)}<pre>'
|
| 73 |
document_image_1 = visualize_bboxes_on_image(
|
| 74 |
-
image
|
| 75 |
-
bboxes
|
| 76 |
-
labels
|
| 77 |
-
cache['document_image_1_features'][annotations['predicted_labels']],
|
| 78 |
cache['document_image_1_features'][annotations['predicted_scores']])],
|
| 79 |
-
bbox_outline_color
|
|
|
|
|
|
|
|
|
|
| 80 |
**visualize_bboxes_on_image_kwargs)
|
| 81 |
document_image_2 = visualize_bboxes_on_image(
|
| 82 |
-
image
|
| 83 |
-
bboxes
|
| 84 |
-
labels
|
| 85 |
-
cache['document_image_2_features'][annotations['predicted_labels']],
|
| 86 |
cache['document_image_2_features'][annotations['predicted_scores']])],
|
| 87 |
-
bbox_outline_color
|
|
|
|
|
|
|
|
|
|
| 88 |
**visualize_bboxes_on_image_kwargs)
|
| 89 |
-
|
| 90 |
-
cache['output_document_image_1_hash'] = str(
|
| 91 |
-
|
|
|
|
|
|
|
| 92 |
|
| 93 |
show_vectors_type = True
|
| 94 |
except Exception as e:
|
| 95 |
message = f'<pre style="{pre_message_style}">{traceback.format_exc()}<pre>'
|
| 96 |
return [
|
| 97 |
-
gr.HTML(message, visible=True),
|
| 98 |
-
document_image_1,
|
| 99 |
document_image_2,
|
| 100 |
gr.Dropdown(visible=show_vectors_type)
|
| 101 |
]
|
| 102 |
-
|
| 103 |
-
|
|
|
|
| 104 |
try:
|
| 105 |
image = None
|
|
|
|
| 106 |
try:
|
| 107 |
if (is_online_file(filename)):
|
| 108 |
-
|
| 109 |
else:
|
| 110 |
-
|
| 111 |
-
|
|
|
|
|
|
|
| 112 |
image = get_RGB_image(filename)
|
| 113 |
return [
|
| 114 |
-
gr.Image(value=image, visible=True),
|
| 115 |
None
|
| 116 |
]
|
| 117 |
-
except:
|
| 118 |
-
error = traceback.format_exc()
|
| 119 |
return [None, gr.HTML(value=error, visible=True)]
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
|
|
|
| 123 |
if image:
|
| 124 |
return [gr.Tabs(selected=0), image, error]
|
| 125 |
else:
|
| 126 |
-
return [gr.Tabs(selected=1), image, error]
|
|
|
|
| 127 |
|
| 128 |
def document_view(document_number: int):
|
| 129 |
-
gr.HTML(value=f'<h4>Load the {"first" if document_number == 1 else "second"} PDF or Document Image<h4>', elem_classes=[
|
|
|
|
| 130 |
with gr.Tabs() as document_tabs:
|
| 131 |
with gr.Tab("From Image", id=0):
|
| 132 |
-
document = gr.Image(
|
| 133 |
-
|
|
|
|
|
|
|
| 134 |
document_preview = gr.UploadButton(
|
| 135 |
-
"Upload PDF or Document Image",
|
| 136 |
-
file_types=["image", ".pdf"],
|
| 137 |
file_count="single")
|
| 138 |
with gr.Tab("From URL", id=1):
|
| 139 |
document_url = gr.Textbox(
|
| 140 |
label=f"Document {document_number} URL",
|
| 141 |
info="Paste a Link/URL to PDF or Document Image",
|
| 142 |
placeholder="https://datasets-server.huggingface.co/.../image.jpg")
|
| 143 |
-
document_url_error_message = gr.HTML(
|
| 144 |
-
|
|
|
|
|
|
|
| 145 |
document_preview.upload(
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
document_url_preview.click(
|
| 150 |
-
fn
|
| 151 |
-
inputs
|
| 152 |
-
outputs
|
| 153 |
return document
|
| 154 |
|
| 155 |
-
|
|
|
|
| 156 |
model: lp.Detectron2LayoutModel = lp.Detectron2LayoutModel(
|
| 157 |
-
config_path
|
| 158 |
-
model_path
|
| 159 |
-
label_map
|
| 160 |
title = 'Document Similarity Search Using Visual Layout Features'
|
| 161 |
description = f"<h2>{title}<h2>"
|
| 162 |
css = '''
|
|
@@ -167,7 +193,7 @@ def app(*, model_path, config_path, debug = False):
|
|
| 167 |
with gr.Blocks(title=title, css=css) as app:
|
| 168 |
with gr.Row():
|
| 169 |
gr.HTML(value=description, elem_classes=['center'])
|
| 170 |
-
with gr.Row(equal_height
|
| 171 |
with gr.Column():
|
| 172 |
document_1_image = document_view(1)
|
| 173 |
with gr.Column():
|
|
@@ -178,22 +204,23 @@ def app(*, model_path, config_path, debug = False):
|
|
| 178 |
submit = gr.Button(value="Get Similarity", variant="primary")
|
| 179 |
with gr.Column():
|
| 180 |
vectors_type = gr.Dropdown(
|
| 181 |
-
choices
|
| 182 |
-
value
|
| 183 |
-
visible
|
| 184 |
-
label
|
| 185 |
-
info
|
| 186 |
-
similarity_output = gr.HTML(
|
|
|
|
| 187 |
reset = gr.Button(value="Reset", variant="secondary")
|
| 188 |
kwargs = {
|
| 189 |
'fn': lambda document_1_image, document_2_image, vectors_type: similarity_fn(
|
| 190 |
-
model,
|
| 191 |
-
document_1_image,
|
| 192 |
-
document_2_image,
|
| 193 |
vectors_type),
|
| 194 |
'inputs': [document_1_image, document_2_image, vectors_type],
|
| 195 |
'outputs': [similarity_output, document_1_image, document_2_image, vectors_type]
|
| 196 |
}
|
| 197 |
submit.click(**kwargs)
|
| 198 |
vectors_type.change(**kwargs)
|
| 199 |
-
return app.launch(debug=debug)
|
|
|
|
| 1 |
import traceback
|
| 2 |
import gradio as gr
|
| 3 |
from utils.get_RGB_image import get_RGB_image, is_online_file, steam_online_file
|
|
|
|
| 4 |
import layoutparser as lp
|
| 5 |
from PIL import Image
|
| 6 |
from utils.get_features import get_features
|
| 7 |
from imagehash import average_hash
|
| 8 |
from sklearn.metrics.pairwise import cosine_similarity
|
| 9 |
from utils.visualize_bboxes_on_image import visualize_bboxes_on_image
|
| 10 |
+
import fitz
|
| 11 |
|
| 12 |
+
label_map = {0: 'Caption', 1: 'Footnote', 2: 'Formula', 3: 'List-item', 4: 'Page-footer',
|
| 13 |
+
5: 'Page-header', 6: 'Picture', 7: 'Section-header', 8: 'Table', 9: 'Text', 10: 'Title'}
|
| 14 |
label_names = list(label_map.values())
|
| 15 |
+
color_map = {'Caption': '#FF0000', 'Footnote': '#00FF00', 'Formula': '#0000FF', 'List-item': '#FF00FF', 'Page-footer': '#FFFF00',
|
| 16 |
+
'Page-header': '#000000', 'Picture': '#FFFFFF', 'Section-header': '#40E0D0', 'Table': '#F28030', 'Text': '#7F00FF', 'Title': '#C0C0C0'}
|
| 17 |
cache = {
|
| 18 |
'output_document_image_1_hash': None,
|
| 19 |
'output_document_image_2_hash': None,
|
|
|
|
| 31 |
'label_rectangle_left_margin': 0,
|
| 32 |
'label_rectangle_top_margin': 0
|
| 33 |
}
|
| 34 |
+
vectors_types = ['vectors', 'weighted_vectors',
|
| 35 |
+
'reduced_vectors', 'weighted_reduced_vectors']
|
| 36 |
+
|
| 37 |
|
| 38 |
def similarity_fn(model: lp.Detectron2LayoutModel, document_image_1: Image.Image, document_image_2: Image.Image, vectors_type: str):
|
| 39 |
message = None
|
| 40 |
annotations = {
|
| 41 |
+
'predicted_bboxes': 'predicted_bboxes' if vectors_type in ['vectors', 'weighted_vectors'] else 'reduced_predicted_bboxes',
|
| 42 |
+
'predicted_scores': 'predicted_scores' if vectors_type in ['vectors', 'weighted_vectors'] else 'reduced_predicted_scores',
|
| 43 |
+
'predicted_labels': 'predicted_labels' if vectors_type in ['vectors', 'weighted_vectors'] else 'reduced_predicted_labels',
|
| 44 |
}
|
| 45 |
show_vectors_type = False
|
| 46 |
try:
|
|
|
|
| 54 |
document_image_1_features = cache['document_image_1_features']
|
| 55 |
document_image_1 = cache['original_document_image_1']
|
| 56 |
else:
|
| 57 |
+
document_image_1_features = get_features(
|
| 58 |
+
document_image_1, model, label_names)
|
| 59 |
cache['document_image_1_features'] = document_image_1_features
|
| 60 |
cache['original_document_image_1'] = document_image_1
|
| 61 |
|
|
|
|
| 63 |
document_image_2_features = cache['document_image_2_features']
|
| 64 |
document_image_2 = cache['original_document_image_2']
|
| 65 |
else:
|
| 66 |
+
document_image_2_features = get_features(
|
| 67 |
+
document_image_2, model, label_names)
|
| 68 |
cache['document_image_2_features'] = document_image_2_features
|
| 69 |
cache['original_document_image_2'] = document_image_2
|
| 70 |
|
| 71 |
[[similarity]] = cosine_similarity(
|
| 72 |
[
|
| 73 |
cache['document_image_1_features'][vectors_type]
|
| 74 |
+
],
|
| 75 |
[
|
| 76 |
cache['document_image_2_features'][vectors_type]
|
| 77 |
])
|
| 78 |
message = f'<pre style="{pre_message_style}">Similarity between the two documents is: {round(similarity, 4)}<pre>'
|
| 79 |
document_image_1 = visualize_bboxes_on_image(
|
| 80 |
+
image=document_image_1,
|
| 81 |
+
bboxes=cache['document_image_1_features'][annotations['predicted_bboxes']],
|
| 82 |
+
labels=[f'{label}, score:{round(score, 2)}' for label, score in zip(
|
| 83 |
+
cache['document_image_1_features'][annotations['predicted_labels']],
|
| 84 |
cache['document_image_1_features'][annotations['predicted_scores']])],
|
| 85 |
+
bbox_outline_color=[
|
| 86 |
+
color_map[label] for label in cache['document_image_1_features'][annotations['predicted_labels']]],
|
| 87 |
+
bbox_fill_color=[
|
| 88 |
+
(color_map[label], 50) for label in cache['document_image_1_features'][annotations['predicted_labels']]],
|
| 89 |
**visualize_bboxes_on_image_kwargs)
|
| 90 |
document_image_2 = visualize_bboxes_on_image(
|
| 91 |
+
image=document_image_2,
|
| 92 |
+
bboxes=cache['document_image_2_features'][annotations['predicted_bboxes']],
|
| 93 |
+
labels=[f'{label}, score:{round(score, 2)}' for label, score in zip(
|
| 94 |
+
cache['document_image_2_features'][annotations['predicted_labels']],
|
| 95 |
cache['document_image_2_features'][annotations['predicted_scores']])],
|
| 96 |
+
bbox_outline_color=[
|
| 97 |
+
color_map[label] for label in cache['document_image_2_features'][annotations['predicted_labels']]],
|
| 98 |
+
bbox_fill_color=[
|
| 99 |
+
(color_map[label], 50) for label in cache['document_image_2_features'][annotations['predicted_labels']]],
|
| 100 |
**visualize_bboxes_on_image_kwargs)
|
| 101 |
+
|
| 102 |
+
cache['output_document_image_1_hash'] = str(
|
| 103 |
+
average_hash(document_image_1))
|
| 104 |
+
cache['output_document_image_2_hash'] = str(
|
| 105 |
+
average_hash(document_image_2))
|
| 106 |
|
| 107 |
show_vectors_type = True
|
| 108 |
except Exception as e:
|
| 109 |
message = f'<pre style="{pre_message_style}">{traceback.format_exc()}<pre>'
|
| 110 |
return [
|
| 111 |
+
gr.HTML(message, visible=True),
|
| 112 |
+
document_image_1,
|
| 113 |
document_image_2,
|
| 114 |
gr.Dropdown(visible=show_vectors_type)
|
| 115 |
]
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def load_image(filename, page=0):
|
| 119 |
try:
|
| 120 |
image = None
|
| 121 |
+
first_error = None
|
| 122 |
try:
|
| 123 |
if (is_online_file(filename)):
|
| 124 |
+
pixmap = fitz.open("pdf", steam_online_file(filename))[page].get_pixmap()
|
| 125 |
else:
|
| 126 |
+
pixmap = fitz.open(filename)[page].get_pixmap()
|
| 127 |
+
image = Image.frombytes("RGB", [pixmap.width, pixmap.height], pixmap.samples)
|
| 128 |
+
except Exception as e:
|
| 129 |
+
first_error = e
|
| 130 |
image = get_RGB_image(filename)
|
| 131 |
return [
|
| 132 |
+
gr.Image(value=image, visible=True),
|
| 133 |
None
|
| 134 |
]
|
| 135 |
+
except Exception as second_error:
|
| 136 |
+
error = f'{traceback.format_exc()}\n\nFirst Error:\n{first_error}\n\nSecond Error:\n{second_error}'
|
| 137 |
return [None, gr.HTML(value=error, visible=True)]
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def preview_url(url, page=0):
|
| 141 |
+
[image, error] = load_image(url, page=page)
|
| 142 |
if image:
|
| 143 |
return [gr.Tabs(selected=0), image, error]
|
| 144 |
else:
|
| 145 |
+
return [gr.Tabs(selected=1), image, error]
|
| 146 |
+
|
| 147 |
|
| 148 |
def document_view(document_number: int):
|
| 149 |
+
gr.HTML(value=f'<h4>Load the {"first" if document_number == 1 else "second"} PDF or Document Image<h4>', elem_classes=[
|
| 150 |
+
'center'])
|
| 151 |
with gr.Tabs() as document_tabs:
|
| 152 |
with gr.Tab("From Image", id=0):
|
| 153 |
+
document = gr.Image(
|
| 154 |
+
type="pil", label=f"Document {document_number}", visible=False, interactive=False, show_download_button=True)
|
| 155 |
+
document_error_message = gr.HTML(
|
| 156 |
+
label="Error Message", visible=False)
|
| 157 |
document_preview = gr.UploadButton(
|
| 158 |
+
"Upload PDF or Document Image",
|
| 159 |
+
file_types=["image", ".pdf"],
|
| 160 |
file_count="single")
|
| 161 |
with gr.Tab("From URL", id=1):
|
| 162 |
document_url = gr.Textbox(
|
| 163 |
label=f"Document {document_number} URL",
|
| 164 |
info="Paste a Link/URL to PDF or Document Image",
|
| 165 |
placeholder="https://datasets-server.huggingface.co/.../image.jpg")
|
| 166 |
+
document_url_error_message = gr.HTML(
|
| 167 |
+
label="Error Message", visible=False)
|
| 168 |
+
document_url_preview = gr.Button(
|
| 169 |
+
value="Preview", variant="primary")
|
| 170 |
document_preview.upload(
|
| 171 |
+
fn=lambda file: load_image(file.name),
|
| 172 |
+
inputs=[document_preview],
|
| 173 |
+
outputs=[document, document_error_message])
|
| 174 |
document_url_preview.click(
|
| 175 |
+
fn=preview_url,
|
| 176 |
+
inputs=[document_url],
|
| 177 |
+
outputs=[document_tabs, document, document_url_error_message])
|
| 178 |
return document
|
| 179 |
|
| 180 |
+
|
| 181 |
+
def app(*, model_path, config_path, debug=False):
|
| 182 |
model: lp.Detectron2LayoutModel = lp.Detectron2LayoutModel(
|
| 183 |
+
config_path=config_path,
|
| 184 |
+
model_path=model_path,
|
| 185 |
+
label_map=label_map)
|
| 186 |
title = 'Document Similarity Search Using Visual Layout Features'
|
| 187 |
description = f"<h2>{title}<h2>"
|
| 188 |
css = '''
|
|
|
|
| 193 |
with gr.Blocks(title=title, css=css) as app:
|
| 194 |
with gr.Row():
|
| 195 |
gr.HTML(value=description, elem_classes=['center'])
|
| 196 |
+
with gr.Row(equal_height=False):
|
| 197 |
with gr.Column():
|
| 198 |
document_1_image = document_view(1)
|
| 199 |
with gr.Column():
|
|
|
|
| 204 |
submit = gr.Button(value="Get Similarity", variant="primary")
|
| 205 |
with gr.Column():
|
| 206 |
vectors_type = gr.Dropdown(
|
| 207 |
+
choices=vectors_types,
|
| 208 |
+
value=vectors_types[0],
|
| 209 |
+
visible=False,
|
| 210 |
+
label="Vectors Type",
|
| 211 |
+
info="Select the Vectors Type to use for Similarity Calculation")
|
| 212 |
+
similarity_output = gr.HTML(
|
| 213 |
+
label="Similarity Score", visible=False)
|
| 214 |
reset = gr.Button(value="Reset", variant="secondary")
|
| 215 |
kwargs = {
|
| 216 |
'fn': lambda document_1_image, document_2_image, vectors_type: similarity_fn(
|
| 217 |
+
model,
|
| 218 |
+
document_1_image,
|
| 219 |
+
document_2_image,
|
| 220 |
vectors_type),
|
| 221 |
'inputs': [document_1_image, document_2_image, vectors_type],
|
| 222 |
'outputs': [similarity_output, document_1_image, document_2_image, vectors_type]
|
| 223 |
}
|
| 224 |
submit.click(**kwargs)
|
| 225 |
vectors_type.change(**kwargs)
|
| 226 |
+
return app.launch(debug=debug)
|
requirements.txt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
PyMuPDF==1.23.26
|
| 2 |
+
scikit-learn==1.3.2
|
| 3 |
+
torch==2.1.0
|
| 4 |
+
torchvision==0.16.0
|
| 5 |
+
tensorflow==2.15.0
|
| 6 |
+
ImageHash==4.3.1
|
| 7 |
+
Pillow==9.5.0
|
| 8 |
+
layoutparser[layoutmodels,ocr]==0.3.4
|
| 9 |
+
detectron2 @ git+https://github.com/facebookresearch/detectron2.git@898507047cf441a1e4be7a729270961c401c4354
|
| 10 |
+
|
utils/get_RGB_image.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
from PIL import Image
|
| 2 |
from urllib.parse import urlparse
|
| 3 |
import requests
|
|
@@ -6,7 +7,7 @@ def is_online_file(url: str) -> bool:
|
|
| 6 |
return urlparse(url).scheme in ["http", "https"]
|
| 7 |
|
| 8 |
def steam_online_file(url: str) -> bytes:
|
| 9 |
-
return requests.get(url, stream=True).
|
| 10 |
|
| 11 |
def get_RGB_image(image_or_path: str | Image.Image) -> bytes:
|
| 12 |
if isinstance(image_or_path, str):
|
|
|
|
| 1 |
+
import io
|
| 2 |
from PIL import Image
|
| 3 |
from urllib.parse import urlparse
|
| 4 |
import requests
|
|
|
|
| 7 |
return urlparse(url).scheme in ["http", "https"]
|
| 8 |
|
| 9 |
def steam_online_file(url: str) -> bytes:
|
| 10 |
+
return io.BytesIO(requests.get(url, stream=True).content)
|
| 11 |
|
| 12 |
def get_RGB_image(image_or_path: str | Image.Image) -> bytes:
|
| 13 |
if isinstance(image_or_path, str):
|
utils/visualize_bboxes_on_image.py
CHANGED
|
@@ -5,115 +5,144 @@ import numpy as np
|
|
| 5 |
import requests
|
| 6 |
from typing import List
|
| 7 |
from functools import cache
|
|
|
|
| 8 |
|
| 9 |
DEFAULTS = {
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
}
|
| 20 |
|
|
|
|
| 21 |
@cache
|
| 22 |
def get_font(path_or_url: str = 'https://github.com/googlefonts/roboto/raw/main/src/hinted/Roboto-Regular.ttf', size: int = DEFAULTS['label_text_size']):
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
def visualize_bboxes_on_image(
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
|
| 79 |
-
if label is not None:
|
| 80 |
-
draw_text_on_image(
|
| 81 |
-
draw,
|
| 82 |
-
[x0, y0],
|
| 83 |
-
label,
|
| 84 |
-
label_text_color,
|
| 85 |
-
label_fill_color,
|
| 86 |
-
label_text_padding,
|
| 87 |
-
label_rectangle_left_margin,
|
| 88 |
-
label_rectangle_top_margin,
|
| 89 |
-
label_text_size,
|
| 90 |
-
font)
|
| 91 |
-
return image
|
| 92 |
|
| 93 |
def draw_text_on_image(
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
|
|
|
|
|
|
|
|
| 5 |
import requests
|
| 6 |
from typing import List
|
| 7 |
from functools import cache
|
| 8 |
+
import matplotlib.colors as colors
|
| 9 |
|
| 10 |
DEFAULTS = {
|
| 11 |
+
'bbox_outline_width': 2,
|
| 12 |
+
# color name or hex code or tuple of RGBA or tuple of RGB or tuple (color_name, alpha)
|
| 13 |
+
# between 0 (fully transparent) and 255 (fully opaque)
|
| 14 |
+
'bbox_outline_color': ('blue', 123),
|
| 15 |
+
# color name or hex code or tuple of RGBA or tuple of RGB or tuple (color_name, alpha)
|
| 16 |
+
# between 0 (fully transparent) and 255 (fully opaque)
|
| 17 |
+
'bbox_fill_color': ('red', 50),
|
| 18 |
+
'label_text_color': "black",
|
| 19 |
+
'label_fill_color': "red",
|
| 20 |
+
'label_text_padding': 0,
|
| 21 |
+
'label_rectangle_left_margin': 0,
|
| 22 |
+
'label_rectangle_top_margin': 0,
|
| 23 |
+
'label_text_size': 12,
|
| 24 |
}
|
| 25 |
|
| 26 |
+
|
| 27 |
@cache
|
| 28 |
def get_font(path_or_url: str = 'https://github.com/googlefonts/roboto/raw/main/src/hinted/Roboto-Regular.ttf', size: int = DEFAULTS['label_text_size']):
|
| 29 |
+
if urlparse(path_or_url).scheme in ["http", "https"]: # Online
|
| 30 |
+
return ImageFont.truetype(requests.get(path_or_url, stream=True).raw, size=size)
|
| 31 |
+
else: # Local
|
| 32 |
+
return ImageFont.truetype(path_or_url, size=size)
|
| 33 |
+
|
| 34 |
+
named_colors_mapping = colors.get_named_colors_mapping()
|
| 35 |
+
@cache
|
| 36 |
+
def get_color(color: str | tuple) -> tuple | str:
|
| 37 |
+
if isinstance(color, tuple):
|
| 38 |
+
if len(color) == 2:
|
| 39 |
+
real_color, alpha = (color[0], int(color[1]))
|
| 40 |
+
if colors.is_color_like(real_color):
|
| 41 |
+
real_color_rgb = colors.hex2color(named_colors_mapping.get(real_color, real_color))
|
| 42 |
+
if len(real_color_rgb) == 3:
|
| 43 |
+
real_color_alpha = (np.array(real_color_rgb, dtype=int) * 255).tolist() + [alpha]
|
| 44 |
+
return tuple(real_color_alpha)
|
| 45 |
+
return color
|
| 46 |
|
| 47 |
def visualize_bboxes_on_image(
|
| 48 |
+
image: Image.Image,
|
| 49 |
+
bboxes: List[List[int]],
|
| 50 |
+
labels: List[str] = None,
|
| 51 |
+
bbox_outline_width=DEFAULTS["bbox_outline_width"],
|
| 52 |
+
bbox_outline_color=DEFAULTS["bbox_outline_color"],
|
| 53 |
+
bbox_fill_color: str | list[tuple | str] = DEFAULTS["bbox_fill_color"],
|
| 54 |
+
label_text_color: str | list[tuple |
|
| 55 |
+
str] = DEFAULTS["label_text_color"],
|
| 56 |
+
label_fill_color=DEFAULTS["label_fill_color"],
|
| 57 |
+
label_text_padding=DEFAULTS["label_text_padding"],
|
| 58 |
+
label_rectangle_left_margin=DEFAULTS["label_rectangle_left_margin"],
|
| 59 |
+
label_rectangle_top_margin=DEFAULTS['label_rectangle_top_margin'],
|
| 60 |
+
label_text_size=DEFAULTS["label_text_size"],
|
| 61 |
+
convert_to_x0y0x1y1=None) -> Image.Image:
|
| 62 |
+
'''
|
| 63 |
+
Visualize bounding boxes on an image
|
| 64 |
+
Args:
|
| 65 |
+
image: Image to visualize
|
| 66 |
+
bboxes: List of bounding boxes
|
| 67 |
+
labels: Titles of the bounding boxes
|
| 68 |
+
bbox_outline_width: Width of the bounding box
|
| 69 |
+
bbox_outline_color: Color of the bounding box
|
| 70 |
+
bbox_fill_color: Fill color of the bounding box
|
| 71 |
+
label_text_color: Color of the label text
|
| 72 |
+
label_fill_color: Color of the label rectangle
|
| 73 |
+
label_text_padding: Padding of the label text
|
| 74 |
+
label_rectangle_left_margin: Left padding of the label rectangle
|
| 75 |
+
label_rectangle_top_margin: Top padding of the label rectangle
|
| 76 |
+
label_text_size: Font size of the label text
|
| 77 |
+
convert_to_x0y0x1y1: Function to convert bounding box to x0y0x1y1 format
|
| 78 |
+
Returns:
|
| 79 |
+
Image: Image annotated with bounding boxes
|
| 80 |
+
'''
|
| 81 |
+
image = image.copy().convert("RGB")
|
| 82 |
+
draw = ImageDraw.Draw(image)
|
| 83 |
+
font = get_font(size=label_text_size)
|
| 84 |
+
labels = (labels or []) + np.full(len(bboxes) -
|
| 85 |
+
len(labels or []), None).tolist()
|
| 86 |
+
bbox_fill_colors = bbox_fill_color if isinstance(bbox_fill_color, list) else [
|
| 87 |
+
bbox_fill_color] * len(bboxes)
|
| 88 |
+
bbox_outline_colors = bbox_outline_color if isinstance(
|
| 89 |
+
bbox_outline_color, list) else [bbox_outline_color] * len(bboxes)
|
| 90 |
+
|
| 91 |
+
for bbox, label, _bbox_fill_color, _bbox_outline_color in zip(bboxes, labels, bbox_fill_colors, bbox_outline_colors):
|
| 92 |
+
x0, y0, x1, y1 = convert_to_x0y0x1y1(
|
| 93 |
+
bbox) if convert_to_x0y0x1y1 is not None else bbox
|
| 94 |
+
_bbox_fill_color = get_color(_bbox_fill_color)
|
| 95 |
+
_bbox_outline_color = get_color(_bbox_outline_color)
|
| 96 |
+
rectangle_image = Image.new('RGBA', image.size)
|
| 97 |
+
rectangle_image_draw = ImageDraw.Draw(rectangle_image)
|
| 98 |
+
rectangle_image_draw.rectangle(
|
| 99 |
+
xy=[x0, y0, x1, y1],
|
| 100 |
+
fill=_bbox_fill_color,
|
| 101 |
+
outline=_bbox_outline_color,
|
| 102 |
+
width=bbox_outline_width)
|
| 103 |
+
image.paste(im=rectangle_image, mask=rectangle_image)
|
| 104 |
|
| 105 |
+
if label is not None:
|
| 106 |
+
draw_text_on_image(
|
| 107 |
+
draw,
|
| 108 |
+
[x0, y0],
|
| 109 |
+
label,
|
| 110 |
+
label_text_color,
|
| 111 |
+
label_fill_color,
|
| 112 |
+
label_text_padding,
|
| 113 |
+
label_rectangle_left_margin,
|
| 114 |
+
label_rectangle_top_margin,
|
| 115 |
+
label_text_size,
|
| 116 |
+
font)
|
| 117 |
+
return image
|
| 118 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
|
| 120 |
def draw_text_on_image(
|
| 121 |
+
image_or_draw: Image.Image | ImageDraw.ImageDraw,
|
| 122 |
+
text_position_xy: List[int],
|
| 123 |
+
label: str,
|
| 124 |
+
label_text_color=DEFAULTS["label_text_color"],
|
| 125 |
+
label_fill_color=DEFAULTS["label_fill_color"],
|
| 126 |
+
label_text_padding=DEFAULTS["label_text_padding"],
|
| 127 |
+
label_rectangle_left_margin=DEFAULTS["label_rectangle_left_margin"],
|
| 128 |
+
label_rectangle_top_margin=DEFAULTS['label_rectangle_top_margin'],
|
| 129 |
+
label_text_size=DEFAULTS["label_text_size"],
|
| 130 |
+
font: ImageFont.FreeTypeFont = None) -> Image.Image:
|
| 131 |
+
is_image = isinstance(image_or_draw, Image.Image)
|
| 132 |
+
image = image_or_draw.copy().convert("RGB") if is_image else None
|
| 133 |
+
font = font or get_font(size=label_text_size)
|
| 134 |
+
x0, y0 = text_position_xy
|
| 135 |
+
text_position = (x0 - label_rectangle_left_margin + label_text_padding,
|
| 136 |
+
y0 - label_rectangle_top_margin + label_text_padding)
|
| 137 |
+
draw = ImageDraw.Draw(image) if is_image else image_or_draw
|
| 138 |
+
_, _, text_bbox_right, text_bbox_bottom = draw.textbbox(
|
| 139 |
+
text_position, label, font=font)
|
| 140 |
+
xy = [
|
| 141 |
+
text_position[0] - label_text_padding,
|
| 142 |
+
text_position[1] - label_text_padding,
|
| 143 |
+
text_bbox_right + label_text_padding + label_text_padding,
|
| 144 |
+
text_bbox_bottom + label_text_padding + label_text_padding
|
| 145 |
+
]
|
| 146 |
+
draw.rectangle(xy, fill=label_fill_color)
|
| 147 |
+
draw.text(text_position, label, font=font, fill=label_text_color)
|
| 148 |
+
return image
|