Hugging Face's logo

Spaces:
whitphx
/
gradio-lite-transformers-js-depth-estimation
Running

App Files Community
whitphx HF Staff commited on
Commit
5edc8b8
·
verified ·
1 Parent(s): 97383d6

Update index.html

Browse files
Files changed (1) hide show
  1. index.html +89 -15
index.html CHANGED
@@ -18,22 +18,75 @@
18
  </style>
19
  </head>
20
  <body>
21
- <gradio-lite>
22
- <gradio-file name="app.py" entrypoint>
23
- from transformers_js import import_transformers_js, as_url
24
  import gradio as gr
 
 
 
 
 
25
 
26
  transformers = await import_transformers_js()
27
  pipeline = transformers.pipeline
28
  depth_estimator = await pipeline('depth-estimation', 'Xenova/depth-anything-small-hf');
29
 
30
 
31
- async def estimate(input_image):
32
- output = await depth_estimator(as_url(input_image))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
 
34
- depth_image = output["depth"].to_pil()
35
 
36
- tensor = output["predicted_depth"]
 
 
 
 
 
 
 
 
 
 
 
 
37
  tensor_data = {
38
  "dims": tensor.dims,
39
  "type": tensor.type,
@@ -41,30 +94,51 @@ async def estimate(input_image):
41
  "size": tensor.size,
42
  }
43
 
44
- return depth_image, tensor_data
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
 
46
  demo = gr.Interface(
47
  fn=estimate,
48
  inputs=[
49
- gr.Image(type="filepath")
 
50
  ],
51
  outputs=[
52
  gr.Image(label="Depth Image"),
 
53
  gr.JSON(label="Tensor"),
54
  ],
55
  examples=[
56
- ["bread_small.png"]
57
  ]
58
  )
59
 
60
  demo.launch()
61
- </gradio-file>
62
 
63
- <gradio-file name="bread_small.png" url="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/bread_small.png" />
64
 
65
- <gradio-requirements>
66
  transformers_js_py
67
- </gradio-requirements>
68
- </gradio-lite>
 
69
  </body>
70
  </html>
 
18
  </style>
19
  </head>
20
  <body>
21
+ <gradio-lite>
22
+ <gradio-file name="app.py" entrypoint>
 
23
  import gradio as gr
24
+ import numpy as np
25
+ import PIL
26
+ import trimesh
27
+ from transformers_js import import_transformers_js, as_url
28
+
29
 
30
  transformers = await import_transformers_js()
31
  pipeline = transformers.pipeline
32
  depth_estimator = await pipeline('depth-estimation', 'Xenova/depth-anything-small-hf');
33
 
34
 
35
+ def depthmap_to_glb_trimesh(depth_map, rgb_image, file_path):
36
+ assert depth_map.shape[:2] == rgb_image.shape[:2], "Depth map and RGB image must have the same dimensions"
37
+
38
+ # Generate vertices and faces
39
+ vertices = []
40
+ colors = []
41
+ faces = []
42
+
43
+ height, width = depth_map.shape
44
+ for y in range(height):
45
+ for x in range(width):
46
+ z = depth_map[y, x]
47
+ vertices.append([x, y, z])
48
+ colors.append(rgb_image[y, x])
49
+
50
+ # Create faces (2 triangles per pixel, except for edges)
51
+ for y in range(height - 1):
52
+ for x in range(width - 1):
53
+ top_left = y * width + x
54
+ top_right = top_left + 1
55
+ bottom_left = top_left + width
56
+ bottom_right = bottom_left + 1
57
+
58
+ faces.append([top_left, bottom_left, top_right])
59
+ faces.append([top_right, bottom_left, bottom_right])
60
+
61
+ # Convert to numpy arrays
62
+ vertices = np.array(vertices, dtype=np.float64)
63
+ colors = np.array(colors, dtype=np.uint8)
64
+ faces = np.array(faces, dtype=np.int32)
65
+
66
+ mesh = trimesh.Trimesh(vertices=vertices, faces=faces, vertex_colors=colors, process=False)
67
+
68
+ # Export to GLB
69
+ mesh.export(file_path, file_type='glb')
70
+
71
+
72
+ def invert_depth(depth_map):
73
+ max_depth = np.max(depth_map)
74
+ return max_depth - depth_map
75
 
 
76
 
77
+ def invert_xy(map):
78
+ return map[::-1, ::-1]
79
+
80
+
81
+ async def estimate(image_path, depth_scale):
82
+ image = PIL.Image.open(image_path)
83
+ image.thumbnail((384, 384)) # Resize the image keeping the aspect ratio
84
+
85
+ predictions = await depth_estimator(as_url(image_path))
86
+
87
+ depth_image = predictions["depth"].to_pil()
88
+
89
+ tensor = predictions["predicted_depth"]
90
  tensor_data = {
91
  "dims": tensor.dims,
92
  "type": tensor.type,
 
94
  "size": tensor.size,
95
  }
96
 
97
+ # Construct the 3D model from the depth map and the RGB image
98
+ depth = predictions["predicted_depth"].numpy()
99
+ depth = invert_depth(depth)
100
+ depth = invert_xy(depth)
101
+
102
+ depth = depth * depth_scale
103
+
104
+ # The model outputs the depth map in a different size than the input image.
105
+ # So we resize the depth map to match the original image size.
106
+ depth = np.array(PIL.Image.fromarray(depth).resize(image.size))
107
+
108
+ image_array = np.asarray(image)
109
+ image_array = invert_xy(image_array)
110
+
111
+ glb_file_path = "output.glb"
112
+ depthmap_to_glb_trimesh(depth, image_array, glb_file_path)
113
+
114
+ return depth_image, glb_file_path, tensor_data
115
+
116
 
117
  demo = gr.Interface(
118
  fn=estimate,
119
  inputs=[
120
+ gr.Image(type="filepath"),
121
+ gr.Slider(minimum=1, maximum=100, value=10, label="Depth Scale")
122
  ],
123
  outputs=[
124
  gr.Image(label="Depth Image"),
125
+ gr.Model3D(label="3D Model"),
126
  gr.JSON(label="Tensor"),
127
  ],
128
  examples=[
129
+ ["bread_small.png"],
130
  ]
131
  )
132
 
133
  demo.launch()
134
+ </gradio-file>
135
 
136
+ <gradio-file name="bread_small.png" url="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/bread_small.png" />
137
 
138
+ <gradio-requirements>
139
  transformers_js_py
140
+ trimesh
141
+ </gradio-requirements>
142
+ </gradio-lite>
143
  </body>
144
  </html>