app.py

import gradio as gr
import cv2
import matplotlib
import numpy as np
import os
from PIL import Image
import spaces
import torch
import tempfile
from gradio_imageslider import ImageSlider
from huggingface_hub import hf_hub_download

from depth_anything_v2.dpt import DepthAnythingV2

css = """
#img-display-container {
    max-height: 100vh;
}
#img-display-input {
    max-height: 80vh;
}
#img-display-output {
    max-height: 80vh;
}
#download {
    height: 62px;
}
"""
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
model_configs = {
    'vits': {'encoder': 'vits', 'features': 64, 'out_channels': [48, 96, 192, 384]},
    'vitb': {'encoder': 'vitb', 'features': 128, 'out_channels': [96, 192, 384, 768]},
    'vitl': {'encoder': 'vitl', 'features': 256, 'out_channels': [256, 512, 1024, 1024]},
    'vitg': {'encoder': 'vitg', 'features': 384, 'out_channels': [1536, 1536, 1536, 1536]}
}
encoder2name = {
    'vits': 'Small',
    'vitb': 'Base',
    'vitl': 'Large',
    'vitg': 'Giant', # we are undergoing company review procedures to release our giant model checkpoint
}
encoder = 'vitl'
model_name = encoder2name[encoder]
model = DepthAnythingV2(**model_configs[encoder])
filepath = hf_hub_download(repo_id=f"depth-anything/Depth-Anything-V2-{model_name}", filename=f"depth_anything_v2_{encoder}.pth", repo_type="model")
state_dict = torch.load(filepath, map_location="cpu")
model.load_state_dict(state_dict)
model = model.to(DEVICE).eval()


def findNormals(gray_depth, format):

    d_im = cv2.cvtColor(cv2.imread(gray_depth).astype(np.uint8), cv2.COLOR_BGR2GRAY)
    zy, zx = np.gradient(d_im)  
    # You may also consider using Sobel to get a joint Gaussian smoothing and differentation
    # to reduce noise
    #zx = cv2.Sobel(d_im, cv2.CV_64F, 1, 0, ksize=5)     
    #zy = cv2.Sobel(d_im, cv2.CV_64F, 0, 1, ksize=5)

    if format == "opengl":
        zy = -zy
        
    normal = np.dstack((np.ones_like(d_im), -zy, -zx))
    n = np.linalg.norm(normal, axis=2)
    normal[:, :, 0] /= n
    normal[:, :, 1] /= n
    normal[:, :, 2] /= n

    # offset and rescale values to be in 0-255
    normal += 1
    normal /= 2
    normal *= 255

    return (normal[:, :, ::-1]).astype(np.uint8)


load_svg="""
async(img, dpt)=>{
  document.getElementById('inimage').outerHTML = '<image id="inimage" crossorigin="anonymous" href="' + img + '" x="0" y="0" height="100%" width="100%" style="filter: url(#displacementFilter)"/>';
  document.getElementById('feimage').outerHTML = '<feImage id="feimage" crossorigin="anonymous" width="100%" height="100%" x="0" y="0" result="10_MAP" href="' + dpt + '"/>';
}
"""


js="""
async()=>{
  document.getElementById('scale').getElementsByTagName('input')[0].addEventListener('input', displace);

  function displace(e) {
    document.getElementById('dmap').setAttributeNS(null, 'scale', ''+e.target.value);
  }
}
"""


title = "# Depth Anything V2"
description = """Unofficial demo for **Depth Anything V2**.
Please refer to their [paper](https://arxiv.org/abs/2406.09414), [project page](https://depth-anything-v2.github.io), and [github](https://github.com/DepthAnything/Depth-Anything-V2) for more details."""

@spaces.GPU
def predict_depth(image):
    return model.infer_image(image)

with gr.Blocks(js=js, css=css) as demo:
    gr.Markdown(title)
    gr.Markdown(description)
    gr.Markdown("### Depth Prediction demo")

    with gr.Row():
        input_image = gr.ImageEditor(label="Input Image", layers=True, sources=('upload', 'clipboard'), show_download_button=True, type="numpy", interactive=True, transforms=(None,), eraser=gr.Eraser(), brush=gr.Brush(default_size=1, colors=['black', '#505050', '#a0a0a0', 'white']), elem_id="img-display-input")
        with gr.Tab("Depth"):
            depth_image_slider = ImageSlider(label="Depth Map with Slider View", elem_id='img-display-output', position=0.5)
            gray_depth_file = gr.File(label="Grayscale depth map", elem_id="download",)
            submit = gr.Button(value="Compute Depth")
        with gr.Tab("Normals"):
            normals_out = gr.Image(label="Normal map", interactive=False)
            format_normals = gr.Radio(choices=["directx", "opengl"])
            find_normals = gr.Button("Find normals")
            find_normals.click(fn=findNormals, inputs=[gray_depth_file, format_normals], outputs=[normals_out])
    
    raw_file = gr.File(label="16-bit raw output (can be considered as disparity)", elem_id="download",)
    cmap = matplotlib.colormaps.get_cmap('Spectral_r')

    def on_submit(img_d):
        image = cv2.cvtColor(img_d["composite"], cv2.COLOR_RGBA2RGB)
        original_image = image.copy()

        h, w = image.shape[:2]

        depth = predict_depth(image[:, :, ::-1])

        raw_depth = Image.fromarray(depth.astype('uint16'))
        tmp_raw_depth = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
        raw_depth.save(tmp_raw_depth.name)

        depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
        depth = depth.astype(np.uint8)
        colored_depth = (cmap(depth)[:, :, :3] * 255).astype(np.uint8)

        gray_depth = Image.fromarray(depth)
        svg = gr.HTML("""
        <svg id='svg'
          width='1024'
          height='512'
          viewBox='0 0 1024 512' 
          xmlns='http://www.w3.org/2000/svg'>
          <style></style>
          <filter id='displacementFilter' color-interpolation-filters='sRGB'>
            <feImage id='feimage' crossorigin='anonymous' width='100%' height='100%' x='0' y='0' result='10_MAP' href='https://freeali.se/panoramera/examples/basic/f0_dmap_.png'/>
            <feGaussianBlur result='blurred' in='10_MAP' stdDeviation='15,15' />
            <feDisplacementMap id='dmap' result='fedm' in2='blurred' in='SourceGraphic' scale='2' xChannelSelector='G' yChannelSelector='A' />
          </filter>
          <image id='inimage' crossorigin='anonymous' href='https://freeali.se/panoramera/examples/basic/f0.jpg' x='0' y='0' height='100%' width='100%' style='filter: url(#displacementFilter)'/>
        </svg>
        """)
        scale = gr.Slider(label='Scale', minimum="-128" maximum="128" value="0" step="1" elem_id="scale")
        gray_depth.change(fn=None, inputs=[original_image, gray_depth], outputs=None, js=load_svg)
        tmp_gray_depth = tempfile.NamedTemporaryFile(suffix='.png', delete=False)
        gray_depth.save(tmp_gray_depth.name)

        return [(original_image, colored_depth), tmp_gray_depth.name, tmp_raw_depth.name]

    submit.click(on_submit, inputs=[input_image], outputs=[depth_image_slider, gray_depth_file, raw_file])

    example_files = os.listdir('assets/drawn_examples')
    example_files.sort()
    example_files = [os.path.join('assets/drawn_examples', filename) for filename in example_files]
    examples = gr.Examples(examples=example_files, inputs=[input_image], outputs=[depth_image_slider, gray_depth_file, raw_file], fn=on_submit)


if __name__ == '__main__':
    demo.queue().launch(share=True)