scenedino/common/io/images.py

from pathlib import Path

import cv2
import numpy as np
import torch
import torch.nn.functional as F
from scenedino.common.cameras.fisheye import project_to_image

from scenedino.common.geometry import azimuth_elevation_to_rotation


class FisheyeToPinholeSampler:
    def __init__(self, K_target, image_size_target) -> None:
        self.K_target = K_target
        self.image_size_target = image_size_target
        x = (
            torch.linspace(-1, 1, image_size_target[1])
            .view(1, -1)
            .expand(image_size_target)
        )
        y = (
            torch.linspace(-1, 1, image_size_target[0])
            .view(-1, 1)
            .expand(image_size_target)
        )
        z = torch.ones_like(x)
        xyz = torch.stack((x, y, z), dim=-1).view(-1, 3)
        self.pts = (torch.inverse(torch.tensor(K_target)) @ xyz.T).T

    def sample_pinhole(
        self, img: torch.Tensor, calib, rotational_offset: torch.Tensor
    ) -> torch.Tensor:
        xyz_pinhole = (rotational_offset @ self.pts.T).T

        xy, _ = project_to_image(xyz_pinhole, calib)

        img = img.unsqueeze(0)
        resampled_img = F.grid_sample(
            img, xy.view(1, *self.image_size_target, 2), align_corners=True
        ).squeeze(0)
        return resampled_img


def load_image(path: Path) -> torch.Tensor:
    return torch.from_numpy(
        cv2.cvtColor(
            cv2.imread(str(path)),
            cv2.COLOR_BGR2RGB,
        ).astype(np.float32)
        / 255
    )


def load_fisheye_image_as_pinhole(
    path: Path,
    resampler: FisheyeToPinholeSampler,
    calib,
    azimuth: float,
    elevation: float,
) -> tuple[torch.Tensor, torch.Tensor]:
    img = load_image(path)
    rotational_offset = azimuth_elevation_to_rotation(azimuth, elevation)
    return resampler.sample_pinhole(img, calib, rotational_offset), rotational_offset