diff --git a/.flake8 b/.flake8
new file mode 100644
index 0000000000000000000000000000000000000000..8c1f66f9d8d970578246875bc8ffdf0e512796d3
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 80
+extend-ignore = E203,E501,E402
+exclude = .git,__pycache__,build,.venv/,third_party
\ No newline at end of file
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000000000000000000000000000000000..3d8953d91e8acf1eff6b1afb8584462afbe7c271
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,38 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..22a4500fc98458744ef0a516ba7fbfae6aad0333
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,28 @@
+build/
+# lib
+bin/
+cmake_modules/
+cmake-build-debug/
+.idea/
+.vscode/
+*.pyc
+flagged
+.ipynb_checkpoints
+__pycache__
+Untitled*
+experiments
+third_party/REKD
+hloc/matchers/dedode.py
+gradio_cached_examples
+*.mp4
+hloc/matchers/quadtree.py
+third_party/QuadTreeAttention
+desktop.ini
+*.egg-info
+output.pkl
+log.txt
+experiments*
+gen_example.py
+datasets/lines/terrace0.JPG
+datasets/lines/terrace1.JPG
+datasets/South-Building*
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..8b32629e5a70144a1549ef5b1f6b6974c192c97f
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,27 @@
+# Use an official conda-based Python image as a parent image
+FROM pytorch/pytorch:2.4.0-cuda12.1-cudnn9-runtime
+LABEL maintainer vincentqyw
+ARG PYTHON_VERSION=3.10.10
+
+# Set the working directory to /code
+WORKDIR /code
+
+# Install Git and Git LFS
+RUN apt-get update && apt-get install -y git-lfs
+RUN git lfs install
+
+# Clone the Git repository
+RUN git clone https://huggingface.co/spaces/Realcat/image-matching-webui /code
+
+RUN conda create -n imw python=${PYTHON_VERSION}
+RUN echo "source activate imw" > ~/.bashrc
+ENV PATH /opt/conda/envs/imw/bin:$PATH
+
+# Make RUN commands use the new environment
+SHELL ["conda", "run", "-n", "imw", "/bin/bash", "-c"]
+RUN pip install --upgrade pip
+RUN pip install -r requirements.txt
+RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y
+
+# Export port
+EXPOSE 7860
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..29f81d812f3e768fa89638d1f72920dbfd1413a8
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..31b8061b6b39c37e248c2c07028ab41b2d506f67
--- /dev/null
+++ b/README.md
@@ -0,0 +1,13 @@
+---
+title: MINIMA
+emoji: 📈
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+sdk_version: 5.9.1
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
diff --git a/api/__init__.py b/api/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..9ded433125b7f9da9db11ec4088f41c5277bcf10
--- /dev/null
+++ b/api/__init__.py
@@ -0,0 +1,42 @@
+import sys
+from typing import List
+from pydantic import BaseModel
+import base64
+import io
+import numpy as np
+from fastapi.exceptions import HTTPException
+from PIL import Image
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parents[1]))
+from hloc import logger
+
+
+class ImagesInput(BaseModel):
+ data: List[str] = []
+ max_keypoints: List[int] = []
+ timestamps: List[str] = []
+ grayscale: bool = False
+ image_hw: List[List[int]] = [[], []]
+ feature_type: int = 0
+ rotates: List[float] = []
+ scales: List[float] = []
+ reference_points: List[List[float]] = []
+ binarize: bool = False
+
+
+def decode_base64_to_image(encoding):
+ if encoding.startswith("data:image/"):
+ encoding = encoding.split(";")[1].split(",")[1]
+ try:
+ image = Image.open(io.BytesIO(base64.b64decode(encoding)))
+ return image
+ except Exception as e:
+ logger.warning(f"API cannot decode image: {e}")
+ raise HTTPException(
+ status_code=500, detail="Invalid encoded image"
+ ) from e
+
+
+def to_base64_nparray(encoding: str) -> np.ndarray:
+ return np.array(decode_base64_to_image(encoding)).astype("uint8")
diff --git a/api/client.py b/api/client.py
new file mode 100644
index 0000000000000000000000000000000000000000..1bf0748eb177ad64bd688e721b5e1c112be9a415
--- /dev/null
+++ b/api/client.py
@@ -0,0 +1,218 @@
+import argparse
+import base64
+import os
+import pickle
+import time
+from typing import Dict, List
+
+import cv2
+import numpy as np
+import requests
+
+ENDPOINT = "http://127.0.0.1:8000"
+if "REMOTE_URL_RAILWAY" in os.environ:
+ ENDPOINT = os.environ["REMOTE_URL_RAILWAY"]
+
+print(f"API ENDPOINT: {ENDPOINT}")
+
+API_VERSION = f"{ENDPOINT}/version"
+API_URL_MATCH = f"{ENDPOINT}/v1/match"
+API_URL_EXTRACT = f"{ENDPOINT}/v1/extract"
+
+
+def read_image(path: str) -> str:
+ """
+ Read an image from a file, encode it as a JPEG and then as a base64 string.
+ Args:
+ path (str): The path to the image to read.
+ Returns:
+ str: The base64 encoded image.
+ """
+ # Read the image from the file
+ img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
+
+ # Encode the image as a png, NO COMPRESSION!!!
+ retval, buffer = cv2.imencode(".png", img)
+
+ # Encode the JPEG as a base64 string
+ b64img = base64.b64encode(buffer).decode("utf-8")
+
+ return b64img
+
+
+def do_api_requests(url=API_URL_EXTRACT, **kwargs):
+ """
+ Helper function to send an API request to the image matching service.
+ Args:
+ url (str): The URL of the API endpoint to use. Defaults to the
+ feature extraction endpoint.
+ **kwargs: Additional keyword arguments to pass to the API.
+ Returns:
+ List[Dict[str, np.ndarray]]: A list of dictionaries containing the
+ extracted features. The keys are "keypoints", "descriptors", and
+ "scores", and the values are ndarrays of shape (N, 2), (N, ?),
+ and (N,), respectively.
+ """
+ # Set up the request body
+ reqbody = {
+ # List of image data base64 encoded
+ "data": [],
+ # List of maximum number of keypoints to extract from each image
+ "max_keypoints": [100, 100],
+ # List of timestamps for each image (not used?)
+ "timestamps": ["0", "1"],
+ # Whether to convert the images to grayscale
+ "grayscale": 0,
+ # List of image height and width
+ "image_hw": [[640, 480], [320, 240]],
+ # Type of feature to extract
+ "feature_type": 0,
+ # List of rotation angles for each image
+ "rotates": [0.0, 0.0],
+ # List of scale factors for each image
+ "scales": [1.0, 1.0],
+ # List of reference points for each image (not used)
+ "reference_points": [[640, 480], [320, 240]],
+ # Whether to binarize the descriptors
+ "binarize": True,
+ }
+ # Update the request body with the additional keyword arguments
+ reqbody.update(kwargs)
+ try:
+ # Send the request
+ r = requests.post(url, json=reqbody)
+ if r.status_code == 200:
+ # Return the response
+ return r.json()
+ else:
+ # Print an error message if the response code is not 200
+ print(f"Error: Response code {r.status_code} - {r.text}")
+ except Exception as e:
+ # Print an error message if an exception occurs
+ print(f"An error occurred: {e}")
+
+
+def send_request_match(path0: str, path1: str) -> Dict[str, np.ndarray]:
+ """
+ Send a request to the API to generate a match between two images.
+ Args:
+ path0 (str): The path to the first image.
+ path1 (str): The path to the second image.
+ Returns:
+ Dict[str, np.ndarray]: A dictionary containing the generated matches.
+ The keys are "keypoints0", "keypoints1", "matches0", and "matches1",
+ and the values are ndarrays of shape (N, 2), (N, 2), (N, 2), and
+ (N, 2), respectively.
+ """
+ files = {"image0": open(path0, "rb"), "image1": open(path1, "rb")}
+ try:
+ # TODO: replace files with post json
+ response = requests.post(API_URL_MATCH, files=files)
+ pred = {}
+ if response.status_code == 200:
+ pred = response.json()
+ for key in list(pred.keys()):
+ pred[key] = np.array(pred[key])
+ else:
+ print(
+ f"Error: Response code {response.status_code} - {response.text}"
+ )
+ finally:
+ files["image0"].close()
+ files["image1"].close()
+ return pred
+
+
+def send_request_extract(
+ input_images: str, viz: bool = False
+) -> List[Dict[str, np.ndarray]]:
+ """
+ Send a request to the API to extract features from an image.
+ Args:
+ input_images (str): The path to the image.
+ Returns:
+ List[Dict[str, np.ndarray]]: A list of dictionaries containing the
+ extracted features. The keys are "keypoints", "descriptors", and
+ "scores", and the values are ndarrays of shape (N, 2), (N, 128),
+ and (N,), respectively.
+ """
+ image_data = read_image(input_images)
+ inputs = {
+ "data": [image_data],
+ }
+ response = do_api_requests(
+ url=API_URL_EXTRACT,
+ **inputs,
+ )
+ # breakpoint()
+ # print("Keypoints detected: {}".format(len(response[0]["keypoints"])))
+
+ # draw matching, debug only
+ if viz:
+ from hloc.utils.viz import plot_keypoints
+ from ui.viz import fig2im, plot_images
+
+ kpts = np.array(response[0]["keypoints_orig"])
+ if "image_orig" in response[0].keys():
+ img_orig = np.array(["image_orig"])
+
+ output_keypoints = plot_images([img_orig], titles="titles", dpi=300)
+ plot_keypoints([kpts])
+ output_keypoints = fig2im(output_keypoints)
+ cv2.imwrite(
+ "demo_match.jpg",
+ output_keypoints[:, :, ::-1].copy(), # RGB -> BGR
+ )
+ return response
+
+
+def get_api_version():
+ try:
+ response = requests.get(API_VERSION).json()
+ print("API VERSION: {}".format(response["version"]))
+ except Exception as e:
+ print(f"An error occurred: {e}")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ description="Send text to stable audio server and receive generated audio."
+ )
+ parser.add_argument(
+ "--image0",
+ required=False,
+ help="Path for the file's melody",
+ default="datasets/sacre_coeur/mapping_rot/02928139_3448003521_rot45.jpg",
+ )
+ parser.add_argument(
+ "--image1",
+ required=False,
+ help="Path for the file's melody",
+ default="datasets/sacre_coeur/mapping_rot/02928139_3448003521_rot90.jpg",
+ )
+ args = parser.parse_args()
+
+ # get api version
+ get_api_version()
+
+ # request match
+ # for i in range(10):
+ # t1 = time.time()
+ # preds = send_request_match(args.image0, args.image1)
+ # t2 = time.time()
+ # print(
+ # "Time cost1: {} seconds, matched: {}".format(
+ # (t2 - t1), len(preds["mmkeypoints0_orig"])
+ # )
+ # )
+
+ # request extract
+ for i in range(1000):
+ t1 = time.time()
+ preds = send_request_extract(args.image0)
+ t2 = time.time()
+ print(f"Time cost2: {(t2 - t1)} seconds")
+
+ # dump preds
+ with open("preds.pkl", "wb") as f:
+ pickle.dump(preds, f)
diff --git a/api/config/api.yaml b/api/config/api.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e98cff20dca8cfd0744132732502bb9313b09bed
--- /dev/null
+++ b/api/config/api.yaml
@@ -0,0 +1,51 @@
+# This file was generated using the `serve build` command on Ray v2.38.0.
+
+proxy_location: EveryNode
+http_options:
+ host: 0.0.0.0
+ port: 8000
+
+grpc_options:
+ port: 9000
+ grpc_servicer_functions: []
+
+logging_config:
+ encoding: TEXT
+ log_level: INFO
+ logs_dir: null
+ enable_access_log: true
+
+applications:
+- name: app1
+ route_prefix: /
+ import_path: api.server:service
+ runtime_env: {}
+ deployments:
+ - name: ImageMatchingService
+ num_replicas: 4
+ ray_actor_options:
+ num_cpus: 2.0
+ num_gpus: 1.0
+
+api:
+ feature:
+ output: feats-superpoint-n4096-rmax1600
+ model:
+ name: superpoint
+ nms_radius: 3
+ max_keypoints: 4096
+ keypoint_threshold: 0.005
+ preprocessing:
+ grayscale: True
+ force_resize: True
+ resize_max: 1600
+ width: 640
+ height: 480
+ dfactor: 8
+ matcher:
+ output: matches-NN-mutual
+ model:
+ name: nearest_neighbor
+ do_mutual_check: True
+ match_threshold: 0.2
+ dense: False
diff --git a/api/server.py b/api/server.py
new file mode 100644
index 0000000000000000000000000000000000000000..442758647154da50c542c0cb543deae01f097740
--- /dev/null
+++ b/api/server.py
@@ -0,0 +1,472 @@
+# server.py
+import warnings
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+import yaml
+
+import ray
+from ray import serve
+
+import cv2
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+from fastapi import FastAPI, File, UploadFile
+from fastapi.responses import JSONResponse
+from PIL import Image
+
+from api import ImagesInput, to_base64_nparray
+from hloc import DEVICE, extract_features, logger, match_dense, match_features
+from hloc.utils.viz import add_text, plot_keypoints
+from ui import get_version
+from ui.utils import filter_matches, get_feature_model, get_model
+from ui.viz import display_matches, fig2im, plot_images
+
+warnings.simplefilter("ignore")
+app = FastAPI()
+if ray.is_initialized():
+ ray.shutdown()
+ray.init(
+ dashboard_port=8265,
+ ignore_reinit_error=True,
+)
+serve.start(
+ http_options={"host": "0.0.0.0", "port": 8000},
+)
+
+
+class ImageMatchingAPI(torch.nn.Module):
+ default_conf = {
+ "ransac": {
+ "enable": True,
+ "estimator": "poselib",
+ "geometry": "Fundamental",
+ "method": "RANSAC",
+ "reproj_threshold": 8,
+ "confidence": 0.99999,
+ "max_iter": 2000,
+ },
+ }
+
+ def __init__(
+ self,
+ conf: dict = {},
+ device: str = "cpu",
+ detect_threshold: float = 0.015,
+ max_keypoints: int = 1024,
+ match_threshold: float = 0.2,
+ ) -> None:
+ """
+ Initializes an instance of the ImageMatchingAPI class.
+ Args:
+ conf (dict): A dictionary containing the configuration parameters.
+ device (str, optional): The device to use for computation. Defaults to "cpu".
+ detect_threshold (float, optional): The threshold for detecting keypoints. Defaults to 0.015.
+ max_keypoints (int, optional): The maximum number of keypoints to extract. Defaults to 1024.
+ match_threshold (float, optional): The threshold for matching keypoints. Defaults to 0.2.
+ Returns:
+ None
+ """
+ super().__init__()
+ self.device = device
+ self.conf = {**self.default_conf, **conf}
+ self._updata_config(detect_threshold, max_keypoints, match_threshold)
+ self._init_models()
+ if device == "cuda":
+ memory_allocated = torch.cuda.memory_allocated(device)
+ memory_reserved = torch.cuda.memory_reserved(device)
+ logger.info(
+ f"GPU memory allocated: {memory_allocated / 1024**2:.3f} MB"
+ )
+ logger.info(
+ f"GPU memory reserved: {memory_reserved / 1024**2:.3f} MB"
+ )
+ self.pred = None
+
+ def parse_match_config(self, conf):
+ if conf["dense"]:
+ return {
+ **conf,
+ "matcher": match_dense.confs.get(
+ conf["matcher"]["model"]["name"]
+ ),
+ "dense": True,
+ }
+ else:
+ return {
+ **conf,
+ "feature": extract_features.confs.get(
+ conf["feature"]["model"]["name"]
+ ),
+ "matcher": match_features.confs.get(
+ conf["matcher"]["model"]["name"]
+ ),
+ "dense": False,
+ }
+
+ def _updata_config(
+ self,
+ detect_threshold: float = 0.015,
+ max_keypoints: int = 1024,
+ match_threshold: float = 0.2,
+ ):
+ self.dense = self.conf["dense"]
+ if self.conf["dense"]:
+ try:
+ self.conf["matcher"]["model"][
+ "match_threshold"
+ ] = match_threshold
+ except TypeError as e:
+ logger.error(e)
+ else:
+ self.conf["feature"]["model"]["max_keypoints"] = max_keypoints
+ self.conf["feature"]["model"][
+ "keypoint_threshold"
+ ] = detect_threshold
+ self.extract_conf = self.conf["feature"]
+
+ self.match_conf = self.conf["matcher"]
+
+ def _init_models(self):
+ # initialize matcher
+ self.matcher = get_model(self.match_conf)
+ # initialize extractor
+ if self.dense:
+ self.extractor = None
+ else:
+ self.extractor = get_feature_model(self.conf["feature"])
+
+ def _forward(self, img0, img1):
+ if self.dense:
+ pred = match_dense.match_images(
+ self.matcher,
+ img0,
+ img1,
+ self.match_conf["preprocessing"],
+ device=self.device,
+ )
+ last_fixed = "{}".format( # noqa: F841
+ self.match_conf["model"]["name"]
+ )
+ else:
+ pred0 = extract_features.extract(
+ self.extractor, img0, self.extract_conf["preprocessing"]
+ )
+ pred1 = extract_features.extract(
+ self.extractor, img1, self.extract_conf["preprocessing"]
+ )
+ pred = match_features.match_images(self.matcher, pred0, pred1)
+ return pred
+
+ def _convert_pred(self, pred):
+ ret = {
+ k: v.cpu().detach()[0].numpy() if isinstance(v, torch.Tensor) else v
+ for k, v in pred.items()
+ }
+ ret = {
+ k: v[0].cpu().detach().numpy() if isinstance(v, list) else v
+ for k, v in ret.items()
+ }
+ return ret
+
+ @torch.inference_mode()
+ def extract(self, img0: np.ndarray, **kwargs) -> Dict[str, np.ndarray]:
+ """Extract features from a single image.
+ Args:
+ img0 (np.ndarray): image
+ Returns:
+ Dict[str, np.ndarray]: feature dict
+ """
+
+ # setting prams
+ self.extractor.conf["max_keypoints"] = kwargs.get("max_keypoints", 512)
+ self.extractor.conf["keypoint_threshold"] = kwargs.get(
+ "keypoint_threshold", 0.0
+ )
+
+ pred = extract_features.extract(
+ self.extractor, img0, self.extract_conf["preprocessing"]
+ )
+ pred = self._convert_pred(pred)
+ # back to origin scale
+ s0 = pred["original_size"] / pred["size"]
+ pred["keypoints_orig"] = (
+ match_features.scale_keypoints(pred["keypoints"] + 0.5, s0) - 0.5
+ )
+ # TODO: rotate back
+ binarize = kwargs.get("binarize", False)
+ if binarize:
+ assert "descriptors" in pred
+ pred["descriptors"] = (pred["descriptors"] > 0).astype(np.uint8)
+ pred["descriptors"] = pred["descriptors"].T # N x DIM
+ return pred
+
+ @torch.inference_mode()
+ def forward(
+ self,
+ img0: np.ndarray,
+ img1: np.ndarray,
+ ) -> Dict[str, np.ndarray]:
+ """
+ Forward pass of the image matching API.
+ Args:
+ img0: A 3D NumPy array of shape (H, W, C) representing the first image.
+ Values are in the range [0, 1] and are in RGB mode.
+ img1: A 3D NumPy array of shape (H, W, C) representing the second image.
+ Values are in the range [0, 1] and are in RGB mode.
+ Returns:
+ A dictionary containing the following keys:
+ - image0_orig: The original image 0.
+ - image1_orig: The original image 1.
+ - keypoints0_orig: The keypoints detected in image 0.
+ - keypoints1_orig: The keypoints detected in image 1.
+ - mkeypoints0_orig: The raw matches between image 0 and image 1.
+ - mkeypoints1_orig: The raw matches between image 1 and image 0.
+ - mmkeypoints0_orig: The RANSAC inliers in image 0.
+ - mmkeypoints1_orig: The RANSAC inliers in image 1.
+ - mconf: The confidence scores for the raw matches.
+ - mmconf: The confidence scores for the RANSAC inliers.
+ """
+ # Take as input a pair of images (not a batch)
+ assert isinstance(img0, np.ndarray)
+ assert isinstance(img1, np.ndarray)
+ self.pred = self._forward(img0, img1)
+ if self.conf["ransac"]["enable"]:
+ self.pred = self._geometry_check(self.pred)
+ return self.pred
+
+ def _geometry_check(
+ self,
+ pred: Dict[str, Any],
+ ) -> Dict[str, Any]:
+ """
+ Filter matches using RANSAC. If keypoints are available, filter by keypoints.
+ If lines are available, filter by lines. If both keypoints and lines are
+ available, filter by keypoints.
+ Args:
+ pred (Dict[str, Any]): dict of matches, including original keypoints.
+ See :func:`filter_matches` for the expected keys.
+ Returns:
+ Dict[str, Any]: filtered matches
+ """
+ pred = filter_matches(
+ pred,
+ ransac_method=self.conf["ransac"]["method"],
+ ransac_reproj_threshold=self.conf["ransac"]["reproj_threshold"],
+ ransac_confidence=self.conf["ransac"]["confidence"],
+ ransac_max_iter=self.conf["ransac"]["max_iter"],
+ )
+ return pred
+
+ def visualize(
+ self,
+ log_path: Optional[Path] = None,
+ ) -> None:
+ """
+ Visualize the matches.
+ Args:
+ log_path (Path, optional): The directory to save the images. Defaults to None.
+ Returns:
+ None
+ """
+ if self.conf["dense"]:
+ postfix = str(self.conf["matcher"]["model"]["name"])
+ else:
+ postfix = "{}_{}".format(
+ str(self.conf["feature"]["model"]["name"]),
+ str(self.conf["matcher"]["model"]["name"]),
+ )
+ titles = [
+ "Image 0 - Keypoints",
+ "Image 1 - Keypoints",
+ ]
+ pred: Dict[str, Any] = self.pred
+ image0: np.ndarray = pred["image0_orig"]
+ image1: np.ndarray = pred["image1_orig"]
+ output_keypoints: np.ndarray = plot_images(
+ [image0, image1], titles=titles, dpi=300
+ )
+ if (
+ "keypoints0_orig" in pred.keys()
+ and "keypoints1_orig" in pred.keys()
+ ):
+ plot_keypoints([pred["keypoints0_orig"], pred["keypoints1_orig"]])
+ text: str = (
+ f"# keypoints0: {len(pred['keypoints0_orig'])} \n"
+ + f"# keypoints1: {len(pred['keypoints1_orig'])}"
+ )
+ add_text(0, text, fs=15)
+ output_keypoints = fig2im(output_keypoints)
+ # plot images with raw matches
+ titles = [
+ "Image 0 - Raw matched keypoints",
+ "Image 1 - Raw matched keypoints",
+ ]
+ output_matches_raw, num_matches_raw = display_matches(
+ pred, titles=titles, tag="KPTS_RAW"
+ )
+ # plot images with ransac matches
+ titles = [
+ "Image 0 - Ransac matched keypoints",
+ "Image 1 - Ransac matched keypoints",
+ ]
+ output_matches_ransac, num_matches_ransac = display_matches(
+ pred, titles=titles, tag="KPTS_RANSAC"
+ )
+ if log_path is not None:
+ img_keypoints_path: Path = log_path / f"img_keypoints_{postfix}.png"
+ img_matches_raw_path: Path = (
+ log_path / f"img_matches_raw_{postfix}.png"
+ )
+ img_matches_ransac_path: Path = (
+ log_path / f"img_matches_ransac_{postfix}.png"
+ )
+ cv2.imwrite(
+ str(img_keypoints_path),
+ output_keypoints[:, :, ::-1].copy(), # RGB -> BGR
+ )
+ cv2.imwrite(
+ str(img_matches_raw_path),
+ output_matches_raw[:, :, ::-1].copy(), # RGB -> BGR
+ )
+ cv2.imwrite(
+ str(img_matches_ransac_path),
+ output_matches_ransac[:, :, ::-1].copy(), # RGB -> BGR
+ )
+ plt.close("all")
+
+
+@serve.deployment(
+ num_replicas=4,
+ ray_actor_options={"num_cpus": 2, "num_gpus": 1}
+)
+@serve.ingress(app)
+class ImageMatchingService:
+ def __init__(self, conf: dict, device: str):
+ self.conf = conf
+ self.api = ImageMatchingAPI(conf=conf, device=device)
+
+ @app.get("/")
+ def root(self):
+ return "Hello, world!"
+
+ @app.get("/version")
+ async def version(self):
+ return {"version": get_version()}
+
+ @app.post("/v1/match")
+ async def match(
+ self, image0: UploadFile = File(...), image1: UploadFile = File(...)
+ ):
+ """
+ Handle the image matching request and return the processed result.
+ Args:
+ image0 (UploadFile): The first image file for matching.
+ image1 (UploadFile): The second image file for matching.
+ Returns:
+ JSONResponse: A JSON response containing the filtered match results
+ or an error message in case of failure.
+ """
+ try:
+ # Load the images from the uploaded files
+ image0_array = self.load_image(image0)
+ image1_array = self.load_image(image1)
+ print('image0_array',image0_array.shape)
+ print('image1_array',image1_array.shape)
+
+ # Perform image matching using the API
+ output = self.api(image0_array, image1_array)
+
+ # Keys to skip in the output
+ skip_keys = ["image0_orig", "image1_orig"]
+
+ # Postprocess the output to filter unwanted data
+ pred = self.postprocess(output, skip_keys)
+
+ # Return the filtered prediction as a JSON response
+ return JSONResponse(content=pred)
+ except Exception as e:
+ # Return an error message with status code 500 in case of exception
+ return JSONResponse(content={"error": str(e)}, status_code=500)
+
+ @app.post("/v1/extract")
+ async def extract(self, input_info: ImagesInput):
+ """
+ Extract keypoints and descriptors from images.
+ Args:
+ input_info: An object containing the image data and options.
+ Returns:
+ A list of dictionaries containing the keypoints and descriptors.
+ """
+ try:
+ preds = []
+ for i, input_image in enumerate(input_info.data):
+ # Load the image from the input data
+ image_array = to_base64_nparray(input_image)
+ # Extract keypoints and descriptors
+ output = self.api.extract(
+ image_array,
+ max_keypoints=input_info.max_keypoints[i],
+ binarize=input_info.binarize,
+ )
+ # Do not return the original image and image_orig
+ # skip_keys = ["image", "image_orig"]
+ skip_keys = []
+
+ # Postprocess the output
+ pred = self.postprocess(output, skip_keys)
+ preds.append(pred)
+ # Return the list of extracted features
+ return JSONResponse(content=preds)
+ except Exception as e:
+ # Return an error message if an exception occurs
+ return JSONResponse(content={"error": str(e)}, status_code=500)
+
+ def load_image(self, file_path: Union[str, UploadFile]) -> np.ndarray:
+ """
+ Reads an image from a file path or an UploadFile object.
+ Args:
+ file_path: A file path or an UploadFile object.
+ Returns:
+ A numpy array representing the image.
+ """
+ if isinstance(file_path, str):
+ file_path = Path(file_path).resolve(strict=False)
+ else:
+ file_path = file_path.file
+ with Image.open(file_path) as img:
+ image_array = np.array(img)
+ return image_array
+
+ def postprocess(
+ self, output: dict, skip_keys: list, binarize: bool = True
+ ) -> dict:
+ pred = {}
+ for key, value in output.items():
+ if key in skip_keys:
+ continue
+ if isinstance(value, np.ndarray):
+ pred[key] = value.tolist()
+ return pred
+
+ def run(self, host: str = "0.0.0.0", port: int = 8001):
+ import uvicorn
+ uvicorn.run(app, host=host, port=port)
+
+
+def read_config(config_path: Path) -> dict:
+ with open(config_path, "r") as f:
+ conf = yaml.safe_load(f)
+ return conf
+
+
+# api server
+conf = read_config(Path(__file__).parent / "config/api.yaml")
+service = ImageMatchingService.bind(conf=conf["api"], device=DEVICE)
+
+# handle = serve.run(service, route_prefix="/")
+# serve run api.server_ray:service
+
+# build to generate config file
+# serve build api.server_ray:service -o api/config/ray.yaml
+# serve run api/config/ray.yaml
diff --git a/api/test/CMakeLists.txt b/api/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e37d4a575e6e3e9d75f8b7a214d32b3e48796cad
--- /dev/null
+++ b/api/test/CMakeLists.txt
@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.10)
+project(imatchui)
+
+set(OpenCV_DIR /usr/include/opencv4)
+find_package(OpenCV REQUIRED)
+
+find_package(Boost REQUIRED COMPONENTS system)
+if(Boost_FOUND)
+ include_directories(${Boost_INCLUDE_DIRS})
+endif()
+
+add_executable(client client.cpp)
+
+target_include_directories(client PRIVATE ${Boost_LIBRARIES} ${OpenCV_INCLUDE_DIRS})
+
+target_link_libraries(client PRIVATE curl jsoncpp b64 ${OpenCV_LIBS})
diff --git a/api/test/build_and_run.sh b/api/test/build_and_run.sh
new file mode 100644
index 0000000000000000000000000000000000000000..e44f6ba9e5d62f94a121e31f39072c469d96e5df
--- /dev/null
+++ b/api/test/build_and_run.sh
@@ -0,0 +1,16 @@
+# g++ main.cpp -I/usr/include/opencv4 -lcurl -ljsoncpp -lb64 -lopencv_core -lopencv_imgcodecs -o main
+# sudo apt-get update
+# sudo apt-get install libboost-all-dev -y
+# sudo apt-get install libcurl4-openssl-dev libjsoncpp-dev libb64-dev libopencv-dev -y
+
+cd build
+cmake ..
+make -j12
+
+echo " ======== RUN DEMO ========"
+
+./client
+
+echo " ======== END DEMO ========"
+
+cd ..
diff --git a/api/test/client.cpp b/api/test/client.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..66476c585d47961b33d650e7cedd167150db72f1
--- /dev/null
+++ b/api/test/client.cpp
@@ -0,0 +1,84 @@
+#include
+
+ Philipp Lindenberger
+ ·
+ Paul-Edouard Sarlin
+ ·
+ Marc Pollefeys
+ ICCV 2023
"),
+ )
+ fig.add_trace(pyramid)
+
+ triangles = np.vstack((i, j, k)).T
+ vertices = np.concatenate(([t], corners))
+ tri_points = np.array([vertices[i] for i in triangles.reshape(-1)])
+ x, y, z = tri_points.T
+
+ pyramid = go.Scatter3d(
+ x=x,
+ y=y,
+ z=z,
+ mode="lines",
+ legendgroup=legendgroup,
+ name=name,
+ line=dict(color=color, width=1),
+ showlegend=False,
+ hovertemplate=text.replace("\n", "
"),
+ )
+ fig.add_trace(pyramid)
+
+
+def plot_camera_colmap(
+ fig: go.Figure,
+ image: pycolmap.Image,
+ camera: pycolmap.Camera,
+ name: Optional[str] = None,
+ **kwargs
+):
+ """Plot a camera frustum from PyCOLMAP objects"""
+ world_t_camera = image.cam_from_world.inverse()
+ plot_camera(
+ fig,
+ world_t_camera.rotation.matrix(),
+ world_t_camera.translation,
+ camera.calibration_matrix(),
+ name=name or str(image.image_id),
+ text=str(image),
+ **kwargs
+ )
+
+
+def plot_cameras(fig: go.Figure, reconstruction: pycolmap.Reconstruction, **kwargs):
+ """Plot a camera as a cone with camera frustum."""
+ for image_id, image in reconstruction.images.items():
+ plot_camera_colmap(
+ fig, image, reconstruction.cameras[image.camera_id], **kwargs
+ )
+
+
+def plot_reconstruction(
+ fig: go.Figure,
+ rec: pycolmap.Reconstruction,
+ max_reproj_error: float = 6.0,
+ color: str = "rgb(0, 0, 255)",
+ name: Optional[str] = None,
+ min_track_length: int = 2,
+ points: bool = True,
+ cameras: bool = True,
+ points_rgb: bool = True,
+ cs: float = 1.0,
+):
+ # Filter outliers
+ bbs = rec.compute_bounding_box(0.001, 0.999)
+ # Filter points, use original reproj error here
+ p3Ds = [
+ p3D
+ for _, p3D in rec.points3D.items()
+ if (
+ (p3D.xyz >= bbs[0]).all()
+ and (p3D.xyz <= bbs[1]).all()
+ and p3D.error <= max_reproj_error
+ and p3D.track.length() >= min_track_length
+ )
+ ]
+ xyzs = [p3D.xyz for p3D in p3Ds]
+ if points_rgb:
+ pcolor = [p3D.color for p3D in p3Ds]
+ else:
+ pcolor = color
+ if points:
+ plot_points(fig, np.array(xyzs), color=pcolor, ps=1, name=name)
+ if cameras:
+ plot_cameras(fig, rec, color=color, legendgroup=name, size=cs)
diff --git a/hloc/visualization.py b/hloc/visualization.py
new file mode 100644
index 0000000000000000000000000000000000000000..cde24aa1760dd955d053d7c82229af17c629e882
--- /dev/null
+++ b/hloc/visualization.py
@@ -0,0 +1,182 @@
+import pickle
+import random
+
+import numpy as np
+import pycolmap
+from matplotlib import cm
+
+from .utils.io import read_image
+from .utils.viz import (
+ add_text,
+ cm_RdGn,
+ plot_images,
+ plot_keypoints,
+ plot_matches,
+)
+
+
+def visualize_sfm_2d(
+ reconstruction,
+ image_dir,
+ color_by="visibility",
+ selected=[],
+ n=1,
+ seed=0,
+ dpi=75,
+):
+ assert image_dir.exists()
+ if not isinstance(reconstruction, pycolmap.Reconstruction):
+ reconstruction = pycolmap.Reconstruction(reconstruction)
+
+ if not selected:
+ image_ids = reconstruction.reg_image_ids()
+ selected = random.Random(seed).sample(image_ids, min(n, len(image_ids)))
+
+ for i in selected:
+ image = reconstruction.images[i]
+ keypoints = np.array([p.xy for p in image.points2D])
+ visible = np.array([p.has_point3D() for p in image.points2D])
+
+ if color_by == "visibility":
+ color = [(0, 0, 1) if v else (1, 0, 0) for v in visible]
+ text = f"visible: {np.count_nonzero(visible)}/{len(visible)}"
+ elif color_by == "track_length":
+ tl = np.array(
+ [
+ (
+ reconstruction.points3D[p.point3D_id].track.length()
+ if p.has_point3D()
+ else 1
+ )
+ for p in image.points2D
+ ]
+ )
+ max_, med_ = np.max(tl), np.median(tl[tl > 1])
+ tl = np.log(tl)
+ color = cm.jet(tl / tl.max()).tolist()
+ text = f"max/median track length: {max_}/{med_}"
+ elif color_by == "depth":
+ p3ids = [p.point3D_id for p in image.points2D if p.has_point3D()]
+ z = np.array(
+ [
+ (image.cam_from_world * reconstruction.points3D[j].xyz)[-1]
+ for j in p3ids
+ ]
+ )
+ z -= z.min()
+ color = cm.jet(z / np.percentile(z, 99.9))
+ text = f"visible: {np.count_nonzero(visible)}/{len(visible)}"
+ keypoints = keypoints[visible]
+ else:
+ raise NotImplementedError(f"Coloring not implemented: {color_by}.")
+
+ name = image.name
+ fig = plot_images([read_image(image_dir / name)], dpi=dpi)
+ plot_keypoints([keypoints], colors=[color], ps=4)
+ add_text(0, text)
+ add_text(0, name, pos=(0.01, 0.01), fs=5, lcolor=None, va="bottom")
+ return fig
+
+
+def visualize_loc(
+ results,
+ image_dir,
+ reconstruction=None,
+ db_image_dir=None,
+ selected=[],
+ n=1,
+ seed=0,
+ prefix=None,
+ **kwargs,
+):
+ assert image_dir.exists()
+
+ with open(str(results) + "_logs.pkl", "rb") as f:
+ logs = pickle.load(f)
+
+ if not selected:
+ queries = list(logs["loc"].keys())
+ if prefix:
+ queries = [q for q in queries if q.startswith(prefix)]
+ selected = random.Random(seed).sample(queries, min(n, len(queries)))
+
+ if reconstruction is not None:
+ if not isinstance(reconstruction, pycolmap.Reconstruction):
+ reconstruction = pycolmap.Reconstruction(reconstruction)
+
+ for qname in selected:
+ loc = logs["loc"][qname]
+ visualize_loc_from_log(
+ image_dir, qname, loc, reconstruction, db_image_dir, **kwargs
+ )
+
+
+def visualize_loc_from_log(
+ image_dir,
+ query_name,
+ loc,
+ reconstruction=None,
+ db_image_dir=None,
+ top_k_db=2,
+ dpi=75,
+):
+ q_image = read_image(image_dir / query_name)
+ if loc.get("covisibility_clustering", False):
+ # select the first, largest cluster if the localization failed
+ loc = loc["log_clusters"][loc["best_cluster"] or 0]
+
+ inliers = np.array(loc["PnP_ret"]["inliers"])
+ mkp_q = loc["keypoints_query"]
+ n = len(loc["db"])
+ if reconstruction is not None:
+ # for each pair of query keypoint and its matched 3D point,
+ # we need to find its corresponding keypoint in each database image
+ # that observes it. We also count the number of inliers in each.
+ kp_idxs, kp_to_3D_to_db = loc["keypoint_index_to_db"]
+ counts = np.zeros(n)
+ dbs_kp_q_db = [[] for _ in range(n)]
+ inliers_dbs = [[] for _ in range(n)]
+ for i, (inl, (p3D_id, db_idxs)) in enumerate(
+ zip(inliers, kp_to_3D_to_db)
+ ):
+ track = reconstruction.points3D[p3D_id].track
+ track = {el.image_id: el.point2D_idx for el in track.elements}
+ for db_idx in db_idxs:
+ counts[db_idx] += inl
+ kp_db = track[loc["db"][db_idx]]
+ dbs_kp_q_db[db_idx].append((i, kp_db))
+ inliers_dbs[db_idx].append(inl)
+ else:
+ # for inloc the database keypoints are already in the logs
+ assert "keypoints_db" in loc
+ assert "indices_db" in loc
+ counts = np.array(
+ [np.sum(loc["indices_db"][inliers] == i) for i in range(n)]
+ )
+
+ # display the database images with the most inlier matches
+ db_sort = np.argsort(-counts)
+ for db_idx in db_sort[:top_k_db]:
+ if reconstruction is not None:
+ db = reconstruction.images[loc["db"][db_idx]]
+ db_name = db.name
+ db_kp_q_db = np.array(dbs_kp_q_db[db_idx])
+ kp_q = mkp_q[db_kp_q_db[:, 0]]
+ kp_db = np.array([db.points2D[i].xy for i in db_kp_q_db[:, 1]])
+ inliers_db = inliers_dbs[db_idx]
+ else:
+ db_name = loc["db"][db_idx]
+ kp_q = mkp_q[loc["indices_db"] == db_idx]
+ kp_db = loc["keypoints_db"][loc["indices_db"] == db_idx]
+ inliers_db = inliers[loc["indices_db"] == db_idx]
+
+ db_image = read_image((db_image_dir or image_dir) / db_name)
+ color = cm_RdGn(inliers_db).tolist()
+ text = f"inliers: {sum(inliers_db)}/{len(inliers_db)}"
+
+ plot_images([q_image, db_image], dpi=dpi)
+ plot_matches(kp_q, kp_db, color, a=0.1)
+ add_text(0, text)
+ opts = dict(pos=(0.01, 0.01), fs=5, lcolor=None, va="bottom")
+ add_text(0, query_name, **opts)
+ add_text(1, db_name, **opts)
diff --git a/log.txt b/log.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ff9422b753c1e1fdf498f019c0638027053f29e2
--- /dev/null
+++ b/log.txt
@@ -0,0 +1,132 @@
+[2024/12/19 15:10:08 hloc INFO] Loading lightglue model, superpoint_minima_lightglue.pth
+[2024/12/19 15:10:08 hloc INFO] Load lightglue model done.
+[2024/12/19 15:10:09 hloc INFO] Loading model using: 1.680s
+[2024/12/19 15:10:09 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:10:10 hloc INFO] Matching images done using: 0.931s
+[2024/12/19 15:10:11 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:10:11 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:10:11 hloc INFO] RANSAC matches done using: 1.024s
+[2024/12/19 15:10:11 hloc INFO] Display matches done using: 0.633s
+[2024/12/19 15:10:12 hloc INFO] TOTAL time: 4.577s
+[2024/12/19 15:10:12 hloc INFO] Dump results done!
+[2024/12/19 15:10:20 hloc INFO] Loading lightglue model, superpoint_lightglue.pth
+[2024/12/19 15:10:22 hloc INFO] Load lightglue model done.
+[2024/12/19 15:10:22 hloc INFO] Loading model using: 1.321s
+[2024/12/19 15:10:22 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:10:22 hloc INFO] Matching images done using: 0.207s
+[2024/12/19 15:10:23 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:10:23 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:10:23 hloc INFO] RANSAC matches done using: 0.882s
+[2024/12/19 15:10:23 hloc INFO] Display matches done using: 0.562s
+[2024/12/19 15:10:24 hloc INFO] TOTAL time: 3.278s
+[2024/12/19 15:10:24 hloc INFO] Dump results done!
+[2024/12/19 15:10:34 hloc INFO] Loading lightglue model, superpoint_minima_lightglue.pth
+[2024/12/19 15:10:34 hloc INFO] Load lightglue model done.
+[2024/12/19 15:10:34 hloc INFO] Loading model using: 0.325s
+[2024/12/19 15:10:34 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:10:34 hloc INFO] Matching images done using: 0.264s
+[2024/12/19 15:10:35 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:10:35 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:10:35 hloc INFO] RANSAC matches done using: 0.961s
+[2024/12/19 15:10:36 hloc INFO] Display matches done using: 0.662s
+[2024/12/19 15:10:36 hloc INFO] TOTAL time: 2.515s
+[2024/12/19 15:10:36 hloc INFO] Dump results done!
+[2024/12/19 15:10:46 hloc INFO] Loading lightglue model, superpoint_lightglue.pth
+[2024/12/19 15:10:47 hloc INFO] Load lightglue model done.
+[2024/12/19 15:10:47 hloc INFO] Loading model using: 1.192s
+[2024/12/19 15:10:47 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:10:47 hloc INFO] Matching images done using: 0.201s
+[2024/12/19 15:10:49 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:10:49 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:10:49 hloc INFO] RANSAC matches done using: 1.362s
+[2024/12/19 15:10:49 hloc INFO] Display matches done using: 0.626s
+[2024/12/19 15:10:50 hloc INFO] TOTAL time: 3.749s
+[2024/12/19 15:10:50 hloc INFO] Dump results done!
+[2024/12/19 15:11:07 hloc INFO] Loading lightglue model, superpoint_minima_lightglue.pth
+[2024/12/19 15:11:07 hloc INFO] Load lightglue model done.
+[2024/12/19 15:11:08 hloc INFO] Loading model using: 0.310s
+[2024/12/19 15:11:08 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:11:08 hloc INFO] Matching images done using: 0.291s
+[2024/12/19 15:11:09 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:11:09 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:11:09 hloc INFO] RANSAC matches done using: 0.972s
+[2024/12/19 15:11:09 hloc INFO] Display matches done using: 0.621s
+[2024/12/19 15:11:10 hloc INFO] TOTAL time: 2.491s
+[2024/12/19 15:11:10 hloc INFO] Dump results done!
+[2024/12/19 15:11:30 hloc INFO] Loading lightglue model, superpoint_lightglue.pth
+[2024/12/19 15:11:30 hloc INFO] Load lightglue model done.
+[2024/12/19 15:11:30 hloc INFO] Loading model using: 0.470s
+[2024/12/19 15:11:30 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:11:30 hloc INFO] Matching images done using: 0.205s
+[2024/12/19 15:11:31 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:11:31 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:11:31 hloc INFO] RANSAC matches done using: 0.955s
+[2024/12/19 15:11:32 hloc INFO] Display matches done using: 0.661s
+[2024/12/19 15:11:32 hloc INFO] TOTAL time: 2.608s
+[2024/12/19 15:11:32 hloc INFO] Dump results done!
+[2024/12/19 15:11:42 hloc INFO] Loading lightglue model, superpoint_minima_lightglue.pth
+[2024/12/19 15:11:42 hloc INFO] Load lightglue model done.
+[2024/12/19 15:11:43 hloc INFO] Loading model using: 0.336s
+[2024/12/19 15:11:43 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:11:43 hloc INFO] Matching images done using: 0.310s
+[2024/12/19 15:11:44 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:11:44 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:11:44 hloc INFO] RANSAC matches done using: 1.001s
+[2024/12/19 15:11:45 hloc INFO] Display matches done using: 0.652s
+[2024/12/19 15:11:45 hloc INFO] TOTAL time: 2.616s
+[2024/12/19 15:11:45 hloc INFO] Dump results done!
+[2024/12/19 15:11:59 hloc INFO] Loading lightglue model, superpoint_lightglue.pth
+[2024/12/19 15:12:01 hloc INFO] Load lightglue model done.
+[2024/12/19 15:12:01 hloc INFO] Loading model using: 1.331s
+[2024/12/19 15:12:01 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:12:01 hloc INFO] Matching images done using: 0.245s
+[2024/12/19 15:12:02 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:12:02 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:12:02 hloc INFO] RANSAC matches done using: 1.416s
+[2024/12/19 15:12:03 hloc INFO] Display matches done using: 1.153s
+[2024/12/19 15:12:04 hloc INFO] TOTAL time: 4.471s
+[2024/12/19 15:12:04 hloc INFO] Dump results done!
+[2024/12/19 15:12:44 hloc INFO] Loading lightglue model, superpoint_minima_lightglue.pth
+[2024/12/19 15:12:44 hloc INFO] Load lightglue model done.
+[2024/12/19 15:12:44 hloc INFO] Loading model using: 0.354s
+[2024/12/19 15:12:44 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:12:44 hloc INFO] Matching images done using: 0.277s
+[2024/12/19 15:12:45 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:12:45 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:12:45 hloc INFO] RANSAC matches done using: 1.012s
+[2024/12/19 15:12:46 hloc INFO] Display matches done using: 0.686s
+[2024/12/19 15:12:47 hloc INFO] TOTAL time: 2.679s
+[2024/12/19 15:12:47 hloc INFO] Dump results done!
+[2024/12/19 15:12:56 hloc INFO] Loading lightglue model, superpoint_lightglue.pth
+[2024/12/19 15:12:57 hloc INFO] Load lightglue model done.
+[2024/12/19 15:12:57 hloc INFO] Loading model using: 1.020s
+[2024/12/19 15:12:57 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:12:58 hloc INFO] Matching images done using: 0.215s
+[2024/12/19 15:12:59 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:12:59 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:12:59 hloc INFO] RANSAC matches done using: 0.991s
+[2024/12/19 15:12:59 hloc INFO] Display matches done using: 0.688s
+[2024/12/19 15:13:00 hloc INFO] TOTAL time: 3.210s
+[2024/12/19 15:13:00 hloc INFO] Dump results done!
+[2024/12/19 15:13:15 hloc INFO] Loading lightglue model, superpoint_minima_lightglue.pth
+[2024/12/19 15:13:15 hloc INFO] Load lightglue model done.
+[2024/12/19 15:13:16 hloc INFO] Loading model using: 0.302s
+[2024/12/19 15:13:16 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:13:16 hloc INFO] Matching images done using: 0.189s
+[2024/12/19 15:13:17 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:13:17 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:13:17 hloc INFO] RANSAC matches done using: 0.988s
+[2024/12/19 15:13:17 hloc INFO] Display matches done using: 0.648s
+[2024/12/19 15:13:18 hloc INFO] TOTAL time: 2.424s
+[2024/12/19 15:13:18 hloc INFO] Dump results done!
+[2024/12/19 15:13:29 hloc INFO] Loading lightglue model, superpoint_lightglue.pth
+[2024/12/19 15:13:30 hloc INFO] Load lightglue model done.
+[2024/12/19 15:13:30 hloc INFO] Loading model using: 0.977s
+[2024/12/19 15:13:30 hloc INFO] Load SuperPoint model done.
+[2024/12/19 15:13:30 hloc INFO] Matching images done using: 0.207s
+[2024/12/19 15:13:31 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Fundamental
+[2024/12/19 15:13:31 hloc INFO] ransac_method: CV2_USAC_MAGSAC, geometry_type: Homography
+[2024/12/19 15:13:31 hloc INFO] RANSAC matches done using: 1.426s
+[2024/12/19 15:13:32 hloc INFO] Display matches done using: 0.601s
+[2024/12/19 15:13:32 hloc INFO] TOTAL time: 3.510s
+[2024/12/19 15:13:32 hloc INFO] Dump results done!
diff --git a/output.pkl b/output.pkl
new file mode 100644
index 0000000000000000000000000000000000000000..c81173c6ba6d0f8df002b2aedf782269fa9b15df
--- /dev/null
+++ b/output.pkl
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faf509c60b0b906ac4e6837b1632a9567d602f04ee069cddc384d617e8d623e5
+size 2792855
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000000000000000000000000000000000000..bd76443f797c026426e49957fe98755ff492c11e
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,37 @@
+[project]
+name = "ImageMatchingWebui"
+description = "Image Matching Webui: A tool for matching images using sota algorithms with a Gradio UI"
+version = "1.0"
+authors = [
+ {name = "vincentqyw"},
+]
+readme = "README.md"
+requires-python = ">=3.8"
+license = {file = "LICENSE"}
+classifiers = [
+ "Programming Language :: Python :: 3",
+ "License :: OSI Approved :: Apache Software License",
+ "Operating System :: OS Independent",
+]
+urls = {Repository = "https://github.com/Vincentqyw/image-matching-webui"}
+dynamic = ["dependencies"]
+
+[project.optional-dependencies]
+dev = ["black", "flake8", "isort"]
+
+[tool.setuptools.packages.find]
+include = ["hloc*", "ui", "api",]
+
+[tool.setuptools.package-data]
+ui = ["*.yaml"]
+api = ["**yaml"]
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
+[tool.black]
+line-length = 80
+
+[tool.isort]
+profile = "black"
+line_length = 80
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ac5c038871f51c9b056bc6d3b8f0fde74a204857
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,44 @@
+e2cnn
+einops
+easydict
+gdown
+gradio==5.9.1
+anyio==4.2.0
+h5py
+huggingface_hub
+imageio
+Jinja2
+kornia
+loguru
+matplotlib==3.8.3
+numpy== 1.26.4
+onnxruntime
+omegaconf
+opencv-python
+opencv-contrib-python
+pandas
+psutil
+plotly
+protobuf
+poselib
+pycolmap==3.11.1
+pytlsd
+PyYAML
+joblib==1.4.2
+# pytorch-lightning==1.4.9
+scikit-image==0.24.0
+# scikit-learn
+scipy==1.13.0
+seaborn==0.13.2
+shapely==2.0.3
+# tensorboardX==2.6.1
+# torchmetrics==0.6.0
+torchvision==0.16.2
+torch==2.1.2
+roma #dust3r
+tqdm
+yacs
+fastapi
+uvicorn
+ray
+ray[serve]
\ No newline at end of file
diff --git a/test_app_cli.py b/test_app_cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..1b6ad18d2d614fac3623c92e713d513b35604228
--- /dev/null
+++ b/test_app_cli.py
@@ -0,0 +1,112 @@
+import sys
+from pathlib import Path
+
+import cv2
+
+from hloc import logger
+from ui.utils import DEVICE, ROOT, get_matcher_zoo, load_config
+
+sys.path.append(str(Path(__file__).parents[1]))
+from api.server import ImageMatchingAPI
+
+
+def test_all(config: dict = None):
+ img_path1 = ROOT / "datasets/sacre_coeur/mapping/02928139_3448003521.jpg"
+ img_path2 = ROOT / "datasets/sacre_coeur/mapping/17295357_9106075285.jpg"
+ image0 = cv2.imread(str(img_path1))[:, :, ::-1] # RGB
+ image1 = cv2.imread(str(img_path2))[:, :, ::-1] # RGB
+
+ matcher_zoo_restored = get_matcher_zoo(config["matcher_zoo"])
+ for k, v in matcher_zoo_restored.items():
+ if image0 is None or image1 is None:
+ logger.error("Error: No images found! Please upload two images.")
+ enable = config["matcher_zoo"][k].get("enable", True)
+ skip_ci = config["matcher_zoo"][k].get("skip_ci", False)
+ if enable and not skip_ci:
+ logger.info(f"Testing {k} ...")
+ api = ImageMatchingAPI(conf=v, device=DEVICE)
+ api(image0, image1)
+ log_path = ROOT / "experiments" / "all"
+ log_path.mkdir(exist_ok=True, parents=True)
+ api.visualize(log_path=log_path)
+ else:
+ logger.info(f"Skipping {k} ...")
+ return 0
+
+
+def test_one():
+ img_path1 = ROOT / "datasets/sacre_coeur/mapping/02928139_3448003521.jpg"
+ img_path2 = ROOT / "datasets/sacre_coeur/mapping/17295357_9106075285.jpg"
+ image0 = cv2.imread(str(img_path1))[:, :, ::-1] # RGB
+ image1 = cv2.imread(str(img_path2))[:, :, ::-1] # RGB
+ # sparse
+ conf = {
+ "feature": {
+ "output": "feats-superpoint-n4096-rmax1600",
+ "model": {
+ "name": "superpoint",
+ "nms_radius": 3,
+ "max_keypoints": 4096,
+ "keypoint_threshold": 0.005,
+ },
+ "preprocessing": {
+ "grayscale": True,
+ "force_resize": True,
+ "resize_max": 1600,
+ "width": 640,
+ "height": 480,
+ "dfactor": 8,
+ },
+ },
+ "matcher": {
+ "output": "matches-NN-mutual",
+ "model": {
+ "name": "nearest_neighbor",
+ "do_mutual_check": True,
+ "match_threshold": 0.2,
+ },
+ },
+ "dense": False,
+ }
+ api = ImageMatchingAPI(conf=conf, device=DEVICE)
+ api(image0, image1)
+ log_path = ROOT / "experiments" / "one"
+ log_path.mkdir(exist_ok=True, parents=True)
+ api.visualize(log_path=log_path)
+
+ # dense
+ conf = {
+ "matcher": {
+ "output": "matches-loftr",
+ "model": {
+ "name": "loftr",
+ "weights": "outdoor",
+ "max_keypoints": 2000,
+ "match_threshold": 0.2,
+ },
+ "preprocessing": {
+ "grayscale": True,
+ "resize_max": 1024,
+ "dfactor": 8,
+ "width": 640,
+ "height": 480,
+ "force_resize": True,
+ },
+ "max_error": 1,
+ "cell_size": 1,
+ },
+ "dense": True,
+ }
+
+ api = ImageMatchingAPI(conf=conf, device=DEVICE)
+ api(image0, image1)
+ log_path = ROOT / "experiments" / "one"
+ log_path.mkdir(exist_ok=True, parents=True)
+ api.visualize(log_path=log_path)
+ return 0
+
+
+if __name__ == "__main__":
+ config = load_config(ROOT / "ui/config.yaml")
+ test_one()
+ test_all(config)
diff --git a/third_party/LightGlue/.flake8 b/third_party/LightGlue/.flake8
new file mode 100644
index 0000000000000000000000000000000000000000..bf3118243cccca0049b5819c8401fa1e14caa14b
--- /dev/null
+++ b/third_party/LightGlue/.flake8
@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203
+exclude = .git,__pycache__,build,.venv/
diff --git a/third_party/LightGlue/.gitattributes b/third_party/LightGlue/.gitattributes
new file mode 100644
index 0000000000000000000000000000000000000000..60404dcd96640d5095b962678b8ede93465c5dfd
--- /dev/null
+++ b/third_party/LightGlue/.gitattributes
@@ -0,0 +1 @@
+*.ipynb linguist-documentation
\ No newline at end of file
diff --git a/third_party/LightGlue/.github/workflows/code-quality.yml b/third_party/LightGlue/.github/workflows/code-quality.yml
new file mode 100644
index 0000000000000000000000000000000000000000..477082a8cce8cab01420f64a63e5f1919bb007ad
--- /dev/null
+++ b/third_party/LightGlue/.github/workflows/code-quality.yml
@@ -0,0 +1,24 @@
+name: Format and Lint Checks
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - '*.py'
+ pull_request:
+ types: [ assigned, opened, synchronize, reopened ]
+jobs:
+ check:
+ name: Format and Lint Checks
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '3.10'
+ cache: 'pip'
+ - run: python -m pip install --upgrade pip
+ - run: python -m pip install .[dev]
+ - run: python -m flake8 .
+ - run: python -m isort . --check-only --diff
+ - run: python -m black . --check --diff
diff --git a/third_party/LightGlue/.gitignore b/third_party/LightGlue/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..24b3fa261ed6648547871b4b079b70920d973408
--- /dev/null
+++ b/third_party/LightGlue/.gitignore
@@ -0,0 +1,166 @@
+/data/
+/outputs/
+/lightglue/weights/
+*-checkpoint.ipynb
+*.pth
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+# For a library or package, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+# This is especially recommended for binary packages to ensure reproducibility, and is more
+# commonly ignored for libraries.
+# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+# in version control.
+# https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+# and can be added to the global gitignore or merged into this file. For a more nuclear
+# option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
diff --git a/third_party/LightGlue/LICENSE b/third_party/LightGlue/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..5e740e057bf9b03c23644b7773b072abc262b7d4
--- /dev/null
+++ b/third_party/LightGlue/LICENSE
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright 2023 ETH Zurich
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/third_party/LightGlue/README.md b/third_party/LightGlue/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..349a016235eca2a82a2c570ba01c3e407154ef99
--- /dev/null
+++ b/third_party/LightGlue/README.md
@@ -0,0 +1,180 @@
+LightGlue ⚡️
+
Local Feature Matching at Light Speed
+
+
+
+ LightGlue is a deep neural network that matches sparse local features across image pairs.
An adaptive mechanism makes it fast for easy pairs (top) and reduces the computational complexity for difficult ones (bottom).
+
+
+
+ LightGlue can adjust its depth (number of layers) and width (number of keypoints) per image pair, with a marginal impact on accuracy.
+
+
+
+ Benchmark results on GPU (RTX 3080). With compilation and adaptivity, LightGlue runs at 150 FPS @ 1024 keypoints and 50 FPS @ 4096 keypoints per image. This is a 4-10x speedup over SuperGlue.
+
+
+
+ Benchmark results on CPU (Intel i7 10700K). LightGlue runs at 20 FPS @ 512 keypoints.
+
+
+ Johan Edstedt + · + Qiyu Sun + · + Georg Bökman + · + Mårten Wadenbäck + · + Michael Felsberg +
++ Paper | + Project Page +
+ + +
+
+
+ RoMa is the robust dense feature matcher capable of estimating pixel-dense warps and reliable certainties for almost any image pair.
+
+
+
+
+
+
+