Extended verison of XFeat cpp with OpenCV

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.22.1)
+cmake_minimum_required(VERSION 3.5)
 project(xfeat_cpp)
 
 # Build type:
@@ -13,6 +13,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 set(CMAKE_PREFIX_PATH "${PROJECT_SOURCE_DIR}/thirdparty/pytorch/torch")
 
 # find libraries
+set(CMAKE_PREFIX_PATH "/usr/lib/libtorch")
 find_package(Torch REQUIRED)
 find_package(OpenCV REQUIRED)
 

Dockerfile

@@ -0,0 +1,62 @@
+FROM ubuntu:20.04
+
+ENV TX=Indian
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update
+
+RUN apt-get install -y git
+
+WORKDIR /
+RUN apt-get install -y libssl-dev cmake build-essential libusb-1.0-0-dev libudev-dev pkg-config libgtk-3-dev libglfw3-dev libgl1-mesa-dev libglu1-mesa-dev at
+RUN git clone https://github.com/IntelRealSense/librealsense.git
+WORKDIR /librealsense
+RUN mkdir build
+WORKDIR /librealsense/build
+RUN cmake ..
+RUN make -j11
+RUN make install
+
+RUN apt-get update
+
+RUN apt-get install -y software-properties-common
+
+RUN add-apt-repository ppa:deadsnakes/ppa
+
+RUN apt-get update
+
+WORKDIR /
+
+RUN apt-get install -y cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev
+RUN apt-get install -y libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev
+RUN apt-get install -y python2.7-dev python3.6-dev python-dev python-numpy python3-numpy
+RUN apt-get install -y libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev
+RUN apt-get install -y libv4l-dev v4l-utils qv4l2 curl 
+RUN apt-get install -y zip unzip
+
+RUN curl -L https://github.com/opencv/opencv/archive/4.1.1.zip -o opencv-4.1.1.zip
+RUN curl -L https://github.com/opencv/opencv_contrib/archive/4.1.1.zip -o opencv_contrib-4.1.1.zip
+RUN unzip opencv-4.1.1.zip
+RUN unzip opencv_contrib-4.1.1.zip
+WORKDIR /opencv-4.1.1/
+
+RUN sed -i 's/include <Eigen\/Core>/include <eigen3\/Eigen\/Core>/g' modules/core/include/opencv2/core/private.hpp
+
+RUN mkdir build
+WORKDIR /opencv-4.1.1/build
+RUN cmake -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib-4.1.1/modules -D WITH_GSTREAMER=ON -D WITH_LIBV4L=ON -D BUILD_opencv_python2=ON -D BUILD_opencv_python3=ON -D BUILD_TESTS=OFF -D BUILD_PERF_TESTS=OFF -D BUILD_EXAMPLES=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local ..
+RUN make -j11
+RUN make install
+RUN export PYTHONPATH=$PYTHONPATH:'$PWD'/python_loader/
+
+RUN apt-get install -y wget
+# Without GPU
+RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.2.0%2Bcpu.zip
+RUN unzip libtorch-cxx11-abi-shared-with-deps-2.2.0+cpu.zip
+# With GPU
+
+RUN mv libtorch /usr/lib/
+RUN apt-get install -y libeigen3-dev
+
+WORKDIR /work/

README.md

@@ -12,11 +12,11 @@ Paper: https://arxiv.org/abs/2404.19174
 
 In this project, the following packages are used. Make sure the right versions of libraries are installed and linked.
 
-1. Tested in Ubuntu 22.04
+1. Tested in Ubuntu 22.04 / 20.04
 2. Nvidia-driver-535 and CUDA Toolkit 12.2
 3. gcc and g++ compilers 11.4.0
-4. CMake 3.22.1
-5. OpenCV 4.5.4
+4. CMake 3.22.1 / 3.5
+5. OpenCV 4.5.4 / 4.1.1
 6. [libtorch](https://github.com/pytorch/pytorch): Please avoid using the pre-built version of libtorch since it will cause linking issues ([CXX11 ABI issue](https://github.com/pytorch/pytorch/issues/13541))
 
 ## Setup
@@ -43,6 +43,18 @@ cd build
 cmake ..
 make -j4
 ```
+To build the project using Docker
+```bash 
+sudo docker build -t xfeat . 
+            or
+./build_docker.sh
+xhost + (in different terminal)
+./run_docker.sh xfeat
+mkdir build 
+cd build
+cmake ..
+make -j4
+```
 
 ## Running
 
@@ -64,6 +76,12 @@ Realtime Matching Example:
 ./build/examples/realtime_demo
 ```
 
+Realsense Matching Example:
+
+```bash
+./build/realsense_demo
+```
+
 ## Bibtex Citation
 
 ```

build_docker.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+sudo docker build -t xfeat .
+
+echo "Name of Docker image is xfeat"

examples/CMakeLists.txt

@@ -1,3 +1,5 @@
+find_package(Threads REQUIRED)
+
 add_executable(match match.cc)
 target_link_libraries(match
     xfeat
@@ -9,5 +11,19 @@ add_executable(realtime_demo realtime_demo.cc)
 target_link_libraries(realtime_demo
     xfeat
     ${THIRD_PARTY_LIBS}
+    Threads::Threads
 )
-set_property(TARGET realtime_demo PROPERTY CXX_STANDARD 17)
+set_property(TARGET realtime_demo PROPERTY CXX_STANDARD 17)
+
+
+add_executable(realsense_demo realsense_demo.cc)
+target_link_libraries(realsense_demo
+    xfeat
+    realsense
+    ${THIRD_PARTY_LIBS}
+    Threads::Threads
+)
+set_property(TARGET realsense_demo PROPERTY CXX_STANDARD 17)
+
+
+

examples/match.cc

@@ -42,7 +42,7 @@ void warp_corners_and_draw_matches(cv::Mat &ref_points, cv::Mat &dst_points, cv:
     }
 
     cv::Mat mask;
-    cv::Mat H = cv::findHomography(ref_points, dst_points, cv::USAC_MAGSAC, 10.0, mask, 1000, 0.994);
+    cv::Mat H = cv::findHomography(ref_points, dst_points, cv::RANSAC, 10.0, mask, 1000, 0.994);
     if (H.empty()) {
         std::cerr << "Homography matrix is empty" << std::endl;
         return;

examples/realsense_demo.cc

@@ -0,0 +1,94 @@
+#include "realsense.h"
+#include "XFeat.h"
+#include "mutex"
+#include "thread"
+#include "unistd.h"
+
+
+char val = 'q';
+std::mutex mut;
+
+void keyboardInputFunction()
+{
+    while(1)
+    {
+        std::cout<<"Press s to set the reference frame: "<<std::endl;
+
+        char tmp;
+        std::cin>>tmp;
+
+        mut.lock();
+        val = tmp;
+        mut.unlock();
+
+        std::cout<<"reference image set"<<std::endl;
+        usleep(100);
+    }
+}
+
+int main()
+{
+    std::thread input_thread(keyboardInputFunction);
+    int top_k = 1000;
+    float detection_threshold = 0.5;
+    bool use_cuda = false; 
+    XFeat::XFDetector detector(top_k, detection_threshold, use_cuda);
+
+    Realsense realsense_camera;
+    InputDevSetup camera_setup;
+    camera_setup.setDefault();
+    realsense_camera.setupInputDevice(camera_setup);
+    bool first_img_set = false;
+
+    cv::Mat ref_img;
+    cv::Mat ref_desc;
+    std::vector<cv::KeyPoint> ref_kps;
+
+    while(1)
+    {
+        ColourFrame frame;
+
+        realsense_camera.startStreaming();
+        realsense_camera.getColourFrame(frame);
+        frame.vis("camera feed");
+        char tmp;
+
+        mut.lock();
+        tmp = val;
+        mut.unlock();
+
+        if(tmp == 's')
+        {
+            mut.lock();
+            val = 'q';
+            mut.unlock();
+
+            ref_img = frame.bgr_frame.clone();
+            first_img_set = true;
+            detector.extractFeatures(ref_img,ref_kps,ref_desc);
+            continue;
+        }
+
+        if(!first_img_set)
+        {
+            continue;
+        }
+
+        cv::imshow("reference image",ref_img);
+
+        cv::Mat desc;
+        std::vector<cv::KeyPoint> kps;
+        std::vector<cv::DMatch> matches;
+
+        detector.extractFeatures(frame.bgr_frame,kps,desc);
+        detector.matchFeatures(ref_desc,desc,matches);
+
+        cv::Mat output_img;
+        cv::drawMatches(ref_img,ref_kps,frame.bgr_frame,kps,matches,output_img);
+
+        cv::imshow("matching frame",output_img);
+    }   
+
+    input_thread.join();
+    return 1;
+}

examples/realtime_demo.cc

@@ -209,7 +209,7 @@ class MatchingDemo
         }
 
         cv::Mat mask;
-        cv::Mat H = cv::findHomography(mkpts_0_cv, mkpts_1_cv, cv::USAC_MAGSAC, 10.0, mask, 1000, 0.994);
+        cv::Mat H = cv::findHomography(mkpts_0_cv, mkpts_1_cv, cv::RANSAC, 10.0, mask, 1000, 0.994);
         if (H.empty()) 
         {
             std::cerr << "Homography matrix is empty" << std::endl;

include/XFeat.h

@@ -17,11 +17,19 @@ namespace XFeat
         void detectAndCompute(torch::Tensor &x, std::unordered_map<std::string, torch::Tensor> &result);
         void match(torch::Tensor &feats1, torch::Tensor &feats2, torch::Tensor &idx0, torch::Tensor &idx1, float _min_cossim=-1.0);
         void match_xfeat(cv::Mat &img1, cv::Mat &img2, cv::Mat &mkpts_0, cv::Mat &mkpts_1);
-        torch::Tensor parseInput(cv::Mat &img);
+        torch::Tensor parseInput(const cv::Mat &img);
         std::tuple<torch::Tensor, double, double> preprocessTensor(torch::Tensor &x);
         cv::Mat tensorToMat(const torch::Tensor &tensor);
+
+        cv::Mat tensorTo1DMat(const torch::Tensor& tensor);
+        std::unordered_map<std::string,at::Tensor> convertToTorch(const cv::Mat &descriptors, const std::vector<cv::KeyPoint> &keypoints);
+        torch::Tensor cvToTorch(const cv::Mat& descriptors);
+        torch::Tensor cvToTorch(const std::vector<cv::KeyPoint>& keypoints);
 
-    private:
+        void extractFeatures(const cv::Mat& img,std::vector<cv::KeyPoint>& kps,cv::Mat& decs);
+        void matchFeatures(const cv::Mat& desc1,const cv::Mat& desc2, std::vector<cv::DMatch>& good_matches);
+
+    private: 
         torch::Tensor getKptsHeatmap(torch::Tensor &kpts, float softmax_temp=1.0);
         torch::Tensor NMS(torch::Tensor &x, float threshold = 0.05, int kernel_size = 5);
         std::string getWeightsPath(std::string weights);