From 274eac1ae30e011ed6714f12502d4c3cfdb43ddb Mon Sep 17 00:00:00 2001 From: Ayush Kumar Date: Mon, 24 Mar 2025 23:04:55 +0530 Subject: [PATCH 1/4] added opencv functions for easy integration --- CMakeLists.txt | 3 +- Dockerfile | 62 +++++++++++++++ README.md | 21 ++++- examples/CMakeLists.txt | 18 ++++- examples/match.cc | 2 +- examples/realsense_demo.cc | 95 ++++++++++++++++++++++ examples/realtime_demo.cc | 2 +- include/XFeat.h | 12 ++- include/frames.h | 159 +++++++++++++++++++++++++++++++++++++ include/realsense.h | 155 ++++++++++++++++++++++++++++++++++++ run_docker.sh | 2 + src/CMakeLists.txt | 12 +++ src/XFeat.cc | 83 ++++++++++++++++++- src/realsense.cc | 116 +++++++++++++++++++++++++++ 14 files changed, 731 insertions(+), 11 deletions(-) create mode 100644 Dockerfile create mode 100644 examples/realsense_demo.cc create mode 100644 include/frames.h create mode 100644 include/realsense.h create mode 100755 run_docker.sh create mode 100644 src/realsense.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index f48e7c0..58500de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.22.1) +cmake_minimum_required(VERSION 3.5) project(xfeat_cpp) # Build type: @@ -13,6 +13,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON) set(CMAKE_PREFIX_PATH "${PROJECT_SOURCE_DIR}/thirdparty/pytorch/torch") # find libraries +set(CMAKE_PREFIX_PATH "/usr/lib/libtorch") find_package(Torch REQUIRED) find_package(OpenCV REQUIRED) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d46d91f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,62 @@ +FROM ubuntu:20.04 + +ENV TX=Indian + +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update + +RUN apt-get install -y git + +WORKDIR / +RUN apt-get install -y libssl-dev cmake build-essential libusb-1.0-0-dev libudev-dev pkg-config libgtk-3-dev libglfw3-dev libgl1-mesa-dev libglu1-mesa-dev at +RUN git clone https://github.com/IntelRealSense/librealsense.git +WORKDIR /librealsense +RUN mkdir build +WORKDIR /librealsense/build +RUN cmake .. +RUN make -j11 +RUN make install + +RUN apt-get update + +RUN apt-get install -y software-properties-common + +RUN add-apt-repository ppa:deadsnakes/ppa + +RUN apt-get update + +WORKDIR / + +RUN apt-get install -y cmake git libgtk2.0-dev pkg-config libavcodec-dev libavformat-dev libswscale-dev +RUN apt-get install -y libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev +RUN apt-get install -y python2.7-dev python3.6-dev python-dev python-numpy python3-numpy +RUN apt-get install -y libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev +RUN apt-get install -y libv4l-dev v4l-utils qv4l2 curl +RUN apt-get install -y zip unzip + +RUN curl -L https://github.com/opencv/opencv/archive/4.1.1.zip -o opencv-4.1.1.zip +RUN curl -L https://github.com/opencv/opencv_contrib/archive/4.1.1.zip -o opencv_contrib-4.1.1.zip +RUN unzip opencv-4.1.1.zip +RUN unzip opencv_contrib-4.1.1.zip +WORKDIR /opencv-4.1.1/ + +RUN sed -i 's/include /include /g' modules/core/include/opencv2/core/private.hpp + +RUN mkdir build +WORKDIR /opencv-4.1.1/build +RUN cmake -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib-4.1.1/modules -D WITH_GSTREAMER=ON -D WITH_LIBV4L=ON -D BUILD_opencv_python2=ON -D BUILD_opencv_python3=ON -D BUILD_TESTS=OFF -D BUILD_PERF_TESTS=OFF -D BUILD_EXAMPLES=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local .. +RUN make -j11 +RUN make install +RUN export PYTHONPATH=$PYTHONPATH:'$PWD'/python_loader/ + +RUN apt-get install -y wget +# Without GPU +RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.2.0%2Bcpu.zip +RUN unzip libtorch-cxx11-abi-shared-with-deps-2.2.0+cpu.zip +# With GPU + +RUN mv libtorch /usr/lib/ +RUN apt-get install -y libeigen3-dev + +WORKDIR /work/ \ No newline at end of file diff --git a/README.md b/README.md index f4a5043..d5e4eb0 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ Paper: https://arxiv.org/abs/2404.19174 In this project, the following packages are used. Make sure the right versions of libraries are installed and linked. -1. Tested in Ubuntu 22.04 +1. Tested in Ubuntu 22.04 / 20.04 2. Nvidia-driver-535 and CUDA Toolkit 12.2 3. gcc and g++ compilers 11.4.0 -4. CMake 3.22.1 -5. OpenCV 4.5.4 +4. CMake 3.22.1 / 3.5 +5. OpenCV 4.5.4 / 4.1.1 6. [libtorch](https://github.com/pytorch/pytorch): Please avoid using the pre-built version of libtorch since it will cause linking issues ([CXX11 ABI issue](https://github.com/pytorch/pytorch/issues/13541)) ## Setup @@ -43,6 +43,15 @@ cd build cmake .. make -j4 ``` +To build the project using Docker +```bash +sudo docker build -t xfeat . +./run_docker.sh xfeat +mkdir build +cd build +cmake .. +make -j4 +``` ## Running @@ -64,6 +73,12 @@ Realtime Matching Example: ./build/examples/realtime_demo ``` +Realsense Matching Example: + +```bash +./build/realsense_demo +``` + ## Bibtex Citation ``` diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 319706f..0f107cf 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,3 +1,5 @@ +find_package(Threads REQUIRED) + add_executable(match match.cc) target_link_libraries(match xfeat @@ -9,5 +11,19 @@ add_executable(realtime_demo realtime_demo.cc) target_link_libraries(realtime_demo xfeat ${THIRD_PARTY_LIBS} + Threads::Threads ) -set_property(TARGET realtime_demo PROPERTY CXX_STANDARD 17) \ No newline at end of file +set_property(TARGET realtime_demo PROPERTY CXX_STANDARD 17) + + +add_executable(realsense_demo realsense_demo.cc) +target_link_libraries(realsense_demo + xfeat + realsense + ${THIRD_PARTY_LIBS} + Threads::Threads +) +set_property(TARGET realsense_demo PROPERTY CXX_STANDARD 17) + + + diff --git a/examples/match.cc b/examples/match.cc index 2aad95c..dfa44eb 100644 --- a/examples/match.cc +++ b/examples/match.cc @@ -42,7 +42,7 @@ void warp_corners_and_draw_matches(cv::Mat &ref_points, cv::Mat &dst_points, cv: } cv::Mat mask; - cv::Mat H = cv::findHomography(ref_points, dst_points, cv::USAC_MAGSAC, 10.0, mask, 1000, 0.994); + cv::Mat H = cv::findHomography(ref_points, dst_points, cv::RANSAC, 10.0, mask, 1000, 0.994); if (H.empty()) { std::cerr << "Homography matrix is empty" << std::endl; return; diff --git a/examples/realsense_demo.cc b/examples/realsense_demo.cc new file mode 100644 index 0000000..d451523 --- /dev/null +++ b/examples/realsense_demo.cc @@ -0,0 +1,95 @@ +#include "realsense.h" +#include "XFeat.h" +#include "mutex" +#include "thread" +#include "unistd.h" + + +char val = 'q'; +std::mutex mut; + +void keyboardInputFunction() +{ + while(1) + { + std::cout<<"Press s to set the reference frame: "<>tmp; + + mut.lock(); + val = tmp; + mut.unlock(); + + std::cout<<"reference image set"< ref_kps; + + while(1) + { + ColourFrame frame; + + realsense_camera.startStreaming(); + realsense_camera.getColourFrame(frame); + cv::imshow("camera_feed",frame.bgr_frame); + char tmp; + + mut.lock(); + tmp = val; + mut.unlock(); + + if(tmp == 's') + { + mut.lock(); + val = 'q'; + mut.unlock(); + + ref_img = frame.bgr_frame.clone(); + first_img_set = true; + detector.extractFeatures(ref_img,ref_kps,ref_desc); + continue; + } + + if(!first_img_set) + { + continue; + } + + cv::imshow("reference image",ref_img); + + cv::Mat desc; + std::vector kps; + std::vector matches; + + detector.extractFeatures(frame.bgr_frame,kps,desc); + detector.matchFeatures(ref_desc,desc,matches); + + cv::Mat output_img; + cv::drawMatches(ref_img,ref_kps,frame.bgr_frame,kps,matches,output_img); + + cv::imshow("matching frame",output_img); + cv::waitKey(1); + } + + input_thread.join(); + return 1; +} \ No newline at end of file diff --git a/examples/realtime_demo.cc b/examples/realtime_demo.cc index 90c9b25..b5ce533 100644 --- a/examples/realtime_demo.cc +++ b/examples/realtime_demo.cc @@ -209,7 +209,7 @@ class MatchingDemo } cv::Mat mask; - cv::Mat H = cv::findHomography(mkpts_0_cv, mkpts_1_cv, cv::USAC_MAGSAC, 10.0, mask, 1000, 0.994); + cv::Mat H = cv::findHomography(mkpts_0_cv, mkpts_1_cv, cv::RANSAC, 10.0, mask, 1000, 0.994); if (H.empty()) { std::cerr << "Homography matrix is empty" << std::endl; diff --git a/include/XFeat.h b/include/XFeat.h index 8bf4861..d309fff 100644 --- a/include/XFeat.h +++ b/include/XFeat.h @@ -17,11 +17,19 @@ namespace XFeat void detectAndCompute(torch::Tensor &x, std::unordered_map &result); void match(torch::Tensor &feats1, torch::Tensor &feats2, torch::Tensor &idx0, torch::Tensor &idx1, float _min_cossim=-1.0); void match_xfeat(cv::Mat &img1, cv::Mat &img2, cv::Mat &mkpts_0, cv::Mat &mkpts_1); - torch::Tensor parseInput(cv::Mat &img); + torch::Tensor parseInput(const cv::Mat &img); std::tuple preprocessTensor(torch::Tensor &x); cv::Mat tensorToMat(const torch::Tensor &tensor); + + cv::Mat tensorTo1DMat(const torch::Tensor& tensor); + std::unordered_map convertToTorch(const cv::Mat &descriptors, const std::vector &keypoints); + torch::Tensor cvToTorch(const cv::Mat& descriptors); + torch::Tensor cvToTorch(const std::vector& keypoints); - private: + void extractFeatures(const cv::Mat& img,std::vector& kps,cv::Mat& decs); + void matchFeatures(const cv::Mat& desc1,const cv::Mat& desc2, std::vector& good_matches); + + private: torch::Tensor getKptsHeatmap(torch::Tensor &kpts, float softmax_temp=1.0); torch::Tensor NMS(torch::Tensor &x, float threshold = 0.05, int kernel_size = 5); std::string getWeightsPath(std::string weights); diff --git a/include/frames.h b/include/frames.h new file mode 100644 index 0000000..e265a04 --- /dev/null +++ b/include/frames.h @@ -0,0 +1,159 @@ +#ifndef FRAMES_H +#define FRAMES_H + +#include "opencv2/opencv.hpp" +#include "opencv2/opencv.hpp" +#include "opencv2/calib3d/calib3d.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/imgcodecs.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/core/utility.hpp" +#include +#include +#include "opencv2/ximgproc.hpp" +#include "opencv2/xfeatures2d.hpp" +#include +#include + +#define DEFAULT_CAM_HEIGHT 480 +#define DEFAULT_CAM_WIDTH 640 +#define FPS 30 + +#define CAM_Fx 617.201 +#define CAM_Fy 617.362 +#define CAM_Cx 324.637 +#define CAM_Cy 242.462 + +#define CAM_K1 0 +#define CAM_K2 0 +#define CAM_P1 0 +#define CAM_P2 0 +#define CAM_K3 0 +#define DEPTH_MAP_FACTOR 1000 +#define CAM_BASE_LENGTH 0.07732 + +/** + * @brief Camera Frame + * + */ +typedef struct +{ + cv::Mat bgr_frame; + cv::Mat gray_frame; + double time = 0; + + void BGR2Gray() + { + cv::cvtColor(bgr_frame,gray_frame,cv::COLOR_BGR2GRAY); + } + + void vis(std::string name) + { + cv::imshow(name,gray_frame); + cv::waitKey(1); + } + + void destroy(std::string name) + { + cv::destroyWindow(name); + } + +}ColourFrame; + +/** + * @brief Depth Frame + * + */ +typedef struct +{ + cv::Mat data; + + void computeDepth(double scale) + { + cv::Mat tmp_depth; + data.convertTo(tmp_depth,CV_64F,scale); + data = tmp_depth.clone(); + } + + double getDepth(int u,int v) + { + return data.at(u,v); + } + +}DepthFrame; + +/** + * @brief Camera parameters + * + */ +typedef struct +{ + cv::Mat intrinsics = cv::Mat::zeros(cv::Size(3,3),CV_64FC1); + cv::Mat dist_coeff = cv::Mat::zeros(cv::Size(5,1),CV_64FC1); + double base_length = 0; + + + void setDefault() + { + intrinsics.at(0,0) = CAM_Fx; + intrinsics.at(0,1) = 0; + intrinsics.at(0,2) = CAM_Cx; + intrinsics.at(1,0) = 0; + intrinsics.at(1,1) = CAM_Fy; + intrinsics.at(1,2) = CAM_Cy; + intrinsics.at(2,0) = 0; + intrinsics.at(2,1) = 0; + intrinsics.at(2,2) = 1; + + dist_coeff.at(0) = CAM_K1; + dist_coeff.at(1) = CAM_K2; + dist_coeff.at(2) = CAM_P1; + dist_coeff.at(3) = CAM_P2; + dist_coeff.at(4) = CAM_K3; + + base_length = CAM_BASE_LENGTH; + } + +}CameraParams; + +/** + * @brief Input device setup parameters + * + */ +typedef struct +{ + int height; + int width; + int fps; + double depth_map_factor; + std::string serial_num_; + + void setDefault() + { + height = DEFAULT_CAM_HEIGHT; + width = DEFAULT_CAM_WIDTH; + fps = FPS; + depth_map_factor = 1.0 / DEPTH_MAP_FACTOR; + } + +}InputDevSetup; + +typedef struct +{ + double x = 0.0; + double y = 0.0; + double z = 0.0; +}SensorData; + +typedef struct +{ + SensorData gyro; + SensorData accel; + + double gyro_time; + double accel_time; +}IMUFrame; + + + +#endif \ No newline at end of file diff --git a/include/realsense.h b/include/realsense.h new file mode 100644 index 0000000..dbc99a1 --- /dev/null +++ b/include/realsense.h @@ -0,0 +1,155 @@ +#ifndef REALSENSE_H +#define REALSENSE_H + +#include "frames.h" +#include "thread" +#include "mutex" +#include "atomic" + +#define D435i //comment this if your camera does not have IMU + +class Realsense +{ + public: + /** + * @brief Construct a new Realsense object + * + */ + Realsense(); + /** + * @brief Destroy the Realsense object + * + */ + ~Realsense(); + + /** + * @brief + * + * @param setup + * @return true + * @return false + */ + bool setupInputDevice(const InputDevSetup& setup); + + /** + * @brief + * + * @return true + * @return false + */ + bool startStreaming(); + + /** + * @brief Get the Colour Frame object + * + * @param frame + * @return true + * @return false + */ + bool getColourFrame(ColourFrame& frame); + + /** + * @brief Get the Depth Frame object + * + * @param depth + * @return true + * @return false + */ + bool getDepthFrame(DepthFrame& depth); + + /** + * @brief + * + * @param imu + * @return true + * @return false + */ + bool getIMUData(IMUFrame& imu); + + /** + * @brief Get the Dev Params object + * + * @return CameraParams& + */ + CameraParams& getDevParams(); + + private: + /** + * @brief + * + */ + InputDevSetup dev_setup_; + /** + * @brief + * + */ + ColourFrame colour_frame_; + /** + * @brief + * + */ + DepthFrame depth_frame_; + /** + * @brief + * + */ + CameraParams dev_params_; + /** + * @brief + * + */ + rs2::pipeline pipe_; + /** + * @brief + * + */ + rs2::config config_; + /** + * @brief + * + */ + rs2::pipeline_profile selection_; + /** + * @brief + * + */ + std::mutex img_mutex_; + /** + * @brief + * + */ + std::mutex imu_mutex_; + /** + * @brief + * + */ + rs2::frameset frameset; + /** + * @brief + * + */ + std::atomic img_arrived_; + /** + * @brief + * + */ + rs2_vector accel_data_; + /** + * @brief + * + */ + rs2_vector gyro_data_; + /** + * @brief + * + */ + double accel_time_; + /** + * @brief + * + */ + double gyro_time_; + +}; + +#endif diff --git a/run_docker.sh b/run_docker.sh new file mode 100755 index 0000000..56939e7 --- /dev/null +++ b/run_docker.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sudo docker run --net=host -e DISPLAY=$DISPLAY -v $(realpath .):/work -w /work -v $HOME/.Xauthority:/home/user/.Xauthority:rw --env="XDG_RUNTIME_DIR" -v ${XDG_RUNTIME_DIR}:${XDG_RUNTIME_DIR}:rw --rm --privileged --tty --volume /dev:/dev -it $1 /bin/bash diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2ab54c3..d5ff775 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,3 +10,15 @@ target_link_libraries( xfeat ${THIRD_PARTY_LIBS} ) + + +add_library(realsense SHARED + realsense.cc +) +target_link_libraries(realsense PUBLIC + realsense2 + ${OpenCV_LIBS} +) +target_include_directories(realsense PUBLIC + include +) diff --git a/src/XFeat.cc b/src/XFeat.cc index f2afec5..bf19c9e 100644 --- a/src/XFeat.cc +++ b/src/XFeat.cc @@ -135,7 +135,7 @@ namespace XFeat mkpts_1 = tensorToMat(mkpts_1_tensor); } - torch::Tensor XFDetector::parseInput(cv::Mat &img) + torch::Tensor XFDetector::parseInput(const cv::Mat &img) { // if the image is grayscale if (img.channels() == 1) @@ -230,7 +230,7 @@ namespace XFeat { // ensure tesnor is on CPU and convert to float torch::Tensor cpu_tensor = tensor.to(torch::kCPU).to(torch::kFloat); - cv::Mat mat(cpu_tensor.size(0), 2, CV_32F); + cv::Mat mat(cpu_tensor.size(0), cpu_tensor.size(1), CV_32F); std::memcpy(mat.data, cpu_tensor.data_ptr(), cpu_tensor.numel() * sizeof(float)); return mat; } @@ -245,4 +245,83 @@ namespace XFeat return static_cast(full_path); } + cv::Mat XFDetector::tensorTo1DMat(const torch::Tensor& tensor) + { + // ensure tesnor is on CPU and convert to float + torch::Tensor cpu_tensor = tensor.to(torch::kCPU).to(torch::kFloat); + // cv::Mat mat(cpu_tensor.size(0),1, CV_32F); + cv::Mat mat(cpu_tensor.size(0),1, CV_32F); + std::memcpy(mat.data, cpu_tensor.data_ptr(), cpu_tensor.numel() * sizeof(float)); + return mat; + } + + std::unordered_map XFDetector::convertToTorch(const cv::Mat &descriptors, const std::vector &keypoints) + { + std::unordered_map output; + output["keypoints"] = cvToTorch(descriptors); + output["descriptors"] = cvToTorch(keypoints); + + return output; + } + + torch::Tensor XFDetector::cvToTorch(const cv::Mat& descriptors) + { + torch::Tensor desc_tensor = torch::from_blob(descriptors.data, {descriptors.rows, descriptors.cols}, torch::kFloat); + desc_tensor = desc_tensor.clone(); + + return desc_tensor; + + } + + torch::Tensor XFDetector::cvToTorch(const std::vector& keypoints) + { + std::vector kpts_vec; + for (const auto &kp : keypoints) + { + kpts_vec.push_back(kp.pt.x); + kpts_vec.push_back(kp.pt.y); + } + torch::Tensor kpts_tensor = torch::from_blob(kpts_vec.data(), {static_cast(keypoints.size()), 2}, torch::kFloat).clone(); + + return kpts_tensor; + } + + void XFDetector::extractFeatures(const cv::Mat& img,std::vector& keypoints,cv::Mat& descs) + { + torch::Tensor tensor_img = this->parseInput(img); + std::unordered_map out; + this->detectAndCompute(tensor_img,out); + cv::Mat kps = this->tensorToMat(out["keypoints"]); + descs = this->tensorToMat(out["descriptors"]); + + keypoints.clear(); + + for(int i = 0; i < kps.rows; i++) + { + cv::KeyPoint kp(kps.at(i,0),kps.at(i,1),0); + keypoints.push_back(kp); + } + } + + void XFDetector::matchFeatures(const cv::Mat& desc1,const cv::Mat& desc2, std::vector& good_matches) + { + torch::Tensor out_tensor_1 = this->cvToTorch(desc1); + torch::Tensor out_tensor_2 = this->cvToTorch(desc2); + + torch::Tensor idx0,idx1; + this->match(out_tensor_1,out_tensor_2,idx0,idx1,0.82); + + cv::Mat index0 = this->tensorTo1DMat(idx0); + cv::Mat index1 = this->tensorTo1DMat(idx1); + + good_matches.clear(); + + for(int i = 0; i < index0.rows; i++) + { + cv::DMatch match(index0.at(i),index1.at(i),0); + good_matches.push_back(match); + } + } + + } // namespace XFeat \ No newline at end of file diff --git a/src/realsense.cc b/src/realsense.cc new file mode 100644 index 0000000..14fa0d2 --- /dev/null +++ b/src/realsense.cc @@ -0,0 +1,116 @@ +#include "realsense.h" + +Realsense::Realsense() +{ + img_arrived_.store(false,std::memory_order_acquire); +} + +Realsense::~Realsense() +{ + +} + +bool Realsense::setupInputDevice(const InputDevSetup& setup) +{ + dev_setup_ = setup; + config_.enable_stream(RS2_STREAM_COLOR,dev_setup_.width,dev_setup_.height,RS2_FORMAT_BGR8,dev_setup_.fps); + config_.enable_stream(RS2_STREAM_DEPTH,dev_setup_.width,dev_setup_.height,RS2_FORMAT_Z16,dev_setup_.fps); + + #ifdef D435i + config_.enable_stream(RS2_STREAM_ACCEL,RS2_FORMAT_MOTION_XYZ32F); + config_.enable_stream(RS2_STREAM_GYRO,RS2_FORMAT_MOTION_XYZ32F); + #endif + + auto imuCallback_ = [&](rs2::frame frame) + { + if(rs2::frameset fs = frame.as()) + { + img_mutex_.lock(); + frameset = fs; + img_mutex_.unlock(); + + img_arrived_.store(true,std::memory_order_acquire); + } + + else if(rs2::motion_frame m_frame = frame.as()) + { + rs2::motion_frame motion = frame.as(); + + if(motion && motion.get_profile().stream_type() == RS2_STREAM_ACCEL && motion.get_profile().format() == RS2_FORMAT_MOTION_XYZ32F) + { + imu_mutex_.lock(); + accel_data_ = motion.get_motion_data(); + imu_mutex_.unlock(); + } + + else if(motion && motion.get_profile().stream_type() == RS2_STREAM_GYRO && motion.get_profile().format() == RS2_FORMAT_MOTION_XYZ32F) + { + imu_mutex_.lock(); + gyro_data_ = motion.get_motion_data(); + imu_mutex_.unlock(); + } + } + }; + + selection_ = pipe_.start(config_,imuCallback_); + dev_params_.setDefault(); + + return true; + +} + +bool Realsense::startStreaming() +{ + while(!img_arrived_.load(std::memory_order_acquire)) + { + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + + img_mutex_.lock(); + rs2::video_frame colour_frame = frameset.get_color_frame(); + rs2::depth_frame depth = frameset.get_depth_frame(); + img_mutex_.unlock(); + + depth_frame_.data = cv::Mat(cv::Size(dev_setup_.width,dev_setup_.height),CV_16U,(void*)depth.get_data(),cv::Mat::AUTO_STEP); + colour_frame_.bgr_frame = cv::Mat(cv::Size(dev_setup_.width,dev_setup_.height),CV_8UC3,(void*)colour_frame.get_data()); + + img_arrived_.store(false,std::memory_order_acquire); + + return true; +} + +bool Realsense::getIMUData(IMUFrame& data) +{ + imu_mutex_.lock(); + data.accel.x = accel_data_.z; + data.accel.y = -accel_data_.x; + data.accel.z = -accel_data_.y; + + data.gyro.x = gyro_data_.z; + data.gyro.y = -gyro_data_.x; + data.gyro.z = -gyro_data_.y; + + data.gyro_time = gyro_time_; + data.accel_time = accel_time_; + imu_mutex_.unlock(); + return true; +} + +bool Realsense::getColourFrame(ColourFrame& frame) +{ + colour_frame_.BGR2Gray(); + frame = colour_frame_; + return true; +} + +bool Realsense::getDepthFrame(DepthFrame& frame) +{ + depth_frame_.computeDepth(dev_setup_.depth_map_factor); + frame = depth_frame_; + return true; +} + +CameraParams& Realsense::getDevParams() +{ + return dev_params_; +} From 30b96c404d907501ef2698a0f2d3eacc70be9d14 Mon Sep 17 00:00:00 2001 From: Ayush Kumar <72160187+ayushkumar8340@users.noreply.github.com> Date: Mon, 24 Mar 2025 23:14:30 +0530 Subject: [PATCH 2/4] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d5e4eb0..fe6490c 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ make -j4 To build the project using Docker ```bash sudo docker build -t xfeat . +xhost + (in different terminal) ./run_docker.sh xfeat mkdir build cd build From b1d8a2a43f67fcd14c469127f854c9c08ce18f02 Mon Sep 17 00:00:00 2001 From: Ayush Kumar Date: Sat, 29 Mar 2025 23:10:10 +0530 Subject: [PATCH 3/4] fixed some bugs working now --- examples/realsense_demo.cc | 5 ++--- include/frames.h | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/examples/realsense_demo.cc b/examples/realsense_demo.cc index d451523..15108ee 100644 --- a/examples/realsense_demo.cc +++ b/examples/realsense_demo.cc @@ -29,7 +29,7 @@ void keyboardInputFunction() int main() { std::thread input_thread(keyboardInputFunction); - int top_k = 4096; + int top_k = 1000; float detection_threshold = 0.5; bool use_cuda = false; XFeat::XFDetector detector(top_k, detection_threshold, use_cuda); @@ -50,7 +50,7 @@ int main() realsense_camera.startStreaming(); realsense_camera.getColourFrame(frame); - cv::imshow("camera_feed",frame.bgr_frame); + frame.vis("camera feed"); char tmp; mut.lock(); @@ -87,7 +87,6 @@ int main() cv::drawMatches(ref_img,ref_kps,frame.bgr_frame,kps,matches,output_img); cv::imshow("matching frame",output_img); - cv::waitKey(1); } input_thread.join(); diff --git a/include/frames.h b/include/frames.h index e265a04..2ba3e86 100644 --- a/include/frames.h +++ b/include/frames.h @@ -50,7 +50,7 @@ typedef struct void vis(std::string name) { cv::imshow(name,gray_frame); - cv::waitKey(1); + cv::waitKey(10); } void destroy(std::string name) From b437597a47a1347d194c100320165a3ccb833996 Mon Sep 17 00:00:00 2001 From: Ayush Kumar Date: Sat, 29 Mar 2025 23:18:36 +0530 Subject: [PATCH 4/4] docker changes --- README.md | 4 +++- build_docker.sh | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100755 build_docker.sh diff --git a/README.md b/README.md index fe6490c..ecb30e9 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,9 @@ make -j4 ``` To build the project using Docker ```bash -sudo docker build -t xfeat . +sudo docker build -t xfeat . + or +./build_docker.sh xhost + (in different terminal) ./run_docker.sh xfeat mkdir build diff --git a/build_docker.sh b/build_docker.sh new file mode 100755 index 0000000..c8bef2d --- /dev/null +++ b/build_docker.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +sudo docker build -t xfeat . + +echo "Name of Docker image is xfeat" \ No newline at end of file