mirrors/opencv_contrib
1
0
Fork 0
Code Pull Requests Packages Releases Activity

vendor: OpenCV 5.0.0 snapshot at 755e50675d97db9b7d449d8bd6b09888646f6c6e

Browse Source
This commit is contained in:
Gitea Mirror Bot
2026-08-22 00:11:13 +08:00
commit 12022378a3
3872 changed files with 2513409 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/ccalib/CMakeLists.txt
+2
View File
@@ -0,0 +1,2 @@
set(the_description "Custom Calibration Pattern")
ocv_define_module(ccalib opencv_core opencv_imgproc opencv_geometry opencv_calib opencv_objdetect opencv_features opencv_xfeatures2d opencv_highgui opencv_imgcodecs WRAP python)
modules/ccalib/README.md
+6
View File
@@ -0,0 +1,6 @@
Custom Calibration Pattern for 3D reconstruction
================================================
1. Custom calibration pattern for 3D reconstruction
2. Omnidirectional camera calibration
3. random pattern calibration object
4. multi-camera calibration
modules/ccalib/include/opencv2/ccalib.hpp
+157
View File
@@ -0,0 +1,157 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2014, OpenCV Foundation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef __OPENCV_CCALIB_HPP__
#define __OPENCV_CCALIB_HPP__
#include <opencv2/core.hpp>
#include <opencv2/features.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/geometry.hpp>
#include <vector>
/** @defgroup ccalib Custom Calibration Pattern for 3D reconstruction
*/
namespace cv{ namespace ccalib{
//! @addtogroup ccalib
//! @{
class CV_EXPORTS CustomPattern : public Algorithm
{
public:
CustomPattern();
virtual ~CustomPattern();
bool create(InputArray pattern, const Size2f boardSize, OutputArray output = noArray());
bool findPattern(InputArray image, OutputArray matched_features, OutputArray pattern_points, const double ratio = 0.7,
const double proj_error = 8.0, const bool refine_position = false, OutputArray out = noArray(),
OutputArray H = noArray(), OutputArray pattern_corners = noArray());
bool isInitialized();
void getPatternPoints(std::vector<KeyPoint>& original_points);
/**<
Returns a vector<Point> of the original points.
*/
double getPixelSize();
/**<
Get the pixel size of the pattern
*/
bool setFeatureDetector(Ptr<FeatureDetector> featureDetector);
bool setDescriptorExtractor(Ptr<DescriptorExtractor> extractor);
bool setDescriptorMatcher(Ptr<DescriptorMatcher> matcher);
Ptr<FeatureDetector> getFeatureDetector();
Ptr<DescriptorExtractor> getDescriptorExtractor();
Ptr<DescriptorMatcher> getDescriptorMatcher();
double calibrate(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints,
Size imageSize, InputOutputArray cameraMatrix, InputOutputArray distCoeffs,
OutputArrayOfArrays rvecs, OutputArrayOfArrays tvecs, int flags = 0,
TermCriteria criteria = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 30, DBL_EPSILON));
/**<
Calls the calirateCamera function with the same inputs.
*/
bool findRt(InputArray objectPoints, InputArray imagePoints, InputArray cameraMatrix, InputArray distCoeffs,
InputOutputArray rvec, InputOutputArray tvec, bool useExtrinsicGuess = false, int flags = SOLVEPNP_ITERATIVE);
bool findRt(InputArray image, InputArray cameraMatrix, InputArray distCoeffs,
InputOutputArray rvec, InputOutputArray tvec, bool useExtrinsicGuess = false, int flags = SOLVEPNP_ITERATIVE);
/**<
Uses solvePnP to find the rotation and translation of the pattern
with respect to the camera frame.
*/
bool findRtRANSAC(InputArray objectPoints, InputArray imagePoints, InputArray cameraMatrix, InputArray distCoeffs,
InputOutputArray rvec, InputOutputArray tvec, bool useExtrinsicGuess = false, int iterationsCount = 100,
float reprojectionError = 8.0, int minInliersCount = 100, OutputArray inliers = noArray(), int flags = SOLVEPNP_ITERATIVE);
bool findRtRANSAC(InputArray image, InputArray cameraMatrix, InputArray distCoeffs,
InputOutputArray rvec, InputOutputArray tvec, bool useExtrinsicGuess = false, int iterationsCount = 100,
float reprojectionError = 8.0, int minInliersCount = 100, OutputArray inliers = noArray(), int flags = SOLVEPNP_ITERATIVE);
/**<
Uses solvePnPRansac()
*/
void drawOrientation(InputOutputArray image, InputArray tvec, InputArray rvec, InputArray cameraMatrix,
InputArray distCoeffs, double axis_length = 3, int axis_width = 2);
/**<
pattern_corners -> projected over the image position of the edges of the pattern.
*/
private:
Mat img_roi;
std::vector<Point2f> obj_corners;
double pxSize;
bool initialized;
Ptr<FeatureDetector> detector;
Ptr<DescriptorExtractor> descriptorExtractor;
Ptr<DescriptorMatcher> descriptorMatcher;
std::vector<KeyPoint> keypoints;
std::vector<Point3f> points3d;
Mat descriptor;
bool init(Mat& image, const float pixel_size, OutputArray output = noArray());
bool findPatternPass(const Mat& image, std::vector<Point2f>& matched_features, std::vector<Point3f>& pattern_points,
Mat& H, std::vector<Point2f>& scene_corners, const double pratio, const double proj_error,
const bool refine_position = false, const Mat& mask = Mat(), OutputArray output = noArray());
void scaleFoundPoints(const double squareSize, const std::vector<KeyPoint>& corners, std::vector<Point3f>& pts3d);
void check_matches(std::vector<Point2f>& matched, const std::vector<Point2f>& pattern, std::vector<DMatch>& good, std::vector<Point3f>& pattern_3d, const Mat& H);
void keypoints2points(const std::vector<KeyPoint>& in, std::vector<Point2f>& out);
void updateKeypointsPos(std::vector<KeyPoint>& in, const std::vector<Point2f>& new_pos);
void refinePointsPos(const Mat& img, std::vector<Point2f>& p);
void refineKeypointsPos(const Mat& img, std::vector<KeyPoint>& kp);
};
//! @}
}} // namespace ccalib, cv
#endif
modules/ccalib/include/opencv2/ccalib/multicalib.hpp
+212
View File
@@ -0,0 +1,212 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2015, Baisheng Lai (laibaisheng@gmail.com), Zhejiang University,
// all rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef __OPENCV_MULTICAMERACALIBRATION_HPP__
#define __OPENCV_MULTICAMERACALIBRATION_HPP__
#include "opencv2/ccalib/randpattern.hpp"
#include "opencv2/ccalib/omnidir.hpp"
#include <string>
#include <iostream>
namespace cv { namespace multicalib {
//! @addtogroup ccalib
//! @{
#define HEAD -1
#define INVALID -2
/** @brief Class for multiple camera calibration that supports pinhole camera and omnidirection camera.
For omnidirectional camera model, please refer to omnidir.hpp in ccalib module.
It first calibrate each camera individually, then a bundle adjustment like optimization is applied to
refine extrinsic parameters. So far, it only support "random" pattern for calibration,
see randomPattern.hpp in ccalib module for details.
Images that are used should be named by "cameraIdx-timestamp.*", several images with the same timestamp
means that they are the same pattern that are photographed. cameraIdx should start from 0.
For more details, please refer to paper
B. Li, L. Heng, K. Kevin and M. Pollefeys, "A Multiple-Camera System
Calibration Toolbox Using A Feature Descriptor-Based Calibration
Pattern", in IROS 2013.
*/
class CV_EXPORTS MultiCameraCalibration
{
public:
enum {
PINHOLE,
OMNIDIRECTIONAL
//FISHEYE
};
// an edge connects a camera and pattern
struct edge
{
int cameraVertex; // vertex index for camera in this edge
int photoVertex; // vertex index for pattern in this edge
int photoIndex; // photo index among photos for this camera
Mat transform; // transform from pattern to camera
edge(int cv, int pv, int pi, Mat trans)
{
cameraVertex = cv;
photoVertex = pv;
photoIndex = pi;
transform = trans;
}
};
struct vertex
{
Mat pose; // relative pose to the first camera. For camera vertex, it is the
// transform from the first camera to this camera, for pattern vertex,
// it is the transform from pattern to the first camera
int timestamp; // timestamp of photo, only available for photo vertex
vertex(Mat po, int ts)
{
pose = po;
timestamp = ts;
}
vertex()
{
pose = Mat::eye(4, 4, CV_32F);
timestamp = -1;
}
};
/* @brief Constructor
@param cameraType camera type, PINHOLE or OMNIDIRECTIONAL
@param nCameras number of cameras
@fileName filename of string list that are used for calibration, the file is generated
by imagelist_creator from OpenCv samples. The first one in the list is the pattern filename.
@patternWidth the physical width of pattern, in user defined unit.
@patternHeight the physical height of pattern, in user defined unit.
@showExtration whether show extracted features and feature filtering.
@nMiniMatches minimal number of matched features for a frame.
@flags Calibration flags
@criteria optimization stopping criteria.
@detector feature detector that detect feature points in pattern and images.
@descriptor feature descriptor.
@matcher feature matcher.
*/
MultiCameraCalibration(int cameraType, int nCameras, const std::string& fileName, float patternWidth,
float patternHeight, int verbose = 0, int showExtration = 0, int nMiniMatches = 20, int flags = 0,
TermCriteria criteria = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 200, 1e-7),
Ptr<FeatureDetector> detector = xfeatures2d::AKAZE::create(xfeatures2d::AKAZE::DESCRIPTOR_MLDB, 0, 3, 0.006f),
Ptr<DescriptorExtractor> descriptor = xfeatures2d::AKAZE::create(xfeatures2d::AKAZE::DESCRIPTOR_MLDB,0, 3, 0.006f),
Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create("BruteForce-L1"));
/* @brief load images
*/
void loadImages();
/* @brief initialize multiple camera calibration. It calibrates each camera individually.
*/
void initialize();
/* @brief optimization extrinsic parameters
*/
double optimizeExtrinsics();
/* @brief run multi-camera camera calibration, it runs loadImage(), initialize() and optimizeExtrinsics()
*/
double run();
/* @brief write camera parameters to file.
*/
void writeParameters(const std::string& filename);
private:
std::vector<std::string> readStringList();
int getPhotoVertex(int timestamp);
void graphTraverse(const Mat& G, int begin, std::vector<int>& order, std::vector<int>& pre);
void findRowNonZero(const Mat& row, Mat& idx);
void computeJacobianExtrinsic(const Mat& extrinsicParams, Mat& JTJ_inv, Mat& JTE);
void computePhotoCameraJacobian(const Mat& rvecPhoto, const Mat& tvecPhoto, const Mat& rvecCamera,
const Mat& tvecCamera, Mat& rvecTran, Mat& tvecTran, const Mat& objectPoints, const Mat& imagePoints, const Mat& K,
const Mat& distort, const Mat& xi, Mat& jacobianPhoto, Mat& jacobianCamera, Mat& E);
void compose_motion(InputArray _om1, InputArray _T1, InputArray _om2, InputArray _T2, Mat& om3, Mat& T3, Mat& dom3dom1,
Mat& dom3dT1, Mat& dom3dom2, Mat& dom3dT2, Mat& dT3dom1, Mat& dT3dT1, Mat& dT3dom2, Mat& dT3dT2);
void JRodriguesMatlab(const Mat& src, Mat& dst);
void dAB(InputArray A, InputArray B, OutputArray dABdA, OutputArray dABdB);
double computeProjectError(Mat& parameters);
void vector2parameters(const Mat& parameters, std::vector<Vec3f>& rvecVertex, std::vector<Vec3f>& tvecVertexs);
void parameters2vector(const std::vector<Vec3f>& rvecVertex, const std::vector<Vec3f>& tvecVertex, Mat& parameters);
int _camType; //PINHOLE, FISHEYE or OMNIDIRECTIONAL
int _nCamera;
int _nMiniMatches;
int _flags;
int _verbose;
double _error;
float _patternWidth, _patternHeight;
TermCriteria _criteria;
std::string _filename;
int _showExtraction;
Ptr<FeatureDetector> _detector;
Ptr<DescriptorExtractor> _descriptor;
Ptr<DescriptorMatcher> _matcher;
std::vector<edge> _edgeList;
std::vector<vertex> _vertexList;
std::vector<std::vector<cv::Mat> > _objectPointsForEachCamera;
std::vector<std::vector<cv::Mat> > _imagePointsForEachCamera;
std::vector<cv::Mat> _cameraMatrix;
std::vector<cv::Mat> _distortCoeffs;
std::vector<cv::Mat> _xi;
std::vector<std::vector<Mat> > _omEachCamera, _tEachCamera;
};
//! @}
}} // namespace multicalib, cv
#endif
modules/ccalib/include/opencv2/ccalib/omnidir.hpp
+315
View File
@@ -0,0 +1,315 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2015, Baisheng Lai (laibaisheng@gmail.com), Zhejiang University,
// all rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef __OPENCV_OMNIDIR_HPP__
#define __OPENCV_OMNIDIR_HPP__
#include "opencv2/core.hpp"
#include "opencv2/core/affine.hpp"
#include <vector>
namespace cv
{
namespace omnidir
{
//! @addtogroup ccalib
//! @{
enum {
CALIB_USE_GUESS = 1,
CALIB_FIX_SKEW = 2,
CALIB_FIX_K1 = 4,
CALIB_FIX_K2 = 8,
CALIB_FIX_P1 = 16,
CALIB_FIX_P2 = 32,
CALIB_FIX_XI = 64,
CALIB_FIX_GAMMA = 128,
CALIB_FIX_CENTER = 256
};
enum{
RECTIFY_PERSPECTIVE = 1,
RECTIFY_CYLINDRICAL = 2,
RECTIFY_LONGLATI = 3,
RECTIFY_STEREOGRAPHIC = 4
};
enum{
XYZRGB = 1,
XYZ = 2
};
/**
* This module was accepted as a GSoC 2015 project for OpenCV, authored by
* Baisheng Lai, mentored by Bo Li.
*/
/** @brief Projects points for omnidirectional camera using CMei's model
@param objectPoints Object points in world coordinate, vector of vector of Vec3f or Mat of
1xN/Nx1 3-channel of type CV_32F and N is the number of points. 64F is also acceptable.
@param imagePoints Output array of image points, vector of vector of Vec2f or
1xN/Nx1 2-channel of type CV_32F. 64F is also acceptable.
@param rvec vector of rotation between world coordinate and camera coordinate, i.e., om
@param tvec vector of translation between pattern coordinate and camera coordinate
@param K Camera matrix \f$K = \vecthreethree{f_x}{s}{c_x}{0}{f_y}{c_y}{0}{0}{_1}\f$.
@param D Input vector of distortion coefficients \f$(k_1, k_2, p_1, p_2)\f$.
@param xi The parameter xi for CMei's model
@param jacobian Optional output 2Nx16 of type CV_64F jacobian matrix, contains the derivatives of
image pixel points wrt parameters including \f$om, T, f_x, f_y, s, c_x, c_y, xi, k_1, k_2, p_1, p_2\f$.
This matrix will be used in calibration by optimization.
The function projects object 3D points of world coordinate to image pixels, parameter by intrinsic
and extrinsic parameters. Also, it optionally compute a by-product: the jacobian matrix containing
contains the derivatives of image pixel points wrt intrinsic and extrinsic parameters.
*/
CV_EXPORTS_W void projectPoints(InputArray objectPoints, OutputArray imagePoints, InputArray rvec, InputArray tvec,
InputArray K, double xi, InputArray D, OutputArray jacobian = noArray());
/** @overload */
CV_EXPORTS void projectPoints(InputArray objectPoints, OutputArray imagePoints, const Affine3d& affine,
InputArray K, double xi, InputArray D, OutputArray jacobian = noArray());
/** @brief Undistort 2D image points for omnidirectional camera using CMei's model
@param distorted Array of distorted image points, vector of Vec2f
or 1xN/Nx1 2-channel Mat of type CV_32F, 64F depth is also acceptable
@param K Camera matrix \f$K = \vecthreethree{f_x}{s}{c_x}{0}{f_y}{c_y}{0}{0}{_1}\f$.
@param D Distortion coefficients \f$(k_1, k_2, p_1, p_2)\f$.
@param xi The parameter xi for CMei's model
@param R Rotation trainsform between the original and object space : 3x3 1-channel, or vector: 3x1/1x3
1-channel or 1x1 3-channel
@param undistorted array of normalized object points, vector of Vec2f/Vec2d or 1xN/Nx1 2-channel Mat with the same
depth of distorted points.
*/
CV_EXPORTS_W void undistortPoints(InputArray distorted, OutputArray undistorted, InputArray K, InputArray D, InputArray xi, InputArray R);
/** @brief Computes undistortion and rectification maps for omnidirectional camera image transform by a rotation R.
It output two maps that are used for cv::remap(). If D is empty then zero distortion is used,
if R or P is empty then identity matrices are used.
@param K Camera matrix \f$K = \vecthreethree{f_x}{s}{c_x}{0}{f_y}{c_y}{0}{0}{_1}\f$, with depth CV_32F or CV_64F
@param D Input vector of distortion coefficients \f$(k_1, k_2, p_1, p_2)\f$, with depth CV_32F or CV_64F
@param xi The parameter xi for CMei's model
@param R Rotation transform between the original and object space : 3x3 1-channel, or vector: 3x1/1x3, with depth CV_32F or CV_64F
@param P New camera matrix (3x3) or new projection matrix (3x4)
@param size Undistorted image size.
@param m1type Type of the first output map that can be CV_32FC1 or CV_16SC2 . See convertMaps()
for details.
@param map1 The first output map.
@param map2 The second output map.
@param flags Flags indicates the rectification type, RECTIFY_PERSPECTIVE, RECTIFY_CYLINDRICAL, RECTIFY_LONGLATI and RECTIFY_STEREOGRAPHIC
are supported.
*/
CV_EXPORTS_W void initUndistortRectifyMap(InputArray K, InputArray D, InputArray xi, InputArray R, InputArray P, const cv::Size& size,
int m1type, OutputArray map1, OutputArray map2, int flags);
/** @brief Undistort omnidirectional images to perspective images
@param distorted The input omnidirectional image.
@param undistorted The output undistorted image.
@param K Camera matrix \f$K = \vecthreethree{f_x}{s}{c_x}{0}{f_y}{c_y}{0}{0}{_1}\f$.
@param D Input vector of distortion coefficients \f$(k_1, k_2, p_1, p_2)\f$.
@param xi The parameter xi for CMei's model.
@param flags Flags indicates the rectification type, RECTIFY_PERSPECTIVE, RECTIFY_CYLINDRICAL, RECTIFY_LONGLATI and RECTIFY_STEREOGRAPHIC
@param Knew Camera matrix of the distorted image. If it is not assigned, it is just K.
@param new_size The new image size. By default, it is the size of distorted.
@param R Rotation matrix between the input and output images. By default, it is identity matrix.
*/
CV_EXPORTS_W void undistortImage(InputArray distorted, OutputArray undistorted, InputArray K, InputArray D, InputArray xi, int flags,
InputArray Knew = cv::noArray(), const Size& new_size = Size(), InputArray R = Mat::eye(3, 3, CV_64F));
/** @brief Perform omnidirectional camera calibration, the default depth of outputs is CV_64F.
@param objectPoints Vector of vector of Vec3f object points in world (pattern) coordinate.
It also can be vector of Mat with size 1xN/Nx1 and type CV_32FC3. Data with depth of 64_F is also acceptable.
@param imagePoints Vector of vector of Vec2f corresponding image points of objectPoints. It must be the same
size and the same type with objectPoints.
@param size Image size of calibration images.
@param K Output calibrated camera matrix.
@param xi Output parameter xi for CMei's model
@param D Output distortion parameters \f$(k_1, k_2, p_1, p_2)\f$
@param rvecs Output rotations for each calibration images
@param tvecs Output translation for each calibration images
@param flags The flags that control calibrate
@param criteria Termination criteria for optimization
@param idx Indices of images that pass initialization, which are really used in calibration. So the size of rvecs is the
same as idx.total().
*/
CV_EXPORTS_W double calibrate(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints, Size size,
InputOutputArray K, InputOutputArray xi, InputOutputArray D, OutputArrayOfArrays rvecs, OutputArrayOfArrays tvecs,
int flags, TermCriteria criteria, OutputArray idx=noArray());
/** @brief Stereo calibration for omnidirectional camera model. It computes the intrinsic parameters for two
cameras and the extrinsic parameters between two cameras. The default depth of outputs is CV_64F.
@param objectPoints Object points in world (pattern) coordinate. Its type is vector<vector<Vec3f> >.
It also can be vector of Mat with size 1xN/Nx1 and type CV_32FC3. Data with depth of 64_F is also acceptable.
@param imagePoints1 The corresponding image points of the first camera, with type vector<vector<Vec2f> >.
It must be the same size and the same type as objectPoints.
@param imagePoints2 The corresponding image points of the second camera, with type vector<vector<Vec2f> >.
It must be the same size and the same type as objectPoints.
@param imageSize1 Image size of calibration images of the first camera.
@param imageSize2 Image size of calibration images of the second camera.
@param K1 Output camera matrix for the first camera.
@param xi1 Output parameter xi of Mei's model for the first camera
@param D1 Output distortion parameters \f$(k_1, k_2, p_1, p_2)\f$ for the first camera
@param K2 Output camera matrix for the first camera.
@param xi2 Output parameter xi of CMei's model for the second camera
@param D2 Output distortion parameters \f$(k_1, k_2, p_1, p_2)\f$ for the second camera
@param rvec Output rotation between the first and second camera
@param tvec Output translation between the first and second camera
@param rvecsL Output rotation for each image of the first camera
@param tvecsL Output translation for each image of the first camera
@param flags The flags that control stereoCalibrate
@param criteria Termination criteria for optimization
@param idx Indices of image pairs that pass initialization, which are really used in calibration. So the size of rvecs is the
same as idx.total().
@
*/
CV_EXPORTS_W double stereoCalibrate(InputOutputArrayOfArrays objectPoints, InputOutputArrayOfArrays imagePoints1, InputOutputArrayOfArrays imagePoints2,
const Size& imageSize1, const Size& imageSize2, InputOutputArray K1, InputOutputArray xi1, InputOutputArray D1, InputOutputArray K2, InputOutputArray xi2,
InputOutputArray D2, OutputArray rvec, OutputArray tvec, OutputArrayOfArrays rvecsL, OutputArrayOfArrays tvecsL, int flags, TermCriteria criteria, OutputArray idx=noArray());
/** @brief Stereo rectification for omnidirectional camera model. It computes the rectification rotations for two cameras
@param R Rotation between the first and second camera
@param T Translation between the first and second camera
@param R1 Output 3x3 rotation matrix for the first camera
@param R2 Output 3x3 rotation matrix for the second camera
*/
CV_EXPORTS_W void stereoRectify(InputArray R, InputArray T, OutputArray R1, OutputArray R2);
/** @brief Stereo 3D reconstruction from a pair of images
@param image1 The first input image
@param image2 The second input image
@param K1 Input camera matrix of the first camera
@param D1 Input distortion parameters \f$(k_1, k_2, p_1, p_2)\f$ for the first camera
@param xi1 Input parameter xi for the first camera for CMei's model
@param K2 Input camera matrix of the second camera
@param D2 Input distortion parameters \f$(k_1, k_2, p_1, p_2)\f$ for the second camera
@param xi2 Input parameter xi for the second camera for CMei's model
@param R Rotation between the first and second camera
@param T Translation between the first and second camera
@param flag Flag of rectification type, RECTIFY_PERSPECTIVE or RECTIFY_LONGLATI
@param numDisparities The parameter 'numDisparities' in StereoSGBM, see StereoSGBM for details.
@param SADWindowSize The parameter 'SADWindowSize' in StereoSGBM, see StereoSGBM for details.
@param disparity Disparity map generated by stereo matching
@param image1Rec Rectified image of the first image
@param image2Rec rectified image of the second image
@param newSize Image size of rectified image, see omnidir::undistortImage
@param Knew New camera matrix of rectified image, see omnidir::undistortImage
@param pointCloud Point cloud of 3D reconstruction, with type CV_64FC3
@param pointType Point cloud type, it can be XYZRGB or XYZ
*/
CV_EXPORTS_W void stereoReconstruct(InputArray image1, InputArray image2, InputArray K1, InputArray D1, InputArray xi1,
InputArray K2, InputArray D2, InputArray xi2, InputArray R, InputArray T, int flag, int numDisparities, int SADWindowSize,
OutputArray disparity, OutputArray image1Rec, OutputArray image2Rec, const Size& newSize = Size(), InputArray Knew = cv::noArray(),
OutputArray pointCloud = cv::noArray(), int pointType = XYZRGB);
namespace internal
{
void initializeCalibration(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints, Size size, OutputArrayOfArrays omAll,
OutputArrayOfArrays tAll, OutputArray K, double& xi, OutputArray idx = noArray());
void initializeStereoCalibration(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints1, InputArrayOfArrays imagePoints2,
const Size& size1, const Size& size2, OutputArray om, OutputArray T, OutputArrayOfArrays omL, OutputArrayOfArrays tL, OutputArray K1, OutputArray D1, OutputArray K2, OutputArray D2,
double &xi1, double &xi2, int flags, OutputArray idx);
void computeJacobian(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints, InputArray parameters, Mat& JTJ_inv, Mat& JTE, int flags,
double epsilon);
void computeJacobianStereo(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints1, InputArrayOfArrays imagePoints2,
InputArray parameters, Mat& JTJ_inv, Mat& JTE, int flags, double epsilon);
void encodeParameters(InputArray K, InputArrayOfArrays omAll, InputArrayOfArrays tAll, InputArray distoaration, double xi, OutputArray parameters);
void encodeParametersStereo(InputArray K1, InputArray K2, InputArray om, InputArray T, InputArrayOfArrays omL, InputArrayOfArrays tL,
InputArray D1, InputArray D2, double xi1, double xi2, OutputArray parameters);
void decodeParameters(InputArray paramsters, OutputArray K, OutputArrayOfArrays omAll, OutputArrayOfArrays tAll, OutputArray distoration, double& xi);
void decodeParametersStereo(InputArray parameters, OutputArray K1, OutputArray K2, OutputArray om, OutputArray T, OutputArrayOfArrays omL,
OutputArrayOfArrays tL, OutputArray D1, OutputArray D2, double& xi1, double& xi2);
void estimateUncertainties(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints, InputArray parameters, Mat& errors, Vec2d& std_error, double& rms, int flags);
void estimateUncertaintiesStereo(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints1, InputArrayOfArrays imagePoints2, InputArray parameters, Mat& errors,
Vec2d& std_error, double& rms, int flags);
double computeMeanReproErr(InputArrayOfArrays imagePoints, InputArrayOfArrays proImagePoints);
double computeMeanReproErr(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints, InputArray K, InputArray D, double xi, InputArrayOfArrays omAll,
InputArrayOfArrays tAll);
double computeMeanReproErrStereo(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints1, InputArrayOfArrays imagePoints2, InputArray K1, InputArray K2,
InputArray D1, InputArray D2, double xi1, double xi2, InputArray om, InputArray T, InputArrayOfArrays omL, InputArrayOfArrays TL);
void subMatrix(const Mat& src, Mat& dst, const std::vector<int>& cols, const std::vector<int>& rows);
void flags2idx(int flags, std::vector<int>& idx, int n);
void flags2idxStereo(int flags, std::vector<int>& idx, int n);
void fillFixed(Mat&G, int flags, int n);
void fillFixedStereo(Mat& G, int flags, int n);
double findMedian(const Mat& row);
Vec3d findMedian3(InputArray mat);
void getInterset(InputArray idx1, InputArray idx2, OutputArray inter1, OutputArray inter2, OutputArray inter_ori);
void compose_motion(InputArray _om1, InputArray _T1, InputArray _om2, InputArray _T2, Mat& om3, Mat& T3, Mat& dom3dom1,
Mat& dom3dT1, Mat& dom3dom2, Mat& dom3dT2, Mat& dT3dom1, Mat& dT3dT1, Mat& dT3dom2, Mat& dT3dT2);
//void JRodriguesMatlab(const Mat& src, Mat& dst);
//void dAB(InputArray A, InputArray B, OutputArray dABdA, OutputArray dABdB);
} // internal
//! @}
} // omnidir
} //cv
#endif
modules/ccalib/include/opencv2/ccalib/randpattern.hpp
+185
View File
@@ -0,0 +1,185 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2015, Baisheng Lai (laibaisheng@gmail.com), Zhejiang University,
// all rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef __OPENCV_RANDOMPATTERN_HPP__
#define __OPENCV_RANDOMPATTERN_HPP__
#include "opencv2/features.hpp"
#include "opencv2/xfeatures2d.hpp"
#include "opencv2/highgui.hpp"
namespace cv { namespace randpattern {
//! @addtogroup ccalib
//! @{
/** @brief Class for finding features points and corresponding 3D in world coordinate of
a "random" pattern, which can be to be used in calibration. It is useful when pattern is
partly occluded or only a part of pattern can be observed in multiple cameras calibration.
The pattern can be generated by RandomPatternGenerator class described in this file.
Please refer to paper
B. Li, L. Heng, K. Kevin and M. Pollefeys, "A Multiple-Camera System
Calibration Toolbox Using A Feature Descriptor-Based Calibration
Pattern", in IROS 2013.
*/
class CV_EXPORTS RandomPatternCornerFinder
{
public:
/* @brief Construct RandomPatternCornerFinder object
@param patternWidth the real width of "random" pattern in a user defined unit.
@param patternHeight the real height of "random" pattern in a user defined unit.
@param nMiniMatch number of minimal matches, otherwise that image is abandoned
@depth depth of output objectPoints and imagePoints, set it to be CV_32F or CV_64F.
@showExtraction whether show feature extraction, 0 for no and 1 for yes.
@detector feature detector to detect feature points in pattern and images.
@descriptor feature descriptor.
@matcher feature matcher.
*/
RandomPatternCornerFinder(float patternWidth, float patternHeight,
int nminiMatch = 20, int depth = CV_32F, int verbose = 0, int showExtraction = 0,
Ptr<FeatureDetector> detector = xfeatures2d::AKAZE::create(xfeatures2d::AKAZE::DESCRIPTOR_MLDB, 0, 3, 0.005f),
Ptr<DescriptorExtractor> descriptor = xfeatures2d::AKAZE::create(xfeatures2d::AKAZE::DESCRIPTOR_MLDB,0, 3, 0.005f),
Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create("BruteForce-L1"));
/* @brief Load pattern image and compute features for pattern
@param patternImage image for "random" pattern generated by RandomPatternGenerator, run it first.
*/
void loadPattern(const cv::Mat &patternImage);
/* @brief Load pattern and features
@param patternImage image for "random" pattern generated by RandomPatternGenerator, run it first.
@param patternKeyPoints keyPoints created from a FeatureDetector.
@param patternDescriptors descriptors created from a DescriptorExtractor.
*/
void loadPattern(const cv::Mat &patternImage, const std::vector<cv::KeyPoint> &patternKeyPoints, const cv::Mat &patternDescriptors);
/* @brief Compute matched object points and image points which are used for calibration
The objectPoints (3D) and imagePoints (2D) are stored inside the class. Run getObjectPoints()
and getImagePoints() to get them.
@param inputImages vector of 8-bit grayscale images containing "random" pattern
that are used for calibration.
*/
void computeObjectImagePoints(std::vector<cv::Mat> inputImages);
//void computeObjectImagePoints2(std::vector<cv::Mat> inputImages);
/* @brief Compute object and image points for a single image. It returns a vector<Mat> that
the first element stores the imagePoints and the second one stores the objectPoints.
@param inputImage single input image for calibration
*/
std::vector<cv::Mat> computeObjectImagePointsForSingle(cv::Mat inputImage);
/* @brief Get object(3D) points
*/
const std::vector<cv::Mat> &getObjectPoints();
/* @brief and image(2D) points
*/
const std::vector<cv::Mat> &getImagePoints();
private:
std::vector<cv::Mat> _objectPonits, _imagePoints;
float _patternWidth, _patternHeight;
cv::Size _patternImageSize;
int _nminiMatch;
int _depth;
int _verbose;
Ptr<FeatureDetector> _detector;
Ptr<DescriptorExtractor> _descriptor;
Ptr<DescriptorMatcher> _matcher;
Mat _descriptorPattern;
std::vector<cv::KeyPoint> _keypointsPattern;
Mat _patternImage;
int _showExtraction;
void keyPoints2MatchedLocation(const std::vector<cv::KeyPoint>& imageKeypoints,
const std::vector<cv::KeyPoint>& patternKeypoints, const std::vector<cv::DMatch> matchces,
cv::Mat& matchedImagelocation, cv::Mat& matchedPatternLocation);
void getFilteredLocation(cv::Mat& imageKeypoints, cv::Mat& patternKeypoints, const cv::Mat mask);
void getObjectImagePoints(const cv::Mat& imageKeypoints, const cv::Mat& patternKeypoints);
void crossCheckMatching( cv::Ptr<DescriptorMatcher>& descriptorMatcher,
const Mat& descriptors1, const Mat& descriptors2,
std::vector<DMatch>& filteredMatches12, int knn=1 );
void drawCorrespondence(const Mat& image1, const std::vector<cv::KeyPoint> keypoint1,
const Mat& image2, const std::vector<cv::KeyPoint> keypoint2, const std::vector<cv::DMatch> matchces,
const Mat& mask1, const Mat& mask2, const int step);
};
/* @brief Class to generate "random" pattern image that are used for RandomPatternCornerFinder
Please refer to paper
B. Li, L. Heng, K. Kevin and M. Pollefeys, "A Multiple-Camera System
Calibration Toolbox Using A Feature Descriptor-Based Calibration
Pattern", in IROS 2013.
*/
class CV_EXPORTS RandomPatternGenerator
{
public:
/* @brief Construct RandomPatternGenerator
@param imageWidth image width of the generated pattern image
@param imageHeight image height of the generated pattern image
*/
RandomPatternGenerator(int imageWidth, int imageHeight);
/* @brief Generate pattern
*/
void generatePattern();
/* @brief Get pattern
*/
cv::Mat getPattern();
private:
cv::Mat _pattern;
int _imageWidth, _imageHeight;
};
//! @}
}} //namespace randpattern, cv
#endif
modules/ccalib/samples/multi_cameras_calibration.cpp
+120
View File
@@ -0,0 +1,120 @@
#include "opencv2/ccalib/omnidir.hpp"
#include "opencv2/ccalib/multicalib.hpp"
#include "opencv2/ccalib/randpattern.hpp"
using namespace std;
using namespace cv;
const char * usage =
"\n example command line for multi-camera calibration by using random pattern \n"
" multi_cameras_calibration -nc 5 -pw 800 -ph 600 -ct 1 -fe 0 -nm 25 -v 0 multi_camera_omnidir.xml \n"
"\n"
" the file multi_camera_omnidir.xml is generated by imagelist_creator as \n"
" imagelist_creator multi_camera_omnidir.xml *.* \n"
" note the first filename in multi_camera_omnidir.xml is the pattern, the rest are photo names,\n"
" photo names should be in form of cameraIdx-timestamp.*, and cameraIdx starts from 0";
static void help()
{
printf("\n This is a sample for multi-camera calibration, so far it only support random pattern,\n"
"see randomPattern.hpp for detail. Pinhole and omnidirectional cameras are both supported, \n"
"for omnidirectional camera, see omnidir.hpp for detail.\n"
"Usage: mutiCamCalib \n"
" -nc <num_camera> # number of cameras \n"
" -pw <pattern_width> # physical width of random pattern \n"
" -ph <pattern_height> # physical height of random pattern \n"
" -ct <camera_type> # camera type, 0 for pinhole and 1 for omnidirectional \n"
" -fe # whether show feature extraction\n"
" -nm # number of minimal matches of an image \n"
" -v # whether show verbose information \n"
" input_data # text file with pattern file names and a list of photo names, the file is generated by imagelist_creator \n");
printf("\n %s", usage);
}
int main(int argc, char** argv)
{
float patternWidth = 0.0f, patternHeight = 0.0f;
int nCamera = 0, nMiniMatches = 0, cameraType = 0;
const char* outputFilename = "multi-camera-results.xml";
const char* inputFilename = 0;
int showFeatureExtraction = 0, verbose = 0;
if (argc < 2)
{
help();
return 1;
}
for (int i = 1; i < argc; ++i)
{
const char* s = argv[i];
if (strcmp( s, "-nc") == 0)
{
if (sscanf( argv[++i], "%u", &nCamera) != 1 || nCamera <= 0)
{
return fprintf(stderr, "Invalid number of cameras \n"), -1;
}
}
else if ( strcmp( s, "-pw" ) == 0 )
{
if (sscanf( argv[++i], "%f", &patternWidth) != 1 || patternWidth <=0 )
{
return fprintf(stderr, "Invalid pattern width \n"), -1;
}
}
else if ( strcmp( s, "-ph" ) == 0 )
{
if (sscanf( argv[++i], "%f", &patternHeight) != 1 || patternHeight <=0 )
{
return fprintf(stderr, "Invalid pattern height \n"), -1;
}
}
else if ( strcmp( s, "-ct" ) == 0 )
{
if (sscanf( argv[++i], "%u", &cameraType) != 1 || (cameraType !=0 && cameraType !=1 && cameraType !=2) )
{
return fprintf(stderr, "Invalid camera type, 0 for pinhole and 1 for omnidirectional \n"), -1;
}
}
else if ( strcmp( s, "-fe" ) == 0 )
{
if (sscanf( argv[++i], "%u", &showFeatureExtraction) != 1 || (showFeatureExtraction !=1 && showFeatureExtraction !=0) )
{
return fprintf(stderr, "Not bool value, set to 0 or 1 \n"), -1;
}
}
else if ( strcmp( s, "-nm" ) == 0 )
{
if (sscanf( argv[++i], "%u", &nMiniMatches) != 1 || nMiniMatches <=0 )
{
return fprintf(stderr, "Invalid number of minimal matches \n"), -1;
}
}
else if ( strcmp( s, "-v" ) == 0 )
{
if (sscanf( argv[++i], "%u", &verbose) != 1 || (verbose !=1 && verbose !=0) )
{
return fprintf(stderr, "verbose is not bool value, set to 0 or 1 \n"), -1;
}
}
else if( s[0] != '-')
{
inputFilename = s;
}
else
{
return fprintf( stderr, "Unknown option %s\n", s ), -1;
}
}
// do multi-camera calibration
multicalib::MultiCameraCalibration multiCalib(cameraType, nCamera, inputFilename, patternWidth, patternHeight, verbose, showFeatureExtraction, nMiniMatches);
multiCalib.loadImages();
multiCalib.initialize();
multiCalib.optimizeExtrinsics();
// the above three lines can be replaced by multiCalib.run();
multiCalib.writeParameters(outputFilename);
}
modules/ccalib/samples/omni_calibration.cpp
+224
View File
@@ -0,0 +1,224 @@
#include "opencv2/ccalib/omnidir.hpp"
#include "opencv2/core.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/geometry.hpp"
#include "opencv2/calib.hpp"
#include "opencv2/objdetect.hpp"
#include "opencv2/highgui.hpp"
#include <vector>
#include <iostream>
#include <string>
#include <time.h>
using namespace cv;
using namespace std;
static void calcChessboardCorners(const Size &boardSize, const Size2d &squareSize, Mat& corners)
{
// corners has type of CV_64FC3
corners.release();
int n = boardSize.width * boardSize.height;
corners.create(n, 1, CV_64FC3);
Vec3d *ptr = corners.ptr<Vec3d>();
for (int i = 0; i < boardSize.height; ++i)
{
for (int j = 0; j < boardSize.width; ++j)
{
ptr[i*boardSize.width + j] = Vec3d(double(j * squareSize.width), double(i * squareSize.height), 0.0);
}
}
}
static bool detecChessboardCorners(const vector<string>& list, vector<string>& list_detected,
vector<Mat>& imagePoints, Size boardSize, Size& imageSize)
{
imagePoints.resize(0);
list_detected.resize(0);
int n_img = (int)list.size();
Mat img;
for(int i = 0; i < n_img; ++i)
{
cout << list[i] << "... ";
Mat points;
img = imread(list[i], IMREAD_GRAYSCALE);
bool found = findChessboardCorners( img, boardSize, points);
if (found)
{
if (points.type() != CV_64FC2)
points.convertTo(points, CV_64FC2);
imagePoints.push_back(points);
list_detected.push_back(list[i]);
}
cout << (found ? "FOUND" : "NO") << endl;
}
if (!img.empty())
imageSize = img.size();
if (imagePoints.size() < 3)
return false;
else
return true;
}
static bool readStringList( const string& filename, vector<string>& l )
{
l.resize(0);
FileStorage fs(filename, FileStorage::READ);
if( !fs.isOpened() )
return false;
FileNode n = fs.getFirstTopLevelNode();
if( n.type() != FileNode::SEQ )
return false;
FileNodeIterator it = n.begin(), it_end = n.end();
for( ; it != it_end; ++it )
l.push_back((string)*it);
return true;
}
static void saveCameraParams( const string & filename, int flags, const Mat& cameraMatrix,
const Mat& distCoeffs, const double xi, const vector<Vec3d>& rvecs, const vector<Vec3d>& tvecs,
vector<string> detec_list, const Mat& idx, const double rms, const vector<Mat>& imagePoints)
{
FileStorage fs( filename, FileStorage::WRITE );
time_t tt;
time( &tt );
struct tm *t2 = localtime( &tt );
char buf[1024];
strftime( buf, sizeof(buf)-1, "%c", t2 );
fs << "calibration_time" << buf;
if ( !rvecs.empty())
fs << "nFrames" << (int)rvecs.size();
if ( flags != 0)
{
sprintf( buf, "flags: %s%s%s%s%s%s%s%s%s",
flags & omnidir::CALIB_USE_GUESS ? "+use_intrinsic_guess" : "",
flags & omnidir::CALIB_FIX_SKEW ? "+fix_skew" : "",
flags & omnidir::CALIB_FIX_K1 ? "+fix_k1" : "",
flags & omnidir::CALIB_FIX_K2 ? "+fix_k2" : "",
flags & omnidir::CALIB_FIX_P1 ? "+fix_p1" : "",
flags & omnidir::CALIB_FIX_P2 ? "+fix_p2" : "",
flags & omnidir::CALIB_FIX_XI ? "+fix_xi" : "",
flags & omnidir::CALIB_FIX_GAMMA ? "+fix_gamma" : "",
flags & omnidir::CALIB_FIX_CENTER ? "+fix_center" : "");
//cvWriteComment( *fs, buf, 0 );
}
fs << "flags" << flags;
fs << "camera_matrix" << cameraMatrix;
fs << "distortion_coefficients" << distCoeffs;
fs << "xi" << xi;
//cvWriteComment( *fs, "names of images that are acturally used in calibration", 0 );
fs << "used_imgs" << "[";
for (int i = 0; i < (int)idx.total(); ++i)
{
fs << detec_list[(int)idx.at<int>(i)];
}
fs << "]";
if ( !rvecs.empty() && !tvecs.empty() )
{
Mat rvec_tvec((int)rvecs.size(), 6, CV_64F);
for (int i = 0; i < (int)rvecs.size(); ++i)
{
Mat(rvecs[i]).reshape(1, 1).copyTo(rvec_tvec(Rect(0, i, 3, 1)));
Mat(tvecs[i]).reshape(1, 1).copyTo(rvec_tvec(Rect(3, i, 3, 1)));
}
//cvWriteComment( *fs, "a set of 6-tuples (rotation vector + translation vector) for each view", 0 );
fs << "extrinsic_parameters" << rvec_tvec;
}
fs << "rms" << rms;
if ( !imagePoints.empty() )
{
Mat imageMat((int)imagePoints.size(), (int)imagePoints[0].total(), CV_64FC2);
for (int i = 0; i < (int)imagePoints.size(); ++i)
{
Mat r = imageMat.row(i).reshape(2, imageMat.cols);
Mat imagei(imagePoints[i]);
imagei.copyTo(r);
}
fs << "image_points" << imageMat;
}
}
int main(int argc, char** argv)
{
cv::CommandLineParser parser(argc, argv,
"{w||board width}"
"{h||board height}"
"{sw|1.0|square width}"
"{sh|1.0|square height}"
"{o|out_camera_params.xml|output file}"
"{fs|false|fix skew}"
"{fp|false|fix principal point at the center}"
"{@input||input file - xml file with a list of the images, created with cpp-example-imagelist_creator tool}"
"{help||show help}"
);
parser.about("This is a sample for omnidirectional camera calibration. Example command line:\n"
" omni_calibration -w=6 -h=9 -sw=80 -sh=80 imagelist.xml \n");
if (parser.has("help") || !parser.has("w") || !parser.has("h"))
{
parser.printMessage();
return 0;
}
Size boardSize(parser.get<int>("w"), parser.get<int>("h"));
Size2d squareSize(parser.get<double>("sw"), parser.get<double>("sh"));
int flags = 0;
if (parser.get<bool>("fs"))
flags |= omnidir::CALIB_FIX_SKEW;
if (parser.get<bool>("fp"))
flags |= omnidir::CALIB_FIX_CENTER;
const string outputFilename = parser.get<string>("o");
const string inputFilename = parser.get<string>(0);
if (!parser.check())
{
parser.printErrors();
return -1;
}
// get image name list
vector<string> image_list, detec_list;
if(!readStringList(inputFilename, image_list))
{
cout << "Can not read imagelist" << endl;
return -1;
}
// find corners in images
// some images may be failed in automatic corner detection, passed cases are in detec_list
cout << "Detecting chessboards (" << image_list.size() << ")" << endl;
vector<Mat> imagePoints;
Size imageSize;
if(!detecChessboardCorners(image_list, detec_list, imagePoints, boardSize, imageSize))
{
cout << "Not enough corner detected images" << endl;
return -1;
}
// calculate object coordinates
vector<Mat> objectPoints;
Mat object;
calcChessboardCorners(boardSize, squareSize, object);
for(int i = 0; i < (int)detec_list.size(); ++i)
objectPoints.push_back(object);
// run calibration, some images are discarded in calibration process because they are failed
// in initialization. Retained image indexes are in idx variable.
Mat K, D, xi, idx;
vector<Vec3d> rvecs, tvecs;
double _xi, rms;
TermCriteria criteria(3, 200, 1e-8);
rms = omnidir::calibrate(objectPoints, imagePoints, imageSize, K, xi, D, rvecs, tvecs, flags, criteria, idx);
_xi = xi.at<double>(0);
cout << "Saving camera params to " << outputFilename << endl;
saveCameraParams(outputFilename, flags, K, D, _xi,
rvecs, tvecs, detec_list, idx, rms, imagePoints);
}
modules/ccalib/samples/omni_stereo_calibration.cpp
+319
View File
@@ -0,0 +1,319 @@
#include "opencv2/ccalib/omnidir.hpp"
#include "opencv2/core.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/geometry.hpp"
#include "opencv2/objdetect.hpp"
#include "opencv2/calib.hpp"
#include <vector>
#include <iostream>
#include <string>
#include <time.h>
using namespace cv;
using namespace std;
const char * usage =
"\n example command line for calibrate a pair of omnidirectional camera.\n"
" omni_stereo_calibration -w 8 -h 6 -sw 2.4399 -sh 2.4399 imagelist_left.xml imagelist_right.xml\n"
" \n"
" the file image_list_1.xml and image_list_2.xml generated by imagelist_creator as\n"
"imagelist_creator image_list_1.xml *.*";
static void help()
{
printf("\n This is a sample for omnidirectional camera calibration.\n"
"Usage: omni_calibration\n"
" -w <board_width> # the number of inner corners per one of board dimension\n"
" -h <board_height> # the number of inner corners per another board dimension\n"
" [-sw <square_width>] # the width of square in some user-defined units (1 by default)\n"
" [-sh <square_height>] # the height of square in some user-defined units (1 by default)\n"
" [-o <out_camera_params>] # the output filename for intrinsic [and extrinsic] parameters\n"
" [-fs <fix_skew>] # fix skew\n"
" [-fp ] # fix the principal point at the center\n"
" input_data_1 # input data - text file with a list of the images of the first camera, which is generated by imagelist_creator"
" input_data_2 # input data - text file with a list of the images of the second camera, which is generated by imagelist_creator"
);
printf("\n %s", usage);
}
static void calcChessboardCorners(Size boardSize, double square_width, double square_height,
Mat& corners)
{
// corners has type of CV_64FC3
corners.release();
int n = boardSize.width * boardSize.height;
corners.create(n, 1, CV_64FC3);
Vec3d *ptr = corners.ptr<Vec3d>();
for (int i = 0; i < boardSize.height; ++i)
{
for (int j = 0; j < boardSize.width; ++j)
{
ptr[i*boardSize.width + j] = Vec3d(double(j * square_width), double(i * square_height), 0.0);
}
}
}
static bool detecChessboardCorners(const vector<string>& list1, vector<string>& list_detected_1,
const vector<string>& list2, vector<string>& list_detected_2,
vector<Mat>& image_points_1, vector<Mat>& image_points_2, Size boardSize, Size& imageSize1, Size& imageSize2)
{
image_points_1.resize(0);
image_points_2.resize(0);
list_detected_1.resize(0);
list_detected_2.resize(0);
int n_img = (int)list1.size();
Mat img_l, img_r;
for(int i = 0; i < n_img; ++i)
{
Mat points_1, points_2;
img_l = imread(list1[i], IMREAD_GRAYSCALE);
img_r = imread(list2[i], IMREAD_GRAYSCALE);
bool found_l = findChessboardCorners( img_l, boardSize, points_1);
bool found_r = findChessboardCorners( img_r, boardSize, points_2);
if (found_l && found_r)
{
if (points_1.type() != CV_64FC2)
points_1.convertTo(points_1, CV_64FC2);
if (points_2.type() != CV_64FC2)
points_2.convertTo(points_2, CV_64FC2);
image_points_1.push_back(points_1);
image_points_2.push_back(points_2);
list_detected_1.push_back(list1[i]);
list_detected_2.push_back(list2[i]);
}
}
if (!img_l.empty())
imageSize1 = img_l.size();
if (!img_r.empty())
{
imageSize2 = img_r.size();
}
if (image_points_1.size() < 3)
return false;
else
return true;
}
static bool readStringList( const string& filename, vector<string>& l )
{
l.resize(0);
FileStorage fs(filename, FileStorage::READ);
if( !fs.isOpened() )
return false;
FileNode n = fs.getFirstTopLevelNode();
if( n.type() != FileNode::SEQ )
return false;
FileNodeIterator it = n.begin(), it_end = n.end();
for( ; it != it_end; ++it )
l.push_back((string)*it);
return true;
}
static void saveCameraParams( const string & filename, const int flags, const Mat& cameraMatrix1, const Mat& cameraMatrix2, const Mat& distCoeffs1,
const Mat& disCoeffs2, const double xi1, const double xi2, const Vec3d rvec, const Vec3d tvec,
const vector<Vec3d>& rvecs, const vector<Vec3d>& tvecs, vector<string> detec_list_1, vector<string> detec_list_2,
const Mat& idx, const double rms, const vector<Mat>& imagePoints1, const vector<Mat>& imagePoints2)
{
FileStorage fs( filename, FileStorage::WRITE );
time_t tt;
time( &tt );
struct tm *t2 = localtime( &tt );
char buf[1024];
strftime( buf, sizeof(buf)-1, "%c", t2 );
fs << "calibration_time" << buf;
if ( !rvecs.empty())
fs << "nFrames" << (int)rvecs.size();
if ( flags != 0)
{
sprintf( buf, "flags: %s%s%s%s%s%s%s%s%s",
flags & omnidir::CALIB_USE_GUESS ? "+use_intrinsic_guess" : "",
flags & omnidir::CALIB_FIX_SKEW ? "+fix_skew" : "",
flags & omnidir::CALIB_FIX_K1 ? "+fix_k1" : "",
flags & omnidir::CALIB_FIX_K2 ? "+fix_k2" : "",
flags & omnidir::CALIB_FIX_P1 ? "+fix_p1" : "",
flags & omnidir::CALIB_FIX_P2 ? "+fix_p2" : "",
flags & omnidir::CALIB_FIX_XI ? "+fix_xi" : "",
flags & omnidir::CALIB_FIX_GAMMA ? "+fix_gamma" : "",
flags & omnidir::CALIB_FIX_CENTER ? "+fix_center" : "");
//cvWriteComment( *fs, buf, 0 );
}
fs << "flags" << flags;
fs << "camera_matrix_1" << cameraMatrix1;
fs << "distortion_coefficients_1" << distCoeffs1;
fs << "xi_1" << xi1;
fs << "camera_matrix_2" << cameraMatrix2;
fs << "distortion_coefficients_2" << disCoeffs2;
fs << "xi_2" << xi2;
Mat om_t(1, 6, CV_64F);
Mat(rvec).reshape(1, 1).copyTo(om_t.colRange(0, 3));
Mat(tvec).reshape(1, 1).copyTo(om_t.colRange(3, 6));
//cvWriteComment( *fs, "6-tuples (rotation vector + translation vector) for each view", 0 );
fs << "extrinsic_parameters" << om_t;
if ( !rvecs.empty() && !tvecs.empty() )
{
Mat rvec_tvec((int)rvecs.size(), 6, CV_64F);
for (int i = 0; i < (int)rvecs.size(); ++i)
{
Mat(rvecs[i]).reshape(1, 1).copyTo(rvec_tvec(Rect(0, i, 3, 1)));
Mat(tvecs[i]).reshape(1, 1).copyTo(rvec_tvec(Rect(3, i, 3, 1)));
}
//cvWriteComment( *fs, "a set of 6-tuples (rotation vector + translation vector) for each view", 0 );
fs << "extrinsic_parameters_1" << rvec_tvec;
}
fs << "rms" << rms;
//cvWriteComment( *fs, "names of images that are acturally used in calibration", 0 );
fs << "used_imgs_1" << "[";
for (int i = 0; i < (int)idx.total(); ++i)
{
fs << detec_list_1[(int)idx.at<int>(i)];
}
fs << "]";
fs << "used_imgs_2" << "[";
for (int i = 0; i < (int)idx.total(); ++i)
{
fs << detec_list_2[(int)idx.at<int>(i)];
}
fs << "]";
if ( !imagePoints1.empty() )
{
Mat imageMat((int)imagePoints1.size(), (int)imagePoints1[0].total(), CV_64FC2);
for (int i = 0; i < (int)imagePoints1.size(); ++i)
{
Mat r = imageMat.row(i).reshape(2, imageMat.cols);
Mat imagei(imagePoints1[i]);
imagei.copyTo(r);
}
fs << "image_points_1" << imageMat;
}
if ( !imagePoints2.empty() )
{
Mat imageMat((int)imagePoints2.size(), (int)imagePoints2[0].total(), CV_64FC2);
for (int i = 0; i < (int)imagePoints2.size(); ++i)
{
Mat r = imageMat.row(i).reshape(2, imageMat.cols);
Mat imagei(imagePoints2[i]);
imagei.copyTo(r);
}
fs << "image_points_2" << imageMat;
}
}
int main(int argc, char** argv)
{
Size boardSize, imageSize1, imageSize2;
int flags = 0;
double square_width = 0.0, square_height = 0.0;
const char* outputFilename = "out_camera_params_stereo.xml";
const char* inputFilename1 = 0;
const char* inputFilename2 = 0;
vector<Mat> objectPoints;
vector<Mat> imagePoints1;
vector<Mat> imagePoints2;
if(argc < 2)
{
help();
return 1;
}
bool fist_flag = true;
for(int i = 1; i < argc; i++)
{
const char* s = argv[i];
if( strcmp( s, "-w") == 0)
{
if( sscanf( argv[++i], "%u", &boardSize.width ) != 1 || boardSize.width <= 0 )
return fprintf( stderr, "Invalid board width\n" ), -1;
}
else if( strcmp( s, "-h" ) == 0 )
{
if( sscanf( argv[++i], "%u", &boardSize.height ) != 1 || boardSize.height <= 0 )
return fprintf( stderr, "Invalid board height\n" ), -1;
}
else if( strcmp( s, "-sw" ) == 0 )
{
if( sscanf( argv[++i], "%lf", &square_width ) != 1 || square_width <= 0 )
return fprintf(stderr, "Invalid square width\n"), -1;
}
else if( strcmp( s, "-sh" ) == 0 )
{
if( sscanf( argv[++i], "%lf", &square_height) != 1 || square_height <= 0 )
return fprintf(stderr, "Invalid square height\n"), -1;
}
else if( strcmp( s, "-o" ) == 0 )
{
outputFilename = argv[++i];
}
else if( strcmp( s, "-fs" ) == 0 )
{
flags |= omnidir::CALIB_FIX_SKEW;
}
else if( strcmp( s, "-fp" ) == 0 )
{
flags |= omnidir::CALIB_FIX_CENTER;
}
else if( s[0] != '-' && fist_flag)
{
fist_flag = false;
inputFilename1 = s;
}
else if( s[0] != '-' && !fist_flag)
{
inputFilename2 = s;
}
else
{
return fprintf( stderr, "Unknown option %s\n", s ), -1;
}
}
// get image name list
vector<string> image_list1, detec_list_1, image_list2, detec_list_2;
if((!readStringList(inputFilename1, image_list1)) || (!readStringList(inputFilename2, image_list2)))
return fprintf( stderr, "Failed to read image list\n"), -1;
// find corners in images
// some images may be failed in automatic corner detection, passed cases are in detec_list
if(!detecChessboardCorners(image_list1, detec_list_1, image_list2, detec_list_2,
imagePoints1, imagePoints2, boardSize, imageSize1, imageSize2))
return fprintf(stderr, "Not enough corner detected images\n"), -1;
// calculate object coordinates
Mat object;
calcChessboardCorners(boardSize, square_width, square_height, object);
for(int i = 0; i < (int)detec_list_1.size(); ++i)
{
objectPoints.push_back(object);
}
// run calibration, some images are discarded in calibration process because they are failed
// in initialization. Retained image indexes are in idx variable.
Mat K1, K2, D1, D2, xi1, xi2, idx;
vector<Vec3d> rvecs, tvecs;
Vec3d rvec, tvec;
double _xi1, _xi2, rms;
TermCriteria criteria(3, 200, 1e-8);
rms = omnidir::stereoCalibrate(objectPoints, imagePoints1, imagePoints2, imageSize1, imageSize2, K1, xi1, D1,
K2, xi2, D2, rvec, tvec, rvecs, tvecs, flags, criteria, idx);
_xi1 = xi1.at<double>(0);
_xi2 = xi2.at<double>(0);
saveCameraParams(outputFilename, flags, K1, K2, D1, D2, _xi1, _xi2, rvec, tvec, rvecs, tvecs,
detec_list_1, detec_list_2, idx, rms, imagePoints1, imagePoints2);
}
modules/ccalib/samples/random_pattern_calibration.cpp
+162
View File
@@ -0,0 +1,162 @@
#include "opencv2/ccalib/randpattern.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/geometry.hpp"
#include "opencv2/calib.hpp"
#include <vector>
#include <iostream>
#include <time.h>
using namespace std;
using namespace cv;
const char * usage =
"\n example command line for calibrate a camera by random pattern. \n"
" random_pattern_calibration -pw 600 -ph 850 -mm 20 image_list.xml \n"
"\n"
" the file image_list.xml is generated by imagelist_creator as\n"
"imagelist_creator image_list.xml *.*";
static void help()
{
printf("\n This is a sample for camera calibration by a random pattern.\n"
"Usage: random_pattern_calibration\n"
" -pw <pattern_width> # the physical width of random pattern\n"
" -ph <pattern_height> # the physical height of random pattern\n"
" -mm <minimal_match> # minimal number of matches\n"
" [-fp ] # fix the principal point at the center \n"
" input_data # input data - text file with a list of the images of the board, which is generated by imagelist_creator"
);
printf("\n %s", usage);
}
static bool readStringList( const string& filename, vector<string>& l )
{
l.resize(0);
FileStorage fs(filename, FileStorage::READ);
if( !fs.isOpened() )
return false;
FileNode n = fs.getFirstTopLevelNode();
if( n.type() != FileNode::SEQ )
return false;
FileNodeIterator it = n.begin(), it_end = n.end();
for( ; it != it_end; ++it )
l.push_back((string)*it);
return true;
}
static void saveCameraParams(const string& filename, Size imageSize, float patternWidth,
float patternHeight, int flags, const Mat& cameraMatrix, const Mat& distCoeffs,
const vector<Mat>& rvecs, const vector<Mat>& tvecs, double rms)
{
FileStorage fs (filename, FileStorage::WRITE );
time_t tt;
time( &tt );
struct tm *t2 = localtime( &tt );
char buf[1024];
strftime( buf, sizeof(buf)-1, "%c", t2 );
fs << "calibration_time" << buf;
if( !rvecs.empty())
fs << "nframes" << (int)rvecs.size();
fs << "image_width" << imageSize.width;
fs << "image_height" << imageSize.height;
fs << "pattern_width" << patternWidth;
fs << "pattern_height" << patternHeight;
fs << "flags" <<flags;
fs << "camera_matrix" << cameraMatrix;
fs << "distortion_coefficients" << distCoeffs;
fs << "rms" << rms;
if( !rvecs.empty() && !tvecs.empty() )
{
CV_Assert(rvecs[0].type() == tvecs[0].type());
Mat bigmat((int)rvecs.size(), 6, rvecs[0].type());
for( int i = 0; i < (int)rvecs.size(); i++ )
{
Mat r = bigmat(Range(i, i+1), Range(0,3));
Mat t = bigmat(Range(i, i+1), Range(3,6));
CV_Assert(rvecs[i].rows == 3 && rvecs[i].cols == 1);
CV_Assert(tvecs[i].rows == 3 && tvecs[i].cols == 1);
//*.t() is MatExpr (not Mat) so we can use assignment operator
r = rvecs[i].t();
t = tvecs[i].t();
}
//cvWriteComment( *fs, "a set of 6-tuples (rotation vector + translation vector) for each view", 0 );
fs << "extrinsic_parameters" << bigmat;
}
}
int main(int argc, char** argv)
{
const char* inputFilename = 0;
const char* outputFilename = "out_camera_params.xml";
vector<string> imglist;
vector<Mat> vecImg;
int flags = 0;
float patternWidth = 0.0f, patternHeight = 0.0f;
int nMiniMatches = 0;
if(argc < 2)
{
help();
return 1;
}
for (int i = 1; i < argc; ++i)
{
const char* s = argv[i];
if(strcmp(s, "-pw") == 0)
{
if(sscanf(argv[++i], "%f", &patternWidth) != 1 || patternWidth <= 0)
return fprintf( stderr, "Invalid pattern width\n"), -1;
}
else if(strcmp(s, "-ph") == 0)
{
if(sscanf(argv[++i], "%f", &patternHeight) != 1 || patternHeight <= 0)
return fprintf( stderr, "Invalid pattern height\n"), -1;
}
else if (strcmp(s, "-mm") == 0)
{
if (sscanf(argv[++i], "%d", &nMiniMatches) != 1 || nMiniMatches < 15)
return fprintf( stderr, "Invalid number of minimal matches or number is too small"), -1;
}
else if( strcmp( s, "-fp" ) == 0 )
{
flags |= CALIB_FIX_PRINCIPAL_POINT;
}
else if( s[0] != '-')
{
inputFilename = s;
}
else
{
return fprintf( stderr, "Unknown option %s\n", s ), -1;
}
}
readStringList(inputFilename, imglist);
// the first image is the pattern
Mat pattern = cv::imread(imglist[0], cv::IMREAD_GRAYSCALE);
for (int i = 1; i < (int)imglist.size(); ++i)
{
Mat img;
img = cv::imread(imglist[i], cv::IMREAD_GRAYSCALE);
vecImg.push_back(img);
}
randpattern::RandomPatternCornerFinder finder(patternWidth, patternHeight, nMiniMatches);
finder.loadPattern(pattern);
finder.computeObjectImagePoints(vecImg);
vector<Mat> objectPoints = finder.getObjectPoints();
vector<Mat> imagePoints = finder.getImagePoints();
Mat K;
Mat D;
vector<Mat> rvec, tvec;
double rms = calibrateCamera(objectPoints, imagePoints, vecImg[0].size(), K, D, rvec, tvec);
saveCameraParams(outputFilename, vecImg[0].size(), patternWidth, patternHeight, flags, K, D, rvec, tvec, rms);
}
modules/ccalib/samples/random_pattern_generator.cpp
+59
View File
@@ -0,0 +1,59 @@
#include "opencv2/ccalib/randpattern.hpp"
using namespace cv;
const char * usage =
"\n example command line for generating a random pattern. \n"
" random_patterng_generator -iw 600 -ih 850 pattern.png\n"
"\n";
static void help()
{
printf("\n This is a sample for generating a random pattern that can be used for calibration.\n"
"Usage: random_patterng_generator\n"
" -iw <image_width> # the width of pattern image\n"
" -ih <image_height> # the height of pattern image\n"
" filename # the filename for pattern image \n"
);
printf("\n %s", usage);
}
int main(int argc, char** argv)
{
const char* filename = 0;
Mat pattern;
int width = 0, height = 0;
if(argc < 2)
{
help();
return 1;
}
for (int i = 1; i < argc; ++i)
{
const char* s = argv[i];
if(strcmp(s, "-iw") == 0)
{
if(sscanf(argv[++i], "%d", &width) != 1 || width <= 0)
return fprintf( stderr, "Invalid pattern image width\n"), -1;
}
else if(strcmp(s, "-ih") == 0)
{
if(sscanf(argv[++i], "%d", &height) != 1 || height <= 0)
return fprintf( stderr, "Invalid pattern image height\n"), -1;
}
else if( s[0] != '-')
{
filename = s;
}
else
{
return fprintf( stderr, "Unknown option %s\n", s ), -1;
}
}
randpattern::RandomPatternGenerator generator(width, height);
generator.generatePattern();
pattern = generator.getPattern();
imwrite(filename, pattern);
}
modules/ccalib/src/ccalib.cpp
+495
View File
@@ -0,0 +1,495 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2014, OpenCV Foundation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef __OPENCV_CCALIB_CPP__
#define __OPENCV_CCALIB_CPP__
#ifdef __cplusplus
#include "precomp.hpp"
#include "opencv2/ccalib.hpp"
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/geometry.hpp>
#include <opencv2/calib.hpp>
#include <opencv2/features.hpp>
#include <vector>
#include <cstring>
namespace cv{ namespace ccalib{
using namespace std;
const int MIN_CONTOUR_AREA_PX = 100;
const float MIN_CONTOUR_AREA_RATIO = 0.2f;
const float MAX_CONTOUR_AREA_RATIO = 5.0f;
const int MIN_POINTS_FOR_H = 10;
const float MAX_PROJ_ERROR_PX = 5.0f;
CustomPattern::CustomPattern()
{
initialized = false;
}
bool CustomPattern::create(InputArray pattern, const Size2f boardSize, OutputArray output)
{
CV_Assert(!pattern.empty() && (boardSize.area() > 0));
Mat img = pattern.getMat();
float pixel_size = (boardSize.width > boardSize.height)? // Choose the longer side for more accurate calculation
float(img.cols) / boardSize.width: // width is longer
float(img.rows) / boardSize.height; // height is longer
return init(img, pixel_size, output);
}
bool CustomPattern::init(Mat& image, const float pixel_size, OutputArray output)
{
image.copyTo(img_roi);
//Setup object corners
obj_corners = std::vector<Point2f>(4);
obj_corners[0] = Point2f(0, 0); obj_corners[1] = Point2f(float(img_roi.cols), 0);
obj_corners[2] = Point2f(float(img_roi.cols), float(img_roi.rows)); obj_corners[3] = Point2f(0, float(img_roi.rows));
if (!detector) // if no detector chosen, use default
{
Ptr<ORB> orb = ORB::create();
orb->setMaxFeatures(2000);
orb->setScaleFactor(1.15);
orb->setNLevels(30);
detector = orb;
}
detector->detect(img_roi, keypoints);
if (keypoints.empty())
{
initialized = false;
return initialized;
}
refineKeypointsPos(img_roi, keypoints);
if (!descriptorExtractor) // if no extractor chosen, use default
descriptorExtractor = ORB::create();
descriptorExtractor->compute(img_roi, keypoints, descriptor);
if (!descriptorMatcher)
descriptorMatcher = DescriptorMatcher::create("BruteForce-Hamming(2)");
// Scale found points by pixelSize
pxSize = pixel_size;
scaleFoundPoints(pxSize, keypoints, points3d);
if (output.needed())
{
Mat out;
drawKeypoints(img_roi, keypoints, out, Scalar(0, 0, 255));
out.copyTo(output);
}
initialized = !keypoints.empty();
return initialized; // initialized if any keypoints are found
}
CustomPattern::~CustomPattern() {}
bool CustomPattern::isInitialized()
{
return initialized;
}
bool CustomPattern::setFeatureDetector(Ptr<FeatureDetector> featureDetector)
{
if (!initialized)
{
this->detector = featureDetector;
return true;
}
else
return false;
}
bool CustomPattern::setDescriptorExtractor(Ptr<DescriptorExtractor> extractor)
{
if (!initialized)
{
this->descriptorExtractor = extractor;
return true;
}
else
return false;
}
bool CustomPattern::setDescriptorMatcher(Ptr<DescriptorMatcher> matcher)
{
if (!initialized)
{
this->descriptorMatcher = matcher;
return true;
}
else
return false;
}
Ptr<FeatureDetector> CustomPattern::getFeatureDetector()
{
return detector;
}
Ptr<DescriptorExtractor> CustomPattern::getDescriptorExtractor()
{
return descriptorExtractor;
}
Ptr<DescriptorMatcher> CustomPattern::getDescriptorMatcher()
{
return descriptorMatcher;
}
void CustomPattern::scaleFoundPoints(const double pixelSize,
const vector<KeyPoint>& corners, vector<Point3f>& pts3d)
{
for (unsigned int i = 0; i < corners.size(); ++i)
{
pts3d.push_back(Point3f(
float(corners[i].pt.x * pixelSize),
float(corners[i].pt.y * pixelSize),
0));
}
}
//Takes a descriptor and turns it into an (x,y) point
void CustomPattern::keypoints2points(const vector<KeyPoint>& in, vector<Point2f>& out)
{
out.clear();
out.reserve(in.size());
for (size_t i = 0; i < in.size(); ++i)
{
out.push_back(in[i].pt);
}
}
void CustomPattern::updateKeypointsPos(vector<KeyPoint>& in, const vector<Point2f>& new_pos)
{
for (size_t i = 0; i < in.size(); ++i)
{
in[i].pt= new_pos[i];
}
}
void CustomPattern::refinePointsPos(const Mat& img, vector<Point2f>& p)
{
Mat gray;
cvtColor(img, gray, COLOR_RGB2GRAY);
cornerSubPix(gray, p, Size(10, 10), Size(-1, -1),
TermCriteria(cv::TermCriteria::MAX_ITER + cv::TermCriteria::EPS, 30, 0.1));
}
void CustomPattern::refineKeypointsPos(const Mat& img, vector<KeyPoint>& kp)
{
vector<Point2f> points;
keypoints2points(kp, points);
refinePointsPos(img, points);
updateKeypointsPos(kp, points);
}
template<typename Tstve>
void deleteStdVecElem(vector<Tstve>& v, int idx)
{
v[idx] = v.back();
v.pop_back();
}
void CustomPattern::check_matches(vector<Point2f>& matched, const vector<Point2f>& pattern, vector<DMatch>& good,
vector<Point3f>& pattern_3d, const Mat& H)
{
vector<Point2f> proj;
perspectiveTransform(pattern, proj, H);
for (uint i = 0; i < proj.size(); ++i)
{
double error = norm(matched[i] - proj[i]);
if (error >= MAX_PROJ_ERROR_PX)
{
deleteStdVecElem(good, i);
deleteStdVecElem(matched, i);
deleteStdVecElem(pattern_3d, i);
}
}
}
bool CustomPattern::findPatternPass(const Mat& image, vector<Point2f>& matched_features, vector<Point3f>& pattern_points,
Mat& H, vector<Point2f>& scene_corners, const double pratio, const double proj_error,
const bool refine_position, const Mat& mask, OutputArray output)
{
if (!initialized) {return false; }
matched_features.clear();
pattern_points.clear();
vector<vector<DMatch> > matches;
vector<KeyPoint> f_keypoints;
Mat f_descriptor;
detector->detect(image, f_keypoints, mask);
if (refine_position) refineKeypointsPos(image, f_keypoints);
descriptorExtractor->compute(image, f_keypoints, f_descriptor);
descriptorMatcher->knnMatch(f_descriptor, descriptor, matches, 2); // k = 2;
vector<DMatch> good_matches;
vector<Point2f> obj_points;
for(int i = 0; i < f_descriptor.rows; ++i)
{
if(matches[i][0].distance < pratio * matches[i][1].distance)
{
const DMatch& dm = matches[i][0];
good_matches.push_back(dm);
// "keypoints1[matches[i].queryIdx] has a corresponding point in keypoints2[matches[i].trainIdx]"
matched_features.push_back(f_keypoints[dm.queryIdx].pt);
pattern_points.push_back(points3d[dm.trainIdx]);
obj_points.push_back(keypoints[dm.trainIdx].pt);
}
}
if (good_matches.size() < MIN_POINTS_FOR_H) return false;
Mat h_mask;
H = findHomography(obj_points, matched_features, RANSAC, proj_error, h_mask);
if (H.empty())
{
// cout << "findHomography() returned empty Mat." << endl;
return false;
}
for(unsigned int i = 0; i < good_matches.size(); ++i)
{
if(!h_mask.data[i])
{
deleteStdVecElem(good_matches, i);
deleteStdVecElem(matched_features, i);
deleteStdVecElem(pattern_points, i);
}
}
if (good_matches.empty()) return false;
size_t numb_elem = good_matches.size();
check_matches(matched_features, obj_points, good_matches, pattern_points, H);
if (good_matches.empty() || numb_elem < good_matches.size()) return false;
// Get the corners from the image
scene_corners = vector<Point2f>(4);
perspectiveTransform(obj_corners, scene_corners, H);
// Check correctnes of H
// Is it a convex hull?
bool cConvex = isContourConvex(scene_corners);
if (!cConvex) return false;
// Is the hull too large or small?
double scene_area = contourArea(scene_corners);
if (scene_area < MIN_CONTOUR_AREA_PX) return false;
double ratio = scene_area/img_roi.size().area();
if ((ratio < MIN_CONTOUR_AREA_RATIO) ||
(ratio > MAX_CONTOUR_AREA_RATIO)) return false;
// Is any of the projected points outside the hull?
for(unsigned int i = 0; i < good_matches.size(); ++i)
{
if(pointPolygonTest(scene_corners, f_keypoints[good_matches[i].queryIdx].pt, false) < 0)
{
deleteStdVecElem(good_matches, i);
deleteStdVecElem(matched_features, i);
deleteStdVecElem(pattern_points, i);
}
}
if (output.needed())
{
Mat out;
drawMatches(image, f_keypoints, img_roi, keypoints, good_matches, out);
// Draw lines between the corners (the mapped object in the scene - image_2 )
line(out, scene_corners[0], scene_corners[1], Scalar(0, 255, 0), 2);
line(out, scene_corners[1], scene_corners[2], Scalar(0, 255, 0), 2);
line(out, scene_corners[2], scene_corners[3], Scalar(0, 255, 0), 2);
line(out, scene_corners[3], scene_corners[0], Scalar(0, 255, 0), 2);
out.copyTo(output);
}
return (!good_matches.empty()); // return true if there are enough good matches
}
bool CustomPattern::findPattern(InputArray image, OutputArray matched_features, OutputArray pattern_points,
const double ratio, const double proj_error, const bool refine_position, OutputArray out,
OutputArray H, OutputArray pattern_corners)
{
CV_Assert(!image.empty() && proj_error > 0);
Mat img = image.getMat();
vector<Point2f> m_ftrs;
vector<Point3f> pattern_pts;
Mat _H;
vector<Point2f> scene_corners;
if (!findPatternPass(img, m_ftrs, pattern_pts, _H, scene_corners, 0.6, proj_error, refine_position))
return false; // pattern not found
Mat mask = Mat::zeros(img.size(), CV_8UC1);
vector<vector<Point> > obj(1);
vector<Point> scorners_int(scene_corners.size());
for (uint i = 0; i < scene_corners.size(); ++i)
scorners_int[i] = (Point)scene_corners[i]; // for drawContours
obj[0] = scorners_int;
drawContours(mask, obj, 0, Scalar(255), FILLED);
// Second pass
Mat output;
if (!findPatternPass(img, m_ftrs, pattern_pts, _H, scene_corners,
ratio, proj_error, refine_position, mask, output))
return false; // pattern not found
Mat(m_ftrs).copyTo(matched_features);
Mat(pattern_pts).copyTo(pattern_points);
if (out.needed()) output.copyTo(out);
if (H.needed()) _H.copyTo(H);
if (pattern_corners.needed()) Mat(scene_corners).copyTo(pattern_corners);
return (!m_ftrs.empty());
}
void CustomPattern::getPatternPoints(std::vector<KeyPoint>& original_points)
{
original_points = keypoints;
}
double CustomPattern::getPixelSize()
{
return pxSize;
}
double CustomPattern::calibrate(InputArrayOfArrays objectPoints, InputArrayOfArrays imagePoints,
Size imageSize, InputOutputArray cameraMatrix, InputOutputArray distCoeffs,
OutputArrayOfArrays rvecs, OutputArrayOfArrays tvecs, int flags,
TermCriteria criteria)
{
return calibrateCamera(objectPoints, imagePoints, imageSize, cameraMatrix, distCoeffs,
rvecs, tvecs, flags, criteria);
}
bool CustomPattern::findRt(InputArray objectPoints, InputArray imagePoints, InputArray cameraMatrix,
InputArray distCoeffs, InputOutputArray rvec, InputOutputArray tvec, bool useExtrinsicGuess, int flags)
{
return solvePnP(objectPoints, imagePoints, cameraMatrix, distCoeffs, rvec, tvec, useExtrinsicGuess, flags);
}
bool CustomPattern::findRt(InputArray image, InputArray cameraMatrix, InputArray distCoeffs,
InputOutputArray rvec, InputOutputArray tvec, bool useExtrinsicGuess, int flags)
{
vector<Point2f> imagePoints;
vector<Point3f> objectPoints;
if (!findPattern(image, imagePoints, objectPoints))
return false;
return solvePnP(objectPoints, imagePoints, cameraMatrix, distCoeffs, rvec, tvec, useExtrinsicGuess, flags);
}
bool CustomPattern::findRtRANSAC(InputArray objectPoints, InputArray imagePoints, InputArray cameraMatrix, InputArray distCoeffs,
InputOutputArray rvec, InputOutputArray tvec, bool useExtrinsicGuess, int iterationsCount,
float reprojectionError, int minInliersCount, OutputArray inliers, int flags)
{
int npoints = imagePoints.getMat().checkVector(2);
CV_Assert(npoints > 0);
double confidence_factor = (double)minInliersCount / (double)npoints;
double confidence = confidence_factor < 0.001 ? 0.001 : confidence_factor > 0.999 ? 0.999 : confidence_factor;
solvePnPRansac(objectPoints, imagePoints, cameraMatrix, distCoeffs, rvec, tvec, useExtrinsicGuess,
iterationsCount, reprojectionError, confidence, inliers, flags);
return true; // for consistency with the other methods
}
bool CustomPattern::findRtRANSAC(InputArray image, InputArray cameraMatrix, InputArray distCoeffs,
InputOutputArray rvec, InputOutputArray tvec, bool useExtrinsicGuess, int iterationsCount,
float reprojectionError, int minInliersCount, OutputArray inliers, int flags)
{
vector<Point2f> imagePoints;
vector<Point3f> objectPoints;
if (!findPattern(image, imagePoints, objectPoints))
return false;
double confidence_factor = (double)minInliersCount / (double)imagePoints.size();
double confidence = confidence_factor < 0.001 ? 0.001 : confidence_factor > 0.999 ? 0.999 : confidence_factor;
solvePnPRansac(objectPoints, imagePoints, cameraMatrix, distCoeffs, rvec, tvec, useExtrinsicGuess,
iterationsCount, reprojectionError, confidence, inliers, flags);
return true;
}
void CustomPattern::drawOrientation(InputOutputArray image, InputArray tvec, InputArray rvec,
InputArray cameraMatrix, InputArray distCoeffs,
double axis_length, int axis_width)
{
Point3f ptrCtr3d = Point3f(float((img_roi.cols * pxSize)/2.0), float((img_roi.rows * pxSize)/2.0), 0);
vector<Point3f> axis(4);
float alen = float(axis_length * pxSize);
axis[0] = ptrCtr3d;
axis[1] = Point3f(alen, 0, 0) + ptrCtr3d;
axis[2] = Point3f(0, alen, 0) + ptrCtr3d;
axis[3] = Point3f(0, 0, -alen) + ptrCtr3d;
vector<Point2f> proj_axis;
projectPoints(axis, rvec, tvec, cameraMatrix, distCoeffs, proj_axis);
Mat img = image.getMat();
line(img, proj_axis[0], proj_axis[1], Scalar(0, 0, 255), axis_width); // red
line(img, proj_axis[0], proj_axis[2], Scalar(0, 255, 0), axis_width); // green
line(img, proj_axis[0], proj_axis[3], Scalar(255, 0, 0), axis_width); // blue
img.copyTo(image);
}
}} // namespace ccalib, cv
#endif // __OPENCV_CCALIB_CPP__
#endif // cplusplus
modules/ccalib/src/multicalib.cpp
+780
View File
@@ -0,0 +1,780 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2015, Baisheng Lai (laibaisheng@gmail.com), Zhejiang University,
// all rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
/**
* This module was accepted as a GSoC 2015 project for OpenCV, authored by
* Baisheng Lai, mentored by Bo Li.
*
* The omnidirectional camera in this module is denoted by the catadioptric
* model. Please refer to Mei's paper for details of the camera model:
*
* C. Mei and P. Rives, "Single view point omnidirectional camera
* calibration from planar grids", in ICRA 2007.
*
* The implementation of the calibration part is based on Li's calibration
* toolbox:
*
* B. Li, L. Heng, K. Kevin and M. Pollefeys, "A Multiple-Camera System
* Calibration Toolbox Using A Feature Descriptor-Based Calibration
* Pattern", in IROS 2013.
*/
#include "precomp.hpp"
#include "opencv2/ccalib/multicalib.hpp"
#include "opencv2/core.hpp"
#include <string>
#include <vector>
#include <queue>
#include <iostream>
namespace cv { namespace multicalib {
MultiCameraCalibration::MultiCameraCalibration(int cameraType, int nCameras, const std::string& fileName,
float patternWidth, float patternHeight, int verbose, int showExtration, int nMiniMatches, int flags, TermCriteria criteria,
Ptr<FeatureDetector> detector, Ptr<DescriptorExtractor> descriptor,
Ptr<DescriptorMatcher> matcher)
{
_camType = cameraType;
_nCamera = nCameras;
_flags = flags;
_nMiniMatches = nMiniMatches;
_filename = fileName;
_patternWidth = patternWidth;
_patternHeight = patternHeight;
_criteria = criteria;
_showExtraction = showExtration;
_objectPointsForEachCamera.resize(_nCamera);
_imagePointsForEachCamera.resize(_nCamera);
_cameraMatrix.resize(_nCamera);
_distortCoeffs.resize(_nCamera);
_xi.resize(_nCamera);
_omEachCamera.resize(_nCamera);
_tEachCamera.resize(_nCamera);
_detector = detector;
_descriptor = descriptor;
_matcher = matcher;
_verbose = verbose;
for (int i = 0; i < _nCamera; ++i)
{
_vertexList.push_back(vertex());
}
}
double MultiCameraCalibration::run()
{
loadImages();
initialize();
double error = optimizeExtrinsics();
return error;
}
std::vector<std::string> MultiCameraCalibration::readStringList()
{
std::vector<std::string> l;
l.resize(0);
FileStorage fs(_filename, FileStorage::READ);
FileNode n = fs.getFirstTopLevelNode();
FileNodeIterator it = n.begin(), it_end = n.end();
for( ; it != it_end; ++it )
l.push_back((std::string)*it);
return l;
}
void MultiCameraCalibration::loadImages()
{
std::vector<std::string> file_list;
file_list = readStringList();
Ptr<FeatureDetector> detector = _detector;
Ptr<DescriptorExtractor> descriptor = _descriptor;
Ptr<DescriptorMatcher> matcher = _matcher;
randpattern::RandomPatternCornerFinder finder(_patternWidth, _patternHeight, _nMiniMatches, CV_32F, _verbose, this->_showExtraction, detector, descriptor, matcher);
Mat pattern = cv::imread(file_list[0]);
finder.loadPattern(pattern);
std::vector<std::vector<std::string> > filesEachCameraFull(_nCamera);
std::vector<std::vector<int> > timestampFull(_nCamera);
std::vector<std::vector<int> > timestampAvailable(_nCamera);
for (int i = 1; i < (int)file_list.size(); ++i)
{
int cameraVertex, timestamp;
std::string filename = file_list[i].substr(0, file_list[i].rfind('.'));
size_t spritPosition1 = filename.rfind('/');
size_t spritPosition2 = filename.rfind('\\');
if (spritPosition1!=std::string::npos)
{
filename = filename.substr(spritPosition1+1, filename.size() - 1);
}
else if(spritPosition2!= std::string::npos)
{
filename = filename.substr(spritPosition2+1, filename.size() - 1);
}
sscanf(filename.c_str(), "%d-%d", &cameraVertex, &timestamp);
filesEachCameraFull[cameraVertex].push_back(file_list[i]);
timestampFull[cameraVertex].push_back(timestamp);
}
// calibrate each camera individually
for (int camera = 0; camera < _nCamera; ++camera)
{
Mat image, cameraMatrix, distortCoeffs;
// find image and object points
for (int imgIdx = 0; imgIdx < (int)filesEachCameraFull[camera].size(); ++imgIdx)
{
image = imread(filesEachCameraFull[camera][imgIdx], IMREAD_GRAYSCALE);
if (!image.empty() && _verbose)
{
std::cout << "open image " << filesEachCameraFull[camera][imgIdx] << " successfully" << std::endl;
}
else if (image.empty() && _verbose)
{
std::cout << "open image" << filesEachCameraFull[camera][imgIdx] << " failed" << std::endl;
}
std::vector<Mat> imgObj = finder.computeObjectImagePointsForSingle(image);
if ((int)imgObj[0].total() > _nMiniMatches)
{
_imagePointsForEachCamera[camera].push_back(imgObj[0]);
_objectPointsForEachCamera[camera].push_back(imgObj[1]);
timestampAvailable[camera].push_back(timestampFull[camera][imgIdx]);
}
else if ((int)imgObj[0].total() <= _nMiniMatches && _verbose)
{
std::cout << "image " << filesEachCameraFull[camera][imgIdx] <<" has too few matched points "<< std::endl;
}
}
// calibrate
Mat idx;
double rms = 0.0;
if (_camType == PINHOLE)
{
rms = cv::calibrateCamera(_objectPointsForEachCamera[camera], _imagePointsForEachCamera[camera],
image.size(), _cameraMatrix[camera], _distortCoeffs[camera], _omEachCamera[camera],
_tEachCamera[camera],_flags);
idx = Mat(1, (int)_omEachCamera[camera].size(), CV_32S);
for (int i = 0; i < (int)idx.total(); ++i)
{
idx.at<int>(i) = i;
}
}
//else if (_camType == FISHEYE)
//{
// rms = cv::fisheye::calibrate(_objectPointsForEachCamera[camera], _imagePointsForEachCamera[camera],
// image.size(), _cameraMatrix[camera], _distortCoeffs[camera], _omEachCamera[camera],
// _tEachCamera[camera], _flags);
// idx = Mat(1, (int)_omEachCamera[camera].size(), CV_32S);
// for (int i = 0; i < (int)idx.total(); ++i)
// {
// idx.at<int>(i) = i;
// }
//}
else if (_camType == OMNIDIRECTIONAL)
{
rms = cv::omnidir::calibrate(_objectPointsForEachCamera[camera], _imagePointsForEachCamera[camera],
image.size(), _cameraMatrix[camera], _xi[camera], _distortCoeffs[camera], _omEachCamera[camera],
_tEachCamera[camera], _flags, TermCriteria(cv::TermCriteria::COUNT + cv::TermCriteria::EPS, 300, 1e-7),
idx);
}
_cameraMatrix[camera].convertTo(_cameraMatrix[camera], CV_32F);
_distortCoeffs[camera].convertTo(_distortCoeffs[camera], CV_32F);
_xi[camera].convertTo(_xi[camera], CV_32F);
//else
//{
// CV_Error_(cv::Error::StsOutOfRange, "Unknown camera type, use PINHOLE or OMNIDIRECTIONAL");
//}
for (int i = 0; i < (int)_omEachCamera[camera].size(); ++i)
{
int cameraVertex, timestamp, photoVertex;
cameraVertex = camera;
timestamp = timestampAvailable[camera][idx.at<int>(i)];
photoVertex = this->getPhotoVertex(timestamp);
if (_omEachCamera[camera][i].type()!=CV_32F)
{
_omEachCamera[camera][i].convertTo(_omEachCamera[camera][i], CV_32F);
}
if (_tEachCamera[camera][i].type()!=CV_32F)
{
_tEachCamera[camera][i].convertTo(_tEachCamera[camera][i], CV_32F);
}
Mat transform = Mat::eye(4, 4, CV_32F);
Mat R, T;
Rodrigues(_omEachCamera[camera][i], R);
T = (_tEachCamera[camera][i]).reshape(1, 3);
R.copyTo(transform.rowRange(0, 3).colRange(0, 3));
T.copyTo(transform.rowRange(0, 3).col(3));
this->_edgeList.push_back(edge(cameraVertex, photoVertex, idx.at<int>(i), transform));
}
std::cout << "initialized for camera " << camera << " rms = " << rms << std::endl;
std::cout << "initialized camera matrix for camera " << camera << " is" << std::endl;
std::cout << _cameraMatrix[camera] << std::endl;
std::cout << "xi for camera " << camera << " is " << _xi[camera] << std::endl;
}
}
int MultiCameraCalibration::getPhotoVertex(int timestamp)
{
int photoVertex = INVALID;
// find in existing photo vertex
for (int i = 0; i < (int)_vertexList.size(); ++i)
{
if (_vertexList[i].timestamp == timestamp)
{
photoVertex = i;
break;
}
}
// add a new photo vertex
if (photoVertex == INVALID)
{
_vertexList.push_back(vertex(Mat::eye(4, 4, CV_32F), timestamp));
photoVertex = (int)_vertexList.size() - 1;
}
return photoVertex;
}
void MultiCameraCalibration::initialize()
{
int nVertices = (int)_vertexList.size();
int nEdges = (int) _edgeList.size();
// build graph
Mat G = Mat::zeros(nVertices, nVertices, CV_32S);
for (int edgeIdx = 0; edgeIdx < nEdges; ++edgeIdx)
{
G.at<int>(this->_edgeList[edgeIdx].cameraVertex, this->_edgeList[edgeIdx].photoVertex) = edgeIdx + 1;
}
G = G + G.t();
// traverse the graph
std::vector<int> pre, order;
graphTraverse(G, 0, order, pre);
for (int i = 0; i < _nCamera; ++i)
{
if (pre[i] == INVALID)
{
std::cout << "camera" << i << "is not connected" << std::endl;
}
}
for (int i = 1; i < (int)order.size(); ++i)
{
int vertexIdx = order[i];
Mat prePose = this->_vertexList[pre[vertexIdx]].pose;
int edgeIdx = G.at<int>(vertexIdx, pre[vertexIdx]) - 1;
Mat transform = this->_edgeList[edgeIdx].transform;
if (vertexIdx < _nCamera)
{
this->_vertexList[vertexIdx].pose = transform * prePose.inv();
this->_vertexList[vertexIdx].pose.convertTo(this->_vertexList[vertexIdx].pose, CV_32F);
if (_verbose)
{
std::cout << "initial pose for camera " << vertexIdx << " is " << std::endl;
std::cout << this->_vertexList[vertexIdx].pose << std::endl;
}
}
else
{
this->_vertexList[vertexIdx].pose = prePose.inv() * transform;
this->_vertexList[vertexIdx].pose.convertTo(this->_vertexList[vertexIdx].pose, CV_32F);
}
}
}
double MultiCameraCalibration::optimizeExtrinsics()
{
// get om, t vector
int nVertex = (int)this->_vertexList.size();
Mat extrinParam(1, (nVertex-1)*6, CV_32F);
int offset = 0;
// the pose of the vertex[0] is eye
for (int i = 1; i < nVertex; ++i)
{
Mat rvec, tvec;
cv::Rodrigues(this->_vertexList[i].pose.rowRange(0,3).colRange(0, 3), rvec);
this->_vertexList[i].pose.rowRange(0,3).col(3).copyTo(tvec);
rvec.reshape(1, 1).copyTo(extrinParam.colRange(offset, offset + 3));
tvec.reshape(1, 1).copyTo(extrinParam.colRange(offset+3, offset +6));
offset += 6;
}
//double error_pre = computeProjectError(extrinParam);
// optimization
const double alpha_smooth = 0.01;
double change = 1;
for(int iter = 0; ; ++iter)
{
if ((_criteria.type == 1 && iter >= _criteria.maxCount) ||
(_criteria.type == 2 && change <= _criteria.epsilon) ||
(_criteria.type == 3 && (change <= _criteria.epsilon || iter >= _criteria.maxCount)))
break;
double alpha_smooth2 = 1 - std::pow(1 - alpha_smooth, (double)iter + 1.0);
Mat JTJ_inv, JTError;
this->computeJacobianExtrinsic(extrinParam, JTJ_inv, JTError);
Mat G = alpha_smooth2*JTJ_inv * JTError;
if (G.depth() == CV_64F)
{
G.convertTo(G, CV_32F);
}
extrinParam = extrinParam + G.reshape(1, 1);
change = norm(G) / norm(extrinParam);
//double error = computeProjectError(extrinParam);
}
double error = computeProjectError(extrinParam);
std::vector<Vec3f> rvecVertex, tvecVertex;
vector2parameters(extrinParam, rvecVertex, tvecVertex);
for (int verIdx = 1; verIdx < (int)_vertexList.size(); ++verIdx)
{
Mat R;
Mat pose = Mat::eye(4, 4, CV_32F);
Rodrigues(rvecVertex[verIdx-1], R);
R.copyTo(pose.colRange(0, 3).rowRange(0, 3));
Mat(tvecVertex[verIdx-1]).reshape(1, 3).copyTo(pose.rowRange(0, 3).col(3));
_vertexList[verIdx].pose = pose;
if (_verbose && verIdx < _nCamera)
{
std::cout << "final camera pose of camera " << verIdx << " is" << std::endl;
std::cout << pose << std::endl;
}
}
return error;
}
void MultiCameraCalibration::computeJacobianExtrinsic(const Mat& extrinsicParams, Mat& JTJ_inv, Mat& JTE)
{
int nParam = (int)extrinsicParams.total();
int nEdge = (int)_edgeList.size();
std::vector<int> pointsLocation(nEdge+1, 0);
for (int edgeIdx = 0; edgeIdx < nEdge; ++edgeIdx)
{
int nPoints = (int)_objectPointsForEachCamera[_edgeList[edgeIdx].cameraVertex][_edgeList[edgeIdx].photoIndex].total();
pointsLocation[edgeIdx+1] = pointsLocation[edgeIdx] + nPoints*2;
}
JTJ_inv = Mat(nParam, nParam, CV_64F);
JTE = Mat(nParam, 1, CV_64F);
Mat J = Mat::zeros(pointsLocation[nEdge], nParam, CV_64F);
Mat E = Mat::zeros(pointsLocation[nEdge], 1, CV_64F);
for (int edgeIdx = 0; edgeIdx < nEdge; ++edgeIdx)
{
int photoVertex = _edgeList[edgeIdx].photoVertex;
int photoIndex = _edgeList[edgeIdx].photoIndex;
int cameraVertex = _edgeList[edgeIdx].cameraVertex;
Mat objectPoints = _objectPointsForEachCamera[cameraVertex][photoIndex];
Mat imagePoints = _imagePointsForEachCamera[cameraVertex][photoIndex];
Mat rvecTran, tvecTran;
Mat R = _edgeList[edgeIdx].transform.rowRange(0, 3).colRange(0, 3);
tvecTran = _edgeList[edgeIdx].transform.rowRange(0, 3).col(3);
cv::Rodrigues(R, rvecTran);
Mat rvecPhoto = extrinsicParams.colRange((photoVertex-1)*6, (photoVertex-1)*6 + 3);
Mat tvecPhoto = extrinsicParams.colRange((photoVertex-1)*6 + 3, (photoVertex-1)*6 + 6);
Mat rvecCamera, tvecCamera;
if (cameraVertex > 0)
{
rvecCamera = extrinsicParams.colRange((cameraVertex-1)*6, (cameraVertex-1)*6 + 3);
tvecCamera = extrinsicParams.colRange((cameraVertex-1)*6 + 3, (cameraVertex-1)*6 + 6);
}
else
{
rvecCamera = Mat::zeros(3, 1, CV_32F);
tvecCamera = Mat::zeros(3, 1, CV_32F);
}
Mat jacobianPhoto, jacobianCamera, error;
computePhotoCameraJacobian(rvecPhoto, tvecPhoto, rvecCamera, tvecCamera, rvecTran, tvecTran,
objectPoints, imagePoints, this->_cameraMatrix[cameraVertex], this->_distortCoeffs[cameraVertex],
this->_xi[cameraVertex], jacobianPhoto, jacobianCamera, error);
if (cameraVertex > 0)
{
jacobianCamera.copyTo(J.rowRange(pointsLocation[edgeIdx], pointsLocation[edgeIdx+1]).
colRange((cameraVertex-1)*6, cameraVertex*6));
}
jacobianPhoto.copyTo(J.rowRange(pointsLocation[edgeIdx], pointsLocation[edgeIdx+1]).
colRange((photoVertex-1)*6, photoVertex*6));
error.copyTo(E.rowRange(pointsLocation[edgeIdx], pointsLocation[edgeIdx+1]));
}
//std::cout << J.t() * J << std::endl;
JTJ_inv = (J.t() * J + 1e-10).inv();
JTE = J.t() * E;
}
void MultiCameraCalibration::computePhotoCameraJacobian(const Mat& rvecPhoto, const Mat& tvecPhoto, const Mat& rvecCamera,
const Mat& tvecCamera, Mat& rvecTran, Mat& tvecTran, const Mat& objectPoints, const Mat& imagePoints, const Mat& K,
const Mat& distort, const Mat& xi, Mat& jacobianPhoto, Mat& jacobianCamera, Mat& E)
{
Mat drvecTran_drecvPhoto, drvecTran_dtvecPhoto,
drvecTran_drvecCamera, drvecTran_dtvecCamera,
dtvecTran_drvecPhoto, dtvecTran_dtvecPhoto,
dtvecTran_drvecCamera, dtvecTran_dtvecCamera;
MultiCameraCalibration::compose_motion(rvecPhoto, tvecPhoto, rvecCamera, tvecCamera, rvecTran, tvecTran,
drvecTran_drecvPhoto, drvecTran_dtvecPhoto, drvecTran_drvecCamera, drvecTran_dtvecCamera,
dtvecTran_drvecPhoto, dtvecTran_dtvecPhoto, dtvecTran_drvecCamera, dtvecTran_dtvecCamera);
if (rvecTran.depth() == CV_64F)
{
rvecTran.convertTo(rvecTran, CV_32F);
}
if (tvecTran.depth() == CV_64F)
{
tvecTran.convertTo(tvecTran, CV_32F);
}
float xif = 0.0f;
if (_camType == OMNIDIRECTIONAL)
{
xif= xi.at<float>(0);
}
Mat imagePoints2, jacobian, dx_drvecCamera, dx_dtvecCamera, dx_drvecPhoto, dx_dtvecPhoto;
if (_camType == PINHOLE)
{
cv::projectPoints(objectPoints, rvecTran, tvecTran, K, distort, imagePoints2, jacobian);
}
//else if (_camType == FISHEYE)
//{
// cv::fisheye::projectPoints(objectPoints, imagePoints2, rvecTran, tvecTran, K, distort, 0, jacobian);
//}
else if (_camType == OMNIDIRECTIONAL)
{
cv::omnidir::projectPoints(objectPoints, imagePoints2, rvecTran, tvecTran, K, xif, distort, jacobian);
}
if (objectPoints.depth() == CV_32F)
{
Mat(imagePoints - imagePoints2).convertTo(E, CV_64FC2);
}
else
{
E = imagePoints - imagePoints2;
}
E = E.reshape(1, (int)imagePoints.total()*2);
dx_drvecCamera = jacobian.colRange(0, 3) * drvecTran_drvecCamera + jacobian.colRange(3, 6) * dtvecTran_drvecCamera;
dx_dtvecCamera = jacobian.colRange(0, 3) * drvecTran_dtvecCamera + jacobian.colRange(3, 6) * dtvecTran_dtvecCamera;
dx_drvecPhoto = jacobian.colRange(0, 3) * drvecTran_drecvPhoto + jacobian.colRange(3, 6) * dtvecTran_drvecPhoto;
dx_dtvecPhoto = jacobian.colRange(0, 3) * drvecTran_dtvecPhoto + jacobian.colRange(3, 6) * dtvecTran_dtvecPhoto;
jacobianCamera = cv::Mat(dx_drvecCamera.rows, 6, CV_64F);
jacobianPhoto = cv::Mat(dx_drvecPhoto.rows, 6, CV_64F);
dx_drvecCamera.copyTo(jacobianCamera.colRange(0, 3));
dx_dtvecCamera.copyTo(jacobianCamera.colRange(3, 6));
dx_drvecPhoto.copyTo(jacobianPhoto.colRange(0, 3));
dx_dtvecPhoto.copyTo(jacobianPhoto.colRange(3, 6));
}
void MultiCameraCalibration::graphTraverse(const Mat& G, int begin, std::vector<int>& order, std::vector<int>& pre)
{
CV_Assert(!G.empty() && G.rows == G.cols);
int nVertex = G.rows;
order.resize(0);
pre.resize(nVertex, INVALID);
pre[begin] = -1;
std::vector<bool> visited(nVertex, false);
std::queue<int> q;
visited[begin] = true;
q.push(begin);
order.push_back(begin);
while(!q.empty())
{
int v = q.front();
q.pop();
Mat idx;
// use my findNonZero maybe
findRowNonZero(G.row(v), idx);
for(int i = 0; i < (int)idx.total(); ++i)
{
int neighbor = idx.at<int>(i);
if (!visited[neighbor])
{
visited[neighbor] = true;
q.push(neighbor);
order.push_back(neighbor);
pre[neighbor] = v;
}
}
}
}
void MultiCameraCalibration::findRowNonZero(const Mat& row, Mat& idx)
{
CV_Assert(!row.empty() && row.rows == 1 && row.channels() == 1);
Mat _row;
std::vector<int> _idx;
row.convertTo(_row, CV_32F);
for (int i = 0; i < (int)row.total(); ++i)
{
if (_row.at<float>(i) != 0)
{
_idx.push_back(i);
}
}
idx.release();
idx.create(1, (int)_idx.size(), CV_32S);
for (int i = 0; i < (int)_idx.size(); ++i)
{
idx.at<int>(i) = _idx[i];
}
}
double MultiCameraCalibration::computeProjectError(Mat& parameters)
{
int nVertex = (int)_vertexList.size();
CV_Assert((int)parameters.total() == (nVertex-1) * 6 && parameters.depth() == CV_32F);
int nEdge = (int)_edgeList.size();
// recompute the transform between photos and cameras
std::vector<edge> edgeList = this->_edgeList;
std::vector<Vec3f> rvecVertex, tvecVertex;
vector2parameters(parameters, rvecVertex, tvecVertex);
float totalError = 0;
int totalNPoints = 0;
for (int edgeIdx = 0; edgeIdx < nEdge; ++edgeIdx)
{
Mat RPhoto, RCamera, TPhoto, TCamera, transform;
int cameraVertex = edgeList[edgeIdx].cameraVertex;
int photoVertex = edgeList[edgeIdx].photoVertex;
int PhotoIndex = edgeList[edgeIdx].photoIndex;
TPhoto = Mat(tvecVertex[photoVertex - 1]).reshape(1, 3);
//edgeList[edgeIdx].transform = Mat::ones(4, 4, CV_32F);
transform = Mat::eye(4, 4, CV_32F);
cv::Rodrigues(rvecVertex[photoVertex-1], RPhoto);
if (cameraVertex == 0)
{
RPhoto.copyTo(transform.rowRange(0, 3).colRange(0, 3));
TPhoto.copyTo(transform.rowRange(0, 3).col(3));
}
else
{
TCamera = Mat(tvecVertex[cameraVertex - 1]).reshape(1, 3);
cv::Rodrigues(rvecVertex[cameraVertex - 1], RCamera);
Mat(RCamera*RPhoto).copyTo(transform.rowRange(0, 3).colRange(0, 3));
Mat(RCamera * TPhoto + TCamera).copyTo(transform.rowRange(0, 3).col(3));
}
transform.copyTo(edgeList[edgeIdx].transform);
Mat rvec, tvec;
cv::Rodrigues(transform.rowRange(0, 3).colRange(0, 3), rvec);
transform.rowRange(0, 3).col(3).copyTo(tvec);
Mat objectPoints, imagePoints, proImagePoints;
objectPoints = this->_objectPointsForEachCamera[cameraVertex][PhotoIndex];
imagePoints = this->_imagePointsForEachCamera[cameraVertex][PhotoIndex];
if (this->_camType == PINHOLE)
{
cv::projectPoints(objectPoints, rvec, tvec, _cameraMatrix[cameraVertex], _distortCoeffs[cameraVertex],
proImagePoints);
}
//else if (this->_camType == FISHEYE)
//{
// cv::fisheye::projectPoints(objectPoints, proImagePoints, rvec, tvec, _cameraMatrix[cameraVertex],
// _distortCoeffs[cameraVertex]);
//}
else if (this->_camType == OMNIDIRECTIONAL)
{
float xi = _xi[cameraVertex].at<float>(0);
cv::omnidir::projectPoints(objectPoints, proImagePoints, rvec, tvec, _cameraMatrix[cameraVertex],
xi, _distortCoeffs[cameraVertex]);
}
Mat error = imagePoints - proImagePoints;
Vec2f* ptr_err = error.ptr<Vec2f>();
for (int i = 0; i < (int)error.total(); ++i)
{
totalError += sqrt(ptr_err[i][0]*ptr_err[i][0] + ptr_err[i][1]*ptr_err[i][1]);
}
totalNPoints += (int)error.total();
}
double meanReProjError = totalError / totalNPoints;
_error = meanReProjError;
return meanReProjError;
}
void MultiCameraCalibration::compose_motion(InputArray _om1, InputArray _T1, InputArray _om2, InputArray _T2, Mat& om3, Mat& T3, Mat& dom3dom1,
Mat& dom3dT1, Mat& dom3dom2, Mat& dom3dT2, Mat& dT3dom1, Mat& dT3dT1, Mat& dT3dom2, Mat& dT3dT2)
{
Mat om1, om2, T1, T2;
_om1.getMat().convertTo(om1, CV_64F);
_om2.getMat().convertTo(om2, CV_64F);
_T1.getMat().reshape(1, 3).convertTo(T1, CV_64F);
_T2.getMat().reshape(1, 3).convertTo(T2, CV_64F);
/*Mat om2 = _om2.getMat();
Mat T1 = _T1.getMat().reshape(1, 3);
Mat T2 = _T2.getMat().reshape(1, 3);*/
//% Rotations:
Mat R1, R2, R3, dR1dom1(9, 3, CV_64FC1), dR2dom2;
cv::Rodrigues(om1, R1, dR1dom1);
cv::Rodrigues(om2, R2, dR2dom2);
/*JRodriguesMatlab(dR1dom1, dR1dom1);
JRodriguesMatlab(dR2dom2, dR2dom2);*/
dR1dom1 = dR1dom1.t();
dR2dom2 = dR2dom2.t();
R3 = R2 * R1;
Mat dR3dR2, dR3dR1;
//dAB(R2, R1, dR3dR2, dR3dR1);
matMulDeriv(R2, R1, dR3dR2, dR3dR1);
Mat dom3dR3;
cv::Rodrigues(R3, om3, dom3dR3);
//JRodriguesMatlab(dom3dR3, dom3dR3);
dom3dR3 = dom3dR3.t();
dom3dom1 = dom3dR3 * dR3dR1 * dR1dom1;
dom3dom2 = dom3dR3 * dR3dR2 * dR2dom2;
dom3dT1 = Mat::zeros(3, 3, CV_64FC1);
dom3dT2 = Mat::zeros(3, 3, CV_64FC1);
//% Translations:
Mat T3t = R2 * T1;
Mat dT3tdR2, dT3tdT1;
//dAB(R2, T1, dT3tdR2, dT3tdT1);
matMulDeriv(R2, T1, dT3tdR2, dT3tdT1);
Mat dT3tdom2 = dT3tdR2 * dR2dom2;
T3 = T3t + T2;
dT3dT1 = dT3tdT1;
dT3dT2 = Mat::eye(3, 3, CV_64FC1);
dT3dom2 = dT3tdom2;
dT3dom1 = Mat::zeros(3, 3, CV_64FC1);
}
void MultiCameraCalibration::vector2parameters(const Mat& parameters, std::vector<Vec3f>& rvecVertex, std::vector<Vec3f>& tvecVertexs)
{
int nVertex = (int)_vertexList.size();
CV_Assert((int)parameters.channels() == 1 && (int)parameters.total() == 6*(nVertex - 1));
CV_Assert(parameters.depth() == CV_32F);
parameters.reshape(1, 1);
rvecVertex.reserve(0);
tvecVertexs.resize(0);
for (int i = 0; i < nVertex - 1; ++i)
{
rvecVertex.push_back(Vec3f(parameters.colRange(i*6, i*6 + 3)));
tvecVertexs.push_back(Vec3f(parameters.colRange(i*6 + 3, i*6 + 6)));
}
}
void MultiCameraCalibration::parameters2vector(const std::vector<Vec3f>& rvecVertex, const std::vector<Vec3f>& tvecVertex, Mat& parameters)
{
CV_Assert(rvecVertex.size() == tvecVertex.size());
int nVertex = (int)rvecVertex.size();
// the pose of the first camera is known
parameters.create(1, 6*(nVertex-1), CV_32F);
for (int i = 0; i < nVertex-1; ++i)
{
Mat(rvecVertex[i]).reshape(1, 1).copyTo(parameters.colRange(i*6, i*6 + 3));
Mat(tvecVertex[i]).reshape(1, 1).copyTo(parameters.colRange(i*6 + 3, i*6 + 6));
}
}
void MultiCameraCalibration::writeParameters(const std::string& filename)
{
FileStorage fs( filename, FileStorage::WRITE );
fs << "nCameras" << _nCamera;
for (int camIdx = 0; camIdx < _nCamera; ++camIdx)
{
std::stringstream tmpStr;
tmpStr << camIdx;
std::string cameraMatrix = "camera_matrix_" + tmpStr.str();
std::string cameraPose = "camera_pose_" + tmpStr.str();
std::string cameraDistortion = "camera_distortion_" + tmpStr.str();
std::string cameraXi = "xi_" + tmpStr.str();
fs << cameraMatrix << _cameraMatrix[camIdx];
fs << cameraDistortion << _distortCoeffs[camIdx];
if (_camType == OMNIDIRECTIONAL)
{
fs << cameraXi << _xi[camIdx].at<float>(0);
}
fs << cameraPose << _vertexList[camIdx].pose;
}
fs << "meanReprojectError" <<_error;
for (int photoIdx = _nCamera; photoIdx < (int)_vertexList.size(); ++photoIdx)
{
std::stringstream tmpStr;
tmpStr << _vertexList[photoIdx].timestamp;
std::string photoTimestamp = "pose_timestamp_" + tmpStr.str();
fs << photoTimestamp << _vertexList[photoIdx].pose;
}
}
}} // namespace multicalib, cv
modules/ccalib/src/omnidir.cpp
+2273
View File
File diff suppressed because it is too large Load Diff
modules/ccalib/src/precomp.hpp
+55
View File
@@ -0,0 +1,55 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#ifndef __OPENCV_PRECOMP_H__
#define __OPENCV_PRECOMP_H__
#include <opencv2/core.hpp>
#include <opencv2/geometry.hpp>
#include <opencv2/calib.hpp>
#include <opencv2/features.hpp>
#include "opencv2/xfeatures2d.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/highgui.hpp"
#include <vector>
#endif
modules/ccalib/src/randpattern.cpp
+410
View File
@@ -0,0 +1,410 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2015, Baisheng Lai (laibaisheng@gmail.com), Zhejiang University,
// all rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
/**
* This module was accepted as a GSoC 2015 project for OpenCV, authored by
* Baisheng Lai, mentored by Bo Li.
* This module implements points extraction from a "random" pattern image.
* It returns 3D object points and 2D image pixels that can be used for calibration.
* Compared with traditional calibration object like chessboard, it is useful when pattern is
* partly occluded or only a part of pattern can be observed in multiple cameras calibration.
* For detail, refer to this paper
* B. Li, L. Heng, K. Kevin and M. Pollefeys, "A Multiple-Camera System
* Calibration Toolbox Using A Feature Descriptor-Based Calibration
* Pattern", in IROS 2013.
*/
#include "precomp.hpp"
#include "opencv2/ccalib/randpattern.hpp"
#include <iostream>
using namespace std;
namespace cv { namespace randpattern {
RandomPatternCornerFinder::RandomPatternCornerFinder(float patternWidth, float patternHeight,
int nminiMatch, int depth, int verbose, int showExtraction, Ptr<FeatureDetector> detector, Ptr<DescriptorExtractor> descriptor,
Ptr<DescriptorMatcher> matcher)
{
_patternHeight = patternHeight;
_patternWidth = patternWidth;
_nminiMatch = nminiMatch;
_objectPonits.resize(0);
_imagePoints.resize(0);
_depth = depth;
_detector = detector;
_descriptor = descriptor;
_matcher = matcher;
_showExtraction = showExtraction;
_verbose = verbose;
}
void RandomPatternCornerFinder::computeObjectImagePoints(std::vector<cv::Mat> inputImages)
{
CV_Assert(!_patternImage.empty());
CV_Assert(inputImages.size() > 0);
int nImages = (int)inputImages.size();
std::vector<Mat> imageObjectPoints;
for (int i = 0; i < nImages; ++i)
{
imageObjectPoints = computeObjectImagePointsForSingle(inputImages[i]);
if ((int)imageObjectPoints[0].total() > _nminiMatch)
{
_imagePoints.push_back(imageObjectPoints[0]);
_objectPonits.push_back(imageObjectPoints[1]);
}
}
}
void RandomPatternCornerFinder::keyPoints2MatchedLocation(const std::vector<cv::KeyPoint>& imageKeypoints,
const std::vector<cv::KeyPoint>& patternKeypoints, const std::vector<cv::DMatch> matchces,
cv::Mat& matchedImagelocation, cv::Mat& matchedPatternLocation)
{
matchedImagelocation.release();
matchedPatternLocation.release();
std::vector<Vec2d> image, pattern;
for (int i = 0; i < (int)matchces.size(); ++i)
{
Point2f imgPt = imageKeypoints[matchces[i].queryIdx].pt;
Point2f patPt = patternKeypoints[matchces[i].trainIdx].pt;
image.push_back(Vec2d(imgPt.x, imgPt.y));
pattern.push_back(Vec2d(patPt.x, patPt.y));
}
Mat(image).convertTo(matchedImagelocation, CV_64FC2);
Mat(pattern).convertTo(matchedPatternLocation, CV_64FC2);
}
void RandomPatternCornerFinder::getFilteredLocation(cv::Mat& imageKeypoints, cv::Mat& patternKeypoints, const cv::Mat mask)
{
Mat tmpKeypoint, tmpPattern;
imageKeypoints.copyTo(tmpKeypoint);
patternKeypoints.copyTo(tmpPattern);
imageKeypoints.release();
patternKeypoints.release();
std::vector<cv::Vec2d> vecKeypoint, vecPattern;
for(int i = 0; i < (int)mask.total(); ++i)
{
if (mask.at<uchar>(i) == 1)
{
vecKeypoint.push_back(tmpKeypoint.at<Vec2d>(i));
vecPattern.push_back(tmpPattern.at<Vec2d>(i));
}
}
Mat(vecKeypoint).convertTo(imageKeypoints, CV_64FC2);
Mat(vecPattern).convertTo(patternKeypoints, CV_64FC2);
}
void RandomPatternCornerFinder::getObjectImagePoints(const cv::Mat& imageKeypoints, const cv::Mat& patternKeypoints)
{
Mat imagePoints_i, objectPoints_i;
int imagePointsType = CV_MAKETYPE(_depth, 2);
int objectPointsType = CV_MAKETYPE(_depth, 3);
imageKeypoints.convertTo(imagePoints_i, imagePointsType);
_imagePoints.push_back(imagePoints_i);
if (patternKeypoints.total()!=CV_64FC2)
patternKeypoints.convertTo(patternKeypoints, CV_64FC2);
std::vector<Vec3d> objectPoints;
for (int i = 0; i < (int)patternKeypoints.total(); ++i)
{
double x = patternKeypoints.at<Vec2d>(i)[0];
double y = patternKeypoints.at<Vec2d>(i)[1];
x = x / _patternImageSize.width * _patternWidth;
y = y / _patternImageSize.height * _patternHeight;
objectPoints.push_back(Vec3d(x, y, 0));
}
Mat(objectPoints).convertTo(objectPoints_i, objectPointsType);
_objectPonits.push_back(objectPoints_i);
}
void RandomPatternCornerFinder::crossCheckMatching( Ptr<DescriptorMatcher>& descriptorMatcher,
const Mat& descriptors1, const Mat& descriptors2,
std::vector<DMatch>& filteredMatches12, int knn )
{
filteredMatches12.clear();
std::vector<std::vector<DMatch> > matches12, matches21;
descriptorMatcher->knnMatch( descriptors1, descriptors2, matches12, knn );
descriptorMatcher->knnMatch( descriptors2, descriptors1, matches21, knn );
for( size_t m = 0; m < matches12.size(); m++ )
{
bool findCrossCheck = false;
for( size_t fk = 0; fk < matches12[m].size(); fk++ )
{
DMatch forward = matches12[m][fk];
for( size_t bk = 0; bk < matches21[forward.trainIdx].size(); bk++ )
{
DMatch backward = matches21[forward.trainIdx][bk];
if( backward.trainIdx == forward.queryIdx )
{
filteredMatches12.push_back(forward);
findCrossCheck = true;
break;
}
}
if( findCrossCheck ) break;
}
}
}
void RandomPatternCornerFinder::drawCorrespondence(const Mat& image1, const std::vector<cv::KeyPoint> keypoint1,
const Mat& image2, const std::vector<cv::KeyPoint> keypoint2, const std::vector<cv::DMatch> matchces,
const Mat& mask1, const Mat& mask2, const int step)
{
Mat img_corr;
if(step == 1)
{
drawMatches(image1, keypoint1, image2, keypoint2, matchces, img_corr);
}
else if(step == 2)
{
std::vector<cv::DMatch> matchesFilter;
for (int i = 0; i < (int)mask1.total(); ++i)
{
if (!mask1.empty() && mask1.at<uchar>(i) == 1)
{
matchesFilter.push_back(matchces[i]);
}
}
drawMatches(image1, keypoint1, image2, keypoint2, matchesFilter, img_corr);
}
else if(step == 3)
{
std::vector<cv::DMatch> matchesFilter;
int j = 0;
for (int i = 0; i < (int)mask1.total(); ++i)
{
if (mask1.at<uchar>(i) == 1)
{
if (!mask2.empty() && mask2.at<uchar>(j) == 1)
{
matchesFilter.push_back(matchces[i]);
}
j++;
}
}
drawMatches(image1, keypoint1, image2, keypoint2, matchesFilter, img_corr);
}
imshow("correspondence", img_corr);
waitKey(0);
}
const std::vector<cv::Mat> &RandomPatternCornerFinder::getObjectPoints()
{
return _objectPonits;
}
const std::vector<cv::Mat> &RandomPatternCornerFinder::getImagePoints()
{
return _imagePoints;
}
void RandomPatternCornerFinder::loadPattern(const cv::Mat &patternImage)
{
_patternImage = patternImage.clone();
if (_patternImage.type()!= CV_8U)
_patternImage.convertTo(_patternImage, CV_8U);
_patternImageSize = _patternImage.size();
_detector->detect(patternImage, _keypointsPattern);
_descriptor->compute(patternImage, _keypointsPattern, _descriptorPattern);
_descriptorPattern.convertTo(_descriptorPattern, CV_32F);
}
void RandomPatternCornerFinder::loadPattern(const cv::Mat &patternImage, const std::vector<cv::KeyPoint> &patternKeyPoints, const cv::Mat &patternDescriptors)
{
CV_Assert((int)patternKeyPoints.size()==patternDescriptors.rows);
CV_Assert(patternDescriptors.cols==_descriptor->descriptorSize());
CV_Assert(patternDescriptors.type()==_descriptor->descriptorType());
_patternImage=patternImage.clone();
if(_patternImage.type()!=CV_8U)
_patternImage.convertTo(_patternImage, CV_8U);
_patternImageSize=patternImage.size();
_keypointsPattern=patternKeyPoints;
_descriptorPattern=patternDescriptors.clone();
_descriptorPattern.convertTo(_descriptorPattern, CV_32F);
}
std::vector<cv::Mat> RandomPatternCornerFinder::computeObjectImagePointsForSingle(cv::Mat inputImage)
{
CV_Assert(!_patternImage.empty());
std::vector<cv::Mat> r(2);
Mat descriptorImage1, descriptorImage2,descriptorImage;
std::vector<cv::KeyPoint> keypointsImage1, keypointsImage2, keypointsImage;
if (inputImage.type()!=CV_8U)
{
inputImage.convertTo(inputImage, CV_8U);
}
Mat imageEquHist;
equalizeHist(inputImage, imageEquHist);
_detector->detect(inputImage, keypointsImage1);
_descriptor->compute(inputImage, keypointsImage1, descriptorImage1);
_detector->detect(imageEquHist, keypointsImage2);
_descriptor->compute(imageEquHist, keypointsImage2, descriptorImage2);
descriptorImage1.convertTo(descriptorImage1, CV_32F);
descriptorImage2.convertTo(descriptorImage2, CV_32F);
// match with pattern
std::vector<DMatch> matchesImgtoPat, matchesImgtoPat1, matchesImgtoPat2;
cv::Mat keypointsImageLocation, keypointsPatternLocation;
crossCheckMatching(this->_matcher, descriptorImage1, this->_descriptorPattern, matchesImgtoPat1, 1);
crossCheckMatching(this->_matcher, descriptorImage2, this->_descriptorPattern, matchesImgtoPat2, 1);
if ((int)matchesImgtoPat1.size() > (int)matchesImgtoPat2.size())
{
matchesImgtoPat = matchesImgtoPat1;
keypointsImage = keypointsImage1;
}
else
{
matchesImgtoPat = matchesImgtoPat2;
keypointsImage = keypointsImage2;
}
keyPoints2MatchedLocation(keypointsImage, this->_keypointsPattern, matchesImgtoPat,
keypointsImageLocation, keypointsPatternLocation);
Mat img_corr;
// innerMask is CV_8U type
Mat innerMask1, innerMask2;
// draw raw correspondence
if(this->_showExtraction)
{
drawCorrespondence(inputImage, keypointsImage, _patternImage, _keypointsPattern, matchesImgtoPat,
innerMask1, innerMask2, 1);
}
if (_verbose)
{
std::cout << "number of matched points " << (int)keypointsImageLocation.total() << std::endl;
}
// outlier remove
findFundamentalMat(keypointsImageLocation, keypointsPatternLocation,
FM_RANSAC, 1, 0.995, innerMask1);
getFilteredLocation(keypointsImageLocation, keypointsPatternLocation, innerMask1);
if (this->_showExtraction)
{
drawCorrespondence(inputImage, keypointsImage, _patternImage, _keypointsPattern, matchesImgtoPat,
innerMask1, innerMask2, 2);
}
findHomography(keypointsImageLocation, keypointsPatternLocation, RANSAC, 30*inputImage.cols/1000, innerMask2);
getFilteredLocation(keypointsImageLocation, keypointsPatternLocation, innerMask2);
if (_verbose)
{
std::cout << "number of filtered points " << (int)keypointsImageLocation.total() << std::endl;
}
// draw filtered correspondence
if (this->_showExtraction)
{
drawCorrespondence(inputImage, keypointsImage, _patternImage, _keypointsPattern, matchesImgtoPat,
innerMask1, innerMask2, 3);
}
std::vector<Vec3d> objectPoints;
int imagePointsType = CV_MAKETYPE(_depth, 2);
int objectPointsType = CV_MAKETYPE(_depth, 3);
keypointsImageLocation.convertTo(r[0], imagePointsType);
for (int i = 0; i < (int)keypointsPatternLocation.total(); ++i)
{
double x = keypointsPatternLocation.at<Vec2d>(i)[0];
double y = keypointsPatternLocation.at<Vec2d>(i)[1];
x = x / _patternImageSize.width * _patternWidth;
y = y / _patternImageSize.height * _patternHeight;
objectPoints.push_back(Vec3d(x, y, 0));
}
Mat(objectPoints).convertTo(r[1], objectPointsType);
return r;
}
RandomPatternGenerator::RandomPatternGenerator(int imageWidth, int imageHeight)
{
_imageWidth = imageWidth;
_imageHeight = imageHeight;
}
void RandomPatternGenerator::generatePattern()
{
Mat pattern = Mat(_imageHeight, _imageWidth, CV_32F, Scalar(0.));
int m = 5;
int count = 0;
while(m < _imageWidth)
{
int n = (int)std::floor(double(_imageHeight) / double(_imageWidth) * double(m)) + 1;
Mat r = Mat(n, m, CV_32F);
cv::randn(r, Scalar::all(0), Scalar::all(1));
cv::resize(r, r, Size(_imageWidth ,_imageHeight));
double min_r, max_r;
minMaxLoc(r, &min_r, &max_r);
r = (r - (float)min_r) / float(max_r - min_r);
pattern += r;
count += 1;
m *= 2;
}
pattern = pattern / count * 255;
pattern.convertTo(pattern, CV_8U);
equalizeHist(pattern, pattern);
pattern.copyTo(_pattern);
}
cv::Mat RandomPatternGenerator::getPattern()
{
return _pattern;
}
}} //namespace randpattern, cv
modules/ccalib/tutorials/data/omni_calib_data.xml
+1146
View File
File diff suppressed because it is too large Load Diff
modules/ccalib/tutorials/data/omni_stereocalib_data.xml
+4683
View File
File diff suppressed because it is too large Load Diff
modules/ccalib/tutorials/img/disparity.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 119 KiB

modules/ccalib/tutorials/img/imgs.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 104 KiB

modules/ccalib/tutorials/img/lines.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 182 KiB

modules/ccalib/tutorials/img/pattern_img.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 105 KiB

modules/ccalib/tutorials/img/pointCloud.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 41 KiB

modules/ccalib/tutorials/img/random_pattern.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 100 KiB

modules/ccalib/tutorials/img/sample.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 74 KiB

modules/ccalib/tutorials/img/sample_rec_cyl.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 93 KiB

modules/ccalib/tutorials/img/sample_rec_log.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 63 KiB

modules/ccalib/tutorials/img/sample_rec_per.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 51 KiB

modules/ccalib/tutorials/img/sample_rec_ste.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 105 KiB

modules/ccalib/tutorials/multi_camera_tutorial.markdown
+45
View File
@@ -0,0 +1,45 @@
Multi-camera Calibration {#tutorial_multi_camera_main}
====================================
This tutorial will show how to use the multiple camera calibration toolbox. This toolbox is based on the usage of "random" pattern calibration object, so the tutorial is mainly two parts: an introduction to "random" pattern and multiple camera calibration.
Random Pattern Calibration Object
-------------------------------
The random pattern is an image that is randomly generated. It is "random" so that it has many feature points. After generating it, print it out and use it as a calibration object. The following two images are random pattern and a photo taken for it.
![image](img/random_pattern.jpg)
![image](img/pattern_img.jpg)
To generate a random pattern, use the class ```cv::randpattern::RandomPatternGenerator``` in ```ccalib``` module. Run it as
```
cv::randpattern::RandomPatternGenerator generator(width, height);
generator.generatePattern();
cv::Mat pattern = generator.getPattern();
```
Here ```width``` and ```height``` are width and height of pattern image. After getting the pattern, print it out and take some photos of it.
Now we can use these images to calibrate camera. First, ```objectPoints``` and ```imagePoints``` need to be detected. Use class ```cv::randpattern::RandomPatternCornerFinder``` to detect them. A sample code can be
```
cv::randpattern::RandomPatternCornerFinder finder(patternWidth, patternHeight, nMiniMatches);
finder.loadPattern(pattern);
finder.computeObjectImagePoints(vecImg);
vector<Mat> objectPoints = finder.getObjectPoints();
vector<Mat> imagePoints = finder.getImagePoints();
```
Here the variables ```patternWidth``` and ```patternHeight``` refer to the physical dimensions of the calibration object in the chosen unit of measurement. ```vecImg``` is a vector of images that stores calibration images.
Second, use calibration functions like ```cv::calibrateCamera``` or ```cv::omnidir::calibrate``` to calibrate camera.
Multiple Cameras Calibration
-------------------------------
Now we move to multiple camera calibration, so far this toolbox must use random pattern object.
To calibrate multiple cameras, we first need to take some photos of random pattern. Of course, to calibrate the extrinsic parameters, one pattern needs to be viewed by multiple cameras (at least two) at the same time. Another thing is that to help the program know which camera and which pattern the photo is taken, the image file should be named as "cameraIdx-timestamp.*". Photos with same timestamp means that they are the same object taken by several cameras. In addition, cameraIdx should start from 0. Some examples of files names are "0-129.png", "0-187.png", "1-187", "2-129".
Then, we can run multiple cameras calibration as
```
cv::multicalib::MultiCameraCalibration multiCalib(cameraType, nCamera, inputFilename,patternWidth, patternHeight, showFeatureExtraction, nMiniMatches);
multiCalib.run();
multiCalib.writeParameters(outputFilename);
```
Here ```cameraType``` indicates the camera type, ```multicalib::MultiCameraCalibration::PINHOLE``` and ```multicalib::MultiCameraCalibration::OMNIDIRECTIONAL``` are supported. For omnidirectional camera, you can refer to ```cv::omnidir``` module for detail. ```nCamera``` is the number of cameras. ```inputFilename``` is the name of a file generated by ```imagelist_creator``` from ```opencv/sample```. It stores names of random pattern and calibration images, the first file name is the name of random pattern. ```patternWidth``` and ```patternHeight``` represents the physical width and height of the pattern. ```showFeatureExtraction``` is a boolean flag that determines whether the feature extraction process is displayed. ```nMiniMatches``` is the minimum number of points that should be detected in each frame, otherwise this frame will be abandoned. ```outputFilename``` is an XML that will store the calibration parameters.
modules/ccalib/tutorials/omnidir_tutorial.markdown
+185
View File
@@ -0,0 +1,185 @@
Omnidirectional Camera Calibration {#tutorial_omnidir_calib_main}
======================
This module includes calibration, rectification and stereo reconstruction of omnidirectional camearas. The camera model is described in this paper:
*C. Mei and P. Rives, Single view point omnidirectional camera calibration from planar grids, in ICRA 2007.*
The model is capable of modeling catadioptric cameras and fisheye cameras, which may both have very large field of view.
The implementation of the calibration part is based on Li's calibration toolbox:
*B. Li, L. Heng, K. Kevin and M. Pollefeys, "A Multiple-Camera System Calibration Toolbox Using A Feature Descriptor-Based Calibration Pattern", in IROS 2013.*
This tutorial will introduce the following parts of omnidirectional camera calibartion module:
- calibrate a single camera.
- calibrate a stereo pair of cameras.
- rectify images so that large distoration is removed.
- reconstruct 3D from two stereo images, with large filed of view.
- comparison with fisheye model in opencv/calib/
Single Camera Calibration
---------------------
The first step to calibrate camera is to get a calibration pattern and take some photos. Several kinds of patterns are supported by OpenCV, like checkerborad and circle grid. A new pattern named random pattern can also be used, you can refer to opencv_contrib/modules/ccalib for more details.
Next step is to extract corners from calibration pattern. For checkerboard, use OpenCV function ```cv::findChessboardCorners```; for circle grid, use ```cv::findCirclesGrid```, for random pattern, use the ```randomPatternCornerFinder``` class in opencv_contrib/modules/ccalib/src/randomPattern.hpp. Save the positions of corners in images in a variable like ```imagePoints```. The type of ```imagePoints``` may be ```std::vector<std::vector<cv::Vec2f>>```, the first vector stores corners in each frame, the second vector stores corners in an individual frame. The type can also be ```std::vector<cv::Mat>``` where the ```cv::Mat``` is ```CV_32FC2```.
Also, the corresponding 3D points in world (pattern) coordinate are required. You can compute they for yourself if you know the physical size of your pattern. Save 3D points in ```objectPoints```, similar to ```imagePoints```, it can be ```std::vector<std::vector<Vec3f>>``` or ```std::vector<cv::Mat>``` where ```cv::Mat``` is of type ```CV_32FC3```. Note the size of ```objectPoints``` and ```imagePoints``` must be the same because they are corresponding to each other.
Another thing you should input is the size of images. The file opencv_contrib/modules/ccalib/tutorial/data/omni_calib_data.xml stores an example of objectPoints, imagePoints and imageSize. Use the following code to load them:
```
cv::FileStorage fs("omni_calib_data.xml", cv::FileStorage::READ);
std::vector<cv::Mat> objectPoints, imagePoints;
cv::Size imgSize;
fs["objectPoints"] >> objectPoints;
fs["imagePoints"] >> imagePoints;
fs["imageSize"] >> imgSize;
```
Then define some variables to store the output parameters and run the calibration function like:
```
cv::Mat K, xi, D, idx;
int flags = 0;
cv::TermCriteria critia(cv::TermCriteria::COUNT + cv::TermCriteria::EPS, 200, 0.0001);
std::vector<cv::Mat> rvecs, tvecs;
double rms = cv::omnidir::calibrate(objectPoints, imagePoints, imgSize, K, xi, D, rvecs, tvecs, flags, critia, idx);
```
`K`, ```xi```, ```D``` are internal parameters and ```rvecs```, ```tvecs``` are external parameters that store the pose of patterns. All of them have depth of ```CV_64F```. The ```xi``` is a single value variable of Mei's model. ```idx``` is a ```CV_32S``` Mat that stores indices of images that are really used in calibration. This is due to some images are failed in the initialization step so they are not used in the final optimization. The returned value *rms* is the root mean square of reprojection errors.
The calibration supports some features, *flags* is a enumeration for some features, including:
- cv::omnidir::CALIB_FIX_SKEW
- cv::omnidir::CALIB_FIX_K1
- cv::omnidir::CALIB_FIX_K2
- cv::omnidir::CALIB_FIX_P1
- cv::omnidir::CALIB_FIX_P2
- cv::omnidir::CALIB_FIX_XI
- cv::omnidir::CALIB_FIX_GAMMA
- cv::omnidir::CALIB_FIX_CENTER
Your can specify ```flags``` to fix parameters during calibration. Use 'plus' operator to set multiple features. For example, ```CALIB_FIX_SKEW+CALIB_FIX_K1``` means fixing skew and K1.
`criteria` is the stopping criteria during optimization, set it to be, for example, cv::TermCriteria(cv::TermCriteria::COUNT + cv::TermCriteria::EPS, 200, 0.0001), which means using 200 iterations and stopping when relative change is smaller than 0.0001.
Stereo Calibration
--------------------------
Stereo calibration is to calibrate two cameras together. The output parameters include camera parameters of two cameras and the relative pose of them. To recover the relative pose, two cameras must observe the same pattern at the same time, so the ```objectPoints``` of two cameras are the same.
Now detect image corners for both cameras as discussed above to get ```imagePoints1``` and ```imagePoints2```. Then compute the shared ```objectPoints```.
An example of of stereo calibration data is stored in opencv_contrib/modules/ccalib/tutorial/data/omni_stereocalib_data.xml. Load the data by
```
cv::FileStorage fs("omni_stereocalib_data.xml", cv::FileStorage::READ);
std::vector<cv::Mat> objectPoints, imagePoints1, imagePoints2;
cv::Size imgSize1, imgSize2;
fs["objectPoints"] >> objectPoints;
fs["imagePoints1"] >> imagePoints1;
fs["imagePoints2"] >> imagePoints2;
fs["imageSize1"] >> imgSize1;
fs["imageSize2"] >> imgSize2;
```
Then do stereo calibration by
```
cv::Mat K1, K2, xi1, xi2, D1, D2;
int flags = 0;
cv::TermCriteria critia(cv::TermCriteria::COUNT + cv::TermCriteria::EPS, 200, 0.0001);
std::vector<cv::Mat> rvecsL, tvecsL;
cv::Mat rvec, tvec;
double rms = cv::omnidir::stereoCalibrate(objectPoints, imagePoints1, imagePoints2, imgSize1, imgSize2, K1, xi1, D1, K2, xi2, D2, rvec, tvec, rvecsL, tvecsL, flags, critia, idx);
```
Here ```rvec``` and ```tvec``` are the transform between the first and the second camera. ```rvecsL``` and ```tvecsL``` are the transforms between patterns and the first camera.
Image Rectificaiton
---------------------------
Omnidirectional images have very large distortion, so it is not compatible with human's eye balls. For better view, rectification can be applied if camera parameters are known. Here is an example of omnidirectional image of 360 degrees of horizontal field of view.
![image](img/sample.jpg)
After rectification, a perspective like view is generated. Here is one example to run image rectification in this module:
```
cv::omnidir::undistortImage(distorted, undistorted, K, D, xi, int flags, Knew, new_size)
```
The variable *distorted* and *undistorted* are the origional image and rectified image perspectively. *K*, *D*, *xi* are camera parameters. *KNew* and *new_size* are the camera matrix and image size for rectified image. *flags* is the rectification type, it can be:
- RECTIFY_PERSPECTIVE: rectify to perspective images, which will lose some filed of view.
- RECTIFY_CYLINDRICAL: rectify to cylindrical images that preserve all view.
- RECTIFY_STEREOGRAPHIC: rectify to stereographic images that may lose a little view.
- RECTIFY_LONGLATI: rectify to longitude-latitude map like a world map of the earth. This rectification can be used to stereo reconstruction but may not be friendly for view. This map is described in paper:
*Li S. Binocular spherical stereo[J]. Intelligent Transportation Systems, IEEE Transactions on, 2008, 9(4): 589-600.*
The following four images are four types of rectified images discribed above:
![image](img/sample_rec_per.jpg)
![image](img/sample_rec_cyl.jpg)
![image](img/sample_rec_ste.jpg)
![image](img/sample_rec_log.jpg)
It can be observed that perspective rectified image perserves only a little field of view and is not goodlooking. Cylindrical rectification preserves all field of view and scene is unnatural only in the middle of bottom. The distortion of stereographic in the middle of bottom is smaller than cylindrical but the distortion of other places are larger, and it can not preserve all field of view. For images with very large distortion, the longitude-latitude rectification does not give a good result, but it is available to make epipolar constraint in a line so that stereo matching can be applied in omnidirectional images.
**Note**: To have a better result, you should carefully choose ```Knew``` and it is related to your camera. In general, a smaller focal length leads to a smaller field of view and vice versa. Here are recommonded settings.
For RECTIFY_PERSPECTIVE
```
Knew = Matx33f(new_size.width/4, 0, new_size.width/2,
0, new_size.height/4, new_size.height/2,
0, 0, 1);
```
For RECTIFY_CYLINDRICAL, RECTIFY_STEREOGRAPHIC, RECTIFY_LONGLATI
```
Knew = Matx33f(new_size.width/3.1415, 0, 0,
0, new_size.height/3.1415, 0,
0, 0, 1);
```
Maybe you need to change ```(u0, v0)``` to get a better view.
Stereo Reconstruction
------------------------------
Stereo reconstruction is to reconstruct 3D points from a calibrated stereo camera pair. It is a basic problem of computer vison. However, for omnidirectional camera, it is not very popular because of the large distortion make it a little difficult. Conventional methods rectify images to perspective ones and do stereo reconstruction in perspective images. However, the last section shows that recifying to perspective images lose too much field of view, which waste the advantage of omnidirectional camera, i.e. large field of view.
The first step of stereo reconstruction is stereo rectification so that epipolar lines are horizontal lines. Here, we use longitude-latitude rectification to preserve all filed of view, or perspective rectification which is available but is not recommended. The second step is stereo matching to get a disparity map. At last, 3D points can be generated from disparity map.
The API of stereo reconstruction for omnidrectional camera is ```omnidir::stereoReconstruct```. Here we use an example to show how it works.
First, calibrate a stereo pair of cameras as described above and get parameters like ```K1```, ```D1```, ```xi1```, ```K2```, ```D2```, ```xi2```, ```rvec```, ```tvec```. Then read two images from the first and second camera respectively, for instance, ```image1``` and ```image2```, which are shown below.
![image](img/imgs.jpg)
Second, run ```omnidir::stereoReconstruct``` like:
```
cv::Size imgSize = img1.size();
int numDisparities = 16*5;
int SADWindowSize = 5;
cv::Mat disMap;
int flag = cv::omnidir::RECTIFY_LONGLATI;
int pointType = omnidir::XYZRGB;
// the range of theta is (0, pi) and the range of phi is (0, pi)
cv::Matx33d KNew(imgSize.width / 3.1415, 0, 0, 0, imgSize.height / 3.1415, 0, 0, 0, 1);
Mat imageRec1, imageRec2, pointCloud;
cv::omnidir::stereoReconstruct(img1, img2, K1, D1, xi1, K2, D2, xi2, R, T, flag, numDisparities, SADWindowSize, disMap, imageRec1, imageRec2, imgSize, KNew, pointCloud);
```
Here variable ```flag``` indicates the recectify type, only ```RECTIFY_LONGLATI```(recommend) and ```RECTIFY_PERSPECTIVE``` make sense. ```numDisparities``` is the max disparity value and ```SADWindowSize``` is the window size of ```cv::StereoSGBM```. ```pointType``` is a flag to define the type of point cloud, ```omnidir::XYZRGB``` each point is a 6-dimensional vector, the first three elements are xyz coordinate, the last three elements are rgb color information. Another type ```omnidir::XYZ``` means each point is 3-dimensional and has only xyz coordiante.
Moreover, ```imageRec1``` and ```imagerec2``` are rectified versions of the first and second images. The epipolar lines of them have the same y-coordinate so that stereo matching becomes easy. Here are an example of them:
![image](img/lines.jpg)
It can be observed that they are well aligned. The variable ```disMap``` is the disparity map computed by ```cv::StereoSGBM``` from ```imageRec1``` and ```imageRec2```. The disparity map of the above two images is:
![image](img/disparity.jpg)
After we have disparity, we can compute 3D location for each pixel. The point cloud is stored in ```pointCloud```, which is a 3-channel or 6-channel ```cv::Mat```. We show the point cloud in the following image.
![image](img/pointCloud.jpg)