mirrors/opencv_contrib
1
0
Fork 0
Code Pull Requests Packages Releases Activity

vendor: OpenCV 5.0.0 snapshot at 755e50675d97db9b7d449d8bd6b09888646f6c6e

Browse Source
This commit is contained in:
Gitea Mirror Bot
2026-08-22 00:11:13 +08:00
commit 12022378a3
3872 changed files with 2513409 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/optflow/CMakeLists.txt
+2
View File
@@ -0,0 +1,2 @@
set(the_description "Optical Flow Algorithms")
ocv_define_module(optflow opencv_core opencv_imgproc opencv_features opencv_geometry opencv_video opencv_ximgproc opencv_imgcodecs opencv_flann WRAP python)
modules/optflow/README.md
+4
View File
@@ -0,0 +1,4 @@
Optical Flow Algorithms
=======================
Algorithms for running and evaluating deepflow, simpleflow, sparsetodenseflow, robust local optical flow and motion templates (silhouette flow).
modules/optflow/doc/optflow.bib
+129
View File
@@ -0,0 +1,129 @@
@article{Bradski00,
title={Motion segmentation and pose recognition with motion history gradients},
author={Bradski, Gary R and Davis, James W},
journal={Machine Vision and Applications},
volume={13},
number={3},
pages={174--184},
year={2002},
publisher={Springer}
}
@inproceedings{Davis97,
title={The representation and recognition of human movement using temporal templates},
author={Davis, James W and Bobick, Aaron F},
booktitle={Computer Vision and Pattern Recognition, 1997. Proceedings., 1997 IEEE Computer Society Conference on},
pages={928--934},
year={1997},
organization={IEEE}
}
@inproceedings{Tao2012,
title={SimpleFlow: A Non-iterative, Sublinear Optical Flow Algorithm},
author={Tao, Michael and Bai, Jiamin and Kohli, Pushmeet and Paris, Sylvain},
booktitle={Computer Graphics Forum},
volume={31},
number={2pt1},
pages={345--353},
year={2012},
organization={Wiley Online Library}
}
@inproceedings{Weinzaepfel2013,
title={DeepFlow: Large displacement optical flow with deep matching},
author={Weinzaepfel, Philippe and Revaud, Jerome and Harchaoui, Zaid and Schmid, Cordelia},
booktitle={Computer Vision (ICCV), 2013 IEEE International Conference on},
pages={1385--1392},
year={2013},
organization={IEEE}
}
@inproceedings{Brox2004,
title={High accuracy optical flow estimation based on a theory for warping},
author={Brox, Thomas and Bruhn, Andr{\'e}s and Papenberg, Nils and Weickert, Joachim},
booktitle={European Conference on Computer Vision (ECCV)},
pages={25--36},
year={2004}
}
@inproceedings{Wulff:CVPR:2015,
title = {Efficient Sparse-to-Dense Optical Flow Estimation using a Learned Basis and Layers},
author = {Wulff, Jonas and Black, Michael J.},
booktitle = { IEEE Conf. on Computer Vision and Pattern Recognition (CVPR) 2015},
month = {June},
year = {2015}
}
@inproceedings{Wang_2016_CVPR,
author = {Wang, Shenlong and Ryan Fanello, Sean and Rhemann, Christoph and Izadi, Shahram and Kohli, Pushmeet},
title = {The Global Patch Collider},
booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},
month = {June},
year = {2016}
}
@inproceedings{Geistert2016,
author = {Jonas Geistert and Tobias Senst and Thomas Sikora},
title = {Robust Local Optical Flow: Dense Motion Vector Field Interpolation},
booktitle = {Picture Coding Symposium},
year = {2016},
pages = {1--5},
}
@inproceedings{Senst2016,
author = {Tobias Senst and Jonas Geistert and Thomas Sikora},
title = {Robust local optical flow: Long-range motions and varying illuminations},
booktitle = {IEEE International Conference on Image Processing},
year = {2016},
pages = {4478--4482},
}
@inproceedings{Senst2014,
author = {Tobias Senst and Thilo Borgmann and Ivo Keller and Thomas Sikora},
title = {Cross based Robust Local Optical Flow},
booktitle = {21th IEEE International Conference on Image Processing},
year = {2014},
pages = {1967--1971},
}
@inproceedings{Senst2013,
author = {Tobias Senst and Jonas Geistert and Ivo Keller and Thomas Sikora},
title = {Robust Local Optical Flow Estimation using Bilinear Equations for Sparse Motion Estimation},
booktitle = {20th IEEE International Conference on Image Processing},
year = {2013},
pages = {2499--2503},
}
@article{Senst2012,
author = {Tobias Senst and Volker Eiselein and Thomas Sikora},
title = {Robust Local Optical Flow for Feature Tracking},
journal = {IEEE Transactions on Circuits and Systems for Video Technology},
year = {2012},
pages = {1377--1387},
volume = {22},
number = {9},
}
@phdthesis{Senst2019,
title={Estimation and analysis of motion in video data},
author={Senst, Tobias},
year={2019},
school={Technical Univerity Berlin},
doi = {10.14279/depositonce-9085},
url = {https://doi.org/10.14279/depositonce-9085}
}
@article{Tibshirani2008,
title={Fast computation of the median by successive binning},
author={Tibshirani, Ryan J},
journal={arXiv preprint arXiv:0806.3301},
year={2008}
}
@inproceedings{Hu2017,
title={Robust interpolation of correspondences for large displacement optical flow},
author={Hu, Yinlin and Li, Yunsong and Song, Rui},
booktitle={IEEE Conference on Computer Vision and Pattern Recognition},
pages={481--489},
year={2017}
}
modules/optflow/include/opencv2/optflow.hpp
+309
View File
@@ -0,0 +1,309 @@
/*
By downloading, copying, installing or using the software you agree to this
license. If you do not agree to this license, do not download, install,
copy or use the software.
License Agreement
For Open Source Computer Vision Library
(3-clause BSD License)
Copyright (C) 2013, OpenCV Foundation, all rights reserved.
Third party copyrights are property of their respective owners.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the names of the copyright holders nor the names of the contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
This software is provided by the copyright holders and contributors "as is" and
any express or implied warranties, including, but not limited to, the implied
warranties of merchantability and fitness for a particular purpose are
disclaimed. In no event shall copyright holders or contributors be liable for
any direct, indirect, incidental, special, exemplary, or consequential damages
(including, but not limited to, procurement of substitute goods or services;
loss of use, data, or profits; or business interruption) however caused
and on any theory of liability, whether in contract, strict liability,
or tort (including negligence or otherwise) arising in any way out of
the use of this software, even if advised of the possibility of such damage.
*/
#ifndef __OPENCV_OPTFLOW_HPP__
#define __OPENCV_OPTFLOW_HPP__
#include "opencv2/core.hpp"
#include "opencv2/video.hpp"
/**
@defgroup optflow Optical Flow Algorithms
Dense optical flow algorithms compute motion for each point:
- cv::optflow::calcOpticalFlowSF
- cv::optflow::createOptFlow_DeepFlow
Motion templates is alternative technique for detecting motion and computing its direction.
See samples/motempl.py.
- cv::motempl::updateMotionHistory
- cv::motempl::calcMotionGradient
- cv::motempl::calcGlobalOrientation
- cv::motempl::segmentMotion
Functions reading and writing .flo files in "Middlebury" format, see: <http://vision.middlebury.edu/flow/code/flow-code/README.txt>
- cv::optflow::readOpticalFlow
- cv::optflow::writeOpticalFlow
*/
#include "opencv2/optflow/pcaflow.hpp"
#include "opencv2/optflow/sparse_matching_gpc.hpp"
#include "opencv2/optflow/rlofflow.hpp"
namespace cv
{
namespace optflow
{
//! @addtogroup optflow
//! @{
/** @overload */
CV_EXPORTS_W void calcOpticalFlowSF( InputArray from, InputArray to, OutputArray flow,
int layers, int averaging_block_size, int max_flow);
/** @brief Calculate an optical flow using "SimpleFlow" algorithm.
@param from First 8-bit 3-channel image.
@param to Second 8-bit 3-channel image of the same size as prev
@param flow computed flow image that has the same size as prev and type CV_32FC2
@param layers Number of layers
@param averaging_block_size Size of block through which we sum up when calculate cost function
for pixel
@param max_flow maximal flow that we search at each level
@param sigma_dist vector smooth spatial sigma parameter
@param sigma_color vector smooth color sigma parameter
@param postprocess_window window size for postprocess cross bilateral filter
@param sigma_dist_fix spatial sigma for postprocess cross bilateralf filter
@param sigma_color_fix color sigma for postprocess cross bilateral filter
@param occ_thr threshold for detecting occlusions
@param upscale_averaging_radius window size for bilateral upscale operation
@param upscale_sigma_dist spatial sigma for bilateral upscale operation
@param upscale_sigma_color color sigma for bilateral upscale operation
@param speed_up_thr threshold to detect point with irregular flow - where flow should be
recalculated after upscale
See @cite Tao2012 . And site of project - <http://graphics.berkeley.edu/papers/Tao-SAN-2012-05/>.
@note
- An example using the simpleFlow algorithm can be found at samples/simpleflow_demo.cpp
*/
CV_EXPORTS_W void calcOpticalFlowSF( InputArray from, InputArray to, OutputArray flow, int layers,
int averaging_block_size, int max_flow,
double sigma_dist, double sigma_color, int postprocess_window,
double sigma_dist_fix, double sigma_color_fix, double occ_thr,
int upscale_averaging_radius, double upscale_sigma_dist,
double upscale_sigma_color, double speed_up_thr );
/** @brief Fast dense optical flow based on PyrLK sparse matches interpolation.
@param from first 8-bit 3-channel or 1-channel image.
@param to second 8-bit 3-channel or 1-channel image of the same size as from
@param flow computed flow image that has the same size as from and CV_32FC2 type
@param grid_step stride used in sparse match computation. Lower values usually
result in higher quality but slow down the algorithm.
@param k number of nearest-neighbor matches considered, when fitting a locally affine
model. Lower values can make the algorithm noticeably faster at the cost of
some quality degradation.
@param sigma parameter defining how fast the weights decrease in the locally-weighted affine
fitting. Higher values can help preserve fine details, lower values can help to get rid
of the noise in the output flow.
@param use_post_proc defines whether the ximgproc::fastGlobalSmootherFilter() is used
for post-processing after interpolation
@param fgs_lambda see the respective parameter of the ximgproc::fastGlobalSmootherFilter()
@param fgs_sigma see the respective parameter of the ximgproc::fastGlobalSmootherFilter()
*/
CV_EXPORTS_W void calcOpticalFlowSparseToDense ( InputArray from, InputArray to, OutputArray flow,
int grid_step = 8, int k = 128, float sigma = 0.05f,
bool use_post_proc = true, float fgs_lambda = 500.0f,
float fgs_sigma = 1.5f );
/** @brief DeepFlow optical flow algorithm implementation.
The class implements the DeepFlow optical flow algorithm described in @cite Weinzaepfel2013 . See
also <http://lear.inrialpes.fr/src/deepmatching/> .
Parameters - class fields - that may be modified after creating a class instance:
- member float alpha
Smoothness assumption weight
- member float delta
Color constancy assumption weight
- member float gamma
Gradient constancy weight
- member float sigma
Gaussian smoothing parameter
- member int minSize
Minimal dimension of an image in the pyramid (next, smaller images in the pyramid are generated
until one of the dimensions reaches this size)
- member float downscaleFactor
Scaling factor in the image pyramid (must be \< 1)
- member int fixedPointIterations
How many iterations on each level of the pyramid
- member int sorIterations
Iterations of Succesive Over-Relaxation (solver)
- member float omega
Relaxation factor in SOR
*/
CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_DeepFlow();
//! Additional interface to the SimpleFlow algorithm - calcOpticalFlowSF()
CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_SimpleFlow();
//! Additional interface to the Farneback's algorithm - calcOpticalFlowFarneback()
CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_Farneback();
//! Additional interface to the SparseToDenseFlow algorithm - calcOpticalFlowSparseToDense()
CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_SparseToDense();
/** @brief "Dual TV L1" Optical Flow Algorithm.
The class implements the "Dual TV L1" optical flow algorithm described in @cite Zach2007 and
@cite Javier2012 .
Here are important members of the class that control the algorithm, which you can set after
constructing the class instance:
- member double tau
Time step of the numerical scheme.
- member double lambda
Weight parameter for the data term, attachment parameter. This is the most relevant
parameter, which determines the smoothness of the output. The smaller this parameter is,
the smoother the solutions we obtain. It depends on the range of motions of the images, so
its value should be adapted to each image sequence.
- member double theta
Weight parameter for (u - v)\^2, tightness parameter. It serves as a link between the
attachment and the regularization terms. In theory, it should have a small value in order
to maintain both parts in correspondence. The method is stable for a large range of values
of this parameter.
- member int nscales
Number of scales used to create the pyramid of images.
- member int warps
Number of warpings per scale. Represents the number of times that I1(x+u0) and grad(
I1(x+u0) ) are computed per scale. This is a parameter that assures the stability of the
method. It also affects the running time, so it is a compromise between speed and
accuracy.
- member double epsilon
Stopping criterion threshold used in the numerical scheme, which is a trade-off between
precision and running time. A small value will yield more accurate solutions at the
expense of a slower convergence.
- member int iterations
Stopping criterion iterations number used in the numerical scheme.
C. Zach, T. Pock and H. Bischof, "A Duality Based Approach for Realtime TV-L1 Optical Flow".
Javier Sanchez, Enric Meinhardt-Llopis and Gabriele Facciolo. "TV-L1 Optical Flow Estimation".
*/
class CV_EXPORTS_W DualTVL1OpticalFlow : public DenseOpticalFlow
{
public:
//! @brief Time step of the numerical scheme
/** @see setTau */
CV_WRAP virtual double getTau() const = 0;
/** @copybrief getTau @see getTau */
CV_WRAP virtual void setTau(double val) = 0;
//! @brief Weight parameter for the data term, attachment parameter
/** @see setLambda */
CV_WRAP virtual double getLambda() const = 0;
/** @copybrief getLambda @see getLambda */
CV_WRAP virtual void setLambda(double val) = 0;
//! @brief Weight parameter for (u - v)^2, tightness parameter
/** @see setTheta */
CV_WRAP virtual double getTheta() const = 0;
/** @copybrief getTheta @see getTheta */
CV_WRAP virtual void setTheta(double val) = 0;
//! @brief coefficient for additional illumination variation term
/** @see setGamma */
CV_WRAP virtual double getGamma() const = 0;
/** @copybrief getGamma @see getGamma */
CV_WRAP virtual void setGamma(double val) = 0;
//! @brief Number of scales used to create the pyramid of images
/** @see setScalesNumber */
CV_WRAP virtual int getScalesNumber() const = 0;
/** @copybrief getScalesNumber @see getScalesNumber */
CV_WRAP virtual void setScalesNumber(int val) = 0;
//! @brief Number of warpings per scale
/** @see setWarpingsNumber */
CV_WRAP virtual int getWarpingsNumber() const = 0;
/** @copybrief getWarpingsNumber @see getWarpingsNumber */
CV_WRAP virtual void setWarpingsNumber(int val) = 0;
//! @brief Stopping criterion threshold used in the numerical scheme, which is a trade-off between precision and running time
/** @see setEpsilon */
CV_WRAP virtual double getEpsilon() const = 0;
/** @copybrief getEpsilon @see getEpsilon */
CV_WRAP virtual void setEpsilon(double val) = 0;
//! @brief Inner iterations (between outlier filtering) used in the numerical scheme
/** @see setInnerIterations */
CV_WRAP virtual int getInnerIterations() const = 0;
/** @copybrief getInnerIterations @see getInnerIterations */
CV_WRAP virtual void setInnerIterations(int val) = 0;
//! @brief Outer iterations (number of inner loops) used in the numerical scheme
/** @see setOuterIterations */
CV_WRAP virtual int getOuterIterations() const = 0;
/** @copybrief getOuterIterations @see getOuterIterations */
CV_WRAP virtual void setOuterIterations(int val) = 0;
//! @brief Use initial flow
/** @see setUseInitialFlow */
CV_WRAP virtual bool getUseInitialFlow() const = 0;
/** @copybrief getUseInitialFlow @see getUseInitialFlow */
CV_WRAP virtual void setUseInitialFlow(bool val) = 0;
//! @brief Step between scales (<1)
/** @see setScaleStep */
CV_WRAP virtual double getScaleStep() const = 0;
/** @copybrief getScaleStep @see getScaleStep */
CV_WRAP virtual void setScaleStep(double val) = 0;
//! @brief Median filter kernel size (1 = no filter) (3 or 5)
/** @see setMedianFiltering */
CV_WRAP virtual int getMedianFiltering() const = 0;
/** @copybrief getMedianFiltering @see getMedianFiltering */
CV_WRAP virtual void setMedianFiltering(int val) = 0;
/** @brief Creates instance of cv::DualTVL1OpticalFlow*/
CV_WRAP static Ptr<DualTVL1OpticalFlow> create(
double tau = 0.25,
double lambda = 0.15,
double theta = 0.3,
int nscales = 5,
int warps = 5,
double epsilon = 0.01,
int innnerIterations = 30,
int outerIterations = 10,
double scaleStep = 0.8,
double gamma = 0.0,
int medianFiltering = 5,
bool useInitialFlow = false);
};
/** @brief Creates instance of cv::DenseOpticalFlow
*/
CV_EXPORTS_W Ptr<DualTVL1OpticalFlow> createOptFlow_DualTVL1();
//! @}
} //optflow
}
#include "opencv2/optflow/motempl.hpp"
#endif
modules/optflow/include/opencv2/optflow/motempl.hpp
+147
View File
@@ -0,0 +1,147 @@
/*
By downloading, copying, installing or using the software you agree to this
license. If you do not agree to this license, do not download, install,
copy or use the software.
License Agreement
For Open Source Computer Vision Library
(3-clause BSD License)
Copyright (C) 2013, OpenCV Foundation, all rights reserved.
Third party copyrights are property of their respective owners.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the names of the copyright holders nor the names of the contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
This software is provided by the copyright holders and contributors "as is" and
any express or implied warranties, including, but not limited to, the implied
warranties of merchantability and fitness for a particular purpose are
disclaimed. In no event shall copyright holders or contributors be liable for
any direct, indirect, incidental, special, exemplary, or consequential damages
(including, but not limited to, procurement of substitute goods or services;
loss of use, data, or profits; or business interruption) however caused
and on any theory of liability, whether in contract, strict liability,
or tort (including negligence or otherwise) arising in any way out of
the use of this software, even if advised of the possibility of such damage.
*/
#ifndef __OPENCV_OPTFLOW_MOTEMPL_HPP__
#define __OPENCV_OPTFLOW_MOTEMPL_HPP__
#include "opencv2/core.hpp"
namespace cv
{
namespace motempl
{
//! @addtogroup optflow
//! @{
/** @brief Updates the motion history image by a moving silhouette.
@param silhouette Silhouette mask that has non-zero pixels where the motion occurs.
@param mhi Motion history image that is updated by the function (single-channel, 32-bit
floating-point).
@param timestamp Current time in milliseconds or other units.
@param duration Maximal duration of the motion track in the same units as timestamp .
The function updates the motion history image as follows:
\f[\texttt{mhi} (x,y)= \forkthree{\texttt{timestamp}}{if \(\texttt{silhouette}(x,y) \ne 0\)}{0}{if \(\texttt{silhouette}(x,y) = 0\) and \(\texttt{mhi} < (\texttt{timestamp} - \texttt{duration})\)}{\texttt{mhi}(x,y)}{otherwise}\f]
That is, MHI pixels where the motion occurs are set to the current timestamp , while the pixels
where the motion happened last time a long time ago are cleared.
The function, together with calcMotionGradient and calcGlobalOrientation , implements a motion
templates technique described in @cite Davis97 and @cite Bradski00 .
*/
CV_EXPORTS_W void updateMotionHistory( InputArray silhouette, InputOutputArray mhi,
double timestamp, double duration );
/** @brief Calculates a gradient orientation of a motion history image.
@param mhi Motion history single-channel floating-point image.
@param mask Output mask image that has the type CV_8UC1 and the same size as mhi . Its non-zero
elements mark pixels where the motion gradient data is correct.
@param orientation Output motion gradient orientation image that has the same type and the same
size as mhi . Each pixel of the image is a motion orientation, from 0 to 360 degrees.
@param delta1 Minimal (or maximal) allowed difference between mhi values within a pixel
neighborhood.
@param delta2 Maximal (or minimal) allowed difference between mhi values within a pixel
neighborhood. That is, the function finds the minimum ( \f$m(x,y)\f$ ) and maximum ( \f$M(x,y)\f$ ) mhi
values over \f$3 \times 3\f$ neighborhood of each pixel and marks the motion orientation at \f$(x, y)\f$
as valid only if
\f[\min ( \texttt{delta1} , \texttt{delta2} ) \le M(x,y)-m(x,y) \le \max ( \texttt{delta1} , \texttt{delta2} ).\f]
@param apertureSize Aperture size of the Sobel operator.
The function calculates a gradient orientation at each pixel \f$(x, y)\f$ as:
\f[\texttt{orientation} (x,y)= \arctan{\frac{d\texttt{mhi}/dy}{d\texttt{mhi}/dx}}\f]
In fact, fastAtan2 and phase are used so that the computed angle is measured in degrees and covers
the full range 0..360. Also, the mask is filled to indicate pixels where the computed angle is
valid.
@note
- (Python) An example on how to perform a motion template technique can be found at
opencv_source_code/samples/python2/motempl.py
*/
CV_EXPORTS_W void calcMotionGradient( InputArray mhi, OutputArray mask, OutputArray orientation,
double delta1, double delta2, int apertureSize = 3 );
/** @brief Calculates a global motion orientation in a selected region.
@param orientation Motion gradient orientation image calculated by the function calcMotionGradient
@param mask Mask image. It may be a conjunction of a valid gradient mask, also calculated by
calcMotionGradient , and the mask of a region whose direction needs to be calculated.
@param mhi Motion history image calculated by updateMotionHistory .
@param timestamp Timestamp passed to updateMotionHistory .
@param duration Maximum duration of a motion track in milliseconds, passed to updateMotionHistory
The function calculates an average motion direction in the selected region and returns the angle
between 0 degrees and 360 degrees. The average direction is computed from the weighted orientation
histogram, where a recent motion has a larger weight and the motion occurred in the past has a
smaller weight, as recorded in mhi .
*/
CV_EXPORTS_W double calcGlobalOrientation( InputArray orientation, InputArray mask, InputArray mhi,
double timestamp, double duration );
/** @brief Splits a motion history image into a few parts corresponding to separate independent motions (for
example, left hand, right hand).
@param mhi Motion history image.
@param segmask Image where the found mask should be stored, single-channel, 32-bit floating-point.
@param boundingRects Vector containing ROIs of motion connected components.
@param timestamp Current time in milliseconds or other units.
@param segThresh Segmentation threshold that is recommended to be equal to the interval between
motion history "steps" or greater.
The function finds all of the motion segments and marks them in segmask with individual values
(1,2,...). It also computes a vector with ROIs of motion connected components. After that the motion
direction for every component can be calculated with calcGlobalOrientation using the extracted mask
of the particular component.
*/
CV_EXPORTS_W void segmentMotion( InputArray mhi, OutputArray segmask,
CV_OUT std::vector<Rect>& boundingRects,
double timestamp, double segThresh );
//! @}
}
}
#endif
modules/optflow/include/opencv2/optflow/pcaflow.hpp
+149
View File
@@ -0,0 +1,149 @@
/*
By downloading, copying, installing or using the software you agree to this
license. If you do not agree to this license, do not download, install,
copy or use the software.
License Agreement
For Open Source Computer Vision Library
(3-clause BSD License)
Copyright (C) 2016, OpenCV Foundation, all rights reserved.
Third party copyrights are property of their respective owners.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the names of the copyright holders nor the names of the contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
This software is provided by the copyright holders and contributors "as is" and
any express or implied warranties, including, but not limited to, the implied
warranties of merchantability and fitness for a particular purpose are
disclaimed. In no event shall copyright holders or contributors be liable for
any direct, indirect, incidental, special, exemplary, or consequential damages
(including, but not limited to, procurement of substitute goods or services;
loss of use, data, or profits; or business interruption) however caused
and on any theory of liability, whether in contract, strict liability,
or tort (including negligence or otherwise) arising in any way out of
the use of this software, even if advised of the possibility of such damage.
*/
/**
* @file pcaflow.hpp
* @author Vladislav Samsonov <vvladxx@gmail.com>
* @brief Implementation of the PCAFlow algorithm from the following paper:
* http://files.is.tue.mpg.de/black/papers/cvpr2015_pcaflow.pdf
*
* @cite Wulff:CVPR:2015
*
* There are some key differences which distinguish this algorithm from the original PCAFlow (see paper):
* - Discrete Cosine Transform basis is used instead of basis extracted with PCA.
* Reasoning: DCT basis has comparable performance and it doesn't require additional storage space.
* Also, this decision helps to avoid overloading the algorithm with a lot of external input.
* - Usage of built-in OpenCV feature tracking instead of libviso.
*/
#ifndef __OPENCV_OPTFLOW_PCAFLOW_HPP__
#define __OPENCV_OPTFLOW_PCAFLOW_HPP__
#include "opencv2/core.hpp"
#include "opencv2/video.hpp"
namespace cv
{
namespace optflow
{
//! @addtogroup optflow
//! @{
/** @brief
* This class can be used for imposing a learned prior on the resulting optical flow.
* Solution will be regularized according to this prior.
* You need to generate appropriate prior file with "learn_prior.py" script beforehand.
*/
class CV_EXPORTS_W PCAPrior
{
private:
Mat L1;
Mat L2;
Mat c1;
Mat c2;
public:
PCAPrior( const char *pathToPrior );
int getPadding() const { return L1.size().height; }
int getBasisSize() const { return L1.size().width; }
void fillConstraints( float *A1, float *A2, float *b1, float *b2 ) const;
};
/** @brief PCAFlow algorithm.
*/
class CV_EXPORTS_W OpticalFlowPCAFlow : public DenseOpticalFlow
{
protected:
const Ptr<const PCAPrior> prior;
const Size basisSize;
const float sparseRate; // (0 .. 0.1)
const float retainedCornersFraction; // [0 .. 1]
const float occlusionsThreshold;
const float dampingFactor;
const float claheClip;
bool useOpenCL;
public:
/** @brief Creates an instance of PCAFlow algorithm.
* @param _prior Learned prior or no prior (default). @see cv::optflow::PCAPrior
* @param _basisSize Number of basis vectors.
* @param _sparseRate Controls density of sparse matches.
* @param _retainedCornersFraction Retained corners fraction.
* @param _occlusionsThreshold Occlusion threshold.
* @param _dampingFactor Regularization term for solving least-squares. It is not related to the prior regularization.
* @param _claheClip Clip parameter for CLAHE.
*/
OpticalFlowPCAFlow( Ptr<const PCAPrior> _prior = Ptr<const PCAPrior>(), const Size _basisSize = Size( 18, 14 ),
float _sparseRate = 0.024, float _retainedCornersFraction = 0.2,
float _occlusionsThreshold = 0.0003, float _dampingFactor = 0.00002, float _claheClip = 14 );
void calc( InputArray I0, InputArray I1, InputOutputArray flow ) CV_OVERRIDE;
void collectGarbage() CV_OVERRIDE;
private:
void findSparseFeatures( UMat &from, UMat &to, std::vector<Point2f> &features,
std::vector<Point2f> &predictedFeatures ) const;
void removeOcclusions( UMat &from, UMat &to, std::vector<Point2f> &features,
std::vector<Point2f> &predictedFeatures ) const;
void getSystem( OutputArray AOut, OutputArray b1Out, OutputArray b2Out, const std::vector<Point2f> &features,
const std::vector<Point2f> &predictedFeatures, const Size size );
void getSystem( OutputArray A1Out, OutputArray A2Out, OutputArray b1Out, OutputArray b2Out,
const std::vector<Point2f> &features, const std::vector<Point2f> &predictedFeatures,
const Size size );
OpticalFlowPCAFlow& operator=( const OpticalFlowPCAFlow& ); // make it non-assignable
};
/** @brief Creates an instance of PCAFlow
*/
CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_PCAFlow();
//! @}
}
}
#endif
modules/optflow/include/opencv2/optflow/rlofflow.hpp
+553
View File
@@ -0,0 +1,553 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef __OPENCV_OPTFLOW_RLOFFLOW_HPP__
#define __OPENCV_OPTFLOW_RLOFFLOW_HPP__
#include "opencv2/core.hpp"
#include "opencv2/video.hpp"
namespace cv
{
namespace optflow
{
//! @addtogroup optflow
//! @{
enum SupportRegionType {
SR_FIXED = 0, /**< Apply a constant support region */
SR_CROSS = 1 /**< Apply a adaptive support region obtained by cross-based segmentation
* as described in @cite Senst2014
*/
};
enum SolverType {
ST_STANDART = 0, /**< Apply standard iterative refinement */
ST_BILINEAR = 1 /**< Apply optimized iterative refinement based bilinear equation solutions
* as described in @cite Senst2013
*/
};
enum InterpolationType
{
INTERP_GEO = 0, /**< Fast geodesic interpolation, see @cite Geistert2016 */
INTERP_EPIC = 1, /**< Edge-preserving interpolation using ximgproc::EdgeAwareInterpolator, see @cite Revaud2015,Geistert2016. */
INTERP_RIC = 2, /**< SLIC based robust interpolation using ximgproc::RICInterpolator, see @cite Hu2017. */
};
/** @brief This is used store and set up the parameters of the robust local optical flow (RLOF) algoritm.
*
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
* proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
* The implementation is derived from optflow::calcOpticalFlowPyrLK().
* This RLOF implementation can be seen as an improved pyramidal iterative Lucas-Kanade and includes
* a set of improving modules. The main improvements in respect to the pyramidal iterative Lucas-Kanade
* are:
* - A more robust redecending M-estimator framework (see @cite Senst2012) to improve the accuracy at
* motion boundaries and appearing and disappearing pixels.
* - an adaptive support region strategies to improve the accuracy at motion boundaries to reduce the
* corona effect, i.e oversmoothing of the PLK at motion/object boundaries. The cross-based segementation
* strategy (SR_CROSS) proposed in @cite Senst2014 uses a simple segmenation approach to obtain the optimal
* shape of the support region.
* - To deal with illumination changes (outdoor sequences and shadow) the intensity constancy assumption
* based optical flow equation has been adopt with the Gennert and Negahdaripour illumination model
* (see @cite Senst2016). This model can be switched on/off with the useIlluminationModel variable.
* - By using a global motion prior initialization (see @cite Senst2016) of the iterative refinement
* the accuracy could be significantly improved for large displacements. This initialization can be
* switched on and of with useGlobalMotionPrior variable.
*
* The RLOF can be computed with the SparseOpticalFlow class or function interface to track a set of features
* or with the DenseOpticalFlow class or function interface to compute dense optical flow.
*
* @see optflow::DenseRLOFOpticalFlow, optflow::calcOpticalFlowDenseRLOF(), optflow::SparseRLOFOpticalFlow, optflow::calcOpticalFlowSparseRLOF()
*/
class CV_EXPORTS_W RLOFOpticalFlowParameter{
public:
RLOFOpticalFlowParameter()
:solverType(ST_BILINEAR)
,supportRegionType(SR_CROSS)
,normSigma0(std::numeric_limits<float>::max())
,normSigma1(std::numeric_limits<float>::max())
,smallWinSize(9)
,largeWinSize(21)
,crossSegmentationThreshold(25)
,maxLevel(4)
,useInitialFlow(false)
,useIlluminationModel(true)
,useGlobalMotionPrior(true)
,maxIteration(30)
,minEigenValue(0.0001f)
,globalMotionRansacThreshold(10)
{}
SolverType solverType;
/**< Variable specifies the iterative refinement strategy. Please consider citing @cite Senst2013 when
* using ST_BILINEAR.
*/
SupportRegionType supportRegionType;
/**< Variable specifies the support region shape extraction or shrinking strategy.
*/
float normSigma0;
/**< &sigma parameter of the shrinked Hampel norm introduced in @cite Senst2012. If
* &sigma = std::numeric_limist<float>::max() the least-square estimator will be used
* instead of the M-estimator. Althoug M-estimator is more robust against outlier in the support
* region the least-square can be fast in computation.
*/
float normSigma1;
/**< &sigma parameter of the shrinked Hampel norm introduced in @cite Senst2012. If
* &sigma = std::numeric_limist<float>::max() the least-square estimator will be used
* instead of the M-estimator. Althoug M-estimator is more robust against outlier in the support
* region the least-square can be fast in computation.
*/
int smallWinSize;
/**< Minimal window size of the support region. This parameter is only used if supportRegionType is SR_CROSS.
*/
int largeWinSize;
/**< Maximal window size of the support region. If supportRegionType is SR_FIXED this gives the exact support
* region size. The speed of the RLOF is related to the applied win sizes. The smaller the window size the lower is the runtime,
* but the more sensitive to noise is the method.
*/
int crossSegmentationThreshold;
/**< Color similarity threshold used by cross-based segmentation following @cite Senst2014 .
* (Only used if supportRegionType is SR_CROSS). With the cross-bassed segmentation
* motion boundaries can be computed more accurately.
*/
int maxLevel;
/**< Maximal number of pyramid level used. The large this value is the more likely it is
* to obtain accurate solutions for long-range motions. The runtime is linear related to
* this parameter.
*/
bool useInitialFlow;
/**< Use next point list as initial values. A good intialization can imporve the algortihm
* accuracy and reduce the runtime by a faster convergence of the iteration refinement.
*/
bool useIlluminationModel;
/**< Use the Gennert and Negahdaripour illumination model instead of the intensity brigthness
* constraint. (proposed in @cite Senst2016 ) This model is defined as follow:
* \f[ I(\mathbf{x},t) + m \cdot I(\mathbf{x},t) + c = I(\mathbf{x},t+1) \f]
* and contains with m and c a multiplicative and additive term which makes the estimate
* more robust against illumination changes. The computational complexity is increased by
* enabling the illumination model.
*/
bool useGlobalMotionPrior;
/**< Use global motion prior initialisation has been introduced in @cite Senst2016 . It
* allows to be more accurate for long-range motion. The computational complexity is
* slightly increased by enabling the global motion prior initialisation.
*/
int maxIteration;
/**< Number of maximal iterations used for the iterative refinement. Lower values can
* reduce the runtime but also the accuracy.
*/
float minEigenValue;
/**< Threshold for the minimal eigenvalue of the gradient matrix defines when to abort the
* iterative refinement.
*/
float globalMotionRansacThreshold;
/**< To apply the global motion prior motion vectors will be computed on a regulary sampled which
* are the basis for Homography estimation using RANSAC. The reprojection threshold is based on
* n-th percentil (given by this value [0 ... 100]) of the motion vectors magnitude.
* See @cite Senst2016 for more details.
*/
//! @brief Enable M-estimator or disable and use least-square estimator.
/** Enables M-estimator by setting sigma parameters to (3.2, 7.0). Disabling M-estimator can reduce
* runtime, while enabling can improve the accuracy.
* @param val If true M-estimator is used. If false least-square estimator is used.
* @see setNormSigma0, setNormSigma1
*/
CV_WRAP void setUseMEstimator(bool val);
CV_WRAP void setSolverType(SolverType val);
CV_WRAP SolverType getSolverType() const;
CV_WRAP void setSupportRegionType(SupportRegionType val);
CV_WRAP SupportRegionType getSupportRegionType() const;
CV_WRAP void setNormSigma0(float val);
CV_WRAP float getNormSigma0() const;
CV_WRAP void setNormSigma1(float val);
CV_WRAP float getNormSigma1() const;
CV_WRAP void setSmallWinSize(int val);
CV_WRAP int getSmallWinSize() const;
CV_WRAP void setLargeWinSize(int val);
CV_WRAP int getLargeWinSize() const;
CV_WRAP void setCrossSegmentationThreshold(int val);
CV_WRAP int getCrossSegmentationThreshold() const;
CV_WRAP void setMaxLevel(int val);
CV_WRAP int getMaxLevel() const;
CV_WRAP void setUseInitialFlow(bool val);
CV_WRAP bool getUseInitialFlow() const;
CV_WRAP void setUseIlluminationModel(bool val);
CV_WRAP bool getUseIlluminationModel() const;
CV_WRAP void setUseGlobalMotionPrior(bool val);
CV_WRAP bool getUseGlobalMotionPrior() const;
CV_WRAP void setMaxIteration(int val);
CV_WRAP int getMaxIteration() const;
CV_WRAP void setMinEigenValue(float val);
CV_WRAP float getMinEigenValue() const;
CV_WRAP void setGlobalMotionRansacThreshold(float val);
CV_WRAP float getGlobalMotionRansacThreshold() const;
//! @brief Creates instance of optflow::RLOFOpticalFlowParameter
CV_WRAP static Ptr<RLOFOpticalFlowParameter> create();
};
/** @brief Fast dense optical flow computation based on robust local optical flow (RLOF) algorithms and sparse-to-dense interpolation
* scheme.
*
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
* proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
* The implementation is derived from optflow::calcOpticalFlowPyrLK().
*
* The sparse-to-dense interpolation scheme allows for fast computation of dense optical flow using RLOF (see @cite Geistert2016).
* For this scheme the following steps are applied:
* -# motion vector seeded at a regular sampled grid are computed. The sparsity of this grid can be configured with setGridStep
* -# (optinally) errornous motion vectors are filter based on the forward backward confidence. The threshold can be configured
* with setForwardBackward. The filter is only applied if the threshold >0 but than the runtime is doubled due to the estimation
* of the backward flow.
* -# Vector field interpolation is applied to the motion vector set to obtain a dense vector field.
*
* For the RLOF configuration see optflow::RLOFOpticalFlowParameter for further details.
* Parameters have been described in @cite Senst2012 @cite Senst2013 @cite Senst2014 and @cite Senst2016.
*
* @note If the grid size is set to (1,1) and the forward backward threshold <= 0 than pixelwise dense optical flow field is
* computed by RLOF without using interpolation.
*
* @note Note that in output, if no correspondences are found between \a I0 and \a I1, the \a flow is set to 0.
* @see optflow::calcOpticalFlowDenseRLOF(), optflow::RLOFOpticalFlowParameter
*/
class CV_EXPORTS_W DenseRLOFOpticalFlow : public DenseOpticalFlow
{
public:
//! @brief Configuration of the RLOF alogrithm.
/**
@see optflow::RLOFOpticalFlowParameter, getRLOFOpticalFlowParameter
*/
CV_WRAP virtual void setRLOFOpticalFlowParameter(Ptr<RLOFOpticalFlowParameter> val) = 0;
/** @copybrief setRLOFOpticalFlowParameter
@see optflow::RLOFOpticalFlowParameter, setRLOFOpticalFlowParameter
*/
CV_WRAP virtual Ptr<RLOFOpticalFlowParameter> getRLOFOpticalFlowParameter() const = 0;
//! @brief Threshold for the forward backward confidence check
/**For each grid point \f$ \mathbf{x} \f$ a motion vector \f$ d_{I0,I1}(\mathbf{x}) \f$ is computed.
* If the forward backward error \f[ EP_{FB} = || d_{I0,I1} + d_{I1,I0} || \f]
* is larger than threshold given by this function then the motion vector will not be used by the following
* vector field interpolation. \f$ d_{I1,I0} \f$ denotes the backward flow. Note, the forward backward test
* will only be applied if the threshold > 0. This may results into a doubled runtime for the motion estimation.
* @see getForwardBackward, setGridStep
*/
CV_WRAP virtual void setForwardBackward(float val) = 0;
/** @copybrief setForwardBackward
@see setForwardBackward
*/
CV_WRAP virtual float getForwardBackward() const = 0;
//! @brief Size of the grid to spawn the motion vectors.
/** For each grid point a motion vector is computed. Some motion vectors will be removed due to the forwatd backward
* threshold (if set >0). The rest will be the base of the vector field interpolation.
* @see getForwardBackward, setGridStep
*/
CV_WRAP virtual Size getGridStep() const = 0;
/** @copybrief getGridStep
* @see getGridStep
*/
CV_WRAP virtual void setGridStep(Size val) = 0;
//! @brief Interpolation used to compute the dense optical flow.
/** Two interpolation algorithms are supported
* - **INTERP_GEO** applies the fast geodesic interpolation, see @cite Geistert2016.
* - **INTERP_EPIC_RESIDUAL** applies the edge-preserving interpolation, see @cite Revaud2015,Geistert2016.
* @see ximgproc::EdgeAwareInterpolator, getInterpolation
*/
CV_WRAP virtual void setInterpolation(InterpolationType val) = 0;
/** @copybrief setInterpolation
* @see ximgproc::EdgeAwareInterpolator, setInterpolation
*/
CV_WRAP virtual InterpolationType getInterpolation() const = 0;
//! @brief see ximgproc::EdgeAwareInterpolator() K value.
/** K is a number of nearest-neighbor matches considered, when fitting a locally affine
* model. Usually it should be around 128. However, lower values would make the interpolation noticeably faster.
* @see ximgproc::EdgeAwareInterpolator, setEPICK
*/
CV_WRAP virtual int getEPICK() const = 0;
/** @copybrief getEPICK
* @see ximgproc::EdgeAwareInterpolator, getEPICK
*/
CV_WRAP virtual void setEPICK(int val) = 0;
//! @brief see ximgproc::EdgeAwareInterpolator() sigma value.
/** Sigma is a parameter defining how fast the weights decrease in the locally-weighted affine
* fitting. Higher values can help preserve fine details, lower values can help to get rid of noise in the
* output flow.
* @see ximgproc::EdgeAwareInterpolator, setEPICSigma
*/
CV_WRAP virtual float getEPICSigma() const = 0;
/** @copybrief getEPICSigma
* @see ximgproc::EdgeAwareInterpolator, getEPICSigma
*/
CV_WRAP virtual void setEPICSigma(float val) = 0;
//! @brief see ximgproc::EdgeAwareInterpolator() lambda value.
/** Lambda is a parameter defining the weight of the edge-aware term in geodesic distance,
* should be in the range of 0 to 1000.
* @see ximgproc::EdgeAwareInterpolator, setEPICSigma
*/
CV_WRAP virtual float getEPICLambda() const = 0;
/** @copybrief getEPICLambda
* @see ximgproc::EdgeAwareInterpolator, getEPICLambda
*/
CV_WRAP virtual void setEPICLambda(float val) = 0;
//! @brief see ximgproc::EdgeAwareInterpolator().
/** Sets the respective fastGlobalSmootherFilter() parameter.
* @see ximgproc::EdgeAwareInterpolator, setFgsLambda
*/
CV_WRAP virtual float getFgsLambda() const = 0;
/** @copybrief getFgsLambda
* @see ximgproc::EdgeAwareInterpolator, ximgproc::fastGlobalSmootherFilter, getFgsLambda
*/
CV_WRAP virtual void setFgsLambda(float val) = 0;
//! @brief see ximgproc::EdgeAwareInterpolator().
/** Sets the respective fastGlobalSmootherFilter() parameter.
* @see ximgproc::EdgeAwareInterpolator, ximgproc::fastGlobalSmootherFilter, setFgsSigma
*/
CV_WRAP virtual float getFgsSigma() const = 0;
/** @copybrief getFgsSigma
* @see ximgproc::EdgeAwareInterpolator, ximgproc::fastGlobalSmootherFilter, getFgsSigma
*/
CV_WRAP virtual void setFgsSigma(float val) = 0;
//! @brief enables ximgproc::fastGlobalSmootherFilter
/**
* @see getUsePostProc
*/
CV_WRAP virtual void setUsePostProc(bool val) = 0;
/** @copybrief setUsePostProc
* @see ximgproc::fastGlobalSmootherFilter, setUsePostProc
*/
CV_WRAP virtual bool getUsePostProc() const = 0;
//! @brief enables VariationalRefinement
/**
* @see getUseVariationalRefinement
*/
CV_WRAP virtual void setUseVariationalRefinement(bool val) = 0;
/** @copybrief setUseVariationalRefinement
* @see ximgproc::fastGlobalSmootherFilter, setUsePostProc
*/
CV_WRAP virtual bool getUseVariationalRefinement() const = 0;
//! @brief Parameter to tune the approximate size of the superpixel used for oversegmentation.
/**
* @see cv::ximgproc::createSuperpixelSLIC, cv::ximgproc::RICInterpolator
*/
CV_WRAP virtual void setRICSPSize(int val) = 0;
/** @copybrief setRICSPSize
* @see setRICSPSize
*/
CV_WRAP virtual int getRICSPSize() const = 0;
/** @brief Parameter to choose superpixel algorithm variant to use:
* - cv::ximgproc::SLICType SLIC segments image using a desired region_size (value: 100)
* - cv::ximgproc::SLICType SLICO will optimize using adaptive compactness factor (value: 101)
* - cv::ximgproc::SLICType MSLIC will optimize using manifold methods resulting in more content-sensitive superpixels (value: 102).
* @see cv::ximgproc::createSuperpixelSLIC, cv::ximgproc::RICInterpolator
*/
CV_WRAP virtual void setRICSLICType(int val) = 0;
/** @copybrief setRICSLICType
* @see setRICSLICType
*/
CV_WRAP virtual int getRICSLICType() const = 0;
//! @brief Creates instance of optflow::DenseRLOFOpticalFlow
/**
* @param rlofParam see optflow::RLOFOpticalFlowParameter
* @param forwardBackwardThreshold see setForwardBackward
* @param gridStep see setGridStep
* @param interp_type see setInterpolation
* @param epicK see setEPICK
* @param epicSigma see setEPICSigma
* @param epicLambda see setEPICLambda
* @param ricSPSize see setRICSPSize
* @param ricSLICType see setRICSLICType
* @param use_post_proc see setUsePostProc
* @param fgsLambda see setFgsLambda
* @param fgsSigma see setFgsSigma
* @param use_variational_refinement see setUseVariationalRefinement
*/
CV_WRAP static Ptr<DenseRLOFOpticalFlow> create(
Ptr<RLOFOpticalFlowParameter> rlofParam = Ptr<RLOFOpticalFlowParameter>(),
float forwardBackwardThreshold = 1.f,
Size gridStep = Size(6, 6),
InterpolationType interp_type = InterpolationType::INTERP_EPIC,
int epicK = 128,
float epicSigma = 0.05f,
float epicLambda = 999.0f,
int ricSPSize = 15,
int ricSLICType = 100,
bool use_post_proc = true,
float fgsLambda = 500.0f,
float fgsSigma = 1.5f,
bool use_variational_refinement = false);
};
/** @brief Class used for calculation sparse optical flow and feature tracking with robust local optical flow (RLOF) algorithms.
*
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
* proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
* The implementation is derived from optflow::calcOpticalFlowPyrLK().
*
* For the RLOF configuration see optflow::RLOFOpticalFlowParameter for further details.
* Parameters have been described in @cite Senst2012, @cite Senst2013, @cite Senst2014 and @cite Senst2016.
*
* @note SIMD parallelization is only available when compiling with SSE4.1.
* @see optflow::calcOpticalFlowSparseRLOF(), optflow::RLOFOpticalFlowParameter
*/
class CV_EXPORTS_W SparseRLOFOpticalFlow : public SparseOpticalFlow
{
public:
/** @copydoc DenseRLOFOpticalFlow::setRLOFOpticalFlowParameter
*/
CV_WRAP virtual void setRLOFOpticalFlowParameter(Ptr<RLOFOpticalFlowParameter> val) = 0;
/** @copybrief setRLOFOpticalFlowParameter
* @see setRLOFOpticalFlowParameter
*/
CV_WRAP virtual Ptr<RLOFOpticalFlowParameter> getRLOFOpticalFlowParameter() const = 0;
//! @brief Threshold for the forward backward confidence check
/** For each feature point a motion vector \f$ d_{I0,I1}(\mathbf{x}) \f$ is computed.
* If the forward backward error \f[ EP_{FB} = || d_{I0,I1} + d_{I1,I0} || \f]
* is larger than threshold given by this function then the status will not be used by the following
* vector field interpolation. \f$ d_{I1,I0} \f$ denotes the backward flow. Note, the forward backward test
* will only be applied if the threshold > 0. This may results into a doubled runtime for the motion estimation.
* @see setForwardBackward
*/
CV_WRAP virtual void setForwardBackward(float val) = 0;
/** @copybrief setForwardBackward
* @see setForwardBackward
*/
CV_WRAP virtual float getForwardBackward() const = 0;
//! @brief Creates instance of SparseRLOFOpticalFlow
/**
* @param rlofParam see setRLOFOpticalFlowParameter
* @param forwardBackwardThreshold see setForwardBackward
*/
CV_WRAP static Ptr<SparseRLOFOpticalFlow> create(
Ptr<RLOFOpticalFlowParameter> rlofParam = Ptr<RLOFOpticalFlowParameter>(),
float forwardBackwardThreshold = 1.f);
};
/** @brief Fast dense optical flow computation based on robust local optical flow (RLOF) algorithms and sparse-to-dense interpolation scheme.
*
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
* proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
* The implementation is derived from optflow::calcOpticalFlowPyrLK().
*
* The sparse-to-dense interpolation scheme allows for fast computation of dense optical flow using RLOF (see @cite Geistert2016).
* For this scheme the following steps are applied:
* -# motion vector seeded at a regular sampled grid are computed. The sparsity of this grid can be configured with setGridStep
* -# (optinally) errornous motion vectors are filter based on the forward backward confidence. The threshold can be configured
* with setForwardBackward. The filter is only applied if the threshold >0 but than the runtime is doubled due to the estimation
* of the backward flow.
* -# Vector field interpolation is applied to the motion vector set to obtain a dense vector field.
*
* @param I0 first 8-bit input image. If The cross-based RLOF is used (by selecting optflow::RLOFOpticalFlowParameter::supportRegionType
* = SupportRegionType::SR_CROSS) image has to be a 8-bit 3 channel image.
* @param I1 second 8-bit input image. If The cross-based RLOF is used (by selecting optflow::RLOFOpticalFlowParameter::supportRegionType
* = SupportRegionType::SR_CROSS) image has to be a 8-bit 3 channel image.
* @param flow computed flow image that has the same size as I0 and type CV_32FC2.
* @param rlofParam see optflow::RLOFOpticalFlowParameter
* @param forwardBackwardThreshold Threshold for the forward backward confidence check.
* For each grid point \f$ \mathbf{x} \f$ a motion vector \f$ d_{I0,I1}(\mathbf{x}) \f$ is computed.
* If the forward backward error \f[ EP_{FB} = || d_{I0,I1} + d_{I1,I0} || \f]
* is larger than threshold given by this function then the motion vector will not be used by the following
* vector field interpolation. \f$ d_{I1,I0} \f$ denotes the backward flow. Note, the forward backward test
* will only be applied if the threshold > 0. This may results into a doubled runtime for the motion estimation.
* @param gridStep Size of the grid to spawn the motion vectors. For each grid point a motion vector is computed.
* Some motion vectors will be removed due to the forwatd backward threshold (if set >0). The rest will be the
* base of the vector field interpolation.
* @param interp_type interpolation method used to compute the dense optical flow. Two interpolation algorithms are
* supported:
* - **INTERP_GEO** applies the fast geodesic interpolation, see @cite Geistert2016.
* - **INTERP_EPIC_RESIDUAL** applies the edge-preserving interpolation, see @cite Revaud2015,Geistert2016.
* @param epicK see ximgproc::EdgeAwareInterpolator sets the respective parameter.
* @param epicSigma see ximgproc::EdgeAwareInterpolator sets the respective parameter.
* @param epicLambda see ximgproc::EdgeAwareInterpolator sets the respective parameter.
* @param ricSPSize see ximgproc::RICInterpolator sets the respective parameter.
* @param ricSLICType see ximgproc::RICInterpolator sets the respective parameter.
* @param use_post_proc enables ximgproc::fastGlobalSmootherFilter() parameter.
* @param fgsLambda sets the respective ximgproc::fastGlobalSmootherFilter() parameter.
* @param fgsSigma sets the respective ximgproc::fastGlobalSmootherFilter() parameter.
* @param use_variational_refinement enables VariationalRefinement
*
* Parameters have been described in @cite Senst2012, @cite Senst2013, @cite Senst2014, @cite Senst2016.
* For the RLOF configuration see optflow::RLOFOpticalFlowParameter for further details.
* @note If the grid size is set to (1,1) and the forward backward threshold <= 0 that the dense optical flow field is purely
* computed with the RLOF.
*
* @note SIMD parallelization is only available when compiling with SSE4.1.
* @note Note that in output, if no correspondences are found between \a I0 and \a I1, the \a flow is set to 0.
*
* @sa optflow::DenseRLOFOpticalFlow, optflow::RLOFOpticalFlowParameter
*/
CV_EXPORTS_W void calcOpticalFlowDenseRLOF(InputArray I0, InputArray I1, InputOutputArray flow,
Ptr<RLOFOpticalFlowParameter> rlofParam = Ptr<RLOFOpticalFlowParameter>(),
float forwardBackwardThreshold = 0, Size gridStep = Size(6, 6),
InterpolationType interp_type = InterpolationType::INTERP_EPIC,
int epicK = 128, float epicSigma = 0.05f, float epicLambda = 100.f,
int ricSPSize = 15, int ricSLICType = 100,
bool use_post_proc = true, float fgsLambda = 500.0f, float fgsSigma = 1.5f,
bool use_variational_refinement = false);
/** @brief Calculates fast optical flow for a sparse feature set using the robust local optical flow (RLOF) similar
* to optflow::calcOpticalFlowPyrLK().
*
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
* proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.
* The implementation is derived from optflow::calcOpticalFlowPyrLK().
*
* @param prevImg first 8-bit input image. If The cross-based RLOF is used (by selecting optflow::RLOFOpticalFlowParameter::supportRegionType
* = SupportRegionType::SR_CROSS) image has to be a 8-bit 3 channel image.
* @param nextImg second 8-bit input image. If The cross-based RLOF is used (by selecting optflow::RLOFOpticalFlowParameter::supportRegionType
* = SupportRegionType::SR_CROSS) image has to be a 8-bit 3 channel image.
* @param prevPts vector of 2D points for which the flow needs to be found; point coordinates must be single-precision
* floating-point numbers.
* @param nextPts output vector of 2D points (with single-precision floating-point coordinates) containing the calculated
* new positions of input features in the second image; when optflow::RLOFOpticalFlowParameter::useInitialFlow variable is true the vector must
* have the same size as in the input and contain the initialization point correspondences.
* @param status output status vector (of unsigned chars); each element of the vector is set to 1 if the flow for the
* corresponding features has passed the forward backward check.
* @param err output vector of errors; each element of the vector is set to the forward backward error for the corresponding feature.
* @param rlofParam see optflow::RLOFOpticalFlowParameter
* @param forwardBackwardThreshold Threshold for the forward backward confidence check. If forewardBackwardThreshold <=0 the forward
*
* @note SIMD parallelization is only available when compiling with SSE4.1.
*
* Parameters have been described in @cite Senst2012, @cite Senst2013, @cite Senst2014 and @cite Senst2016.
* For the RLOF configuration see optflow::RLOFOpticalFlowParameter for further details.
*/
CV_EXPORTS_W void calcOpticalFlowSparseRLOF(InputArray prevImg, InputArray nextImg,
InputArray prevPts, InputOutputArray nextPts,
OutputArray status, OutputArray err,
Ptr<RLOFOpticalFlowParameter> rlofParam = Ptr<RLOFOpticalFlowParameter>(),
float forwardBackwardThreshold = 0);
//! Additional interface to the Dense RLOF algorithm - optflow::calcOpticalFlowDenseRLOF()
CV_EXPORTS_W Ptr<DenseOpticalFlow> createOptFlow_DenseRLOF();
//! Additional interface to the Sparse RLOF algorithm - optflow::calcOpticalFlowSparseRLOF()
CV_EXPORTS_W Ptr<SparseOpticalFlow> createOptFlow_SparseRLOF();
//! @}
} // namespace
} // namespace
#endif
modules/optflow/include/opencv2/optflow/sparse_matching_gpc.hpp
+372
View File
@@ -0,0 +1,372 @@
/*
By downloading, copying, installing or using the software you agree to this
license. If you do not agree to this license, do not download, install,
copy or use the software.
License Agreement
For Open Source Computer Vision Library
(3-clause BSD License)
Copyright (C) 2016, OpenCV Foundation, all rights reserved.
Third party copyrights are property of their respective owners.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the names of the copyright holders nor the names of the contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
This software is provided by the copyright holders and contributors "as is" and
any express or implied warranties, including, but not limited to, the implied
warranties of merchantability and fitness for a particular purpose are
disclaimed. In no event shall copyright holders or contributors be liable for
any direct, indirect, incidental, special, exemplary, or consequential damages
(including, but not limited to, procurement of substitute goods or services;
loss of use, data, or profits; or business interruption) however caused
and on any theory of liability, whether in contract, strict liability,
or tort (including negligence or otherwise) arising in any way out of
the use of this software, even if advised of the possibility of such damage.
*/
/**
* @file sparse_matching_gpc.hpp
* @author Vladislav Samsonov <vvladxx@gmail.com>
* @brief Implementation of the Global Patch Collider.
*
* Implementation of the Global Patch Collider algorithm from the following paper:
* http://research.microsoft.com/en-us/um/people/pkohli/papers/wfrik_cvpr2016.pdf
*
* @cite Wang_2016_CVPR
*/
#ifndef __OPENCV_OPTFLOW_SPARSE_MATCHING_GPC_HPP__
#define __OPENCV_OPTFLOW_SPARSE_MATCHING_GPC_HPP__
#include "opencv2/core.hpp"
#include "opencv2/imgproc.hpp"
namespace cv
{
namespace optflow
{
//! @addtogroup optflow
//! @{
struct CV_EXPORTS_W GPCPatchDescriptor
{
static const unsigned nFeatures = 18; //!< number of features in a patch descriptor
Vec< double, nFeatures > feature;
double dot( const Vec< double, nFeatures > &coef ) const;
void markAsSeparated() { feature[0] = std::numeric_limits< double >::quiet_NaN(); }
bool isSeparated() const { return cvIsNaN( feature[0] ) != 0; }
};
struct CV_EXPORTS_W GPCPatchSample
{
GPCPatchDescriptor ref;
GPCPatchDescriptor pos;
GPCPatchDescriptor neg;
void getDirections( bool &refdir, bool &posdir, bool &negdir, const Vec< double, GPCPatchDescriptor::nFeatures > &coef, double rhs ) const;
};
typedef std::vector< GPCPatchSample > GPCSamplesVector;
/** @brief Descriptor types for the Global Patch Collider.
*/
enum GPCDescType
{
GPC_DESCRIPTOR_DCT = 0, //!< Better quality but slow
GPC_DESCRIPTOR_WHT //!< Worse quality but much faster
};
/** @brief Class encapsulating training samples.
*/
class CV_EXPORTS_W GPCTrainingSamples
{
private:
GPCSamplesVector samples;
int descriptorType;
public:
/** @brief This function can be used to extract samples from a pair of images and a ground truth flow.
* Sizes of all the provided vectors must be equal.
*/
static Ptr< GPCTrainingSamples > create( const std::vector< String > &imagesFrom, const std::vector< String > &imagesTo,
const std::vector< String > &gt, int descriptorType );
static Ptr< GPCTrainingSamples > create( InputArrayOfArrays imagesFrom, InputArrayOfArrays imagesTo, InputArrayOfArrays gt,
int descriptorType );
size_t size() const { return samples.size(); }
int type() const { return descriptorType; }
operator GPCSamplesVector &() { return samples; }
};
/** @brief Class encapsulating training parameters.
*/
struct GPCTrainingParams
{
unsigned maxTreeDepth; //!< Maximum tree depth to stop partitioning.
int minNumberOfSamples; //!< Minimum number of samples in the node to stop partitioning.
int descriptorType; //!< Type of descriptors to use.
bool printProgress; //!< Print progress to stdout.
GPCTrainingParams( unsigned _maxTreeDepth = 20, int _minNumberOfSamples = 3, GPCDescType _descriptorType = GPC_DESCRIPTOR_DCT,
bool _printProgress = true )
: maxTreeDepth( _maxTreeDepth ), minNumberOfSamples( _minNumberOfSamples ), descriptorType( _descriptorType ),
printProgress( _printProgress )
{
CV_Assert( check() );
}
bool check() const { return maxTreeDepth > 1 && minNumberOfSamples > 1; }
};
/** @brief Class encapsulating matching parameters.
*/
struct GPCMatchingParams
{
bool useOpenCL; //!< Whether to use OpenCL to speed up the matching.
GPCMatchingParams( bool _useOpenCL = false ) : useOpenCL( _useOpenCL ) {}
GPCMatchingParams( const GPCMatchingParams &params ) : useOpenCL( params.useOpenCL ) {}
};
/** @brief Class for individual tree.
*/
class CV_EXPORTS_W GPCTree : public Algorithm
{
public:
struct Node
{
Vec< double, GPCPatchDescriptor::nFeatures > coef; //!< Hyperplane coefficients
double rhs; //!< Bias term of the hyperplane
unsigned left;
unsigned right;
bool operator==( const Node &n ) const { return coef == n.coef && rhs == n.rhs && left == n.left && right == n.right; }
};
private:
typedef GPCSamplesVector::iterator SIter;
std::vector< Node > nodes;
GPCTrainingParams params;
bool trainNode( size_t nodeId, SIter begin, SIter end, unsigned depth );
public:
void train( GPCTrainingSamples &samples, const GPCTrainingParams params = GPCTrainingParams() );
void write( FileStorage &fs ) const CV_OVERRIDE;
void read( const FileNode &fn ) CV_OVERRIDE;
unsigned findLeafForPatch( const GPCPatchDescriptor &descr ) const;
static Ptr< GPCTree > create() { return makePtr< GPCTree >(); }
bool operator==( const GPCTree &t ) const { return nodes == t.nodes; }
int getDescriptorType() const { return params.descriptorType; }
};
template < int T > class GPCForest : public Algorithm
{
private:
struct Trail
{
unsigned leaf[T]; //!< Inside which leaf of the tree 0..T the patch fell?
Point2i coord; //!< Patch coordinates.
bool operator==( const Trail &trail ) const { return memcmp( leaf, trail.leaf, sizeof( leaf ) ) == 0; }
bool operator<( const Trail &trail ) const
{
for ( int i = 0; i < T - 1; ++i )
if ( leaf[i] != trail.leaf[i] )
return leaf[i] < trail.leaf[i];
return leaf[T - 1] < trail.leaf[T - 1];
}
};
class ParallelTrailsFilling : public ParallelLoopBody
{
private:
const GPCForest *forest;
const std::vector< GPCPatchDescriptor > *descr;
std::vector< Trail > *trails;
ParallelTrailsFilling &operator=( const ParallelTrailsFilling & );
public:
ParallelTrailsFilling( const GPCForest *_forest, const std::vector< GPCPatchDescriptor > *_descr, std::vector< Trail > *_trails )
: forest( _forest ), descr( _descr ), trails( _trails ){};
void operator()( const Range &range ) const CV_OVERRIDE
{
for ( int t = range.start; t < range.end; ++t )
for ( size_t i = 0; i < descr->size(); ++i )
trails->at( i ).leaf[t] = forest->tree[t].findLeafForPatch( descr->at( i ) );
}
};
GPCTree tree[T];
public:
/** @brief Train the forest using one sample set for every tree.
* Please, consider using the next method instead of this one for better quality.
*/
void train( GPCTrainingSamples &samples, const GPCTrainingParams params = GPCTrainingParams() )
{
for ( int i = 0; i < T; ++i )
tree[i].train( samples, params );
}
/** @brief Train the forest using individual samples for each tree.
* It is generally better to use this instead of the first method.
*/
void train( const std::vector< String > &imagesFrom, const std::vector< String > &imagesTo, const std::vector< String > &gt,
const GPCTrainingParams params = GPCTrainingParams() )
{
for ( int i = 0; i < T; ++i )
{
Ptr< GPCTrainingSamples > samples =
GPCTrainingSamples::create( imagesFrom, imagesTo, gt, params.descriptorType ); // Create training set for the tree
tree[i].train( *samples, params );
}
}
void train( InputArrayOfArrays imagesFrom, InputArrayOfArrays imagesTo, InputArrayOfArrays gt,
const GPCTrainingParams params = GPCTrainingParams() )
{
for ( int i = 0; i < T; ++i )
{
Ptr< GPCTrainingSamples > samples =
GPCTrainingSamples::create( imagesFrom, imagesTo, gt, params.descriptorType ); // Create training set for the tree
tree[i].train( *samples, params );
}
}
void write( FileStorage &fs ) const CV_OVERRIDE
{
fs << "ntrees" << T << "trees"
<< "[";
for ( int i = 0; i < T; ++i )
{
fs << "{";
tree[i].write( fs );
fs << "}";
}
fs << "]";
}
void read( const FileNode &fn ) CV_OVERRIDE
{
CV_Assert( T <= (int)fn["ntrees"] );
FileNodeIterator it = fn["trees"].begin();
for ( int i = 0; i < T; ++i, ++it )
tree[i].read( *it );
}
/** @brief Find correspondences between two images.
* @param[in] imgFrom First image in a sequence.
* @param[in] imgTo Second image in a sequence.
* @param[out] corr Output vector with pairs of corresponding points.
* @param[in] params Additional matching parameters for fine-tuning.
*/
void findCorrespondences( InputArray imgFrom, InputArray imgTo, std::vector< std::pair< Point2i, Point2i > > &corr,
const GPCMatchingParams params = GPCMatchingParams() ) const;
static Ptr< GPCForest > create() { return makePtr< GPCForest >(); }
};
class CV_EXPORTS_W GPCDetails
{
public:
static void dropOutliers( std::vector< std::pair< Point2i, Point2i > > &corr );
static void getAllDescriptorsForImage( const Mat *imgCh, std::vector< GPCPatchDescriptor > &descr, const GPCMatchingParams &mp,
int type );
static void getCoordinatesFromIndex( size_t index, Size sz, int &x, int &y );
};
template < int T >
void GPCForest< T >::findCorrespondences( InputArray imgFrom, InputArray imgTo, std::vector< std::pair< Point2i, Point2i > > &corr,
const GPCMatchingParams params ) const
{
CV_Assert( imgFrom.channels() == 3 );
CV_Assert( imgTo.channels() == 3 );
Mat from, to;
imgFrom.getMat().convertTo( from, CV_32FC3 );
imgTo.getMat().convertTo( to, CV_32FC3 );
cvtColor( from, from, COLOR_BGR2YCrCb );
cvtColor( to, to, COLOR_BGR2YCrCb );
Mat fromCh[3], toCh[3];
split( from, fromCh );
split( to, toCh );
std::vector< GPCPatchDescriptor > descr;
GPCDetails::getAllDescriptorsForImage( fromCh, descr, params, tree[0].getDescriptorType() );
std::vector< Trail > trailsFrom( descr.size() ), trailsTo( descr.size() );
for ( size_t i = 0; i < descr.size(); ++i )
GPCDetails::getCoordinatesFromIndex( i, from.size(), trailsFrom[i].coord.x, trailsFrom[i].coord.y );
parallel_for_( Range( 0, T ), ParallelTrailsFilling( this, &descr, &trailsFrom ) );
descr.clear();
GPCDetails::getAllDescriptorsForImage( toCh, descr, params, tree[0].getDescriptorType() );
for ( size_t i = 0; i < descr.size(); ++i )
GPCDetails::getCoordinatesFromIndex( i, to.size(), trailsTo[i].coord.x, trailsTo[i].coord.y );
parallel_for_( Range( 0, T ), ParallelTrailsFilling( this, &descr, &trailsTo ) );
std::sort( trailsFrom.begin(), trailsFrom.end() );
std::sort( trailsTo.begin(), trailsTo.end() );
for ( size_t i = 0; i < trailsFrom.size(); ++i )
{
bool uniq = true;
while ( i + 1 < trailsFrom.size() && trailsFrom[i] == trailsFrom[i + 1] )
++i, uniq = false;
if ( uniq )
{
typename std::vector< Trail >::const_iterator lb = std::lower_bound( trailsTo.begin(), trailsTo.end(), trailsFrom[i] );
if ( lb != trailsTo.end() && *lb == trailsFrom[i] && ( ( lb + 1 ) == trailsTo.end() || !( *lb == *( lb + 1 ) ) ) )
corr.push_back( std::make_pair( trailsFrom[i].coord, lb->coord ) );
}
}
GPCDetails::dropOutliers( corr );
}
//! @}
} // namespace optflow
CV_EXPORTS void write( FileStorage &fs, const String &name, const optflow::GPCTree::Node &node );
CV_EXPORTS void read( const FileNode &fn, optflow::GPCTree::Node &node, optflow::GPCTree::Node );
} // namespace cv
#endif
modules/optflow/perf/opencl/perf_optflow_dualTVL1.cpp
+230
View File
@@ -0,0 +1,230 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
// Fangfang Bai, fangfang@multicorewareinc.com
// Jin Ma, jin@multicorewareinc.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "../perf_precomp.hpp"
#include "opencv2/ts/ocl_perf.hpp"
#ifdef HAVE_OPENCL
namespace opencv_test {
namespace ocl {
#undef VISUALIZE_FLOW
//#define VISUALIZE_FLOW 1
#ifdef VISUALIZE_FLOW
// [TODO] move it to opencv_video/tracking.hpp
// Standard visualization of the optical flow, according to Scharstein/Middlebury (ICCV 2007).
// flow — input flow of CV_32FC2 type (field of motion vectors)
// dst — output visual optical flow representation of CV_8UC3 type
// maxFlow — max flow magnitude or (if <=0, it's computed automatically; use with care,
// because it 'boosts' flow when there is almost no one).
// pixels where the motion vector magnitude is equal or exceeds maxFlow will be
// painted with maximum saturation.
// eps — maxFlow threshold for 'no motion/still scene' cases
// computedMaxFlow — the optional output value to hold the computed maxFlow, to let user
// gradually calibrate maxFlow parameter.
static void tsVisualizeFlow(InputArray flowarr, OutputArray dstarr,
float maxFlow0 = -1.f, float eps = 1e-3f,
float* computedMaxFlow=nullptr)
{
Mat flow = flowarr.getMat();
CV_Assert(flow.type() == CV_32FC2);
// --- color ring (55 colors, RGB) ---
// sectors: RY=15, YG=6, GC=4, CB=11, BM=13, MR=6
constexpr int RY=15, YG=6, GC=4, CB=11, BM=13, MR=6;
constexpr int NCOLS = RY+YG+GC+CB+BM+MR; // 55
std::array<Vec3b, NCOLS+1> cwheel_; // RGB
int k = 0;
for (int i=0;i<RY;i++,k++) cwheel_[k]=Vec3b(0, uint8_t(255*i/RY), 255);
for (int i=0;i<YG;i++,k++) cwheel_[k]=Vec3b(0, 255, uint8_t(255-255*i/YG));
for (int i=0;i<GC;i++,k++) cwheel_[k]=Vec3b(uint8_t(255*i/GC), 255, 0);
for (int i=0;i<CB;i++,k++) cwheel_[k]=Vec3b(255, uint8_t(255-255*i/CB), 0);
for (int i=0;i<BM;i++,k++) cwheel_[k]=Vec3b(255, 0, uint8_t(255*i/BM));
for (int i=0;i<MR;i++,k++) cwheel_[k]=Vec3b(uint8_t(255-255*i/MR), 0, 255);
cwheel_[NCOLS] = cwheel_[0];
std::vector<float> maxvals(flow.rows);
// --- compute max flow automatically ---
if (maxFlow0 <= 0.0f) {
parallel_for_(cv::Range(0, flow.rows), [&](const cv::Range& range) {
for (int y = range.start; y < range.end; y++) {
float maxval = 0.f;
const cv::Vec2f* row = flow.ptr<cv::Vec2f>(y);
for (int x = 0; x < flow.cols; x++) {
float dx = row[x][0], dy = row[x][1];
float mag = std::hypot(dx, dy);
maxval = std::max(maxval, mag);
}
maxvals[y] = maxval;
}
});
maxFlow0 = 0.f;
for (int y = 0; y < flow.rows; y++)
maxFlow0 = std::max(maxFlow0, maxvals[y]);
}
maxFlow0 = maxFlow0 > eps ? maxFlow0 : 1.f;
dstarr.create(flow.size(), CV_8UC3);
Mat dst = dstarr.getMat();
// paint the optical flow map
parallel_for_(cv::Range(0, flow.rows), [&](const cv::Range& range) {
const Vec3b* cwheel = cwheel_.data();
float maxval = 0.f, maxflow = maxFlow0;
for (int y = range.start; y < range.end; y++) {
const cv::Vec2f* src = flow.ptr<cv::Vec2f>(y);
cv::Vec3b* out = dst.ptr<cv::Vec3b>(y);
for (int x = 0; x < flow.cols; x++) {
float dx = src[x][0];
float dy = src[x][1];
float mag = std::hypot(dx, dy);
maxval = std::max(maxval, mag);
mag = std::min(mag / maxflow, 1.f);
// compute the color from angle
float a = std::atan2(-dy, -dx) / static_cast<float>(M_PI);
float f = (a + 1.0f) * 0.5f * (NCOLS - 1);
int idx = (int)f;
f -= idx;
// 'white' means no motion,
// the stronger the motion the more saturated the corresponding pixel is.
Vec3b clr;
for (int c = 0; c < 3; c++) {
float chval = cwheel[idx][c] * (1.f - f) + cwheel[idx + 1][c] * f;
chval = 255.f - mag * (255.f - chval);
clr[c] = saturate_cast<uint8_t>(chval);
}
out[x] = clr;
}
maxvals[y] = maxval;
}
});
if (computedMaxFlow) {
maxFlow0 = 0.f;
for (int y = 0; y < flow.rows; y++)
maxFlow0 = std::max(maxFlow0, maxvals[y]);
*computedMaxFlow = maxFlow0;
}
}
#endif
///////////// OpticalFlow Dual TVL1 ////////////////////////
typedef tuple< tuple<int, double>, bool> OpticalFlowDualTVL1Params;
typedef TestBaseWithParam<OpticalFlowDualTVL1Params> OpticalFlowDualTVL1Fixture;
OCL_PERF_TEST_P(OpticalFlowDualTVL1Fixture, OpticalFlowDualTVL1,
::testing::Combine(
::testing::Values(make_tuple<int, double>(-1, 0.3),
make_tuple<int, double>(3, 0.5)),
::testing::Bool()
)
)
{
Mat frame0 = imread(getDataPath("cv/optflow/RubberWhale1.png"), IMREAD_GRAYSCALE);
ASSERT_FALSE(frame0.empty()) << "can't load RubberWhale1.png";
Mat frame1 = imread(getDataPath("cv/optflow/RubberWhale2.png"), IMREAD_GRAYSCALE);
ASSERT_FALSE(frame1.empty()) << "can't load RubberWhale2.png";
const Size srcSize = frame0.size();
const OpticalFlowDualTVL1Params params = GetParam();
const tuple<int, double> filteringScale = get<0>(params);
const int medianFiltering = get<0>(filteringScale);
const double scaleStep = get<1>(filteringScale);
const bool useInitFlow = get<1>(params);
//double eps = 0.9;
UMat uFrame0; frame0.copyTo(uFrame0);
UMat uFrame1; frame1.copyTo(uFrame1);
UMat uFlow(srcSize, CV_32FC2);
declare.in(uFrame0, uFrame1, WARMUP_READ).out(uFlow, WARMUP_READ);
//create algorithm
Ptr<DualTVL1OpticalFlow> alg = createOptFlow_DualTVL1();
//set parameters
alg->setScaleStep(scaleStep);
alg->setMedianFiltering(medianFiltering);
if (useInitFlow)
{
//calculate initial flow as result of optical flow
alg->calc(uFrame0, uFrame1, uFlow);
}
//set flag to use initial flow
alg->setUseInitialFlow(useInitFlow);
OCL_TEST_CYCLE()
alg->calc(uFrame0, uFrame1, uFlow);
#ifdef VISUALIZE_FLOW
imshow("frame0", uFrame0);
UMat framediff;
absdiff(uFrame0, uFrame1, framediff);
imshow("framediff", framediff);
Mat flow8u;
tsVisualizeFlow(uFlow, flow8u);
imshow("uFlow", flow8u);
waitKey();
#endif
//SANITY_CHECK(uFlow, eps, ERROR_RELATIVE);
SANITY_CHECK_NOTHING();
}
}
} // namespace opencv_test::ocl
#endif // HAVE_OPENCL
modules/optflow/perf/perf_deepflow.cpp
+32
View File
@@ -0,0 +1,32 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "perf_precomp.hpp"
namespace opencv_test { namespace {
typedef tuple<Size> DFParams;
typedef TestBaseWithParam<DFParams> DenseOpticalFlow_DeepFlow;
PERF_TEST_P(DenseOpticalFlow_DeepFlow, perf, Values(szVGA, sz720p))
{
DFParams params = GetParam();
Size sz = get<0>(params);
Mat frame1(sz, CV_8U);
Mat frame2(sz, CV_8U);
Mat flow;
randu(frame1, 0, 255);
randu(frame2, 0, 255);
TEST_CYCLE_N(1)
{
Ptr<DenseOpticalFlow> algo = createOptFlow_DeepFlow();
algo->calc(frame1, frame2, flow);
}
SANITY_CHECK_NOTHING();
}
}} // namespace
modules/optflow/perf/perf_main.cpp
+6
View File
@@ -0,0 +1,6 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "perf_precomp.hpp"
CV_PERF_TEST_MAIN(optflow)
modules/optflow/perf/perf_precomp.hpp
+15
View File
@@ -0,0 +1,15 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef __OPENCV_PERF_PRECOMP_HPP__
#define __OPENCV_PERF_PRECOMP_HPP__
#include "opencv2/ts.hpp"
#include "opencv2/optflow.hpp"
namespace opencv_test {
using namespace perf;
using namespace cv::optflow;
}
#endif
modules/optflow/perf/perf_rlof.cpp
+74
View File
@@ -0,0 +1,74 @@
#include "perf_precomp.hpp"
namespace opencv_test { namespace {
typedef tuple<std::string, std::string, bool> ST_SR_IM_Sparse_t;
typedef TestBaseWithParam<ST_SR_IM_Sparse_t> ST_SR_IM_Sparse;
PERF_TEST_P(ST_SR_IM_Sparse, OpticalFlow_SparseRLOF,
testing::Combine(
testing::Values<std::string>("ST_BILINEAR", "ST_STANDART"),
testing::Values<std::string>("SR_CROSS", "SR_FIXED"),
testing::Values(true, false))
)
{
Mat frame1 = imread(getDataPath("cv/optflow/RubberWhale1.png"));
Mat frame2 = imread(getDataPath("cv/optflow/RubberWhale2.png"));
ASSERT_FALSE(frame1.empty());
ASSERT_FALSE(frame2.empty());
vector<Point2f> prevPts, currPts;
for (int r = 0; r < frame1.rows; r += 10)
{
for (int c = 0; c < frame1.cols; c += 10)
{
prevPts.push_back(Point2f(static_cast<float>(c), static_cast<float>(r)));
}
}
vector<uchar> status(prevPts.size());
vector<float> err(prevPts.size());
Ptr<RLOFOpticalFlowParameter> param = Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter);
if (get<0>(GetParam()) == "ST_BILINEAR")
param->solverType = ST_BILINEAR;
if (get<0>(GetParam()) == "ST_STANDART")
param->solverType = ST_STANDART;
if (get<1>(GetParam()) == "SR_CROSS")
param->supportRegionType = SR_CROSS;
if (get<1>(GetParam()) == "SR_FIXED")
param->supportRegionType = SR_FIXED;
param->useIlluminationModel = get<2>(GetParam());
PERF_SAMPLE_BEGIN()
calcOpticalFlowSparseRLOF(frame1, frame2, prevPts, currPts, status, err, param, 1.f);
PERF_SAMPLE_END()
SANITY_CHECK_NOTHING();
}
typedef tuple<std::string, int> INTERP_GRID_Dense_t;
typedef TestBaseWithParam<INTERP_GRID_Dense_t> INTERP_GRID_Dense;
PERF_TEST_P(INTERP_GRID_Dense, OpticalFlow_DenseRLOF,
testing::Combine(
testing::Values<std::string>("INTERP_EPIC", "INTERP_GEO", "INTERP_RIC"),
testing::Values<int>(4,10))
)
{
Mat flow;
Mat frame1 = imread(getDataPath("cv/optflow/RubberWhale1.png"));
Mat frame2 = imread(getDataPath("cv/optflow/RubberWhale1.png"));
ASSERT_FALSE(frame1.empty());
ASSERT_FALSE(frame2.empty());
Ptr<RLOFOpticalFlowParameter> param = Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter);;
Ptr< DenseRLOFOpticalFlow> algo = DenseRLOFOpticalFlow::create();
InterpolationType interp_type = INTERP_EPIC;
if (get<0>(GetParam()) == "INTERP_EPIC")
interp_type = INTERP_EPIC;
if (get<0>(GetParam()) == "INTERP_GEO")
interp_type = INTERP_GEO;
if (get<0>(GetParam()) == "INTERP_RIC")
interp_type = INTERP_RIC;
PERF_SAMPLE_BEGIN()
calcOpticalFlowDenseRLOF(frame1, frame2,flow, param, 1.0f, Size(get<1>(GetParam()), get<1>(GetParam())), interp_type);
PERF_SAMPLE_END()
SANITY_CHECK_NOTHING();
}
}} // namespace
modules/optflow/perf/perf_tvl1optflow.cpp
+31
View File
@@ -0,0 +1,31 @@
#include "perf_precomp.hpp"
namespace opencv_test { namespace {
using namespace perf;
typedef TestBaseWithParam< std::pair<string, string> > ImagePair;
std::pair<string, string> impair(const char* im1, const char* im2)
{
return std::make_pair(string(im1), string(im2));
}
PERF_TEST_P(ImagePair, OpticalFlowDual_TVL1, testing::Values(impair("cv/optflow/RubberWhale1.png", "cv/optflow/RubberWhale2.png")))
{
declare.time(260);
Mat frame1 = imread(getDataPath(GetParam().first), IMREAD_GRAYSCALE);
Mat frame2 = imread(getDataPath(GetParam().second), IMREAD_GRAYSCALE);
ASSERT_FALSE(frame1.empty());
ASSERT_FALSE(frame2.empty());
Mat flow;
Ptr<DenseOpticalFlow> tvl1 = createOptFlow_DualTVL1();
TEST_CYCLE() tvl1->calc(frame1, frame2, flow);
SANITY_CHECK_NOTHING();
}
}} // namespace
modules/optflow/samples/gpc_evaluate.cpp
+174
View File
@@ -0,0 +1,174 @@
#include "opencv2/core/ocl.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/optflow.hpp"
#include <fstream>
#include <iostream>
#include <stdio.h>
/* This tool finds correspondences between two images using Global Patch Collider
* and calculates error using provided ground truth flow.
*
* It will look for the file named "forest.yml.gz" with a learned forest.
* You can obtain the "forest.yml.gz" either by manually training it using another tool with *_train suffix
* or by downloading one of the files trained on some publicly available dataset from here:
*
* https://drive.google.com/open?id=0B7Hb8cfuzrIIZDFscXVYd0NBNFU
*/
using namespace cv;
const String keys = "{help h ? | | print this message}"
"{@image1 |<none> | image1}"
"{@image2 |<none> | image2}"
"{@groundtruth |<none> | path to the .flo file}"
"{@output | | output to a file instead of displaying, output image path}"
"{g gpu | | use OpenCL}"
"{f forest |forest.yml.gz| path to the forest.yml.gz}";
const int nTrees = 5;
static double normL2( const Point2f &v ) { return sqrt( v.x * v.x + v.y * v.y ); }
static Vec3d getFlowColor( const Point2f &f, const bool logScale = true, const double scaleDown = 5 )
{
if ( f.x == 0 && f.y == 0 )
return Vec3d( 0, 0, 1 );
double radius = normL2( f );
if ( logScale )
radius = log( radius + 1 );
radius /= scaleDown;
radius = std::min( 1.0, radius );
double angle = ( atan2( -f.y, -f.x ) + CV_PI ) * 180 / CV_PI;
return Vec3d( angle, radius, 1 );
}
static void displayFlow( InputArray _flow, OutputArray _img )
{
const Size sz = _flow.size();
Mat flow = _flow.getMat();
_img.create( sz, CV_32FC3 );
Mat img = _img.getMat();
for ( int i = 0; i < sz.height; ++i )
for ( int j = 0; j < sz.width; ++j )
img.at< Vec3f >( i, j ) = getFlowColor( flow.at< Point2f >( i, j ) );
cvtColor( img, img, COLOR_HSV2BGR );
}
static bool fileProbe( const char *name ) { return std::ifstream( name ).good(); }
int main( int argc, const char **argv )
{
CommandLineParser parser( argc, argv, keys );
parser.about( "Global Patch Collider evaluation tool" );
if ( parser.has( "help" ) )
{
parser.printMessage();
return 0;
}
String fromPath = parser.get< String >( 0 );
String toPath = parser.get< String >( 1 );
String gtPath = parser.get< String >( 2 );
String outPath = parser.get< String >( 3 );
const bool useOpenCL = parser.has( "gpu" );
String forestDumpPath = parser.get< String >( "forest" );
if ( !parser.check() )
{
parser.printErrors();
return 1;
}
if ( !fileProbe( forestDumpPath.c_str() ) )
{
std::cerr << "Can't open the file with a trained model: `" << forestDumpPath
<< "`.\nYou can obtain this file either by manually training the model using another tool with *_train suffix or by "
"downloading one of the files trained on some publicly available dataset from "
"here:\nhttps://drive.google.com/open?id=0B7Hb8cfuzrIIZDFscXVYd0NBNFU"
<< std::endl;
return 1;
}
ocl::setUseOpenCL( useOpenCL );
Ptr< optflow::GPCForest< nTrees > > forest = Algorithm::load< optflow::GPCForest< nTrees > >( forestDumpPath );
Mat from = imread( fromPath );
Mat to = imread( toPath );
Mat gt = readOpticalFlow( gtPath );
std::vector< std::pair< Point2i, Point2i > > corr;
TickMeter meter;
meter.start();
forest->findCorrespondences( from, to, corr, optflow::GPCMatchingParams( useOpenCL ) );
meter.stop();
std::cout << "Found " << corr.size() << " matches." << std::endl;
std::cout << "Time: " << meter.getTimeSec() << " sec." << std::endl;
double error = 0;
int totalCorrectFlowVectors = 0;
Mat dispErr = Mat::zeros( from.size(), CV_32FC3 );
dispErr = Scalar( 0, 0, 1 );
Mat disp = Mat::zeros( from.size(), CV_32FC3 );
disp = Scalar( 0, 0, 1 );
for ( size_t i = 0; i < corr.size(); ++i )
{
const Point2f a = corr[i].first;
const Point2f b = corr[i].second;
const Point2f gtDisplacement = gt.at< Point2f >( corr[i].first.y, corr[i].first.x );
// Check that flow vector is correct
if (!cvIsNaN(gtDisplacement.x) && !cvIsNaN(gtDisplacement.y) && gtDisplacement.x < 1e9 && gtDisplacement.y < 1e9)
{
const Point2f c = a + gtDisplacement;
error += normL2( b - c );
circle( dispErr, a, 3, getFlowColor( b - c, false, 32 ), -1 );
++totalCorrectFlowVectors;
}
circle( disp, a, 3, getFlowColor( b - a ), -1 );
}
if (totalCorrectFlowVectors)
error /= totalCorrectFlowVectors;
std::cout << "Average endpoint error: " << error << " px." << std::endl;
cvtColor( disp, disp, COLOR_HSV2BGR );
cvtColor( dispErr, dispErr, COLOR_HSV2BGR );
Mat dispGroundTruth;
displayFlow( gt, dispGroundTruth );
if ( outPath.length() )
{
putText( disp, "Sparse matching: Global Patch Collider", Point2i( 24, 40 ), FONT_HERSHEY_DUPLEX, 1, Vec3b( 1, 0, 0 ), 2, LINE_AA );
char buf[256];
sprintf( buf, "Average EPE: %.2f", error );
putText( disp, buf, Point2i( 24, 80 ), FONT_HERSHEY_DUPLEX, 1, Vec3b( 1, 0, 0 ), 2, LINE_AA );
sprintf( buf, "Number of matches: %u", (unsigned)corr.size() );
putText( disp, buf, Point2i( 24, 120 ), FONT_HERSHEY_DUPLEX, 1, Vec3b( 1, 0, 0 ), 2, LINE_AA );
disp *= 255;
imwrite( outPath, disp );
return 0;
}
namedWindow( "Correspondences", WINDOW_AUTOSIZE );
imshow( "Correspondences", disp );
namedWindow( "Error", WINDOW_AUTOSIZE );
imshow( "Error", dispErr );
namedWindow( "Ground truth", WINDOW_AUTOSIZE );
imshow( "Ground truth", dispGroundTruth );
waitKey( 0 );
return 0;
}
modules/optflow/samples/gpc_train.cpp
+66
View File
@@ -0,0 +1,66 @@
#include "opencv2/optflow.hpp"
#include <iostream>
/* This tool trains the forest for the Global Patch Collider and stores output to the "forest.yml.gz".
*/
using namespace cv;
const String keys = "{help h ? | | print this message}"
"{max-tree-depth | | Maximum tree depth to stop partitioning}"
"{min-samples | | Minimum number of samples in the node to stop partitioning}"
"{descriptor-type|0 | Descriptor type. Set to 0 for quality, 1 for speed.}"
"{print-progress | | Set to 0 to enable quiet mode, set to 1 to print progress}"
"{f forest |forest.yml.gz| Path where to store resulting forest. It is recommended to use .yml.gz extension.}";
const int nTrees = 5;
static void fillInputImagesFromCommandLine( std::vector< String > &img1, std::vector< String > &img2, std::vector< String > &gt, int argc,
const char **argv )
{
for ( int i = 1, j = 0; i < argc; ++i )
{
if ( argv[i][0] == '-' )
continue;
if ( j % 3 == 0 )
img1.push_back( argv[i] );
if ( j % 3 == 1 )
img2.push_back( argv[i] );
if ( j % 3 == 2 )
gt.push_back( argv[i] );
++j;
}
}
int main( int argc, const char **argv )
{
CommandLineParser parser( argc, argv, keys );
parser.about( "Global Patch Collider training tool" );
std::vector< String > img1, img2, gt;
optflow::GPCTrainingParams params;
if ( parser.has( "max-tree-depth" ) )
params.maxTreeDepth = parser.get< unsigned >( "max-tree-depth" );
if ( parser.has( "min-samples" ) )
params.minNumberOfSamples = parser.get< unsigned >( "min-samples" );
if ( parser.has( "descriptor-type" ) )
params.descriptorType = parser.get< int >( "descriptor-type" );
if ( parser.has( "print-progress" ) )
params.printProgress = parser.get< unsigned >( "print-progress" ) != 0;
fillInputImagesFromCommandLine( img1, img2, gt, argc, argv );
if ( parser.has( "help" ) || img1.size() != img2.size() || img1.size() != gt.size() || img1.size() == 0 )
{
std::cerr << "\nUsage: " << argv[0] << " [params] ImageFrom1 ImageTo1 GroundTruth1 ... ImageFromN ImageToN GroundTruthN\n" << std::endl;
parser.printMessage();
return 1;
}
Ptr< optflow::GPCForest< nTrees > > forest = optflow::GPCForest< nTrees >::create();
forest->train( img1, img2, gt, params );
forest->save( parser.get< String >( "forest" ) );
return 0;
}
modules/optflow/samples/gpc_train_middlebury.py
+58
View File
@@ -0,0 +1,58 @@
import argparse
import glob
import os
import subprocess
def execute(cmd):
popen = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for stdout_line in iter(popen.stdout.readline, ''):
print(stdout_line.rstrip())
for stderr_line in iter(popen.stderr.readline, ''):
print(stderr_line.rstrip())
popen.stdout.close()
popen.stderr.close()
return_code = popen.wait()
if return_code != 0:
raise subprocess.CalledProcessError(return_code, cmd)
def main():
parser = argparse.ArgumentParser(
description='Train Global Patch Collider using Middlebury dataset')
parser.add_argument(
'--bin_path',
help='Path to the training executable (example_optflow_gpc_train)',
required=True)
parser.add_argument('--dataset_path',
help='Path to the directory with frames',
required=True)
parser.add_argument('--gt_path',
help='Path to the directory with ground truth flow',
required=True)
parser.add_argument('--descriptor_type',
help='Descriptor type',
type=int,
default=0)
args = parser.parse_args()
seq = glob.glob(os.path.join(args.dataset_path, '*'))
seq.sort()
input_files = []
for s in seq:
if os.path.isdir(s):
seq_name = os.path.basename(s)
frames = glob.glob(os.path.join(s, 'frame*.png'))
frames.sort()
assert (len(frames) == 2)
assert (os.path.basename(frames[0]) == 'frame10.png')
assert (os.path.basename(frames[1]) == 'frame11.png')
gt_flow = os.path.join(args.gt_path, seq_name, 'flow10.flo')
if os.path.isfile(gt_flow):
input_files += [frames[0], frames[1], gt_flow]
execute([args.bin_path, '--descriptor-type=%d' % args.descriptor_type] + input_files)
if __name__ == '__main__':
main()
modules/optflow/samples/gpc_train_sintel.py
+60
View File
@@ -0,0 +1,60 @@
import argparse
import glob
import os
import subprocess
FRAME_DIST = 2
assert (FRAME_DIST >= 1)
def execute(cmd):
popen = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for stdout_line in iter(popen.stdout.readline, ''):
print(stdout_line.rstrip())
for stderr_line in iter(popen.stderr.readline, ''):
print(stderr_line.rstrip())
popen.stdout.close()
popen.stderr.close()
return_code = popen.wait()
if return_code != 0:
raise subprocess.CalledProcessError(return_code, cmd)
def main():
parser = argparse.ArgumentParser(
description='Train Global Patch Collider using MPI Sintel dataset')
parser.add_argument(
'--bin_path',
help='Path to the training executable (example_optflow_gpc_train)',
required=True)
parser.add_argument('--dataset_path',
help='Path to the directory with frames',
required=True)
parser.add_argument('--gt_path',
help='Path to the directory with ground truth flow',
required=True)
parser.add_argument('--descriptor_type',
help='Descriptor type',
type=int,
default=0)
args = parser.parse_args()
seq = glob.glob(os.path.join(args.dataset_path, '*'))
seq.sort()
input_files = []
for s in seq:
seq_name = os.path.basename(s)
frames = glob.glob(os.path.join(s, 'frame*.png'))
frames.sort()
for i in range(0, len(frames) - 1, FRAME_DIST):
gt_flow = os.path.join(args.gt_path, seq_name,
os.path.basename(frames[i])[0:-4] + '.flo')
assert (os.path.isfile(gt_flow))
input_files += [frames[i], frames[i + 1], gt_flow]
execute([args.bin_path, '--descriptor-type=%d' % args.descriptor_type] + input_files)
if __name__ == '__main__':
main()
modules/optflow/samples/motempl.cpp
+168
View File
@@ -0,0 +1,168 @@
#include "opencv2/optflow.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/videoio.hpp"
#include "opencv2/highgui.hpp"
#include <time.h>
#include <stdio.h>
#include <ctype.h>
using namespace cv;
using namespace std;
using namespace cv::motempl;
static void help(void)
{
printf(
"\nThis program demonstrated the use of motion templates -- basically using the gradients\n"
"of thresholded layers of decaying frame differencing. New movements are stamped on top with floating system\n"
"time code and motions too old are thresholded away. This is the 'motion history file'. The program reads from the camera of your choice or from\n"
"a file. Gradients of motion history are used to detect direction of motion etc\n"
"Usage :\n"
"./motempl [camera number 0-n or file name, default is camera 0]\n"
);
}
// various tracking parameters (in seconds)
const double MHI_DURATION = 5;
const double MAX_TIME_DELTA = 0.5;
const double MIN_TIME_DELTA = 0.05;
// number of cyclic frame buffer used for motion detection
// (should, probably, depend on FPS)
// ring image buffer
vector<Mat> buf;
int last = 0;
// temporary images
Mat mhi, orient, mask, segmask, zplane;
vector<Rect> regions;
// parameters:
// img - input video frame
// dst - resultant motion picture
// args - optional parameters
static void update_mhi(const Mat& img, Mat& dst, int diff_threshold)
{
double timestamp = (double)clock() / CLOCKS_PER_SEC; // get current time in seconds
Size size = img.size();
int i, idx1 = last;
Rect comp_rect;
double count;
double angle;
Point center;
double magnitude;
Scalar color;
// allocate images at the beginning or
// reallocate them if the frame size is changed
if (mhi.size() != size)
{
mhi = Mat::zeros(size, CV_32F);
zplane = Mat::zeros(size, CV_8U);
buf[0] = Mat::zeros(size, CV_8U);
buf[1] = Mat::zeros(size, CV_8U);
}
cvtColor(img, buf[last], COLOR_BGR2GRAY); // convert frame to grayscale
int idx2 = (last + 1) % 2; // index of (last - (N-1))th frame
last = idx2;
Mat silh = buf[idx2];
absdiff(buf[idx1], buf[idx2], silh); // get difference between frames
threshold(silh, silh, diff_threshold, 1, THRESH_BINARY); // and threshold it
updateMotionHistory(silh, mhi, timestamp, MHI_DURATION); // update MHI
// convert MHI to blue 8u image
mhi.convertTo(mask, CV_8U, 255. / MHI_DURATION, (MHI_DURATION - timestamp)*255. / MHI_DURATION);
Mat planes[] = { mask, zplane, zplane };
merge(planes, 3, dst);
// calculate motion gradient orientation and valid orientation mask
calcMotionGradient(mhi, mask, orient, MAX_TIME_DELTA, MIN_TIME_DELTA, 3);
// segment motion: get sequence of motion components
// segmask is marked motion components map. It is not used further
regions.clear();
segmentMotion(mhi, segmask, regions, timestamp, MAX_TIME_DELTA);
// iterate through the motion components,
// One more iteration (i == -1) corresponds to the whole image (global motion)
for (i = -1; i < (int)regions.size(); i++) {
if (i < 0) { // case of the whole image
comp_rect = Rect(0, 0, size.width, size.height);
color = Scalar(255, 255, 255);
magnitude = 100;
}
else { // i-th motion component
comp_rect = regions[i];
if (comp_rect.width + comp_rect.height < 100) // reject very small components
continue;
color = Scalar(0, 0, 255);
magnitude = 30;
}
// select component ROI
Mat silh_roi = silh(comp_rect);
Mat mhi_roi = mhi(comp_rect);
Mat orient_roi = orient(comp_rect);
Mat mask_roi = mask(comp_rect);
// calculate orientation
angle = calcGlobalOrientation(orient_roi, mask_roi, mhi_roi, timestamp, MHI_DURATION);
angle = 360.0 - angle; // adjust for images with top-left origin
count = norm(silh_roi, NORM_L1);; // calculate number of points within silhouette ROI
// check for the case of little motion
if (count < comp_rect.width*comp_rect.height * 0.05)
continue;
// draw a clock with arrow indicating the direction
center = Point((comp_rect.x + comp_rect.width / 2),
(comp_rect.y + comp_rect.height / 2));
circle(img, center, cvRound(magnitude*1.2), color, 3, 16, 0);
line(img, center, Point(cvRound(center.x + magnitude*cos(angle*CV_PI / 180)),
cvRound(center.y - magnitude*sin(angle*CV_PI / 180))), color, 3, 16, 0);
}
}
int main(int argc, char** argv)
{
VideoCapture cap;
help();
if (argc == 1 || (argc == 2 && strlen(argv[1]) == 1 && isdigit(argv[1][0])))
cap.open(argc == 2 ? argv[1][0] - '0' : 0);
else if (argc == 2)
cap.open(argv[1]);
if (!cap.isOpened())
{
printf("Could not initialize video capture\n");
return 0;
}
buf.resize(2);
Mat image, motion;
for (;;)
{
cap >> image;
if (image.empty())
break;
update_mhi(image, motion, 30);
imshow("Image", image);
imshow("Motion", motion);
if (waitKey(10) >= 0)
break;
}
return 0;
}
modules/optflow/samples/motempl.py
Executable
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python
import numpy as np
import cv2 as cv
MHI_DURATION = 0.5
DEFAULT_THRESHOLD = 32
MAX_TIME_DELTA = 0.25
MIN_TIME_DELTA = 0.05
# (empty) trackbar callback
def nothing(dummy):
pass
def draw_motion_comp(vis, rect, angle, color):
x, y, w, h = rect
cv.rectangle(vis, (x, y), (x+w, y+h), (0, 255, 0))
r = min(w//2, h//2)
cx, cy = x+w//2, y+h//2
angle = angle*np.pi/180
cv.circle(vis, (cx, cy), r, color, 3)
cv.line(vis, (cx, cy), (int(cx+np.cos(angle)*r), int(cy+np.sin(angle)*r)), color, 3)
if __name__ == '__main__':
import sys
try:
video_src = sys.argv[1]
except:
video_src = 0
cv.namedWindow('motempl')
visuals = ['input', 'frame_diff', 'motion_hist', 'grad_orient']
cv.createTrackbar('visual', 'motempl', 2, len(visuals)-1, nothing)
cv.createTrackbar('threshold', 'motempl', DEFAULT_THRESHOLD, 255, nothing)
cam = cv.VideoCapture(video_src)
if not cam.isOpened():
print("could not open video_src " + str(video_src) + " !\n")
sys.exit(1)
ret, frame = cam.read()
if ret == False:
print("could not read from " + str(video_src) + " !\n")
sys.exit(1)
h, w = frame.shape[:2]
prev_frame = frame.copy()
motion_history = np.zeros((h, w), np.float32)
hsv = np.zeros((h, w, 3), np.uint8)
hsv[:,:,1] = 255
while True:
ret, frame = cam.read()
if ret == False:
break
frame_diff = cv.absdiff(frame, prev_frame)
gray_diff = cv.cvtColor(frame_diff, cv.COLOR_BGR2GRAY)
thrs = cv.getTrackbarPos('threshold', 'motempl')
ret, motion_mask = cv.threshold(gray_diff, thrs, 1, cv.THRESH_BINARY)
timestamp = cv.getTickCount() / cv.getTickFrequency()
cv.motempl.updateMotionHistory(motion_mask, motion_history, timestamp, MHI_DURATION)
mg_mask, mg_orient = cv.motempl.calcMotionGradient( motion_history, MAX_TIME_DELTA, MIN_TIME_DELTA, apertureSize=5 )
seg_mask, seg_bounds = cv.motempl.segmentMotion(motion_history, timestamp, MAX_TIME_DELTA)
visual_name = visuals[cv.getTrackbarPos('visual', 'motempl')]
if visual_name == 'input':
vis = frame.copy()
elif visual_name == 'frame_diff':
vis = frame_diff.copy()
elif visual_name == 'motion_hist':
vis = np.uint8(np.clip((motion_history-(timestamp-MHI_DURATION)) / MHI_DURATION, 0, 1)*255)
vis = cv.cvtColor(vis, cv.COLOR_GRAY2BGR)
elif visual_name == 'grad_orient':
hsv[:,:,0] = mg_orient/2
hsv[:,:,2] = mg_mask*255
vis = cv.cvtColor(hsv, cv.COLOR_HSV2BGR)
for i, rect in enumerate([(0, 0, w, h)] + list(seg_bounds)):
x, y, rw, rh = rect
area = rw*rh
if area < 64**2:
continue
silh_roi = motion_mask [y:y+rh,x:x+rw]
orient_roi = mg_orient [y:y+rh,x:x+rw]
mask_roi = mg_mask [y:y+rh,x:x+rw]
mhi_roi = motion_history[y:y+rh,x:x+rw]
if cv.norm(silh_roi, cv.NORM_L1) < area*0.05:
continue
angle = cv.motempl.calcGlobalOrientation(orient_roi, mask_roi, mhi_roi, timestamp, MHI_DURATION)
color = ((255, 0, 0), (0, 0, 255))[i == 0]
draw_motion_comp(vis, rect, angle, color)
cv.putText(vis, visual_name, (20, 20), cv.FONT_HERSHEY_PLAIN, 1.0, (200,0,0))
cv.imshow('motempl', vis)
prev_frame = frame.copy()
if 0xFF & cv.waitKey(5) == 27:
break
# cleanup the camera and close any open windows
cam.release()
cv.destroyAllWindows()
modules/optflow/samples/optical_flow_benchmark.py
+268
View File
@@ -0,0 +1,268 @@
#!/usr/bin/env python
from __future__ import print_function
import os, sys, shutil
import argparse
import json, re
from subprocess import check_output
import datetime
import matplotlib.pyplot as plt
def load_json(path):
f = open(path, "r")
data = json.load(f)
return data
def save_json(obj, path):
tmp_file = path + ".bak"
f = open(tmp_file, "w")
json.dump(obj, f, indent=2)
f.flush()
os.fsync(f.fileno())
f.close()
try:
os.rename(tmp_file, path)
except:
os.remove(path)
os.rename(tmp_file, path)
def parse_evaluation_result(input_str, i):
res = {}
res['frame_number'] = i + 1
res['error'] = {}
regex = "([A-Za-z. \\[\\].0-9]+):[ ]*([0-9]*\.[0-9]+|[0-9]+)"
for elem in re.findall(regex,input_str):
if "Time" in elem[0]:
res['time'] = float(elem[1])
elif "Average" in elem[0]:
res['error']['average'] = float(elem[1])
elif "deviation" in elem[0]:
res['error']['std'] = float(elem[1])
else:
res['error'][elem[0]] = float(elem[1])
return res
def evaluate_sequence(sequence, algorithm, dataset, executable, img_files, gt_files,
state, state_path):
if "eval_results" not in state[dataset][algorithm][-1].keys():
state[dataset][algorithm][-1]["eval_results"] = {}
elif sequence in state[dataset][algorithm][-1]["eval_results"].keys():
return
res = []
for i in range(len(img_files) - 1):
sys.stdout.write("Algorithm: %-20s Sequence: %-10s Done: [%3d/%3d]\r" %
(algorithm, sequence, i, len(img_files) - 1)),
sys.stdout.flush()
res_string = check_output([executable, img_files[i], img_files[i + 1],
algorithm, gt_files[i]])
res.append(parse_evaluation_result(res_string, i))
state[dataset][algorithm][-1]["eval_results"][sequence] = res
save_json(state, state_path)
#############################DATSET DEFINITIONS################################
def evaluate_mpi_sintel(source_dir, algorithm, evaluation_executable, state, state_path):
evaluation_result = {}
img_dir = os.path.join(source_dir, 'mpi_sintel', 'training', 'final')
gt_dir = os.path.join(source_dir, 'mpi_sintel', 'training', 'flow')
sequences = [f for f in os.listdir(img_dir)
if os.path.isdir(os.path.join(img_dir, f))]
for seq in sequences:
img_files = sorted([os.path.join(img_dir, seq, f)
for f in os.listdir(os.path.join(img_dir, seq))
if f.endswith(".png")])
gt_files = sorted([os.path.join(gt_dir, seq, f)
for f in os.listdir(os.path.join(gt_dir, seq))
if f.endswith(".flo")])
evaluation_result[seq] = evaluate_sequence(seq, algorithm, 'mpi_sintel',
evaluation_executable, img_files, gt_files, state, state_path)
return evaluation_result
def evaluate_middlebury(source_dir, algorithm, evaluation_executable, state, state_path):
evaluation_result = {}
img_dir = os.path.join(source_dir, 'middlebury', 'other-data')
gt_dir = os.path.join(source_dir, 'middlebury', 'other-gt-flow')
sequences = [f for f in os.listdir(gt_dir)
if os.path.isdir(os.path.join(gt_dir, f))]
for seq in sequences:
img_files = sorted([os.path.join(img_dir, seq, f)
for f in os.listdir(os.path.join(img_dir, seq))
if f.endswith(".png")])
gt_files = sorted([os.path.join(gt_dir, seq, f)
for f in os.listdir(os.path.join(gt_dir, seq))
if f.endswith(".flo")])
evaluation_result[seq] = evaluate_sequence(seq, algorithm, 'middlebury',
evaluation_executable, img_files, gt_files, state, state_path)
return evaluation_result
dataset_eval_functions = {
"mpi_sintel": evaluate_mpi_sintel,
"middlebury": evaluate_middlebury
}
###############################################################################
def create_dir(dir):
if not os.path.exists(dir):
os.makedirs(dir)
def parse_sequence(input_str):
if len(input_str) == 0:
return []
else:
return [o.strip() for o in input_str.split(",") if o]
def build_chart(dst_folder, state, dataset):
fig = plt.figure(figsize=(16, 10))
markers = ["o", "s", "h", "^", "D"]
marker_idx = 0
colors = ["b", "g", "r"]
color_idx = 0
for algo in state[dataset].keys():
for eval_instance in state[dataset][algo]:
name = algo + "--" + eval_instance["timestamp"]
average_time = 0.0
average_error = 0.0
num_elem = 0
for seq in eval_instance["eval_results"].keys():
for frame in eval_instance["eval_results"][seq]:
average_time += frame["time"]
average_error += frame["error"]["average"]
num_elem += 1
average_time /= num_elem
average_error /= num_elem
marker_style = colors[color_idx] + markers[marker_idx]
color_idx += 1
if color_idx >= len(colors):
color_idx = 0
marker_idx += 1
if marker_idx >= len(markers):
marker_idx = 0
plt.gca().plot([average_time], [average_error],
marker_style,
markersize=14,
label=name)
plt.gca().set_ylabel('Average Endpoint Error (EPE)', fontsize=20)
plt.gca().set_xlabel('Average Runtime (seconds per frame)', fontsize=20)
plt.gca().set_xscale("log")
plt.gca().set_title('Evaluation on ' + dataset, fontsize=20)
plt.gca().legend()
fig.savefig(os.path.join(dst_folder, "evaluation_results_" + dataset + ".png"),
bbox_inches='tight')
plt.close()
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Optical flow benchmarking script',
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
"bin_path",
default="./optflow-example-optical_flow_evaluation",
help="Path to the optical flow evaluation executable")
parser.add_argument(
"-a",
"--algorithms",
metavar="ALGORITHMS",
default="",
help=("Comma-separated list of optical-flow algorithms to evaluate "
"(example: -a farneback,tvl1,deepflow). Note that previously "
"evaluated algorithms are also included in the output charts"))
parser.add_argument(
"-d",
"--datasets",
metavar="DATASETS",
default="mpi_sintel",
help=("Comma-separated list of datasets for evaluation (currently only "
"'mpi_sintel' and 'middlebury' are supported)"))
parser.add_argument(
"-f",
"--dataset_folder",
metavar="DATASET_FOLDER",
default="./OF_datasets",
help=("Path to a folder containing datasets. To enable evaluation on "
"MPI Sintel dataset, please download it using the following links: "
"http://files.is.tue.mpg.de/sintel/MPI-Sintel-training_images.zip and "
"http://files.is.tue.mpg.de/sintel/MPI-Sintel-training_extras.zip and "
"unzip these archives into the 'mpi_sintel' folder. To enable evaluation "
"on the Middlebury dataset use the following links: "
"http://vision.middlebury.edu/flow/data/comp/zip/other-color-twoframes.zip, "
"http://vision.middlebury.edu/flow/data/comp/zip/other-gt-flow.zip. "
"These should be unzipped into 'middlebury' folder"))
parser.add_argument(
"-o",
"--out",
metavar="OUT_DIR",
default="./OF_evaluation_results",
help="Output directory where to store benchmark results")
parser.add_argument(
"-s",
"--state",
metavar="STATE_JSON",
default="./OF_evaluation_state.json",
help=("Path to a json file that stores the current evaluation state and "
"previous evaluation results"))
args, other_args = parser.parse_known_args()
if not os.path.isfile(args.bin_path):
print("Error: " + args.bin_path + " does not exist")
sys.exit(1)
if not os.path.exists(args.dataset_folder):
print("Error: " + args.dataset_folder + (" does not exist. Please, correctly "
"specify the -f parameter"))
sys.exit(1)
state = {}
if os.path.isfile(args.state):
state = load_json(args.state)
algorithm_list = parse_sequence(args.algorithms)
dataset_list = parse_sequence(args.datasets)
for dataset in dataset_list:
if dataset not in dataset_eval_functions.keys():
print("Error: unsupported dataset " + dataset)
sys.exit(1)
if dataset not in os.listdir(args.dataset_folder):
print("Error: " + os.path.join(args.dataset_folder, dataset) + (" does not exist. "
"Please, download the dataset and follow the naming conventions "
"(use -h for more information)"))
sys.exit(1)
for dataset in dataset_list:
if dataset not in state.keys():
state[dataset] = {}
for algorithm in algorithm_list:
if algorithm in state[dataset].keys():
last_eval_instance = state[dataset][algorithm][-1]
if "finished" not in last_eval_instance.keys():
print(("Continuing an unfinished evaluation of " +
algorithm + " started at " + last_eval_instance["timestamp"]))
else:
state[dataset][algorithm].append({"timestamp":
datetime.datetime.now().strftime("%Y-%m-%d--%H-%M")})
else:
state[dataset][algorithm] = [{"timestamp":
datetime.datetime.now().strftime("%Y-%m-%d--%H-%M")}]
save_json(state, args.state)
dataset_eval_functions[dataset](args.dataset_folder, algorithm, args.bin_path,
state, args.state)
state[dataset][algorithm][-1]["finished"] = True
save_json(state, args.state)
save_json(state, args.state)
create_dir(args.out)
for dataset in dataset_list:
build_chart(args.out, state, dataset)
modules/optflow/samples/optical_flow_evaluation.cpp
+411
View File
@@ -0,0 +1,411 @@
#include "opencv2/highgui.hpp"
#include "opencv2/video.hpp"
#include "opencv2/optflow.hpp"
#include "opencv2/core/ocl.hpp"
#include <fstream>
#include <limits>
using namespace std;
using namespace cv;
using namespace optflow;
const String keys = "{help h usage ? | | print this message }"
"{@image1 | | image1 }"
"{@image2 | | image2 }"
"{@algorithm | | [farneback, simpleflow, tvl1, deepflow, sparsetodenseflow, RLOF_EPIC, RLOF_RIC, pcaflow, DISflow_ultrafast, DISflow_fast, DISflow_medium] }"
"{@groundtruth | | path to the .flo file (optional), Middlebury format }"
"{m measure |endpoint| error measure - [endpoint or angular] }"
"{r region |all | region to compute stats about [all, discontinuities, untextured] }"
"{d display | | display additional info images (pauses program execution) }"
"{g gpu | | use OpenCL}"
"{prior | | path to a prior file for PCAFlow}";
inline bool isFlowCorrect( const Point2f u )
{
return !cvIsNaN(u.x) && !cvIsNaN(u.y) && (fabs(u.x) < 1e9) && (fabs(u.y) < 1e9);
}
inline bool isFlowCorrect( const Point3f u )
{
return !cvIsNaN(u.x) && !cvIsNaN(u.y) && !cvIsNaN(u.z) && (fabs(u.x) < 1e9) && (fabs(u.y) < 1e9)
&& (fabs(u.z) < 1e9);
}
static Mat endpointError( const Mat_<Point2f>& flow1, const Mat_<Point2f>& flow2 )
{
Mat result(flow1.size(), CV_32FC1);
for ( int i = 0; i < flow1.rows; ++i )
{
for ( int j = 0; j < flow1.cols; ++j )
{
const Point2f u1 = flow1(i, j);
const Point2f u2 = flow2(i, j);
if ( isFlowCorrect(u1) && isFlowCorrect(u2) )
{
const Point2f diff = u1 - u2;
result.at<float>(i, j) = sqrt((float)diff.ddot(diff)); //distance
} else
result.at<float>(i, j) = std::numeric_limits<float>::quiet_NaN();
}
}
return result;
}
static Mat angularError( const Mat_<Point2f>& flow1, const Mat_<Point2f>& flow2 )
{
Mat result(flow1.size(), CV_32FC1);
for ( int i = 0; i < flow1.rows; ++i )
{
for ( int j = 0; j < flow1.cols; ++j )
{
const Point2f u1_2d = flow1(i, j);
const Point2f u2_2d = flow2(i, j);
const Point3f u1(u1_2d.x, u1_2d.y, 1);
const Point3f u2(u2_2d.x, u2_2d.y, 1);
if ( isFlowCorrect(u1) && isFlowCorrect(u2) )
result.at<float>(i, j) = acos((float)(u1.ddot(u2) / norm(u1) * norm(u2)));
else
result.at<float>(i, j) = std::numeric_limits<float>::quiet_NaN();
}
}
return result;
}
// what fraction of pixels have errors higher than given threshold?
static float stat_RX( Mat errors, float threshold, Mat mask )
{
CV_Assert(errors.size() == mask.size());
CV_Assert(mask.depth() == CV_8U);
int count = 0, all = 0;
for ( int i = 0; i < errors.rows; ++i )
{
for ( int j = 0; j < errors.cols; ++j )
{
if ( mask.at<char>(i, j) != 0 )
{
++all;
if ( errors.at<float>(i, j) > threshold )
++count;
}
}
}
return (float)count / all;
}
static float stat_AX( Mat hist, int cutoff_count, float max_value )
{
int counter = 0;
int bin = 0;
int bin_count = hist.rows;
while ( bin < bin_count && counter < cutoff_count )
{
counter += (int) hist.at<float>(bin, 0);
++bin;
}
return (float) bin / bin_count * max_value;
}
static void calculateStats( Mat errors, Mat mask = Mat(), bool display_images = false )
{
float R_thresholds[] = { 0.5f, 1.f, 2.f, 5.f, 10.f };
float A_thresholds[] = { 0.5f, 0.75f, 0.95f };
if ( mask.empty() )
mask = Mat::ones(errors.size(), CV_8U);
CV_Assert(errors.size() == mask.size());
CV_Assert(mask.depth() == CV_8U);
//displaying the mask
if(display_images)
{
namedWindow( "Region mask", WINDOW_AUTOSIZE );
imshow( "Region mask", mask );
}
//mean and std computation
Scalar s_mean, s_std;
float mean, std;
meanStdDev(errors, s_mean, s_std, mask);
mean = (float)s_mean[0];
std = (float)s_std[0];
printf("Average: %.2f\nStandard deviation: %.2f\n", mean, std);
//RX stats - displayed in percent
float R;
int R_thresholds_count = sizeof(R_thresholds) / sizeof(float);
for ( int i = 0; i < R_thresholds_count; ++i )
{
R = stat_RX(errors, R_thresholds[i], mask);
printf("R%.1f: %.2f%%\n", R_thresholds[i], R * 100);
}
//AX stats
double max_value;
minMaxLoc(errors, NULL, &max_value, NULL, NULL, mask);
Mat hist;
const int n_images = 1;
const int channels[] = { 0 };
const int n_dimensions = 1;
const int hist_bins[] = { 1024 };
const float iranges[] = { 0, (float) max_value };
const float* ranges[] = { iranges };
const bool uniform = true;
const bool accumulate = false;
calcHist(&errors, n_images, channels, mask, hist, n_dimensions, hist_bins, ranges, uniform,
accumulate);
int all_pixels = countNonZero(mask);
int cutoff_count;
float A;
int A_thresholds_count = sizeof(A_thresholds) / sizeof(float);
for ( int i = 0; i < A_thresholds_count; ++i )
{
cutoff_count = (int) (floor(A_thresholds[i] * all_pixels + 0.5f));
A = stat_AX(hist, cutoff_count, (float) max_value);
printf("A%.2f: %.2f\n", A_thresholds[i], A);
}
}
static Mat flowToDisplay(const Mat flow)
{
Mat flow_split[2];
Mat magnitude, angle;
Mat hsv_split[3], hsv, rgb;
split(flow, flow_split);
cartToPolar(flow_split[0], flow_split[1], magnitude, angle, true);
normalize(magnitude, magnitude, 0, 1, NORM_MINMAX);
hsv_split[0] = angle; // already in degrees - no normalization needed
hsv_split[1] = Mat::ones(angle.size(), angle.type());
hsv_split[2] = magnitude;
merge(hsv_split, 3, hsv);
cvtColor(hsv, rgb, COLOR_HSV2BGR);
return rgb;
}
int main( int argc, char** argv )
{
CommandLineParser parser(argc, argv, keys);
parser.about("OpenCV optical flow evaluation app");
if ( parser.has("help") || argc < 4 )
{
parser.printMessage();
printf("EXAMPLES:\n");
printf("./example_optflow_optical_flow_evaluation im1.png im2.png farneback -d \n");
printf("\t - compute flow field between im1 and im2 with farneback's method and display it");
printf("./example_optflow_optical_flow_evaluation im1.png im2.png simpleflow groundtruth.flo \n");
printf("\t - compute error statistics given the groundtruth; all pixels, endpoint error measure");
printf("./example_optflow_optical_flow_evaluation im1.png im2.png farneback groundtruth.flo -m=angular -r=untextured \n");
printf("\t - as before, but with changed error measure and stats computed only about \"untextured\" areas");
printf("\n\n Flow file format description: http://vision.middlebury.edu/flow/code/flow-code/README.txt\n\n");
return 0;
}
String i1_path = parser.get<String>(0);
String i2_path = parser.get<String>(1);
String method = parser.get<String>(2);
String groundtruth_path = parser.get<String>(3);
String error_measure = parser.get<String>("measure");
String region = parser.get<String>("region");
bool display_images = parser.has("display");
const bool useGpu = parser.has("gpu");
if ( !parser.check() )
{
parser.printErrors();
return 0;
}
cv::ocl::setUseOpenCL(useGpu);
printf("OpenCL Enabled: %u\n", useGpu && cv::ocl::haveOpenCL());
Mat i1, i2;
Mat_<Point2f> flow, ground_truth;
Mat computed_errors;
i1 = imread(i1_path, 1);
i2 = imread(i2_path, 1);
if ( !i1.data || !i2.data )
{
printf("No image data \n");
return -1;
}
if ( i1.size() != i2.size() || i1.channels() != i2.channels() )
{
printf("Dimension mismatch between input images\n");
return -1;
}
// 8-bit images expected by all algorithms
if ( i1.depth() != CV_8U )
i1.convertTo(i1, CV_8U);
if ( i2.depth() != CV_8U )
i2.convertTo(i2, CV_8U);
if ( (method == "farneback" || method == "tvl1" || method == "deepflow" || method == "DISflow_ultrafast" || method == "DISflow_fast" || method == "DISflow_medium") && i1.channels() == 3 )
{ // 1-channel images are expected
cvtColor(i1, i1, COLOR_BGR2GRAY);
cvtColor(i2, i2, COLOR_BGR2GRAY);
} else if ( method == "simpleflow" && i1.channels() == 1 )
{ // 3-channel images expected
cvtColor(i1, i1, COLOR_GRAY2BGR);
cvtColor(i2, i2, COLOR_GRAY2BGR);
}
flow = Mat(i1.size[0], i1.size[1], CV_32FC2);
Ptr<DenseOpticalFlow> algorithm;
if ( method == "farneback" )
algorithm = createOptFlow_Farneback();
else if ( method == "simpleflow" )
algorithm = createOptFlow_SimpleFlow();
else if ( method == "tvl1" )
algorithm = createOptFlow_DualTVL1();
else if ( method == "deepflow" )
algorithm = createOptFlow_DeepFlow();
else if ( method == "sparsetodenseflow" )
algorithm = createOptFlow_SparseToDense();
else if (method == "RLOF_EPIC")
{
algorithm = createOptFlow_DenseRLOF();
Ptr<DenseRLOFOpticalFlow> rlof = algorithm.dynamicCast< DenseRLOFOpticalFlow>();
rlof->setInterpolation(INTERP_EPIC);
rlof->setForwardBackward(1.f);
}
else if (method == "RLOF_RIC")
{
algorithm = createOptFlow_DenseRLOF();
Ptr<DenseRLOFOpticalFlow> rlof = algorithm.dynamicCast< DenseRLOFOpticalFlow>();;
rlof->setInterpolation(INTERP_RIC);
rlof->setForwardBackward(1.f);
}
else if ( method == "pcaflow" ) {
if ( parser.has("prior") ) {
String prior = parser.get<String>("prior");
printf("Using prior file: %s\n", prior.c_str());
algorithm = makePtr<OpticalFlowPCAFlow>(makePtr<PCAPrior>(prior.c_str()));
}
else
algorithm = createOptFlow_PCAFlow();
}
else if ( method == "DISflow_ultrafast" )
algorithm = DISOpticalFlow::create(DISOpticalFlow::PRESET_ULTRAFAST);
else if (method == "DISflow_fast")
algorithm = DISOpticalFlow::create(DISOpticalFlow::PRESET_FAST);
else if (method == "DISflow_medium")
algorithm = DISOpticalFlow::create(DISOpticalFlow::PRESET_MEDIUM);
else
{
printf("Wrong method!\n");
parser.printMessage();
return -1;
}
double startTick, time;
startTick = (double) getTickCount(); // measure time
if (useGpu)
algorithm->calc(i1, i2, flow.getUMat(ACCESS_RW));
else
algorithm->calc(i1, i2, flow);
time = ((double) getTickCount() - startTick) / getTickFrequency();
printf("\nTime [s]: %.3f\n", time);
if(display_images)
{
Mat flow_image = flowToDisplay(flow);
namedWindow( "Computed flow", WINDOW_AUTOSIZE );
imshow( "Computed flow", flow_image );
}
if ( !groundtruth_path.empty() )
{ // compare to ground truth
ground_truth = readOpticalFlow(groundtruth_path);
if ( flow.size() != ground_truth.size() || flow.channels() != 2
|| ground_truth.channels() != 2 )
{
printf("Dimension mismatch between the computed flow and the provided ground truth\n");
return -1;
}
if ( error_measure == "endpoint" )
computed_errors = endpointError(flow, ground_truth);
else if ( error_measure == "angular" )
computed_errors = angularError(flow, ground_truth);
else
{
printf("Invalid error measure! Available options: endpoint, angular\n");
return -1;
}
Mat mask;
if( region == "all" )
mask = Mat::ones(ground_truth.size(), CV_8U) * 255;
else if ( region == "discontinuities" )
{
Mat truth_merged, grad_x, grad_y, gradient;
vector<Mat> truth_split;
split(ground_truth, truth_split);
truth_merged = truth_split[0] + truth_split[1];
Sobel( truth_merged, grad_x, CV_16S, 1, 0, -1, 1, 0, BORDER_REPLICATE );
grad_x = abs(grad_x);
Sobel( truth_merged, grad_y, CV_16S, 0, 1, 1, 1, 0, BORDER_REPLICATE );
grad_y = abs(grad_y);
addWeighted(grad_x, 0.5, grad_y, 0.5, 0, gradient); //approximation!
Scalar s_mean;
s_mean = mean(gradient);
double threshold = s_mean[0]; // threshold value arbitrary
mask = gradient > threshold;
dilate(mask, mask, Mat::ones(9, 9, CV_8U));
}
else if ( region == "untextured" )
{
Mat i1_grayscale, grad_x, grad_y, gradient;
if( i1.channels() == 3 )
cvtColor(i1, i1_grayscale, COLOR_BGR2GRAY);
else
i1_grayscale = i1;
Sobel( i1_grayscale, grad_x, CV_16S, 1, 0, 7 );
grad_x = abs(grad_x);
Sobel( i1_grayscale, grad_y, CV_16S, 0, 1, 7 );
grad_y = abs(grad_y);
addWeighted(grad_x, 0.5, grad_y, 0.5, 0, gradient); //approximation!
GaussianBlur(gradient, gradient, Size(5,5), 1, 1);
Scalar s_mean;
s_mean = mean(gradient);
// arbitrary threshold value used - could be determined statistically from the image?
double threshold = 1000;
mask = gradient < threshold;
dilate(mask, mask, Mat::ones(3, 3, CV_8U));
}
else
{
printf("Invalid region selected! Available options: all, discontinuities, untextured");
return -1;
}
//masking out NaNs and incorrect GT values
Mat truth_split[2];
split(ground_truth, truth_split);
Mat abs_mask = Mat((abs(truth_split[0]) < 1e9) & (abs(truth_split[1]) < 1e9));
Mat nan_mask = Mat((truth_split[0]==truth_split[0]) & (truth_split[1] == truth_split[1]));
bitwise_and(abs_mask, nan_mask, nan_mask);
bitwise_and(nan_mask, mask, mask); //including the selected region
if(display_images) // display difference between computed and GT flow
{
Mat difference = ground_truth - flow;
Mat masked_difference;
difference.copyTo(masked_difference, mask);
Mat flow_image = flowToDisplay(masked_difference);
namedWindow( "Error map", WINDOW_AUTOSIZE );
imshow( "Error map", flow_image );
}
printf("Using %s error measure\n", error_measure.c_str());
calculateStats(computed_errors, mask, display_images);
}
if(display_images) // wait for the user to see all the images
waitKey(0);
return 0;
}
modules/optflow/samples/pcaflow_demo.cpp
+172
View File
@@ -0,0 +1,172 @@
#include "opencv2/core/ocl.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/optflow.hpp"
#include <fstream>
#include <iostream>
#include <stdio.h>
using namespace cv;
using optflow::OpticalFlowPCAFlow;
using optflow::PCAPrior;
const String keys = "{help h ? | | print this message}"
"{@image1 |<none>| image1}"
"{@image2 |<none>| image2}"
"{@groundtruth |<none>| path to the .flo file}"
"{@prior |<none>| path to a prior file for PCAFlow}"
"{@output |<none>| output image path}"
"{g gpu | | use OpenCL}";
static double normL2( const Point2f &v ) { return sqrt( v.x * v.x + v.y * v.y ); }
static bool fileProbe( const char *name ) { return std::ifstream( name ).good(); }
static Vec3d getFlowColor( const Point2f &f, const bool logScale = true, const double scaleDown = 5 )
{
if ( f.x == 0 && f.y == 0 )
return Vec3d( 0, 0, 1 );
double radius = normL2( f );
if ( logScale )
radius = log( radius + 1 );
radius /= scaleDown;
radius = std::min( 1.0, radius );
double angle = ( atan2( -f.y, -f.x ) + CV_PI ) * 180 / CV_PI;
return Vec3d( angle, radius, 1 );
}
static void displayFlow( InputArray _flow, OutputArray _img )
{
const Size sz = _flow.size();
Mat flow = _flow.getMat();
_img.create( sz, CV_32FC3 );
Mat img = _img.getMat();
for ( int i = 0; i < sz.height; ++i )
for ( int j = 0; j < sz.width; ++j )
img.at< Vec3f >( i, j ) = getFlowColor( flow.at< Point2f >( i, j ) );
cvtColor( img, img, COLOR_HSV2BGR );
}
static bool isFlowCorrect( const Point2f &u )
{
return !cvIsNaN( u.x ) && !cvIsNaN( u.y ) && ( fabs( u.x ) < 1e9 ) && ( fabs( u.y ) < 1e9 );
}
static double calcEPE( const Mat &f1, const Mat &f2 )
{
double sum = 0;
Size sz = f1.size();
size_t cnt = 0;
for ( int i = 0; i < sz.height; ++i )
for ( int j = 0; j < sz.width; ++j )
if ( isFlowCorrect( f1.at< Point2f >( i, j ) ) && isFlowCorrect( f2.at< Point2f >( i, j ) ) )
{
sum += normL2( f1.at< Point2f >( i, j ) - f2.at< Point2f >( i, j ) );
++cnt;
}
return sum / cnt;
}
static void displayResult( Mat &i1, Mat &i2, Mat &gt, Ptr< DenseOpticalFlow > &algo, OutputArray _img, const char *descr,
const bool useGpu = false )
{
Mat flow( i1.size[0], i1.size[1], CV_32FC2 );
TickMeter meter;
meter.start();
if ( useGpu )
algo->calc( i1, i2, flow.getUMat( ACCESS_RW ) );
else
algo->calc( i1, i2, flow );
meter.stop();
displayFlow( flow, _img );
Mat img = _img.getMat();
putText( img, descr, Point2i( 24, 40 ), FONT_HERSHEY_DUPLEX, 1, Vec3b( 1, 0, 0 ), 2, LINE_AA );
char buf[256];
sprintf( buf, "Average EPE: %.2f", calcEPE( flow, gt ) );
putText( img, buf, Point2i( 24, 80 ), FONT_HERSHEY_DUPLEX, 1, Vec3b( 1, 0, 0 ), 2, LINE_AA );
sprintf( buf, "Time: %.2fs", meter.getTimeSec() );
putText( img, buf, Point2i( 24, 120 ), FONT_HERSHEY_DUPLEX, 1, Vec3b( 1, 0, 0 ), 2, LINE_AA );
}
static void displayGT( InputArray _flow, OutputArray _img, const char *descr )
{
displayFlow( _flow, _img );
Mat img = _img.getMat();
putText( img, descr, Point2i( 24, 40 ), FONT_HERSHEY_DUPLEX, 1, Vec3b( 1, 0, 0 ), 2, LINE_AA );
}
int main( int argc, const char **argv )
{
CommandLineParser parser( argc, argv, keys );
parser.about( "PCAFlow demonstration" );
if ( parser.has( "help" ) )
{
parser.printMessage();
return 0;
}
String img1 = parser.get< String >( 0 );
String img2 = parser.get< String >( 1 );
String groundtruth = parser.get< String >( 2 );
String prior = parser.get< String >( 3 );
String outimg = parser.get< String >( 4 );
const bool useGpu = parser.has( "gpu" );
if ( !parser.check() )
{
parser.printErrors();
return 1;
}
if ( !fileProbe( prior.c_str() ) )
{
std::cerr << "Can't open the file with prior! Check the provided path: " << prior << std::endl;
return 1;
}
cv::ocl::setUseOpenCL( useGpu );
Mat i1 = imread( img1 );
Mat i2 = imread( img2 );
Mat gt = readOpticalFlow( groundtruth );
Mat i1g, i2g;
cvtColor( i1, i1g, COLOR_BGR2GRAY );
cvtColor( i2, i2g, COLOR_BGR2GRAY );
Mat pcaflowDisp, pcaflowpriDisp, farnebackDisp, gtDisp;
{
Ptr< DenseOpticalFlow > pcaflow = makePtr< OpticalFlowPCAFlow >( makePtr< PCAPrior >( prior.c_str() ) );
displayResult( i1, i2, gt, pcaflow, pcaflowpriDisp, "PCAFlow with prior", useGpu );
}
{
Ptr< DenseOpticalFlow > pcaflow = makePtr< OpticalFlowPCAFlow >();
displayResult( i1, i2, gt, pcaflow, pcaflowDisp, "PCAFlow without prior", useGpu );
}
{
Ptr< DenseOpticalFlow > farneback = optflow::createOptFlow_Farneback();
displayResult( i1g, i2g, gt, farneback, farnebackDisp, "Farneback", useGpu );
}
displayGT( gt, gtDisp, "Ground truth" );
Mat disp1, disp2;
vconcat( pcaflowpriDisp, farnebackDisp, disp1 );
vconcat( pcaflowDisp, gtDisp, disp2 );
hconcat( disp1, disp2, disp1 );
disp1 *= 255;
imwrite( outimg, disp1 );
return 0;
}
modules/optflow/samples/simpleflow_demo.cpp
+222
View File
@@ -0,0 +1,222 @@
#include "opencv2/optflow.hpp"
#include <opencv2/core/utility.hpp>
#include "opencv2/imgproc.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include <cstdio>
#include <iostream>
using namespace cv;
using namespace cv::optflow;
using namespace std;
#define APP_NAME "simpleflow_demo : "
static void help()
{
// print a welcome message, and the OpenCV version
printf("This is a demo of SimpleFlow optical flow algorithm,\n"
"Using OpenCV version %s\n\n", CV_VERSION);
printf("Usage: simpleflow_demo frame1 frame2 output_flow"
"\nApplication will write estimated flow "
"\nbetween 'frame1' and 'frame2' in binary format"
"\ninto file 'output_flow'"
"\nThen one can use code from http://vision.middlebury.edu/flow/data/"
"\nto convert flow in binary file to image\n");
}
// binary file format for flow data specified here:
// http://vision.middlebury.edu/flow/data/
static void writeOpticalFlowToFile(const Mat& flow, FILE* file) {
int cols = flow.cols;
int rows = flow.rows;
fprintf(file, "PIEH");
if (fwrite(&cols, sizeof(int), 1, file) != 1 ||
fwrite(&rows, sizeof(int), 1, file) != 1) {
printf(APP_NAME "writeOpticalFlowToFile : problem writing header\n");
exit(1);
}
for (int i= 0; i < rows; ++i) {
for (int j = 0; j < cols; ++j) {
Vec2f flow_at_point = flow.at<Vec2f>(i, j);
if (fwrite(&(flow_at_point[0]), sizeof(float), 1, file) != 1 ||
fwrite(&(flow_at_point[1]), sizeof(float), 1, file) != 1) {
printf(APP_NAME "writeOpticalFlowToFile : problem writing data\n");
exit(1);
}
}
}
}
static void run(int argc, char** argv) {
if (argc < 3) {
printf(APP_NAME "Wrong number of command line arguments for mode `run`: %d (expected %d)\n",
argc, 3);
exit(1);
}
Mat frame1 = imread(argv[0]);
Mat frame2 = imread(argv[1]);
if (frame1.empty()) {
printf(APP_NAME "Image #1 : %s cannot be read\n", argv[0]);
exit(1);
}
if (frame2.empty()) {
printf(APP_NAME "Image #2 : %s cannot be read\n", argv[1]);
exit(1);
}
if (frame1.rows != frame2.rows && frame1.cols != frame2.cols) {
printf(APP_NAME "Images should be of equal sizes\n");
exit(1);
}
if (frame1.type() != 16 || frame2.type() != 16) {
printf(APP_NAME "Images should be of equal type CV_8UC3\n");
exit(1);
}
printf(APP_NAME "Read two images of size [rows = %d, cols = %d]\n",
frame1.rows, frame1.cols);
Mat flow;
float start = (float)getTickCount();
calcOpticalFlowSF(frame1, frame2,
flow,
3, 2, 4, 4.1, 25.5, 18, 55.0, 25.5, 0.35, 18, 55.0, 25.5, 10);
printf(APP_NAME "calcOpticalFlowSF : %lf sec\n", (getTickCount() - start) / getTickFrequency());
FILE* file = fopen(argv[2], "wb");
if (file == NULL) {
printf(APP_NAME "Unable to open file '%s' for writing\n", argv[2]);
exit(1);
}
printf(APP_NAME "Writing to file\n");
writeOpticalFlowToFile(flow, file);
fclose(file);
}
static bool readOpticalFlowFromFile(FILE* file, Mat& flow) {
char header[5];
if (fread(header, 1, 4, file) < 4 && (string)header != "PIEH") {
return false;
}
int cols, rows;
if (fread(&cols, sizeof(int), 1, file) != 1||
fread(&rows, sizeof(int), 1, file) != 1) {
return false;
}
flow = Mat::zeros(rows, cols, CV_32FC2);
for (int i = 0; i < rows; ++i) {
for (int j = 0; j < cols; ++j) {
Vec2f flow_at_point;
if (fread(&(flow_at_point[0]), sizeof(float), 1, file) != 1 ||
fread(&(flow_at_point[1]), sizeof(float), 1, file) != 1) {
return false;
}
flow.at<Vec2f>(i, j) = flow_at_point;
}
}
return true;
}
static bool isFlowCorrect(float u) {
return !cvIsNaN(u) && (fabs(u) < 1e9);
}
static float calc_rmse(Mat flow1, Mat flow2) {
float sum = 0;
int counter = 0;
const int rows = flow1.rows;
const int cols = flow1.cols;
for (int y = 0; y < rows; ++y) {
for (int x = 0; x < cols; ++x) {
Vec2f flow1_at_point = flow1.at<Vec2f>(y, x);
Vec2f flow2_at_point = flow2.at<Vec2f>(y, x);
float u1 = flow1_at_point[0];
float v1 = flow1_at_point[1];
float u2 = flow2_at_point[0];
float v2 = flow2_at_point[1];
if (isFlowCorrect(u1) && isFlowCorrect(u2) && isFlowCorrect(v1) && isFlowCorrect(v2)) {
sum += (u1-u2)*(u1-u2) + (v1-v2)*(v1-v2);
counter++;
}
}
}
return (float)sqrt(sum / (1e-9 + counter));
}
static void eval(int argc, char** argv) {
if (argc < 2) {
printf(APP_NAME "Wrong number of command line arguments for mode `eval` : %d (expected %d)\n",
argc, 2);
exit(1);
}
Mat flow1, flow2;
FILE* flow_file_1 = fopen(argv[0], "rb");
if (flow_file_1 == NULL) {
printf(APP_NAME "Cannot open file with first flow : %s\n", argv[0]);
exit(1);
}
if (!readOpticalFlowFromFile(flow_file_1, flow1)) {
printf(APP_NAME "Cannot read flow data from file %s\n", argv[0]);
exit(1);
}
fclose(flow_file_1);
FILE* flow_file_2 = fopen(argv[1], "rb");
if (flow_file_2 == NULL) {
printf(APP_NAME "Cannot open file with first flow : %s\n", argv[1]);
exit(1);
}
if (!readOpticalFlowFromFile(flow_file_2, flow2)) {
printf(APP_NAME "Cannot read flow data from file %s\n", argv[1]);
exit(1);
}
fclose(flow_file_2);
float rmse = calc_rmse(flow1, flow2);
printf("%lf\n", rmse);
}
int main(int argc, char** argv) {
if (argc < 2) {
printf(APP_NAME "Mode is not specified\n");
help();
exit(1);
}
string mode = (string)argv[1];
int new_argc = argc - 2;
char** new_argv = &argv[2];
if ("run" == mode) {
run(new_argc, new_argv);
} else if ("eval" == mode) {
eval(new_argc, new_argv);
} else if ("help" == mode)
help();
else {
printf(APP_NAME "Unknown mode : %s\n", argv[1]);
help();
}
return 0;
}
modules/optflow/samples/tvl1_optical_flow.cpp
+206
View File
@@ -0,0 +1,206 @@
#include <iostream>
#include <fstream>
#include <opencv2/core/utility.hpp>
#include "opencv2/video.hpp"
#include "opencv2/optflow.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
using namespace cv;
using namespace std;
using namespace optflow;
inline bool isFlowCorrect(Point2f u)
{
return !cvIsNaN(u.x) && !cvIsNaN(u.y) && fabs(u.x) < 1e9 && fabs(u.y) < 1e9;
}
static Vec3b computeColor(float fx, float fy)
{
static bool first = true;
// relative lengths of color transitions:
// these are chosen based on perceptual similarity
// (e.g. one can distinguish more shades between red and yellow
// than between yellow and green)
const int RY = 15;
const int YG = 6;
const int GC = 4;
const int CB = 11;
const int BM = 13;
const int MR = 6;
const int NCOLS = RY + YG + GC + CB + BM + MR;
static Vec3i colorWheel[NCOLS];
if (first)
{
int k = 0;
for (int i = 0; i < RY; ++i, ++k)
colorWheel[k] = Vec3i(255, 255 * i / RY, 0);
for (int i = 0; i < YG; ++i, ++k)
colorWheel[k] = Vec3i(255 - 255 * i / YG, 255, 0);
for (int i = 0; i < GC; ++i, ++k)
colorWheel[k] = Vec3i(0, 255, 255 * i / GC);
for (int i = 0; i < CB; ++i, ++k)
colorWheel[k] = Vec3i(0, 255 - 255 * i / CB, 255);
for (int i = 0; i < BM; ++i, ++k)
colorWheel[k] = Vec3i(255 * i / BM, 0, 255);
for (int i = 0; i < MR; ++i, ++k)
colorWheel[k] = Vec3i(255, 0, 255 - 255 * i / MR);
first = false;
}
const float rad = sqrt(fx * fx + fy * fy);
const float a = atan2(-fy, -fx) / (float)CV_PI;
const float fk = (a + 1.0f) / 2.0f * (NCOLS - 1);
const int k0 = static_cast<int>(fk);
const int k1 = (k0 + 1) % NCOLS;
const float f = fk - k0;
Vec3b pix;
for (int b = 0; b < 3; b++)
{
const float col0 = colorWheel[k0][b] / 255.f;
const float col1 = colorWheel[k1][b] / 255.f;
float col = (1 - f) * col0 + f * col1;
if (rad <= 1)
col = 1 - rad * (1 - col); // increase saturation with radius
else
col *= .75; // out of range
pix[2 - b] = static_cast<uchar>(255.f * col);
}
return pix;
}
static void drawOpticalFlow(const Mat_<Point2f>& flow, Mat& dst, float maxmotion = -1)
{
dst.create(flow.size(), CV_8UC3);
dst.setTo(Scalar::all(0));
// determine motion range:
float maxrad = maxmotion;
if (maxmotion <= 0)
{
maxrad = 1;
for (int y = 0; y < flow.rows; ++y)
{
for (int x = 0; x < flow.cols; ++x)
{
Point2f u = flow(y, x);
if (!isFlowCorrect(u))
continue;
maxrad = max(maxrad, sqrt(u.x * u.x + u.y * u.y));
}
}
}
for (int y = 0; y < flow.rows; ++y)
{
for (int x = 0; x < flow.cols; ++x)
{
Point2f u = flow(y, x);
if (isFlowCorrect(u))
dst.at<Vec3b>(y, x) = computeColor(u.x / maxrad, u.y / maxrad);
}
}
}
// binary file format for flow data specified here:
// http://vision.middlebury.edu/flow/data/
static void writeOpticalFlowToFile(const Mat_<Point2f>& flow, const string& fileName)
{
static const char FLO_TAG_STRING[] = "PIEH";
ofstream file(fileName.c_str(), ios_base::binary);
file << FLO_TAG_STRING;
file.write((const char*) &flow.cols, sizeof(int));
file.write((const char*) &flow.rows, sizeof(int));
for (int i = 0; i < flow.rows; ++i)
{
for (int j = 0; j < flow.cols; ++j)
{
const Point2f u = flow(i, j);
file.write((const char*) &u.x, sizeof(float));
file.write((const char*) &u.y, sizeof(float));
}
}
}
int main(int argc, const char* argv[])
{
cv::CommandLineParser parser(argc, argv, "{help h || show help message}"
"{ @frame0 | | frame 0}{ @frame1 | | frame 1}{ @output | | output flow}");
if (parser.has("help"))
{
parser.printMessage();
return 0;
}
string frame0_name = parser.get<string>("@frame0");
string frame1_name = parser.get<string>("@frame1");
string file = parser.get<string>("@output");
if (frame0_name.empty() || frame1_name.empty() || file.empty())
{
cerr << "Usage : " << argv[0] << " [<frame0>] [<frame1>] [<output_flow>]" << endl;
return -1;
}
Mat frame0 = imread(frame0_name, IMREAD_GRAYSCALE);
Mat frame1 = imread(frame1_name, IMREAD_GRAYSCALE);
if (frame0.empty())
{
cerr << "Can't open image [" << parser.get<string>("frame0") << "]" << endl;
return -1;
}
if (frame1.empty())
{
cerr << "Can't open image [" << parser.get<string>("frame1") << "]" << endl;
return -1;
}
if (frame1.size() != frame0.size())
{
cerr << "Images should be of equal sizes" << endl;
return -1;
}
Mat_<Point2f> flow;
Ptr<DualTVL1OpticalFlow> tvl1 = DualTVL1OpticalFlow::create();
const double start = (double)getTickCount();
tvl1->calc(frame0, frame1, flow);
const double timeSec = (getTickCount() - start) / getTickFrequency();
cout << "calcOpticalFlowDual_TVL1 : " << timeSec << " sec" << endl;
Mat out;
drawOpticalFlow(flow, out);
if (!file.empty())
writeOpticalFlowToFile(flow, file);
imshow("Flow", out);
waitKey();
return 0;
}
modules/optflow/src/deepflow.cpp
+176
View File
@@ -0,0 +1,176 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
namespace cv
{
namespace optflow
{
class OpticalFlowDeepFlow: public DenseOpticalFlow
{
public:
OpticalFlowDeepFlow();
void calc( InputArray I0, InputArray I1, InputOutputArray flow ) CV_OVERRIDE;
void collectGarbage() CV_OVERRIDE;
protected:
float sigma; // Gaussian smoothing parameter
int minSize; // minimal dimension of an image in the pyramid
float downscaleFactor; // scaling factor in the pyramid
int fixedPointIterations; // during each level of the pyramid
int sorIterations; // iterations of SOR
float alpha; // smoothness assumption weight
float delta; // color constancy weight
float gamma; // gradient constancy weight
float omega; // relaxation factor in SOR
int maxLayers; // max amount of layers in the pyramid
int interpolationType;
private:
std::vector<Mat> buildPyramid( const Mat& src );
};
OpticalFlowDeepFlow::OpticalFlowDeepFlow()
{
// parameters
sigma = 0.6f;
minSize = 25;
downscaleFactor = 0.95f;
fixedPointIterations = 5;
sorIterations = 25;
alpha = 1.0f;
delta = 0.5f;
gamma = 5.0f;
omega = 1.6f;
//consts
interpolationType = INTER_LINEAR;
maxLayers = 200;
}
std::vector<Mat> OpticalFlowDeepFlow::buildPyramid( const Mat& src )
{
std::vector<Mat> pyramid;
pyramid.push_back(src);
Mat prev = pyramid[0];
for( int i = 0; i < this->maxLayers; ++i)
{
Mat next; //TODO: filtering at each level?
Size nextSize((int) (prev.cols * downscaleFactor + 0.5f),
(int) (prev.rows * downscaleFactor + 0.5f));
if( nextSize.height <= minSize || nextSize.width <= minSize)
break;
resize(prev, next,
nextSize, 0, 0,
interpolationType);
pyramid.push_back(next);
prev = next;
}
return pyramid;
}
void OpticalFlowDeepFlow::calc( InputArray _I0, InputArray _I1, InputOutputArray _flow )
{
Mat I0temp = _I0.getMat();
Mat I1temp = _I1.getMat();
CV_Assert(I0temp.size() == I1temp.size());
CV_Assert(I0temp.type() == I1temp.type());
CV_Assert(I0temp.channels() == 1);
// TODO: currently only grayscale - data term could be computed in color version as well...
Mat I0, I1;
I0temp.convertTo(I0, CV_32F);
I1temp.convertTo(I1, CV_32F);
_flow.create(I0.size(), CV_32FC2);
Mat W = _flow.getMat(); // if any data present - will be discarded
// pre-smooth images
int kernelLen = ((int)floor(3 * sigma) * 2) + 1;
Size kernelSize(kernelLen, kernelLen);
GaussianBlur(I0, I0, kernelSize, sigma);
GaussianBlur(I1, I1, kernelSize, sigma);
// build down-sized pyramids
std::vector<Mat> pyramid_I0 = buildPyramid(I0);
std::vector<Mat> pyramid_I1 = buildPyramid(I1);
int levelCount = (int) pyramid_I0.size();
// initialize the first version of flow estimate to zeros
Size smallestSize = pyramid_I0[levelCount - 1].size();
W = Mat::zeros(smallestSize, CV_32FC2);
for ( int level = levelCount - 1; level >= 0; --level )
{ //iterate through all levels, beginning with the most coarse
Ptr<VariationalRefinement> var = VariationalRefinement::create();
var->setAlpha(4 * alpha);
var->setDelta(delta / 3);
var->setGamma(gamma / 3);
var->setFixedPointIterations(fixedPointIterations);
var->setSorIterations(sorIterations);
var->setOmega(omega);
var->calc(pyramid_I0[level], pyramid_I1[level], W);
if ( level > 0 ) //not the last level
{
Mat temp;
Size newSize = pyramid_I0[level - 1].size();
resize(W, temp, newSize, 0, 0, interpolationType); //resize calculated flow
W = temp * (1.0f / downscaleFactor); //scale values
}
}
W.copyTo(_flow);
}
void OpticalFlowDeepFlow::collectGarbage() {}
Ptr<DenseOpticalFlow> createOptFlow_DeepFlow() { return makePtr<OpticalFlowDeepFlow>(); }
}//optflow
}//cv
modules/optflow/src/interfaces.cpp
+217
View File
@@ -0,0 +1,217 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
#include "opencv2/core.hpp"
#include "opencv2/video.hpp"
#include "opencv2/optflow.hpp"
namespace cv
{
namespace optflow
{
class OpticalFlowSimpleFlow : public DenseOpticalFlow
{
public:
OpticalFlowSimpleFlow();
void calc(InputArray I0, InputArray I1, InputOutputArray flow) CV_OVERRIDE;
void collectGarbage() CV_OVERRIDE;
protected:
int layers;
int averaging_radius;
int max_flow;
double sigma_dist;
double sigma_color;
int postprocess_window;
double sigma_dist_fix;
double sigma_color_fix;
double occ_thr;
int upscale_averaging_radius;
double upscale_sigma_dist;
double upscale_sigma_color;
double speed_up_thr;
};
OpticalFlowSimpleFlow::OpticalFlowSimpleFlow()
{
// values from the example app
layers = 3;
averaging_radius = 2;
max_flow = 4;
// values from the default function parameters
sigma_dist = 4.1;
sigma_color = 25.5;
postprocess_window = 18;
sigma_dist_fix = 55.0;
sigma_color_fix = 25.5;
occ_thr = 0.35;
upscale_averaging_radius = 18;
upscale_sigma_dist = 55.0;
upscale_sigma_color = 25.5;
speed_up_thr = 10;
}
void OpticalFlowSimpleFlow::calc(InputArray I0, InputArray I1, InputOutputArray flow)
{
optflow::calcOpticalFlowSF(I0, I1, flow, layers, averaging_radius, max_flow, sigma_dist, sigma_color,
postprocess_window, sigma_dist_fix, sigma_color_fix, occ_thr,
upscale_averaging_radius, upscale_sigma_dist, upscale_sigma_color, speed_up_thr);
}
void OpticalFlowSimpleFlow::collectGarbage()
{
}
//CV_INIT_ALGORITHM(OpticalFlowSimpleFlow, "DenseOpticalFlow.SimpleFlow"),
// obj.info()->addParam(obj, "layers", obj.layers);
// obj.info()->addParam(obj, "averaging_radius", obj.averaging_radius);
// obj.info()->addParam(obj, "max_flow", obj.max_flow);
// obj.info()->addParam(obj, "sigma_dist", obj.sigma_dist);
// obj.info()->addParam(obj, "sigma_color", obj.sigma_color);
// obj.info()->addParam(obj, "postprocess_window", obj.postprocess_window);
// obj.info()->addParam(obj, "sigma_dist_fix", obj.sigma_dist_fix);
// obj.info()->addParam(obj, "sigma_color_fix", obj.sigma_color_fix);
// obj.info()->addParam(obj, "occ_thr", obj.occ_thr);
// obj.info()->addParam(obj, "upscale_averaging_radius", obj.upscale_averaging_radius);
// obj.info()->addParam(obj, "upscale_sigma_dist", obj.upscale_sigma_dist);
// obj.info()->addParam(obj, "upscale_sigma_color", obj.upscale_sigma_color);
// obj.info()->addParam(obj, "speed_up_thr", obj.speed_up_thr))
Ptr<DenseOpticalFlow> createOptFlow_SimpleFlow()
{
return makePtr<OpticalFlowSimpleFlow>();
}
class OpticalFlowFarneback : public DenseOpticalFlow
{
public:
OpticalFlowFarneback();
void calc(InputArray I0, InputArray I1, InputOutputArray flow) CV_OVERRIDE;
void collectGarbage() CV_OVERRIDE;
protected:
int numLevels;
double pyrScale;
bool fastPyramids;
int winSize;
int numIters;
int polyN;
double polySigma;
int flags;
};
OpticalFlowFarneback::OpticalFlowFarneback()
{
// values copied from the FarnebackOpticalFlow class
numLevels = 5;
pyrScale = 0.5;
fastPyramids = false;
winSize = 13;
numIters = 10;
polyN = 5;
polySigma = 1.1;
flags = 0;
}
void OpticalFlowFarneback::calc(InputArray I0, InputArray I1, InputOutputArray flow)
{
calcOpticalFlowFarneback(I0, I1, flow, pyrScale, numLevels, winSize, numIters, polyN, polySigma, flags);
}
void OpticalFlowFarneback::collectGarbage()
{
}
//CV_INIT_ALGORITHM(OpticalFlowFarneback, "DenseOpticalFlow.Farneback",
// obj.info()->addParam(obj, "numLevels", obj.numLevels);
// obj.info()->addParam(obj, "pyrScale", obj.pyrScale);
// obj.info()->addParam(obj, "fastPyramids", obj.fastPyramids);
// obj.info()->addParam(obj, "winSize", obj.winSize);
// obj.info()->addParam(obj, "numIters", obj.numIters);
// obj.info()->addParam(obj, "polyN", obj.polyN);
// obj.info()->addParam(obj, "polySigma", obj.polySigma);
// obj.info()->addParam(obj, "flags", obj.flags))
Ptr<DenseOpticalFlow> createOptFlow_Farneback()
{
return makePtr<OpticalFlowFarneback>();
}
class OpticalFlowSparseToDense : public DenseOpticalFlow
{
public:
OpticalFlowSparseToDense(int _grid_step, int _k, float _sigma, bool _use_post_proc, float _fgs_lambda, float _fgs_sigma);
void calc(InputArray I0, InputArray I1, InputOutputArray flow) CV_OVERRIDE;
void collectGarbage() CV_OVERRIDE;
protected:
int grid_step;
int k;
float sigma;
bool use_post_proc;
float fgs_lambda;
float fgs_sigma;
};
OpticalFlowSparseToDense::OpticalFlowSparseToDense(int _grid_step, int _k, float _sigma, bool _use_post_proc, float _fgs_lambda, float _fgs_sigma)
{
grid_step = _grid_step;
k = _k;
sigma = _sigma;
use_post_proc = _use_post_proc;
fgs_lambda = _fgs_lambda;
fgs_sigma = _fgs_sigma;
}
void OpticalFlowSparseToDense::calc(InputArray I0, InputArray I1, InputOutputArray flow)
{
calcOpticalFlowSparseToDense(I0,I1,flow,grid_step,k,sigma,use_post_proc,fgs_lambda,fgs_sigma);
}
void OpticalFlowSparseToDense::collectGarbage() {}
Ptr<DenseOpticalFlow> createOptFlow_SparseToDense()
{
return makePtr<OpticalFlowSparseToDense>(8,128,0.05f,true,500.0f,1.5f);
}
}
}
modules/optflow/src/learn_prior.py
+166
View File
@@ -0,0 +1,166 @@
#!/usr/bin/env python
import os
import sys
import numpy as np
import cv2 as cv
import struct
import argparse
from math import sqrt
argparser = argparse.ArgumentParser(
description='''Use this script to generate prior for using with PCAFlow.
Basis size here must match corresponding parameter in the PCAFlow.
Gamma should be selected experimentally.''')
argparser.add_argument('-f',
'--files',
nargs='+',
help='List of optical flow .flo files for learning. You can pass a directory here and it will be scanned recursively for .flo files.',
required=True)
argparser.add_argument('-o',
'--output',
help='Output file for prior',
required=True)
argparser.add_argument('--width',
type=int,
help='Size of the basis first dimension',
required=True,
default=18)
argparser.add_argument('--height',
type=int,
help='Size of the basis second dimension',
required=True,
default=14)
argparser.add_argument(
'-g',
'--gamma',
type=float,
help='Amount of regularization. The greater this parameter, the bigger will be an impact of the regularization.',
required=True)
args = argparser.parse_args()
basis_size = (args.height, args.width)
gamma = args.gamma
def find_flo(pp):
f = []
for p in pp:
if os.path.isfile(p):
f.append(p)
else:
for root, subdirs, files in os.walk(p):
f += map(lambda x: os.path.join(root, x),
filter(lambda x: x.split('.')[-1] == 'flo', files))
return list(set(f))
def load_flo(flo):
with open(flo, 'rb') as f:
magic = np.fromfile(f, np.float32, count=1)[0]
if 202021.25 != magic:
print('Magic number incorrect. Invalid .flo file')
else:
w = np.fromfile(f, np.int32, count=1)[0]
h = np.fromfile(f, np.int32, count=1)[0]
print('Reading %dx%d flo file %s' % (w, h, flo))
data = np.fromfile(f, np.float32, count=2 * w * h)
# Reshape data into 3D array (columns, rows, bands)
flow = np.reshape(data, (h, w, 2))
return flow[:, :, 0], flow[:, :, 1]
def get_w(m):
s = m.shape
w = cv.dct(m)
w *= 2.0 / sqrt(s[0] * s[1])
#w[0,0] *= 0.5
w[:, 0] *= sqrt(0.5)
w[0, :] *= sqrt(0.5)
w = w[0:basis_size[0], 0:basis_size[1]].transpose().flatten()
return w
w1 = []
w2 = []
for flo in find_flo(args.files):
x, y = load_flo(flo)
w1.append(get_w(x))
w2.append(get_w(y))
w1mean = sum(w1) / len(w1)
w2mean = sum(w2) / len(w2)
for i in xrange(len(w1)):
w1[i] -= w1mean
for i in xrange(len(w2)):
w2[i] -= w2mean
Q1 = sum([w1[i].reshape(-1, 1).dot(w1[i].reshape(1, -1))
for i in xrange(len(w1))]) / len(w1)
Q2 = sum([w2[i].reshape(-1, 1).dot(w2[i].reshape(1, -1))
for i in xrange(len(w2))]) / len(w2)
Q1 = np.matrix(Q1)
Q2 = np.matrix(Q2)
if len(w1) > 1:
while True:
try:
L1 = np.linalg.cholesky(Q1)
break
except np.linalg.LinAlgError:
mev = min(np.linalg.eig(Q1)[0]).real
assert (mev < 0)
print('Q1', mev)
if -mev < 1e-6:
mev = -1e-6
Q1 += (-mev * 1.000001) * np.identity(Q1.shape[0])
while True:
try:
L2 = np.linalg.cholesky(Q2)
break
except np.linalg.LinAlgError:
mev = min(np.linalg.eig(Q2)[0]).real
assert (mev < 0)
print('Q2', mev)
if -mev < 1e-6:
mev = -1e-6
Q2 += (-mev * 1.000001) * np.identity(Q2.shape[0])
else:
L1 = np.identity(Q1.shape[0])
L2 = np.identity(Q2.shape[0])
L1 = np.linalg.inv(L1) * gamma
L2 = np.linalg.inv(L2) * gamma
assert (L1.shape == L2.shape)
assert (L1.shape[0] == L1.shape[1])
f = open(args.output, 'wb')
f.write(struct.pack('I', L1.shape[0]))
f.write(struct.pack('I', L1.shape[1]))
for i in xrange(L1.shape[0]):
for j in xrange(L1.shape[1]):
f.write(struct.pack('f', L1[i, j]))
for i in xrange(L2.shape[0]):
for j in xrange(L2.shape[1]):
f.write(struct.pack('f', L2[i, j]))
b1 = L1.dot(w1mean.reshape(-1, 1))
b2 = L2.dot(w2mean.reshape(-1, 1))
assert (L1.shape[0] == b1.shape[0])
for i in xrange(b1.shape[0]):
f.write(struct.pack('f', b1[i, 0]))
for i in xrange(b2.shape[0]):
f.write(struct.pack('f', b2[i, 0]))
f.close()
modules/optflow/src/motempl.cpp
+420
View File
@@ -0,0 +1,420 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// Intel License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000, Intel Corporation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of Intel Corporation may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
#include "opencv2/core/utility.hpp"
#include "opencv2/core/hal/hal.hpp"
#include "opencv2/core/private.hpp"
#include "opencl_kernels_optflow.hpp"
namespace cv {
namespace motempl {
using std::vector;
#ifdef HAVE_OPENCL
static bool ocl_updateMotionHistory( InputArray _silhouette, InputOutputArray _mhi,
float timestamp, float delbound )
{
ocl::Kernel k("updateMotionHistory", ocl::optflow::updatemotionhistory_oclsrc);
if (k.empty())
return false;
UMat silh = _silhouette.getUMat(), mhi = _mhi.getUMat();
k.args(ocl::KernelArg::ReadOnlyNoSize(silh), ocl::KernelArg::ReadWrite(mhi),
timestamp, delbound);
size_t globalsize[2] = { (size_t)silh.cols, (size_t)silh.rows };
return k.run(2, globalsize, NULL, false);
}
#endif
void updateMotionHistory( InputArray _silhouette, InputOutputArray _mhi,
double timestamp, double duration )
{
CV_Assert( _silhouette.type() == CV_8UC1 && _mhi.type() == CV_32FC1 );
CV_Assert( _silhouette.sameSize(_mhi) );
float ts = (float)timestamp;
float delbound = (float)(timestamp - duration);
CV_OCL_RUN(_mhi.isUMat() && _mhi.dims() <= 2,
ocl_updateMotionHistory(_silhouette, _mhi, ts, delbound))
Mat silh = _silhouette.getMat(), mhi = _mhi.getMat();
Size size = silh.size();
#if defined(HAVE_IPP)
int silhstep = (int)silh.step, mhistep = (int)mhi.step;
#endif
if( silh.isContinuous() && mhi.isContinuous() )
{
size.width *= size.height;
size.height = 1;
#if defined(HAVE_IPP)
silhstep = (int)silh.total();
mhistep = (int)mhi.total() * sizeof(Ipp32f);
#endif
}
#if defined(HAVE_IPP)
IppStatus status = ippiUpdateMotionHistory_8u32f_C1IR((const Ipp8u *)silh.data, silhstep, (Ipp32f *)mhi.data, mhistep,
ippiSize(size.width, size.height), (Ipp32f)timestamp, (Ipp32f)duration);
if (status >= 0)
return;
#endif
#if CV_SSE2
volatile bool useSIMD = checkHardwareSupport(CV_CPU_SSE2);
#endif
for(int y = 0; y < size.height; y++ )
{
const uchar* silhData = silh.ptr<uchar>(y);
float* mhiData = mhi.ptr<float>(y);
int x = 0;
#if CV_SSE2
if( useSIMD )
{
__m128 ts4 = _mm_set1_ps(ts), db4 = _mm_set1_ps(delbound);
for( ; x <= size.width - 8; x += 8 )
{
__m128i z = _mm_setzero_si128();
__m128i s = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)(silhData + x)), z);
__m128 s0 = _mm_cvtepi32_ps(_mm_unpacklo_epi16(s, z)), s1 = _mm_cvtepi32_ps(_mm_unpackhi_epi16(s, z));
__m128 v0 = _mm_loadu_ps(mhiData + x), v1 = _mm_loadu_ps(mhiData + x + 4);
__m128 fz = _mm_setzero_ps();
v0 = _mm_and_ps(v0, _mm_cmpge_ps(v0, db4));
v1 = _mm_and_ps(v1, _mm_cmpge_ps(v1, db4));
__m128 m0 = _mm_and_ps(_mm_xor_ps(v0, ts4), _mm_cmpneq_ps(s0, fz));
__m128 m1 = _mm_and_ps(_mm_xor_ps(v1, ts4), _mm_cmpneq_ps(s1, fz));
v0 = _mm_xor_ps(v0, m0);
v1 = _mm_xor_ps(v1, m1);
_mm_storeu_ps(mhiData + x, v0);
_mm_storeu_ps(mhiData + x + 4, v1);
}
}
#endif
for( ; x < size.width; x++ )
{
float val = mhiData[x];
val = silhData[x] ? ts : val < delbound ? 0 : val;
mhiData[x] = val;
}
}
}
void calcMotionGradient( InputArray _mhi, OutputArray _mask,
OutputArray _orientation,
double delta1, double delta2,
int aperture_size )
{
Mat mhi = _mhi.getMat();
Size size = mhi.size();
_mask.create(size, CV_8U);
_orientation.create(size, CV_32F);
Mat mask = _mask.getMat();
Mat orient = _orientation.getMat();
if( aperture_size < 3 || aperture_size > 7 || (aperture_size & 1) == 0 )
CV_Error( Error::StsOutOfRange, "aperture_size must be 3, 5 or 7" );
if( delta1 <= 0 || delta2 <= 0 )
CV_Error( Error::StsOutOfRange, "both delta's must be positive" );
if( mhi.type() != CV_32FC1 )
CV_Error( Error::StsUnsupportedFormat,
"MHI must be single-channel floating-point images" );
if( orient.data == mhi.data )
{
_orientation.release();
_orientation.create(size, CV_32F);
orient = _orientation.getMat();
}
if( delta1 > delta2 )
std::swap(delta1, delta2);
float gradient_epsilon = 1e-4f * aperture_size * aperture_size;
float min_delta = (float)delta1;
float max_delta = (float)delta2;
Mat dX_min, dY_max;
// calc Dx and Dy
Sobel( mhi, dX_min, CV_32F, 1, 0, aperture_size, 1, 0, BORDER_REPLICATE );
Sobel( mhi, dY_max, CV_32F, 0, 1, aperture_size, 1, 0, BORDER_REPLICATE );
int x, y;
if( mhi.isContinuous() && orient.isContinuous() && mask.isContinuous() )
{
size.width *= size.height;
size.height = 1;
}
// calc gradient
for( y = 0; y < size.height; y++ )
{
const float* dX_min_row = dX_min.ptr<float>(y);
const float* dY_max_row = dY_max.ptr<float>(y);
float* orient_row = orient.ptr<float>(y);
uchar* mask_row = mask.ptr<uchar>(y);
cv::hal::fastAtan2(dY_max_row, dX_min_row, orient_row, size.width, true);
// make orientation zero where the gradient is very small
for( x = 0; x < size.width; x++ )
{
float dY = dY_max_row[x];
float dX = dX_min_row[x];
if( std::abs(dX) < gradient_epsilon && std::abs(dY) < gradient_epsilon )
{
mask_row[x] = (uchar)0;
orient_row[x] = 0.f;
}
else
mask_row[x] = (uchar)1;
}
}
erode( mhi, dX_min, noArray(), Point(-1,-1), (aperture_size-1)/2, BORDER_REPLICATE );
dilate( mhi, dY_max, noArray(), Point(-1,-1), (aperture_size-1)/2, BORDER_REPLICATE );
// mask off pixels which have little motion difference in their neighborhood
for( y = 0; y < size.height; y++ )
{
const float* dX_min_row = dX_min.ptr<float>(y);
const float* dY_max_row = dY_max.ptr<float>(y);
float* orient_row = orient.ptr<float>(y);
uchar* mask_row = mask.ptr<uchar>(y);
for( x = 0; x < size.width; x++ )
{
float d0 = dY_max_row[x] - dX_min_row[x];
if( mask_row[x] == 0 || d0 < min_delta || max_delta < d0 )
{
mask_row[x] = (uchar)0;
orient_row[x] = 0.f;
}
}
}
}
double calcGlobalOrientation( InputArray _orientation, InputArray _mask,
InputArray _mhi, double /*timestamp*/,
double duration )
{
Mat orient = _orientation.getMat(), mask = _mask.getMat(), mhi = _mhi.getMat();
Size size = mhi.size();
CV_Assert( mask.type() == CV_8U && orient.type() == CV_32F && mhi.type() == CV_32F );
CV_Assert( mask.size() == size && orient.size() == size );
CV_Assert( duration > 0 );
int histSize = 12;
float _ranges[] = { 0.f, 360.f };
const float* ranges = _ranges;
Mat hist;
calcHist(&orient, 1, 0, mask, hist, 1, &histSize, &ranges);
// find the maximum index (the dominant orientation)
Point baseOrientPt;
minMaxLoc(hist, 0, 0, 0, &baseOrientPt);
float fbaseOrient = (baseOrientPt.x + baseOrientPt.y)*360.f/histSize;
// override timestamp with the maximum value in MHI
double timestamp = 0;
minMaxLoc( mhi, 0, &timestamp, 0, 0, mask );
// find the shift relative to the dominant orientation as weighted sum of relative angles
float a = (float)(254. / 255. / duration);
float b = (float)(1. - timestamp * a);
float delbound = (float)(timestamp - duration);
if( mhi.isContinuous() && mask.isContinuous() && orient.isContinuous() )
{
size.width *= size.height;
size.height = 1;
}
/*
a = 254/(255*dt)
b = 1 - t*a = 1 - 254*t/(255*dur) =
(255*dt - 254*t)/(255*dt) =
(dt - (t - dt)*254)/(255*dt);
--------------------------------------------------------
ax + b = 254*x/(255*dt) + (dt - (t - dt)*254)/(255*dt) =
(254*x + dt - (t - dt)*254)/(255*dt) =
((x - (t - dt))*254 + dt)/(255*dt) =
(((x - (t - dt))/dt)*254 + 1)/255 = (((x - low_time)/dt)*254 + 1)/255
*/
float shiftOrient = 0, shiftWeight = 0;
for( int y = 0; y < size.height; y++ )
{
const float* mhiptr = mhi.ptr<float>(y);
const float* oriptr = orient.ptr<float>(y);
const uchar* maskptr = mask.ptr<uchar>(y);
for( int x = 0; x < size.width; x++ )
{
if( maskptr[x] != 0 && mhiptr[x] > delbound )
{
/*
orient in 0..360, base_orient in 0..360
-> (rel_angle = orient - base_orient) in -360..360.
rel_angle is translated to -180..180
*/
float weight = mhiptr[x] * a + b;
float relAngle = oriptr[x] - fbaseOrient;
relAngle += (relAngle < -180 ? 360 : 0);
relAngle += (relAngle > 180 ? -360 : 0);
if( fabs(relAngle) < 45 )
{
shiftOrient += weight * relAngle;
shiftWeight += weight;
}
}
}
}
// add the dominant orientation and the relative shift
if( shiftWeight == 0 )
shiftWeight = 0.01f;
fbaseOrient += shiftOrient / shiftWeight;
fbaseOrient -= (fbaseOrient < 360 ? 0 : 360);
fbaseOrient += (fbaseOrient >= 0 ? 0 : 360);
return fbaseOrient;
}
void segmentMotion(InputArray _mhi, OutputArray _segmask,
vector<Rect>& boundingRects,
double timestamp, double segThresh)
{
Mat mhi = _mhi.getMat();
_segmask.create(mhi.size(), CV_32F);
Mat segmask = _segmask.getMat();
segmask = Scalar::all(0);
CV_Assert( mhi.type() == CV_32F );
CV_Assert( segThresh >= 0 );
Mat mask = Mat::zeros( mhi.rows + 2, mhi.cols + 2, CV_8UC1 );
int x, y;
// protect zero mhi pixels from floodfill.
for( y = 0; y < mhi.rows; y++ )
{
const float* mhiptr = mhi.ptr<float>(y);
uchar* maskptr = mask.ptr<uchar>(y+1) + 1;
for( x = 0; x < mhi.cols; x++ )
{
if( mhiptr[x] == 0 )
maskptr[x] = 1;
}
}
float ts = (float)timestamp;
float comp_idx = 1.f;
for( y = 0; y < mhi.rows; y++ )
{
float* mhiptr = mhi.ptr<float>(y);
uchar* maskptr = mask.ptr<uchar>(y+1) + 1;
for( x = 0; x < mhi.cols; x++ )
{
if( mhiptr[x] == ts && maskptr[x] == 0 )
{
Rect cc;
floodFill( mhi, mask, Point(x,y), Scalar::all(0),
&cc, Scalar::all(segThresh), Scalar::all(segThresh),
FLOODFILL_MASK_ONLY + 2*256 + 4 );
for( int y1 = 0; y1 < cc.height; y1++ )
{
float* segmaskptr = segmask.ptr<float>(cc.y + y1) + cc.x;
uchar* maskptr1 = mask.ptr<uchar>(cc.y + y1 + 1) + cc.x + 1;
for( int x1 = 0; x1 < cc.width; x1++ )
{
if( maskptr1[x1] > 1 )
{
maskptr1[x1] = 1;
segmaskptr[x1] = comp_idx;
}
}
}
comp_idx += 1.f;
boundingRects.push_back(cc);
}
}
}
}
}
}
/* End of file. */
modules/optflow/src/opencl/optical_flow_tvl1.cl
+378
View File
@@ -0,0 +1,378 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
// Jin Ma jin@multicorewareinc.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
__kernel void centeredGradientKernel(__global const float* src_ptr, int src_col, int src_row, int src_step,
__global float* dx, __global float* dy, int d_step)
{
int x = get_global_id(0);
int y = get_global_id(1);
if((x < src_col)&&(y < src_row))
{
int src_x1 = (x + 1) < (src_col -1)? (x + 1) : (src_col - 1);
int src_x2 = (x - 1) > 0 ? (x -1) : 0;
dx[y * d_step+ x] = 0.5f * (src_ptr[y * src_step + src_x1] - src_ptr[y * src_step+ src_x2]);
int src_y1 = (y+1) < (src_row - 1) ? (y + 1) : (src_row - 1);
int src_y2 = (y - 1) > 0 ? (y - 1) : 0;
dy[y * d_step+ x] = 0.5f * (src_ptr[src_y1 * src_step + x] - src_ptr[src_y2 * src_step+ x]);
}
}
inline float bicubicCoeff(float x_)
{
float x = fabs(x_);
if (x <= 1.0f)
return x * x * (1.5f * x - 2.5f) + 1.0f;
else if (x < 2.0f)
return x * (x * (-0.5f * x + 2.5f) - 4.0f) + 2.0f;
else
return 0.0f;
}
__kernel void warpBackwardKernel(__global const float* I0, int I0_step, int I0_col, int I0_row,
image2d_t tex_I1, image2d_t tex_I1x, image2d_t tex_I1y,
__global const float* u1, int u1_step,
__global const float* u2,
__global float* I1w,
__global float* I1wx, /*int I1wx_step,*/
__global float* I1wy, /*int I1wy_step,*/
__global float* grad, /*int grad_step,*/
__global float* rho,
int I1w_step,
int u2_step,
int u1_offset_x,
int u1_offset_y,
int u2_offset_x,
int u2_offset_y)
{
int x = get_global_id(0);
int y = get_global_id(1);
if(x < I0_col&&y < I0_row)
{
//float u1Val = u1(y, x);
float u1Val = u1[(y + u1_offset_y) * u1_step + x + u1_offset_x];
//float u2Val = u2(y, x);
float u2Val = u2[(y + u2_offset_y) * u2_step + x + u2_offset_x];
float wx = x + u1Val;
float wy = y + u2Val;
int xmin = ceil(wx - 2.0f);
int xmax = floor(wx + 2.0f);
int ymin = ceil(wy - 2.0f);
int ymax = floor(wy + 2.0f);
float sum = 0.0f;
float sumx = 0.0f;
float sumy = 0.0f;
float wsum = 0.0f;
sampler_t sampleri = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;
for (int cy = ymin; cy <= ymax; ++cy)
{
for (int cx = xmin; cx <= xmax; ++cx)
{
float w = bicubicCoeff(wx - cx) * bicubicCoeff(wy - cy);
//sum += w * tex2D(tex_I1 , cx, cy);
int2 cood = (int2)(cx, cy);
sum += w * read_imagef(tex_I1, sampleri, cood).x;
//sumx += w * tex2D(tex_I1x, cx, cy);
sumx += w * read_imagef(tex_I1x, sampleri, cood).x;
//sumy += w * tex2D(tex_I1y, cx, cy);
sumy += w * read_imagef(tex_I1y, sampleri, cood).x;
wsum += w;
}
}
float coeff = 1.0f / wsum;
float I1wVal = sum * coeff;
float I1wxVal = sumx * coeff;
float I1wyVal = sumy * coeff;
I1w[y * I1w_step + x] = I1wVal;
I1wx[y * I1w_step + x] = I1wxVal;
I1wy[y * I1w_step + x] = I1wyVal;
float Ix2 = I1wxVal * I1wxVal;
float Iy2 = I1wyVal * I1wyVal;
// store the |Grad(I1)|^2
grad[y * I1w_step + x] = Ix2 + Iy2;
// compute the constant part of the rho function
float I0Val = I0[y * I0_step + x];
rho[y * I1w_step + x] = I1wVal - I1wxVal * u1Val - I1wyVal * u2Val - I0Val;
}
}
inline float readImage(__global const float *image, int x, int y, int rows, int cols, int elemCntPerRow)
{
int i0 = clamp(x, 0, cols - 1);
int j0 = clamp(y, 0, rows - 1);
return image[j0 * elemCntPerRow + i0];
}
__kernel void warpBackwardKernelNoImage2d(__global const float* I0, int I0_step, int I0_col, int I0_row,
__global const float* tex_I1, __global const float* tex_I1x, __global const float* tex_I1y,
__global const float* u1, int u1_step,
__global const float* u2,
__global float* I1w,
__global float* I1wx, /*int I1wx_step,*/
__global float* I1wy, /*int I1wy_step,*/
__global float* grad, /*int grad_step,*/
__global float* rho,
int I1w_step,
int u2_step,
int I1_step,
int I1x_step)
{
int x = get_global_id(0);
int y = get_global_id(1);
if(x < I0_col&&y < I0_row)
{
//float u1Val = u1(y, x);
float u1Val = u1[y * u1_step + x];
//float u2Val = u2(y, x);
float u2Val = u2[y * u2_step + x];
float wx = x + u1Val;
float wy = y + u2Val;
int xmin = ceil(wx - 2.0f);
int xmax = floor(wx + 2.0f);
int ymin = ceil(wy - 2.0f);
int ymax = floor(wy + 2.0f);
float sum = 0.0f;
float sumx = 0.0f;
float sumy = 0.0f;
float wsum = 0.0f;
for (int cy = ymin; cy <= ymax; ++cy)
{
for (int cx = xmin; cx <= xmax; ++cx)
{
float w = bicubicCoeff(wx - cx) * bicubicCoeff(wy - cy);
int2 cood = (int2)(cx, cy);
sum += w * readImage(tex_I1, cood.x, cood.y, I0_col, I0_row, I1_step);
sumx += w * readImage(tex_I1x, cood.x, cood.y, I0_col, I0_row, I1x_step);
sumy += w * readImage(tex_I1y, cood.x, cood.y, I0_col, I0_row, I1x_step);
wsum += w;
}
}
float coeff = 1.0f / wsum;
float I1wVal = sum * coeff;
float I1wxVal = sumx * coeff;
float I1wyVal = sumy * coeff;
I1w[y * I1w_step + x] = I1wVal;
I1wx[y * I1w_step + x] = I1wxVal;
I1wy[y * I1w_step + x] = I1wyVal;
float Ix2 = I1wxVal * I1wxVal;
float Iy2 = I1wyVal * I1wyVal;
// store the |Grad(I1)|^2
grad[y * I1w_step + x] = Ix2 + Iy2;
// compute the constant part of the rho function
float I0Val = I0[y * I0_step + x];
rho[y * I1w_step + x] = I1wVal - I1wxVal * u1Val - I1wyVal * u2Val - I0Val;
}
}
__kernel void estimateDualVariablesKernel(__global const float* u1, int u1_col, int u1_row, int u1_step,
__global const float* u2,
__global float* p11, int p11_step,
__global float* p12,
__global float* p21,
__global float* p22,
float taut,
int u2_step,
int u1_offset_x,
int u1_offset_y,
int u2_offset_x,
int u2_offset_y)
{
int x = get_global_id(0);
int y = get_global_id(1);
if(x < u1_col && y < u1_row)
{
int src_x1 = (x + 1) < (u1_col - 1) ? (x + 1) : (u1_col - 1);
float u1x = u1[(y + u1_offset_y) * u1_step + src_x1 + u1_offset_x] - u1[(y + u1_offset_y) * u1_step + x + u1_offset_x];
int src_y1 = (y + 1) < (u1_row - 1) ? (y + 1) : (u1_row - 1);
float u1y = u1[(src_y1 + u1_offset_y) * u1_step + x + u1_offset_x] - u1[(y + u1_offset_y) * u1_step + x + u1_offset_x];
int src_x2 = (x + 1) < (u1_col - 1) ? (x + 1) : (u1_col - 1);
float u2x = u2[(y + u2_offset_y) * u2_step + src_x2 + u2_offset_x] - u2[(y + u2_offset_y) * u2_step + x + u2_offset_x];
int src_y2 = (y + 1) < (u1_row - 1) ? (y + 1) : (u1_row - 1);
float u2y = u2[(src_y2 + u2_offset_y) * u2_step + x + u2_offset_x] - u2[(y + u2_offset_y) * u2_step + x + u2_offset_x];
float g1 = hypot(u1x, u1y);
float g2 = hypot(u2x, u2y);
float ng1 = 1.0f + taut * g1;
float ng2 = 1.0f + taut * g2;
p11[y * p11_step + x] = (p11[y * p11_step + x] + taut * u1x) / ng1;
p12[y * p11_step + x] = (p12[y * p11_step + x] + taut * u1y) / ng1;
p21[y * p11_step + x] = (p21[y * p11_step + x] + taut * u2x) / ng2;
p22[y * p11_step + x] = (p22[y * p11_step + x] + taut * u2y) / ng2;
}
}
inline float divergence(__global const float* v1, __global const float* v2, int y, int x, int v1_step, int v2_step)
{
if (x > 0 && y > 0)
{
float v1x = v1[y * v1_step + x] - v1[y * v1_step + x - 1];
float v2y = v2[y * v2_step + x] - v2[(y - 1) * v2_step + x];
return v1x + v2y;
}
else
{
if (y > 0)
return v1[y * v1_step + 0] + v2[y * v2_step + 0] - v2[(y - 1) * v2_step + 0];
else
{
if (x > 0)
return v1[0 * v1_step + x] - v1[0 * v1_step + x - 1] + v2[0 * v2_step + x];
else
return v1[0 * v1_step + 0] + v2[0 * v2_step + 0];
}
}
}
__kernel void estimateUKernel(__global const float* I1wx, int I1wx_col, int I1wx_row, int I1wx_step,
__global const float* I1wy, /*int I1wy_step,*/
__global const float* grad, /*int grad_step,*/
__global const float* rho_c, /*int rho_c_step,*/
__global const float* p11, /*int p11_step,*/
__global const float* p12, /*int p12_step,*/
__global const float* p21, /*int p21_step,*/
__global const float* p22, /*int p22_step,*/
__global float* u1, int u1_step,
__global float* u2,
__global float* error, float l_t, float theta, int u2_step,
int u1_offset_x,
int u1_offset_y,
int u2_offset_x,
int u2_offset_y,
char calc_error)
{
int x = get_global_id(0);
int y = get_global_id(1);
if(x < I1wx_col && y < I1wx_row)
{
float I1wxVal = I1wx[y * I1wx_step + x];
float I1wyVal = I1wy[y * I1wx_step + x];
float gradVal = grad[y * I1wx_step + x];
float u1OldVal = u1[(y + u1_offset_y) * u1_step + x + u1_offset_x];
float u2OldVal = u2[(y + u2_offset_y) * u2_step + x + u2_offset_x];
float rho = rho_c[y * I1wx_step + x] + (I1wxVal * u1OldVal + I1wyVal * u2OldVal);
// estimate the values of the variable (v1, v2) (thresholding operator TH)
float d1 = 0.0f;
float d2 = 0.0f;
if (rho < -l_t * gradVal)
{
d1 = l_t * I1wxVal;
d2 = l_t * I1wyVal;
}
else if (rho > l_t * gradVal)
{
d1 = -l_t * I1wxVal;
d2 = -l_t * I1wyVal;
}
else if (gradVal > 1.192092896e-07f)
{
float fi = -rho / gradVal;
d1 = fi * I1wxVal;
d2 = fi * I1wyVal;
}
float v1 = u1OldVal + d1;
float v2 = u2OldVal + d2;
// compute the divergence of the dual variable (p1, p2)
float div_p1 = divergence(p11, p12, y, x, I1wx_step, I1wx_step);
float div_p2 = divergence(p21, p22, y, x, I1wx_step, I1wx_step);
// estimate the values of the optical flow (u1, u2)
float u1NewVal = v1 + theta * div_p1;
float u2NewVal = v2 + theta * div_p2;
u1[(y + u1_offset_y) * u1_step + x + u1_offset_x] = u1NewVal;
u2[(y + u2_offset_y) * u2_step + x + u2_offset_x] = u2NewVal;
if(calc_error)
{
float n1 = (u1OldVal - u1NewVal) * (u1OldVal - u1NewVal);
float n2 = (u2OldVal - u2NewVal) * (u2OldVal - u2NewVal);
error[y * I1wx_step + x] = n1 + n2;
}
}
}
modules/optflow/src/opencl/sparse_matching_gpc.cl
+69
View File
@@ -0,0 +1,69 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2016, Itseez, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
// @Authors
// Vladislav Samsonov, vvladxx@gmail.com
__kernel void getPatchDescriptor(
__global const uchar* imgCh0, int ic0step, int ic0off,
__global const uchar* imgCh1, int ic1step, int ic1off,
__global const uchar* imgCh2, int ic2step, int ic2off,
__global uchar* out, int outstep, int outoff,
const int gh, const int gw, const int PR )
{
const int i = get_global_id(0);
const int j = get_global_id(1);
if (i >= gh || j >= gw)
return;
__global double* desc = (__global double*)(out + (outstep * (i * gw + j) + outoff));
const int patchRadius = PR * 2;
float patch[PATCH_RADIUS_DOUBLED][PATCH_RADIUS_DOUBLED];
for (int i0 = 0; i0 < patchRadius; ++i0) {
__global const float* ch0Row = (__global const float*)(imgCh0 + (ic0step * (i + i0) + ic0off + j * sizeof(float)));
for (int j0 = 0; j0 < patchRadius; ++j0)
patch[i0][j0] = ch0Row[j0];
}
#pragma unroll
for (int n0 = 0; n0 < 4; ++n0) {
#pragma unroll
for (int n1 = 0; n1 < 4; ++n1) {
double sum = 0;
for (int i0 = 0; i0 < patchRadius; ++i0)
for (int j0 = 0; j0 < patchRadius; ++j0)
sum += patch[i0][j0] * cos(CV_PI * (i0 + 0.5) * n0 / patchRadius) * cos(CV_PI * (j0 + 0.5) * n1 / patchRadius);
desc[n0 * 4 + n1] = sum / PR;
}
}
for (int k = 0; k < 4; ++k) {
desc[k] *= SQRT2_INV;
desc[k * 4] *= SQRT2_INV;
}
double sum = 0;
for (int i0 = 0; i0 < patchRadius; ++i0) {
__global const float* ch1Row = (__global const float*)(imgCh1 + (ic1step * (i + i0) + ic1off + j * sizeof(float)));
for (int j0 = 0; j0 < patchRadius; ++j0)
sum += ch1Row[j0];
}
desc[16] = sum / patchRadius;
sum = 0;
for (int i0 = 0; i0 < patchRadius; ++i0) {
__global const float* ch2Row = (__global const float*)(imgCh2 + (ic2step * (i + i0) + ic2off + j * sizeof(float)));
for (int j0 = 0; j0 < patchRadius; ++j0)
sum += ch2Row[j0];
}
desc[17] = sum / patchRadius;
}
modules/optflow/src/opencl/updatemotionhistory.cl
+27
View File
@@ -0,0 +1,27 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2014, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
__kernel void updateMotionHistory(__global const uchar * silh, int silh_step, int silh_offset,
__global uchar * mhiptr, int mhi_step, int mhi_offset, int mhi_rows, int mhi_cols,
float timestamp, float delbound)
{
int x = get_global_id(0);
int y = get_global_id(1);
if (x < mhi_cols && y < mhi_rows)
{
int silh_index = mad24(y, silh_step, silh_offset + x);
int mhi_index = mad24(y, mhi_step, mhi_offset + x * (int)sizeof(float));
silh += silh_index;
__global float * mhi = (__global float *)(mhiptr + mhi_index);
float val = mhi[0];
val = silh[0] ? timestamp : val < delbound ? 0 : val;
mhi[0] = val;
}
}
modules/optflow/src/pcaflow.cpp
+528
View File
@@ -0,0 +1,528 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
#include "opencv2/ximgproc/edge_filter.hpp"
/* Disable "from double to float" and "from size_t to int" warnings.
* Fixing these would make the code look ugly by introducing explicit cast all around.
* Here these warning are pointless anyway.
*/
#ifdef _MSC_VER
#pragma warning( disable : 4305 4244 4267 4838 )
#endif
#ifdef __clang__
#pragma clang diagnostic ignored "-Wshorten-64-to-32"
#endif
namespace cv
{
namespace optflow
{
namespace
{
#ifndef M_SQRT2
const float M_SQRT2 = 1.41421356237309504880;
#endif
template <typename T> inline int mathSign( T val ) { return ( T( 0 ) < val ) - ( val < T( 0 ) ); }
/* Stable symmetric Householder reflection that gives c and s such that
* [ c s ][a] = [d],
* [ s -c ][b] [0]
*
* Output:
* c -- cosine(theta), where theta is the implicit angle of rotation
* (counter-clockwise) in a plane-rotation
* s -- sine(theta)
* r -- two-norm of [a; b]
*/
inline void symOrtho( double a, double b, double &c, double &s, double &r )
{
if ( b == 0 )
{
c = mathSign( a );
s = 0;
r = std::abs( a );
}
else if ( a == 0 )
{
c = 0;
s = mathSign( b );
r = std::abs( b );
}
else if ( std::abs( b ) > std::abs( a ) )
{
const double tau = a / b;
s = mathSign( b ) / std::sqrt( 1 + tau * tau );
c = s * tau;
r = b / s;
}
else
{
const double tau = b / a;
c = mathSign( a ) / std::sqrt( 1 + tau * tau );
s = c * tau;
r = a / c;
}
}
/* Iterative LSQR algorithm for solving least squares problems.
*
* [1] Paige, C. C. and M. A. Saunders,
* LSQR: An Algorithm for Sparse Linear Equations And Sparse Least Squares
* ACM Trans. Math. Soft., Vol.8, 1982, pp. 43-71.
*
* Solves the following problem:
* argmin_x ||Ax - b|| + damp||x||
*
* Output:
* x -- approximate solution
*/
void solveLSQR( const Mat &A, const Mat &b, OutputArray xOut, const double damp = 0.0, const unsigned iter_lim = 10 )
{
const int n = A.size().width;
CV_Assert( A.size().height == b.size().height );
CV_Assert( A.type() == CV_32F );
CV_Assert( b.type() == CV_32F );
xOut.create( n, 1, CV_32F );
Mat v( n, 1, CV_32F, 0.0f );
Mat u = b;
Mat x = xOut.getMat();
x = Mat::zeros( x.size(), x.type() );
double alfa = 0;
double beta = cv::norm( u, NORM_L2 );
Mat w( n, 1, CV_32F, 0.0f );
const Mat AT = A.t();
if ( beta > 0 )
{
u *= 1 / beta;
v = AT * u;
alfa = cv::norm( v, NORM_L2 );
}
if ( alfa > 0 )
{
v *= 1 / alfa;
w = v.clone();
}
double rhobar = alfa;
double phibar = beta;
if ( alfa * beta == 0 )
return;
for ( unsigned itn = 0; itn < iter_lim; ++itn )
{
u *= -alfa;
u += A * v;
beta = cv::norm( u, NORM_L2 );
if ( beta > 0 )
{
u *= 1 / beta;
v *= -beta;
v += AT * u;
alfa = cv::norm( v, NORM_L2 );
if ( alfa > 0 )
v *= 1 / alfa;
}
double rhobar1 = sqrt( rhobar * rhobar + damp * damp );
double cs1 = rhobar / rhobar1;
phibar = cs1 * phibar;
double cs, sn, rho;
symOrtho( rhobar1, beta, cs, sn, rho );
double theta = sn * alfa;
rhobar = -cs * alfa;
double phi = cs * phibar;
phibar = sn * phibar;
double t1 = phi / rho;
double t2 = -theta / rho;
x += t1 * w;
w *= t2;
w += v;
}
}
inline void _cpu_fillDCTSampledPoints( float *row, const Point2f &p, const Size &basisSize, const Size &size )
{
for ( int n1 = 0; n1 < basisSize.width; ++n1 )
for ( int n2 = 0; n2 < basisSize.height; ++n2 )
row[n1 * basisSize.height + n2] =
cosf( ( n1 * CV_PI / size.width ) * ( p.x + 0.5 ) ) * cosf( ( n2 * CV_PI / size.height ) * ( p.y + 0.5 ) );
}
ocl::ProgramSource _ocl_fillDCTSampledPointsSource(
"__kernel void fillDCTSampledPoints(__global const uchar* features, int fstep, int foff, __global "
"uchar* A, int Astep, int Aoff, int fs, int bsw, int bsh, int sw, int sh) {"
"const int i = get_global_id(0);"
"const int n1 = get_global_id(1);"
"const int n2 = get_global_id(2);"
"if (i >= fs || n1 >= bsw || n2 >= bsh) return;"
"__global const float2* f = (__global const float2*)(features + (fstep * i + foff));"
"__global float* a = (__global float*)(A + (Astep * i + Aoff + (n1 * bsh + n2) * sizeof(float)));"
"const float2 p = f[0];"
"const float pi = 3.14159265358979323846;"
"a[0] = cos((n1 * pi / sw) * (p.x + 0.5)) * cos((n2 * pi / sh) * (p.y + 0.5));"
"}" );
void applyCLAHE( UMat &img, float claheClip )
{
Ptr<CLAHE> clahe = createCLAHE();
clahe->setClipLimit( claheClip );
clahe->apply( img, img );
}
void reduceToFlow( const Mat &w1, const Mat &w2, Mat &flow, const Size &basisSize )
{
const Size size = flow.size();
Mat flowX( size, CV_32F, 0.0f );
Mat flowY( size, CV_32F, 0.0f );
const float mult = sqrt( static_cast<float>(size.area()) ) * 0.5;
for ( int i = 0; i < basisSize.width; ++i )
for ( int j = 0; j < basisSize.height; ++j )
{
flowX.at<float>( j, i ) = w1.at<float>( i * basisSize.height + j ) * mult;
flowY.at<float>( j, i ) = w2.at<float>( i * basisSize.height + j ) * mult;
}
for ( int i = 0; i < basisSize.height; ++i )
{
flowX.at<float>( i, 0 ) *= M_SQRT2;
flowY.at<float>( i, 0 ) *= M_SQRT2;
}
for ( int i = 0; i < basisSize.width; ++i )
{
flowX.at<float>( 0, i ) *= M_SQRT2;
flowY.at<float>( 0, i ) *= M_SQRT2;
}
dct( flowX, flowX, DCT_INVERSE );
dct( flowY, flowY, DCT_INVERSE );
for ( int i = 0; i < size.height; ++i )
for ( int j = 0; j < size.width; ++j )
flow.at<Point2f>( i, j ) = Point2f( flowX.at<float>( i, j ), flowY.at<float>( i, j ) );
}
}
void OpticalFlowPCAFlow::findSparseFeatures( UMat &from, UMat &to, std::vector<Point2f> &features,
std::vector<Point2f> &predictedFeatures ) const
{
Size size = from.size();
const unsigned maxFeatures = size.area() * sparseRate;
goodFeaturesToTrack( from, features, maxFeatures * retainedCornersFraction, 0.005, 3 );
// Add points along the grid if not enough features
if ( maxFeatures > features.size() )
{
const unsigned missingPoints = maxFeatures - features.size();
const unsigned blockSize = sqrt( (float)size.area() / missingPoints );
for ( int x = blockSize / 2; x < size.width; x += blockSize )
for ( int y = blockSize / 2; y < size.height; y += blockSize )
features.push_back( Point2f( x, y ) );
}
std::vector<uchar> predictedStatus;
std::vector<float> predictedError;
calcOpticalFlowPyrLK( from, to, features, predictedFeatures, predictedStatus, predictedError );
size_t j = 0;
for ( size_t i = 0; i < features.size(); ++i )
{
if ( predictedStatus[i] )
{
features[j] = features[i];
predictedFeatures[j] = predictedFeatures[i];
++j;
}
}
features.resize( j );
predictedFeatures.resize( j );
}
void OpticalFlowPCAFlow::removeOcclusions( UMat &from, UMat &to, std::vector<Point2f> &features,
std::vector<Point2f> &predictedFeatures ) const
{
std::vector<uchar> predictedStatus;
std::vector<float> predictedError;
std::vector<Point2f> backwardFeatures;
calcOpticalFlowPyrLK( to, from, predictedFeatures, backwardFeatures, predictedStatus, predictedError );
size_t j = 0;
const float threshold = occlusionsThreshold * sqrt( static_cast<float>(from.size().area()) );
for ( size_t i = 0; i < predictedFeatures.size(); ++i )
{
if ( predictedStatus[i] )
{
Point2f flowDiff = features[i] - backwardFeatures[i];
if ( flowDiff.dot( flowDiff ) <= threshold )
{
features[j] = features[i];
predictedFeatures[j] = predictedFeatures[i];
++j;
}
}
}
features.resize( j );
predictedFeatures.resize( j );
}
void OpticalFlowPCAFlow::getSystem( OutputArray AOut, OutputArray b1Out, OutputArray b2Out,
const std::vector<Point2f> &features, const std::vector<Point2f> &predictedFeatures,
const Size size )
{
AOut.create( features.size(), basisSize.area(), CV_32F );
b1Out.create( features.size(), 1, CV_32F );
b2Out.create( features.size(), 1, CV_32F );
if ( useOpenCL )
{
UMat A = AOut.getUMat();
Mat b1 = b1Out.getMat();
Mat b2 = b2Out.getMat();
ocl::Kernel kernel( "fillDCTSampledPoints", _ocl_fillDCTSampledPointsSource );
CV_Assert(basisSize.width > 0 && basisSize.height > 0);
size_t globSize[] = {features.size(), (size_t)basisSize.width, (size_t)basisSize.height};
kernel
.args( cv::ocl::KernelArg::ReadOnlyNoSize( Mat( features ).getUMat( ACCESS_READ ) ),
cv::ocl::KernelArg::WriteOnlyNoSize( A ), (int)features.size(), (int)basisSize.width,
(int)basisSize.height, (int)size.width, (int)size.height )
.run( 3, globSize, 0, true );
for ( size_t i = 0; i < features.size(); ++i )
{
const Point2f flow = predictedFeatures[i] - features[i];
b1.at<float>( i ) = flow.x;
b2.at<float>( i ) = flow.y;
}
}
else
{
Mat A = AOut.getMat();
Mat b1 = b1Out.getMat();
Mat b2 = b2Out.getMat();
for ( size_t i = 0; i < features.size(); ++i )
{
_cpu_fillDCTSampledPoints( A.ptr<float>( i ), features[i], basisSize, size );
const Point2f flow = predictedFeatures[i] - features[i];
b1.at<float>( i ) = flow.x;
b2.at<float>( i ) = flow.y;
}
}
}
void OpticalFlowPCAFlow::getSystem( OutputArray A1Out, OutputArray A2Out, OutputArray b1Out, OutputArray b2Out,
const std::vector<Point2f> &features, const std::vector<Point2f> &predictedFeatures,
const Size size )
{
CV_Assert( prior->getBasisSize() == basisSize.area() );
A1Out.create( features.size() + prior->getPadding(), basisSize.area(), CV_32F );
A2Out.create( features.size() + prior->getPadding(), basisSize.area(), CV_32F );
b1Out.create( features.size() + prior->getPadding(), 1, CV_32F );
b2Out.create( features.size() + prior->getPadding(), 1, CV_32F );
if ( useOpenCL )
{
UMat A = A1Out.getUMat();
Mat b1 = b1Out.getMat();
Mat b2 = b2Out.getMat();
ocl::Kernel kernel( "fillDCTSampledPoints", _ocl_fillDCTSampledPointsSource );
CV_Assert(basisSize.width > 0 && basisSize.height > 0);
size_t globSize[] = {features.size(), (size_t)basisSize.width, (size_t)basisSize.height};
kernel
.args( cv::ocl::KernelArg::ReadOnlyNoSize( Mat( features ).getUMat( ACCESS_READ ) ),
cv::ocl::KernelArg::WriteOnlyNoSize( A ), (int)features.size(), (int)basisSize.width,
(int)basisSize.height, (int)size.width, (int)size.height )
.run( 3, globSize, 0, true );
for ( size_t i = 0; i < features.size(); ++i )
{
const Point2f flow = predictedFeatures[i] - features[i];
b1.at<float>( i ) = flow.x;
b2.at<float>( i ) = flow.y;
}
}
else
{
Mat A1 = A1Out.getMat();
Mat b1 = b1Out.getMat();
Mat b2 = b2Out.getMat();
for ( size_t i = 0; i < features.size(); ++i )
{
_cpu_fillDCTSampledPoints( A1.ptr<float>( i ), features[i], basisSize, size );
const Point2f flow = predictedFeatures[i] - features[i];
b1.at<float>( i ) = flow.x;
b2.at<float>( i ) = flow.y;
}
}
Mat A1 = A1Out.getMat();
Mat A2 = A2Out.getMat();
Mat b1 = b1Out.getMat();
Mat b2 = b2Out.getMat();
memcpy( A2.ptr<float>(), A1.ptr<float>(), features.size() * basisSize.area() * sizeof( float ) );
prior->fillConstraints( A1.ptr<float>( features.size(), 0 ), A2.ptr<float>( features.size(), 0 ),
b1.ptr<float>( features.size(), 0 ), b2.ptr<float>( features.size(), 0 ) );
}
void OpticalFlowPCAFlow::calc( InputArray I0, InputArray I1, InputOutputArray flowOut )
{
const Size size = I0.size();
CV_Assert( size == I1.size() );
UMat from, to;
if ( I0.channels() == 3 )
{
cvtColor( I0, from, COLOR_BGR2GRAY );
from.convertTo( from, CV_8U );
}
else
{
I0.getMat().convertTo( from, CV_8U );
}
if ( I1.channels() == 3 )
{
cvtColor( I1, to, COLOR_BGR2GRAY );
to.convertTo( to, CV_8U );
}
else
{
I1.getMat().convertTo( to, CV_8U );
}
CV_Assert( from.channels() == 1 );
CV_Assert( to.channels() == 1 );
const Mat fromOrig = from.getMat( ACCESS_READ ).clone();
useOpenCL = flowOut.isUMat() && ocl::useOpenCL();
applyCLAHE( from, claheClip );
applyCLAHE( to, claheClip );
std::vector<Point2f> features, predictedFeatures;
findSparseFeatures( from, to, features, predictedFeatures );
removeOcclusions( from, to, features, predictedFeatures );
flowOut.create( size, CV_32FC2 );
Mat flow = flowOut.getMat();
Mat w1, w2;
if ( prior.get() )
{
Mat A1, A2, b1, b2;
getSystem( A1, A2, b1, b2, features, predictedFeatures, size );
solveLSQR( A1, b1, w1, dampingFactor * size.area() );
solveLSQR( A2, b2, w2, dampingFactor * size.area() );
}
else
{
Mat A, b1, b2;
getSystem( A, b1, b2, features, predictedFeatures, size );
solveLSQR( A, b1, w1, dampingFactor * size.area() );
solveLSQR( A, b2, w2, dampingFactor * size.area() );
}
Mat flowSmall( ( size / 8 ) * 2, CV_32FC2 );
reduceToFlow( w1, w2, flowSmall, basisSize );
resize( flowSmall, flow, size, 0, 0, INTER_LINEAR );
ximgproc::fastGlobalSmootherFilter( fromOrig, flow, flow, 500, 2 );
}
OpticalFlowPCAFlow::OpticalFlowPCAFlow( Ptr<const PCAPrior> _prior, const Size _basisSize, float _sparseRate,
float _retainedCornersFraction, float _occlusionsThreshold,
float _dampingFactor, float _claheClip )
: prior( _prior ), basisSize( _basisSize ), sparseRate( _sparseRate ),
retainedCornersFraction( _retainedCornersFraction ), occlusionsThreshold( _occlusionsThreshold ),
dampingFactor( _dampingFactor ), claheClip( _claheClip ), useOpenCL( false )
{
CV_Assert( sparseRate > 0 && sparseRate <= 0.1 );
CV_Assert( retainedCornersFraction >= 0 && retainedCornersFraction <= 1.0 );
CV_Assert( occlusionsThreshold > 0 );
}
void OpticalFlowPCAFlow::collectGarbage() {}
Ptr<DenseOpticalFlow> createOptFlow_PCAFlow() { return makePtr<OpticalFlowPCAFlow>(); }
PCAPrior::PCAPrior( const char *pathToPrior )
{
FILE *f = fopen( pathToPrior, "rb" );
CV_Assert( f );
unsigned n = 0, m = 0;
CV_Assert( fread( &n, sizeof( n ), 1, f ) == 1 );
CV_Assert( fread( &m, sizeof( m ), 1, f ) == 1 );
L1.create( n, m, CV_32F );
L2.create( n, m, CV_32F );
c1.create( n, 1, CV_32F );
c2.create( n, 1, CV_32F );
CV_Assert( fread( L1.ptr<float>(), n * m * sizeof( float ), 1, f ) == 1 );
CV_Assert( fread( L2.ptr<float>(), n * m * sizeof( float ), 1, f ) == 1 );
CV_Assert( fread( c1.ptr<float>(), n * sizeof( float ), 1, f ) == 1 );
CV_Assert( fread( c2.ptr<float>(), n * sizeof( float ), 1, f ) == 1 );
fclose( f );
}
void PCAPrior::fillConstraints( float *A1, float *A2, float *b1, float *b2 ) const
{
memcpy( A1, L1.ptr<float>(), L1.size().area() * sizeof( float ) );
memcpy( A2, L2.ptr<float>(), L2.size().area() * sizeof( float ) );
memcpy( b1, c1.ptr<float>(), c1.size().area() * sizeof( float ) );
memcpy( b2, c2.ptr<float>(), c2.size().area() * sizeof( float ) );
}
}
}
modules/optflow/src/precomp.hpp
+56
View File
@@ -0,0 +1,56 @@
/*
By downloading, copying, installing or using the software you agree to this
license. If you do not agree to this license, do not download, install,
copy or use the software.
License Agreement
For Open Source Computer Vision Library
(3-clause BSD License)
Copyright (C) 2013, OpenCV Foundation, all rights reserved.
Third party copyrights are property of their respective owners.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the names of the copyright holders nor the names of the contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
This software is provided by the copyright holders and contributors "as is" and
any express or implied warranties, including, but not limited to, the implied
warranties of merchantability and fitness for a particular purpose are
disclaimed. In no event shall copyright holders or contributors be liable for
any direct, indirect, incidental, special, exemplary, or consequential damages
(including, but not limited to, procurement of substitute goods or services;
loss of use, data, or profits; or business interruption) however caused
and on any theory of liability, whether in contract, strict liability,
or tort (including negligence or otherwise) arising in any way out of
the use of this software, even if advised of the possibility of such damage.
*/
#ifndef __OPENCV_OPTFLOW_PRECOMP_HPP__
#define __OPENCV_OPTFLOW_PRECOMP_HPP__
#include <opencv2/optflow.hpp>
#include <opencv2/video.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/features.hpp>
#include "opencv2/core/utility.hpp"
#include "opencv2/core/private.hpp"
#include "opencv2/core/ocl.hpp"
#include <algorithm>
#include <cmath>
#endif
modules/optflow/src/rlof/berlof_invoker.hpp
+2454
View File
File diff suppressed because it is too large Load Diff
modules/optflow/src/rlof/geo_interpolation.cpp
+450
View File
@@ -0,0 +1,450 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// This functions have been contributed by Jonas Geisters <geistert@nue.tu-berlin.de>
#include "../precomp.hpp"
#include "geo_interpolation.hpp"
#include <string>
#include <map>
namespace cv {
namespace optflow {
struct Graph_helper {
std::vector<int> mem;
int e_size;
Graph_helper(int k, int num_nodes) {
e_size = (2 * k + 1);
mem.resize(e_size * num_nodes, 0);
}
inline int size(int id) {
int r_addr = id * (e_size);
return mem[r_addr];
}
inline int * data(int id) {
int r_addr = id * (e_size)+1;
return &mem[r_addr];
}
inline void add(int id, std::pair<float, int> data) {
int r_addr = id * (e_size);
int size = ++mem[r_addr];
r_addr += 2 * size - 1;//== 1 + 2*(size-1);
*(float*)&mem[r_addr] = data.first;
mem[r_addr + 1] = data.second;
}
inline bool color_in_target(int id, int color) {
int r_addr = id * (e_size);
int size = mem[r_addr];
r_addr += 2;
for (int i = 0; i < size; i++) {
if (mem[r_addr] == color) {
return true;
}
r_addr += 2;
}
return false;
}
};
Mat sgeo_dist(const Mat& gra, int y, int x, float max, Mat &prev)
{
std::vector <Point2f> points;
points.push_back(Point2f(static_cast<float>(x), static_cast<float>(y)));
return sgeo_dist(gra, points, max, prev);
}
Mat sgeo_dist(const Mat& gra, const std::vector<Point2f> & points, float max, Mat &prev)
{
int Dx[] = { -1,0,1,-1,1,-1,0,1 };
int Dy[] = { -1,-1,-1,0,0,1,1,1 };
Mat dm(gra.rows, gra.cols, CV_32F, Scalar(max));
prev = Mat(gra.rows, gra.cols, CV_8U, Scalar(255));
std::multimap<float, Vec2i > not_visited_with_value;
for (auto i = points.begin(); i != points.end(); i++)
{
int y = static_cast<int>(i->y);
int x = static_cast<int>(i->x);
not_visited_with_value.insert(std::pair<float, Vec2i >(0.f, Vec2i(y, x)));
dm.at<float>(y, x) = 0;
}
bool done = false;
while (!done)
{
if (not_visited_with_value.begin() == not_visited_with_value.end()) {
done = true;
break;
}
std::multimap<float, Vec2i >::iterator current_it = not_visited_with_value.begin();
std::pair<float, Vec2i > current_p = *current_it;
not_visited_with_value.erase(current_it);
int y = current_p.second[0];
int x = current_p.second[1];
float cur_d = current_p.first;
if (dm.at<float>(y, x) != cur_d) {
continue;
}
Vec8f gra_e = gra.at<Vec8f>(y, x);
for (int i = 0; i < 8; i++) {
if (gra_e[i] < 0) {
continue;
}
int dx = Dx[i];
int dy = Dy[i];
if (dm.at<float>(y + dy, x + dx) > cur_d + gra_e[i]) {
dm.at<float>(y + dy, x + dx) = cur_d + gra_e[i];
prev.at<uchar>(y + dy, x + dx) = static_cast<uchar>(7 - i);
not_visited_with_value.insert(std::pair<float, Vec2i >(cur_d + gra_e[i], Vec2i(y + dy, x + dx)));
}
}
}
return dm;
}
Mat interpolate_irregular_nn_raster(const std::vector<Point2f> & prevPoints,
const std::vector<Point2f> & nextPoints,
const std::vector<uchar> & status,
const Mat & i1)
{
Mat gra = getGraph(i1, 0.1f);
int Dx[] = { -1,0,1,-1,1,-1,0,1 };
int Dy[] = { -1,-1,-1,0,0,1,1,1 };
int max_rounds = 10;
Mat dirt = Mat(gra.rows, gra.cols, CV_8U, Scalar(0));
Mat quellknoten = Mat(gra.rows, gra.cols, CV_32S, Scalar(-1));
Mat dist = Mat(gra.rows, gra.cols, CV_32F, Scalar(std::numeric_limits<float>::max()));
/*
* assign quellknoten ids.
*/
for (int i = 0; i < static_cast<int>(prevPoints.size()); i++)
{
int x = (int)prevPoints[i].x;
int y = (int)prevPoints[i].y;
if (status[i] == 0)
continue;
dirt.at<uchar>(y, x) = 1;
dist.at<float>(y, x) = 0;
quellknoten.at<int>(y, x) = i;
}
bool clean = true;
bool done = false;
int x = 0;
int y = 0;
int rounds = 0;
while (!done) {
/*
* Update x and y
* on even rounds go rasterscanorder , on odd round inverse rasterscanorder
*/
if (rounds % 2 == 0) {
x++;
if (x >= gra.cols) {
x = 0;
y++;
if (y >= gra.rows) {
y = 0;
rounds++;
y = gra.rows - 1;
x = gra.cols - 1;
if (rounds >= max_rounds || clean) {
done = true;
break;
}
}
}
}
else {
x--;
if (x < 0) {
x = gra.cols - 1;
y--;
if (y < 0) {
y = gra.rows - 1;
rounds++;
y = 0;
x = 0;
if (rounds >= max_rounds || clean) {
done = true;
break;
}
}
}
}
if (dirt.at<uchar>(y, x) == 0) {
continue;
}
dirt.at<uchar>(y, x) = 0;
float c_dist = dist.at<float>(y, x);
Vec8f gra_e = gra.at<Vec8f>(y, x);
for (int i = 0; i < 8; i++) {
int tx = Dx[i];
int ty = Dy[i];
if (ty == 0 && tx == 0) {
continue;
}
if (x + tx < 0 || x + tx >= gra.cols) {
continue;
}
if (y + ty < 0 || y + ty >= gra.rows) {
continue;
}
if (c_dist > dist.at<float>(y + ty, x + tx)) {
if (c_dist > dist.at<float>(y + ty, x + tx) + gra_e[i]) {
quellknoten.at<int>(y, x) = quellknoten.at<int>(y + ty, x + tx);
dist.at<float>(y, x) = dist.at<float>(y + ty, x + tx) + gra_e[i];
dirt.at<uchar>(y, x) = 1;
clean = false;
}
}
else {
if (c_dist + gra_e[i] < dist.at<float>(y + ty, x + tx)) {
quellknoten.at<int>(y + ty, x + tx) = quellknoten.at<int>(y, x);
dist.at<float>(y + ty, x + tx) = dist.at<float>(y, x) + gra_e[i];
dirt.at<uchar>(y + ty, x + tx) = 1;
clean = false;
}
}
}
}
Mat nnFlow(i1.rows, i1.cols, CV_32FC2, Scalar(0));
for (y = 0; y < i1.rows; y++) {
for (x = 0; x < i1.cols; x++) {
int id = quellknoten.at<int>(y, x);
if (id != -1)
{
nnFlow.at<Point2f>(y, x) = nextPoints[id] - prevPoints[id];
}
}
}
return nnFlow;
}
Mat interpolate_irregular_knn(
const std::vector<Point2f> & _prevPoints,
const std::vector<Point2f> & _nextPoints,
const std::vector<uchar> & status,
const Mat &color_img,
int k,
float pixeldistance)
{
Mat in(color_img.rows, color_img.cols, CV_32FC2);
Mat mask = Mat::zeros(color_img.rows, color_img.cols, CV_8UC1);
for (unsigned n = 0; n < _prevPoints.size(); n++)
{
if (_prevPoints[n].x >= 0 && _prevPoints[n].y >= 0 && _prevPoints[n].x < color_img.cols && _prevPoints[n].y < color_img.rows)
{
in.at<Point2f>(_prevPoints[n]) = _nextPoints[n] - _prevPoints[n];
mask.at<uchar>(_prevPoints[n]) = status[n];
}
}
int Dx[] = { -1,0,1,-1,1,-1,0,1 };
int Dy[] = { -1,-1,-1,0,0,1,1,1 };
Mat gra = getGraph(color_img, pixeldistance);
Mat nnFlow(in.rows, in.cols, CV_32FC2, Scalar(0));
std::multimap<float, Vec2i > my_agents; // <arrivaltim , < target, color >>
Graph_helper graph_helper(k, in.rows*in.cols); //< arrivaltime, color>
int color = 0;
std::vector<Vec2i> flow_point_list;
for (int y = 0; y < in.rows; y++) {
for (int x = 0; x < in.cols; x++) {
if (mask.at<uchar>(y, x) > 0) {
flow_point_list.push_back(Vec2i(y, x));
nnFlow.at<Vec2f>(y, x) = in.at<Vec2f>(y, x);
int v_id = (y * in.cols + x);
graph_helper.add(v_id, std::pair<float, int>(0.f, color));
Vec8f gra_e = gra.at<Vec8f>(y, x);
for (int i = 0; i < 8; i++) {
if (gra_e[i] < 0)
continue;
int dx = Dx[i];
int dy = Dy[i];
int target = (y + dy) * in.cols + (x + dx);
Vec2i agent(target, color);
my_agents.insert(std::pair<float, Vec2i >(gra_e[i], agent));
}
color++;
}
}
}
bool done = false;
while (!done) {
if (my_agents.size() == 0) {
done = true;
break;
}
std::multimap<float, Vec2i >::iterator current_it = my_agents.begin();
std::pair<float, Vec2i > current_p = *current_it;
my_agents.erase(current_it);
int target = current_p.second[0];
color = current_p.second[1];
float arriv_time = current_p.first;
Vec8f gra_e = gra.at<Vec8f>(target);// (y*cols+x)
if (graph_helper.size(target) >= k) {
continue;
}
bool color_found_in_target = graph_helper.color_in_target(target, color);
if (color_found_in_target) {
continue;
}
graph_helper.add(target, std::pair<float, int>(arriv_time, color));
for (int i = 0; i < 8; i++) {
if (gra_e[i] < 0)
continue;
int dx = Dx[i];
int dy = Dy[i];
int new_target = target + dx + (dy*in.cols);
if (graph_helper.size(new_target) >= k) {
continue;
}
color_found_in_target = graph_helper.color_in_target(new_target, color);
if (color_found_in_target) {
continue;
}
Vec2i new_agent(new_target, color);
my_agents.insert(std::pair<float, Vec2i >(arriv_time + gra_e[i], new_agent));
}
}
Mat ret(in.rows, in.cols*k, CV_32FC2);
for (int y = 0; y < in.rows; y++) {
for (int x = 0; x < in.cols; x++) {
for (int i = 0; i < k; i++) {
float dist = *((float*)(graph_helper.data(y*in.cols + x) + 2 * i));
float id = *((float*)(graph_helper.data(y*in.cols + x) + 2 * i + 1));
ret.at<Vec2f>(y, k*x + i) = Vec2f(dist, id);
}
}
}
return ret;
}
Mat getGraph(const Mat &image, float edge_length)
{
int Dx[] = { -1,0,1,-1,1,-1,0,1 };
int Dy[] = { -1,-1,-1,0,0,1,1,1 };
Mat gra(image.rows, image.cols, CV_32FC(8));
for (int y = 0; y < gra.rows; y++) {
for (int x = 0; x < gra.cols; x++) {
for (int i = 0; i < 8; i++) {
int dx = Dx[i];
int dy = Dy[i];
gra.at<Vec8f>(y, x)[i] = -1;
if (x + dx < 0 || y + dy < 0 || x + dx >= gra.cols || y + dy >= gra.rows) {
continue;
}
if (i < 4) {
int si = 7 - i;
gra.at<Vec8f>(y, x)[i] = gra.at<Vec8f>(y + dy, x + dx)[si];
}
else {
float p1 = dx * dx*edge_length*edge_length + dy * dy*edge_length*edge_length;
float p2 = static_cast<float>(image.at<Vec3b>(y, x)[0] - image.at<Vec3b>(y + dy, x + dx)[0]);
float p3 = static_cast<float>(image.at<Vec3b>(y, x)[1] - image.at<Vec3b>(y + dy, x + dx)[1]);
float p4 = static_cast<float>(image.at<Vec3b>(y, x)[2] - image.at<Vec3b>(y + dy, x + dx)[2]);
gra.at<Vec8f>(y, x)[i] = sqrt(p1 + p2 * p2 + p3 * p3 + p4 * p4);
}
}
}
}
return gra;
}
Mat interpolate_irregular_nn(
const std::vector<Point2f> & _prevPoints,
const std::vector<Point2f> & _nextPoints,
const std::vector<uchar> & status,
const Mat &color_img,
float pixeldistance)
{
int Dx[] = { -1,0,1,-1,1,-1,0,1 };
int Dy[] = { -1,-1,-1,0,0,1,1,1 };
std::vector<Point2f> prevPoints, nextPoints;
std::map<std::pair<float, float>, std::pair<float, float>> flowMap;
for (unsigned n = 0; n < _prevPoints.size(); n++)
{
if (status[n] != 0)
{
flowMap.insert(std::make_pair(
std::make_pair(_prevPoints[n].x, _prevPoints[n].y),
std::make_pair(_nextPoints[n].x, _nextPoints[n].y)));
prevPoints.push_back(_prevPoints[n]);
nextPoints.push_back(_nextPoints[n]);
}
}
Mat gra = getGraph(color_img, pixeldistance);
Mat prev;
Mat geo_dist = sgeo_dist(gra, prevPoints, std::numeric_limits<float>::max(), prev);
Mat nnFlow = Mat::zeros(color_img.size(), CV_32FC2);
for (int y = 0; y < nnFlow.rows; y++)
{
for (int x = 0; x < nnFlow.cols; x++)
{
int cx = x;
int cy = y;
while (prev.at<uchar>(cy, cx) != 255)
{
int i = prev.at<uchar>(cy, cx);
cx += Dx[i];
cy += Dy[i];
}
auto val = flowMap[std::make_pair(static_cast<float>(cx), static_cast<float>(cy))];
nnFlow.at<Vec2f>(y, x) = Vec2f(val.first - cx, val.second - cy);
}
}
return nnFlow;
}
}} // namespace
modules/optflow/src/rlof/geo_interpolation.hpp
+35
View File
@@ -0,0 +1,35 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef _GEO_INTERPOLATION_HPP_
#define _GEO_INTERPOLATION_HPP_
namespace cv {
namespace optflow {
typedef Vec<float, 8> Vec8f;
Mat getGraph(const Mat & image, float edge_length);
Mat sgeo_dist(const Mat& gra, int y, int x, float max, Mat &prev);
Mat sgeo_dist(const Mat& gra, const std::vector<Point2f> & points, float max, Mat &prev);
Mat interpolate_irregular_nw(const Mat &in, const Mat &mask, const Mat &color_img, float max_d, float bandwidth, float pixeldistance);
Mat interpolate_irregular_nn(
const std::vector<Point2f> & prevPoints,
const std::vector<Point2f> & nextPoints,
const std::vector<uchar> & status,
const Mat &color_img,
float pixeldistance);
Mat interpolate_irregular_knn(
const std::vector<Point2f> & _prevPoints,
const std::vector<Point2f> & _nextPoints,
const std::vector<uchar> & status,
const Mat &color_img,
int k,
float pixeldistance);
Mat interpolate_irregular_nn_raster(const std::vector<Point2f> & prevPoints,
const std::vector<Point2f> & nextPoints,
const std::vector<uchar> & status,
const Mat & i1);
}} // namespace
#endif
modules/optflow/src/rlof/plk_invoker.hpp
+794
View File
@@ -0,0 +1,794 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef _PLK_INVOKER_HPP_
#define _PLK_INVOKER_HPP_
#include "rlof_invokerbase.hpp"
namespace cv {
namespace optflow {
namespace plk {
// implementierung ohne SSE
namespace radial {
class TrackerInvoker : public cv::ParallelLoopBody
{
public:
TrackerInvoker(
const Mat& _prevImg,
const Mat& _prevDeriv,
const Mat& _nextImg,
const Mat& _rgbPrevImg,
const Mat& _rgbNextImg,
const Point2f* _prevPts,
Point2f* _nextPts,
uchar* _status,
float* _err,
Point2f* _gainVecs,
int _level,
int _maxLevel,
int _winSize[2],
int _maxIteration,
bool _useInitialFlow,
int _supportRegionType,
float _minEigenValue,
int _crossSegmentationThreshold)
{
prevImg = &_prevImg;
prevDeriv = &_prevDeriv;
nextImg = &_nextImg;
rgbPrevImg = &_rgbPrevImg;
rgbNextImg = &_rgbNextImg;
prevPts = _prevPts;
nextPts = _nextPts;
status = _status;
err = _err;
gainVecs = _gainVecs;
minWinSize = _winSize[0];
maxWinSize = _winSize[1];
criteria.maxCount = _maxIteration;
criteria.epsilon = 0.01;
level = _level;
maxLevel = _maxLevel;
windowType = _supportRegionType;
minEigThreshold = _minEigenValue;
useInitialFlow = _useInitialFlow;
crossSegmentationThreshold = _crossSegmentationThreshold;
}
void operator()(const cv::Range& range) const CV_OVERRIDE
{
cv::Size winSize;
cv::Point2f halfWin;
const Mat& I = *prevImg;
const Mat& J = *nextImg;
const Mat& derivI = *prevDeriv;
const Mat& BI = *rgbPrevImg;
winSize = cv::Size(maxWinSize,maxWinSize);
int winMaskwidth = roundUp(winSize.width, 16);
cv::Mat winMaskMatBuf(winMaskwidth, winMaskwidth, tCVMaskType);
winMaskMatBuf.setTo(1);
const float FLT_SCALE = (1.f/(1 << 16));
int cn = I.channels(), cn2 = cn*2;
int winbufwidth = roundUp(winSize.width, 16);
cv::Size winBufSize(winbufwidth,winbufwidth);
cv::Matx44f invTensorMat;
Vec4f mismatchMat;
Vec4f resultMat;
cv::AutoBuffer<deriv_type> _buf(winBufSize.area()*(cn + cn2));
int derivDepth = DataType<deriv_type>::depth;
Mat IWinBuf(winBufSize, CV_MAKETYPE(derivDepth, cn), (deriv_type*)_buf.data());
Mat derivIWinBuf(winBufSize, CV_MAKETYPE(derivDepth, cn2), (deriv_type*)_buf.data() + winBufSize.area()*cn);
for( int ptidx = range.start; ptidx < range.end; ptidx++ )
{
Point2f prevPt = prevPts[ptidx]*(float)(1./(1 << level));
Point2f nextPt;
if( level == maxLevel )
{
if( useInitialFlow )
{
nextPt = nextPts[ptidx]*(float)(1./(1 << level));
}
else
nextPt = prevPt;
}
else
nextPt = nextPts[ptidx]*2.f;
nextPts[ptidx] = nextPt;
Point2i iprevPt, inextPt;
iprevPt.x = cvFloor(prevPt.x);
iprevPt.y = cvFloor(prevPt.y);
int winArea = maxWinSize * maxWinSize;
cv::Mat winMaskMat(winMaskMatBuf, cv::Rect(0,0, maxWinSize,maxWinSize));
winMaskMatBuf.setTo(0);
if( calcWinMaskMat(BI, windowType, iprevPt,
winMaskMat,winSize,halfWin,winArea,
minWinSize,maxWinSize) == false)
continue;
halfWin = Point2f(static_cast<float>(maxWinSize) ,static_cast<float>(maxWinSize) ) - halfWin;
prevPt += halfWin;
iprevPt.x = cvFloor(prevPt.x);
iprevPt.y = cvFloor(prevPt.y);
if( iprevPt.x < 0 || iprevPt.x >= derivI.cols - winSize.width ||
iprevPt.y < 0 || iprevPt.y >= derivI.rows - winSize.height - 1)
{
if( level == 0 )
{
if( status )
status[ptidx] = 3;
if( err )
err[ptidx] = 0;
}
continue;
}
float a = prevPt.x - iprevPt.x;
float b = prevPt.y - iprevPt.y;
const int W_BITS = 14;
int iw00 = cvRound((1.f - a)*(1.f - b)*(1 << W_BITS));
int iw01 = cvRound(a*(1.f - b)*(1 << W_BITS));
int iw10 = cvRound((1.f - a)*b*(1 << W_BITS));
int iw11 = (1 << W_BITS) - iw00 - iw01 - iw10;
float A11 = 0, A12 = 0, A22 = 0;
// tensor
float sumIx = 0;
float sumIy = 0;
float sumI = 0;
float sumW = 0;
float w1 = 0, w2 = 0; // -IyI
float dI = 0; // I^2
float D = 0;
copyWinBuffers(iw00, iw01, iw10, iw11, winSize, I, derivI, winMaskMat, IWinBuf, derivIWinBuf, iprevPt);
cv::Point2f backUpNextPt = nextPt;
nextPt += halfWin;
Point2f prevDelta(0,0); //relates to h(t-1)
Point2f prevGain(1,0);
cv::Point2f gainVec = gainVecs[ptidx];
cv::Point2f backUpGain = gainVec;
int j;
for( j = 0; j < criteria.maxCount; j++ )
{
status[ptidx] = static_cast<uchar>(j);
inextPt.x = cvFloor(nextPt.x);
inextPt.y = cvFloor(nextPt.y);
if( inextPt.x < 0 || inextPt.x >= J.cols - winSize.width ||
inextPt.y < 0 || inextPt.y >= J.rows - winSize.height - 1)
{
if( level == 0 && status )
status[ptidx] = 3;
break;
}
a = nextPt.x - inextPt.x;
b = nextPt.y - inextPt.y;
iw00 = cvRound((1.f - a)*(1.f - b)*(1 << W_BITS));
iw01 = cvRound(a*(1.f - b)*(1 << W_BITS));
iw10 = cvRound((1.f - a)*b*(1 << W_BITS));
iw11 = (1 << W_BITS) - iw00 - iw01 - iw10;
float b1,b2,b3,b4;
b1 = 0;
b2 = 0;
b3 = 0;
b4 = 0;
if( j == 0 )
{
// tensor
w1 = 0; // -IxI
w2 = 0; // -IyI
dI = 0; // I^2
sumIx = 0;
sumIy = 0;
sumI = 0;
sumW = 0;
A11 = 0;
A12 = 0;
A22 = 0;
}
#if CV_SIMD128
v_int16x8 vqw0 = v_int16x8((short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01));
v_int16x8 vqw1 = v_int16x8((short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11));
v_float32x4 vqb0 = v_setzero_f32(), vqb1 = v_setzero_f32(), vqb2 = v_setzero_f32(), vqb3 = v_setzero_f32();
v_float32x4 vsumW1 = v_setzero_f32(), vsumW2 = v_setzero_f32();
v_float32x4 vsumIy = v_setzero_f32(), vsumIx = v_setzero_f32(), vsumI = v_setzero_f32(), vsumDI = v_setzero_f32();
v_float32x4 vAxx = v_setzero_f32(), vAxy = v_setzero_f32(), vAyy = v_setzero_f32();
v_int32x4 vdelta = v_setall_s32(1 << (W_BITS - 5 - 1));
v_int16x8 vmax_val_16 = v_setall_s16(std::numeric_limits<unsigned short>::max());
float gainVal = gainVec.x > 0 ? gainVec.x : -gainVec.x;
int bitShift = gainVec.x == 0 ? 1 : cvCeil(log(200.f / gainVal) / log(2.f));
v_int16x8 vgain_value = v_setall_s16(static_cast<short>(gainVec.x * (float)(1 << bitShift)));
v_int16x8 vconst_value = v_setall_s16(static_cast<short>(gainVec.y));
#endif
for(int y = 0; y < winSize.height; y++ )
{
const uchar* Jptr = J.ptr<uchar>(y + inextPt.y, inextPt.x*cn);
const uchar* Jptr1 = J.ptr<uchar>(y + inextPt.y + 1, inextPt.x*cn);
const short* Iptr = IWinBuf.ptr<short>(y, 0);
const short* dIptr = derivIWinBuf.ptr<short>(y, 0);
const tMaskType* maskPtr = winMaskMat.ptr<tMaskType>(y, 0);
#if CV_SIMD128
for(int x = 0 ; x <= winSize.width*cn; x += 8, dIptr += 8*2 )
{
v_int16x8 vI = v_reinterpret_as_s16(v_load(Iptr + x)), diff0, diff1, diff2;
v_int16x8 v00 = v_reinterpret_as_s16(v_load_expand(Jptr + x));
v_int16x8 v01 = v_reinterpret_as_s16(v_load_expand(Jptr + x + cn));
v_int16x8 v10 = v_reinterpret_as_s16(v_load_expand(Jptr1 + x));
v_int16x8 v11 = v_reinterpret_as_s16(v_load_expand(Jptr1 + x + cn));
v_int16x8 vmask = v_mul(v_reinterpret_as_s16(v_load_expand(maskPtr + x)), vmax_val_16);
v_int32x4 t0, t1;
v_int16x8 t00, t01, t10, t11;
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
//subpixel interpolation
t0 = v_add(v_dotprod(t00, vqw0, vdelta), v_dotprod(t10, vqw1));
t1 = v_add(v_dotprod(t01, vqw0, vdelta), v_dotprod(t11, vqw1));
t0 = v_shr(t0, W_BITS - 5);
t1 = v_shr(t1, W_BITS - 5);
// diff = J - I
diff0 = v_sub(v_pack(t0, t1), vI);
// I*gain.x + gain.x
v_mul_expand(vI, vgain_value, t0, t1);
diff0 = v_add(v_add(diff0, v_pack(v_shr(t0, bitShift), v_shr(t1, bitShift))), vconst_value);
diff0 = v_and(diff0, vmask);
v_zip(diff0, diff0, diff2, diff1);
v_int32x4 diff0_0;
v_int32x4 diff0_1;
v_expand(diff0, diff0_0, diff0_1);
v_int16x8 vIxy_0 = v_reinterpret_as_s16(v_load(dIptr)); // Ix0 Iy0 Ix1 Iy1 ...
v_int16x8 vIxy_1 = v_reinterpret_as_s16(v_load(dIptr + 8));
v_zip(vIxy_0, vIxy_1, v10, v11);
v_zip(diff2, diff1, v00, v01);
vqb0 = v_add(vqb0, v_cvt_f32(v_dotprod(v00, v10)));
vqb1 = v_add(vqb1, v_cvt_f32(v_dotprod(v01, v11)));
v_int32x4 vI0, vI1;
v_expand(vI, vI0, vI1);
vqb2 = v_add(vqb2, v_cvt_f32(v_mul(diff0_0, vI0)));
vqb2 = v_add(vqb2, v_cvt_f32(v_mul(diff0_1, vI1)));
vqb3 = v_add(vqb3, v_cvt_f32(diff0_0));
vqb3 = v_add(vqb3, v_cvt_f32(diff0_1));
if (j == 0)
{
v00 = v_reinterpret_as_s16(v_interleave_pairs(v_reinterpret_as_s32(v_interleave_pairs(vIxy_0))));
v_expand(v00, t1, t0);
v_float32x4 vI_ps = v_cvt_f32(vI0);
v_float32x4 fy = v_cvt_f32(t0);
v_float32x4 fx = v_cvt_f32(t1);
vAyy = v_muladd(fy, fy, vAyy);
vAxy = v_muladd(fx, fy, vAxy);
vAxx = v_muladd(fx, fx, vAxx);
// sumIx und sumIy
vsumIx = v_add(vsumIx, fx);
vsumIy = v_add(vsumIy, fy);
vsumW1 = v_add(vsumW1, v_mul(vI_ps, fx));
vsumW2 = v_add(vsumW2, v_mul(vI_ps, fy));
// sumI
vsumI = v_add(vsumI, vI_ps);
// sumDI
vsumDI = v_add(vsumDI, v_mul(vI_ps, vI_ps));
v01 = v_reinterpret_as_s16(v_interleave_pairs(v_reinterpret_as_s32(v_interleave_pairs(vIxy_1))));
v_expand(v01, t1, t0);
vI_ps = v_cvt_f32(vI1);
fy = v_cvt_f32(t0);
fx = v_cvt_f32(t1);
vAyy = v_muladd(fy, fy, vAyy);
vAxy = v_muladd(fx, fy, vAxy);
vAxx = v_muladd(fx, fx, vAxx);
// sumIx und sumIy
vsumIx = v_add(vsumIx, fx);
vsumIy = v_add(vsumIy, fy);
vsumW1 = v_add(vsumW1, v_mul(vI_ps, fx));
vsumW2 = v_add(vsumW2, v_mul(vI_ps, fy));
// sumI
vsumI = v_add(vsumI, vI_ps);
// sumDI
vsumDI = v_add(vsumDI, v_mul(vI_ps, vI_ps));
}
}
#else
for(int x = 0 ; x < winSize.width*cn; x++, dIptr += 2 )
{
if( maskPtr[x] == 0)
continue;
int J_val = CV_DESCALE(Jptr[x]*iw00 + Jptr[x+cn]*iw01 +
Jptr1[x]*iw10 + Jptr1[x+cn]*iw11,
W_BITS-5);
int diff = static_cast<int>(J_val - Iptr[x] + Iptr[x] * gainVec.x + gainVec.y);
b1 += (float)(diff*dIptr[0]) * FLT_RESCALE;
b2 += (float)(diff*dIptr[1]) * FLT_RESCALE;
b3 += (float)(diff) * Iptr[x] * FLT_RESCALE;
b4 += (float)(diff);
// compute the Gradient Matrice
if( j == 0 )
{
A11 += (float)(dIptr[0]*dIptr[0]);
A12 += (float)(dIptr[0]*dIptr[1]);
A22 += (float)(dIptr[1]*dIptr[1]);
dI += Iptr[x] * Iptr[x] * FLT_RESCALE;
float dx = static_cast<float>(dIptr[0]) * FLT_RESCALE;
float dy = static_cast<float>(dIptr[1]) * FLT_RESCALE;
sumIx += dx;
sumIy += dy;
w1 += dx * Iptr[x];
w2 += dy * Iptr[x];
sumI += Iptr[x] * FLT_RESCALE;
//sumW += FLT_RESCALE;
}
}
#endif
}
if( j == 0 )
{
#if CV_SIMD128
w1 = v_reduce_sum(vsumW1);
w2 = v_reduce_sum(vsumW2);
dI = v_reduce_sum(vsumDI);
sumI = v_reduce_sum(vsumI);
sumIx = v_reduce_sum(vsumIx);
sumIy = v_reduce_sum(vsumIy);
A11 = v_reduce_sum(vAxx);
A12 = v_reduce_sum(vAxy);
A22 = v_reduce_sum(vAyy);
#endif
sumIx *= -FLT_SCALE;
sumIy *= -FLT_SCALE;
sumI *=FLT_SCALE;
sumW = winArea * FLT_SCALE;
w1 *= -FLT_SCALE;
w2 *= -FLT_SCALE;
dI *= FLT_SCALE;
A11 *= FLT_SCALE;
A12 *= FLT_SCALE;
A22 *= FLT_SCALE;
}
#if CV_SIMD128
float CV_DECL_ALIGNED(16) bbuf[4];
v_store_aligned(bbuf, v_add(vqb0, vqb1));
b1 = bbuf[0] + bbuf[2];
b2 = bbuf[1] + bbuf[3];
b3 = v_reduce_sum(vqb2);
b4 = v_reduce_sum(vqb3);
#endif
mismatchMat(0) = b1 * FLT_SCALE;
mismatchMat(1) = b2 * FLT_SCALE;
mismatchMat(2) = -b3 * FLT_SCALE;
mismatchMat(3) = -b4 * FLT_SCALE;
D = - A12*A12*sumI*sumI + dI*sumW*A12*A12 + 2*A12*sumI*sumIx*w2 + 2*A12*sumI*sumIy*w1
- 2*dI*A12*sumIx*sumIy - 2*sumW*A12*w1*w2 + A11*A22*sumI*sumI - 2*A22*sumI*sumIx*w1
- 2*A11*sumI*sumIy*w2 - sumIx*sumIx*w2*w2 + A22*dI*sumIx*sumIx + 2*sumIx*sumIy*w1*w2
- sumIy*sumIy*w1*w1 + A11*dI*sumIy*sumIy + A22*sumW*w1*w1 + A11*sumW*w2*w2 - A11*A22*dI*sumW;
float minEig = (A22 + A11 - std::sqrt((A11-A22)*(A11-A22) +
4.f*A12*A12))/(2*winArea);
if( minEig < minEigThreshold )
{
if( (level == 0 && status) || std::abs(D) < FLT_EPSILON)
status[ptidx] = 0;
if( level > 0)
{
nextPts[ptidx] = backUpNextPt;
gainVecs[ptidx] = backUpGain;
}
break;
}
D = (1.f / D);
invTensorMat(0,0) = (A22*sumI*sumI - 2*sumI*sumIy*w2 + dI*sumIy*sumIy + sumW*w2*w2 - A22*dI*sumW)* D;
invTensorMat(0,1) = (A12*dI*sumW - A12*sumI * sumI - dI*sumIx*sumIy + sumI*sumIx*w2 + sumI*sumIy*w1 - sumW*w1*w2)* D;
invTensorMat(0,2) = (A12*sumI*sumIy - sumIy*sumIy*w1 - A22*sumI*sumIx - A12*sumW*w2 + A22*sumW*w1 + sumIx*sumIy*w2)* D;
invTensorMat(0,3) = (A22*dI*sumIx - A12*dI*sumIy - sumIx*w2*w2 + A12*sumI*w2 - A22*sumI*w1 + sumIy*w1*w2) * D;
invTensorMat(1,0) = invTensorMat(0,1);
invTensorMat(1,1) = (A11*sumI * sumI - 2*sumI*sumIx*w1 + dI*sumIx * sumIx + sumW*w1*w1 - A11*dI*sumW) * D;
invTensorMat(1,2) = (A12*sumI*sumIx - A11*sumI*sumIy - sumIx * sumIx*w2 + A11*sumW*w2 - A12*sumW*w1 + sumIx*sumIy*w1) * D;
invTensorMat(1,3) = (A11*dI*sumIy - sumIy*w1*w1 - A12*dI*sumIx - A11*sumI*w2 + A12*sumI*w1 + sumIx*w1*w2)* D;
invTensorMat(2,0) = invTensorMat(0,2);
invTensorMat(2,1) = invTensorMat(1,2);
invTensorMat(2,2) = (sumW*A12*A12 - 2*A12*sumIx*sumIy + A22*sumIx*sumIx + A11*sumIy*sumIy - A11*A22*sumW)* D;
invTensorMat(2,3) = (A11*A22*sumI - A12*A12*sumI - A11*sumIy*w2 + A12*sumIx*w2 + A12*sumIy*w1 - A22*sumIx*w1)* D;
invTensorMat(3,0) = invTensorMat(0,3);
invTensorMat(3,1) = invTensorMat(1,3);
invTensorMat(3,2) = invTensorMat(2,3);
invTensorMat(3,3) = (dI*A12*A12 - 2*A12*w1*w2 + A22*w1*w1 + A11*w2*w2 - A11*A22*dI)* D;
resultMat = invTensorMat * mismatchMat;
// 0.08 -12.10
Point2f delta(-resultMat(0), -resultMat(1));
Point2f deltaGain(-resultMat(2), -resultMat(3));
if( j == 0)
prevGain = deltaGain;
nextPt += delta;
nextPts[ptidx] = nextPt - halfWin;
gainVecs[ptidx]= gainVec;
if( delta.ddot(delta) <= criteria.epsilon)
break;
if ((
std::abs(delta.x - prevDelta.x) < 0.01 &&
std::abs(delta.y - prevDelta.y) < 0.01
) || (
delta.ddot(delta) <= 0.001 &&
std::abs(prevGain.x - deltaGain.x) < 0.01
))
{
nextPts[ptidx] -= delta*0.5f;
gainVecs[ptidx] -= deltaGain* 0.5f;
break;
}
if(j > 0 && std::abs(delta.x - prevDelta.x) < 0.01 &&
std::abs(delta.y - prevDelta.y) < 0.01)
{
nextPts[ptidx] -= delta*0.5f;
break;
}
prevDelta = delta;
prevGain = deltaGain;
}
}
}
const Mat* prevImg;
const Mat* nextImg;
const Mat* prevDeriv;
const Mat* rgbPrevImg;
const Mat* rgbNextImg;
const Point2f* prevPts;
Point2f* nextPts;
uchar* status;
cv::Point2f* gainVecs; // gain vector x -> multiplier y -> offset
float* err;
int maxWinSize;
int minWinSize;
TermCriteria criteria;
int level;
int maxLevel;
int windowType;
float minEigThreshold;
bool useInitialFlow;
int crossSegmentationThreshold;
};
} // namespace
namespace ica {
class TrackerInvoker : public cv::ParallelLoopBody
{
public:
TrackerInvoker(
const Mat& _prevImg,
const Mat& _prevDeriv,
const Mat& _nextImg,
const Mat& _rgbPrevImg,
const Mat& _rgbNextImg,
const Point2f* _prevPts,
Point2f* _nextPts,
uchar* _status,
float* _err,
int _level,
int _maxLevel,
int _winSize[2],
int _maxIteration,
bool _useInitialFlow,
int _supportRegionType,
int _crossSegmentationThreshold,
float _minEigenValue)
{
prevImg = &_prevImg;
prevDeriv = &_prevDeriv;
nextImg = &_nextImg;
rgbPrevImg = &_rgbPrevImg;
rgbNextImg = &_rgbNextImg;
prevPts = _prevPts;
nextPts = _nextPts;
status = _status;
err = _err;
minWinSize = _winSize[0];
maxWinSize = _winSize[1];
criteria.maxCount = _maxIteration;
criteria.epsilon = 0.01;
level = _level;
maxLevel = _maxLevel;
windowType = _supportRegionType;
minEigThreshold = _minEigenValue;
useInitialFlow = _useInitialFlow;
crossSegmentationThreshold = _crossSegmentationThreshold;
}
void operator()(const cv::Range& range) const CV_OVERRIDE
{
cv::Size winSize;
cv::Point2f halfWin;
const Mat& I = *prevImg;
const Mat& J = *nextImg;
const Mat& derivI = *prevDeriv;
const Mat& BI = *rgbPrevImg;
winSize = cv::Size(maxWinSize,maxWinSize);
int winMaskwidth = roundUp(winSize.width, 8) * 2;
cv::Mat winMaskMatBuf(winMaskwidth, winMaskwidth, tCVMaskType);
winMaskMatBuf.setTo(1);
const float FLT_SCALE = (1.f/(1 << 20)); // 20
int j, cn = I.channels(), cn2 = cn*2;
int winbufwidth = roundUp(winSize.width, 8);
cv::Size winBufSize(winbufwidth,winbufwidth);
std::vector<short> _buf(winBufSize.area()*(cn + cn2));
Mat IWinBuf(winBufSize, CV_MAKETYPE(CV_16S, cn), &_buf[0]);
Mat derivIWinBuf(winBufSize, CV_MAKETYPE(CV_16S, cn2), &_buf[winBufSize.area()*cn]);
for( int ptidx = range.start; ptidx < range.end; ptidx++ )
{
Point2f prevPt = prevPts[ptidx]*(float)(1./(1 << level));
Point2f nextPt;
if( level == maxLevel )
{
if( useInitialFlow )
nextPt = nextPts[ptidx]*(float)(1./(1 << level));
else
nextPt = prevPt;
}
else
nextPt = nextPts[ptidx]*2.f;
nextPts[ptidx] = nextPt;
Point2i iprevPt, inextPt;
iprevPt.x = cvFloor(prevPt.x);
iprevPt.y = cvFloor(prevPt.y);
int winArea = maxWinSize * maxWinSize;
cv::Mat winMaskMat(winMaskMatBuf, cv::Rect(0,0, maxWinSize,maxWinSize));
if( calcWinMaskMat(BI, windowType, iprevPt,
winMaskMat,winSize,halfWin,winArea,
minWinSize,maxWinSize) == false)
continue;
halfWin = Point2f(static_cast<float>(maxWinSize), static_cast<float>(maxWinSize)) - halfWin;
prevPt += halfWin;
iprevPt.x = cvFloor(prevPt.x);
iprevPt.y = cvFloor(prevPt.y);
if( iprevPt.x < 0 || iprevPt.x >= derivI.cols - winSize.width ||
iprevPt.y < 0 || iprevPt.y >= derivI.rows - winSize.height - 1)
{
if( level == 0 )
{
if( status )
status[ptidx] = 3;
if( err )
err[ptidx] = 0;
}
continue;
}
float a = prevPt.x - iprevPt.x;
float b = prevPt.y - iprevPt.y;
const int W_BITS = 14;
int iw00 = cvRound((1.f - a)*(1.f - b)*(1 << W_BITS));
int iw01 = cvRound(a*(1.f - b)*(1 << W_BITS));
int iw10 = cvRound((1.f - a)*b*(1 << W_BITS));
int iw11 = (1 << W_BITS) - iw00 - iw01 - iw10;
float A11 = 0, A12 = 0, A22 = 0;
// extract the patch from the first image, compute covariation matrix of derivatives
int x, y;
copyWinBuffers(iw00, iw01, iw10, iw11, winSize, I, derivI, winMaskMat, IWinBuf, derivIWinBuf, A11, A22, A12, iprevPt);
float D = A11*A22 - A12*A12;
float minEig = (A22 + A11 - std::sqrt((A11-A22)*(A11-A22) +
4.f*A12*A12))/(2 * winArea);
if( err )
err[ptidx] = (float)minEig;
if( minEig < minEigThreshold || D < FLT_EPSILON )
{
if( level == 0 && status )
status[ptidx] = 0;
continue;
}
D = 1.f/D;
nextPt += halfWin;
Point2f prevDelta(0,0); //relates to h(t-1)
for( j = 0; j < criteria.maxCount; j++ )
{
status[ptidx] = static_cast<uchar>(j);
inextPt.x = cvFloor(nextPt.x);
inextPt.y = cvFloor(nextPt.y);
if( inextPt.x < 0 || inextPt.x >= J.cols - winSize.width ||
inextPt.y < 0 || inextPt.y >= J.rows - winSize.height - 1)
{
if( level == 0 && status )
status[ptidx] = 3;
break;
}
a = nextPt.x - inextPt.x;
b = nextPt.y - inextPt.y;
iw00 = cvRound((1.f - a)*(1.f - b)*(1 << W_BITS));
iw01 = cvRound(a*(1.f - b)*(1 << W_BITS));
iw10 = cvRound((1.f - a)*b*(1 << W_BITS));
iw11 = (1 << W_BITS) - iw00 - iw01 - iw10;
float b1 = 0, b2 = 0;
#if CV_SIMD128
v_int16x8 vqw0 = v_int16x8((short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01));
v_int16x8 vqw1 = v_int16x8((short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11));
v_float32x4 vqb0 = v_setzero_f32(), vqb1 = v_setzero_f32();
v_int16x8 vmax_val_16 = v_setall_s16(std::numeric_limits<unsigned short>::max());
v_int32x4 vdelta = v_setall_s32(1 << (W_BITS - 5 - 1));
#endif
for( y = 0; y < winSize.height; y++ )
{
const uchar* Jptr = J.ptr<uchar>(y + inextPt.y, inextPt.x*cn);
const uchar* Jptr1 = J.ptr<uchar>(y + inextPt.y + 1, inextPt.x*cn);
const short* Iptr = IWinBuf.ptr<short>(y, 0);
const short* dIptr = derivIWinBuf.ptr<short>(y, 0);
x = 0;
#if CV_SIMD128
const tMaskType* maskPtr = winMaskMat.ptr<tMaskType>(y, 0);
for( ; x <= winSize.width*cn; x += 8, dIptr += 8*2 )
{
v_int16x8 diff0 = v_reinterpret_as_s16(v_load(Iptr + x)), diff1, diff2;
v_int16x8 v00 = v_reinterpret_as_s16(v_load_expand(Jptr + x));
v_int16x8 v01 = v_reinterpret_as_s16(v_load_expand(Jptr + x + cn));
v_int16x8 v10 = v_reinterpret_as_s16(v_load_expand(Jptr1 + x));
v_int16x8 v11 = v_reinterpret_as_s16(v_load_expand(Jptr1 + x + cn));
v_int16x8 vmask = v_mul(v_reinterpret_as_s16(v_load_expand(maskPtr + x)), vmax_val_16);
v_int32x4 t0, t1;
v_int16x8 t00, t01, t10, t11;
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_add(v_dotprod(t00, vqw0, vdelta), v_dotprod(t10, vqw1));
t1 = v_add(v_dotprod(t01, vqw0, vdelta), v_dotprod(t11, vqw1));
t0 = v_shr(t0, W_BITS - 5);
t1 = v_shr(t1, W_BITS - 5);
diff0 = v_sub(v_pack(t0, t1), diff0);
diff0 = v_and(diff0, vmask);
v_zip(diff0, diff0, diff2, diff1); // It0 It0 It1 It1 ...
v_int16x8 vIxy_0 = v_reinterpret_as_s16(v_load(dIptr)); // Ix0 Iy0 Ix1 Iy1 ...
v_int16x8 vIxy_1 = v_reinterpret_as_s16(v_load(dIptr + 8));
v_zip(vIxy_0, vIxy_1, v10, v11);
v_zip(diff2, diff1, v00, v01);
vqb0 = v_add(vqb0, v_cvt_f32(v_dotprod(v00, v10)));
vqb1 = v_add(vqb1, v_cvt_f32(v_dotprod(v01, v11)));
}
#else
for( ; x < winSize.width*cn; x++, dIptr += 2 )
{
if( dIptr[0] == 0 && dIptr[1] == 0)
continue;
int diff = CV_DESCALE(Jptr[x]*iw00 + Jptr[x+cn]*iw01 +
Jptr1[x]*iw10 + Jptr1[x+cn]*iw11,
W_BITS-5) - Iptr[x];
b1 += (float)(diff*dIptr[0]) * FLT_RESCALE;
b2 += (float)(diff*dIptr[1]) * FLT_RESCALE;
}
#endif
}
#if CV_SIMD128
float CV_DECL_ALIGNED(16) bbuf[4];
v_store_aligned(bbuf, v_add(vqb0, vqb1));
b1 = bbuf[0] + bbuf[2];
b2 = bbuf[1] + bbuf[3];
#endif
b1 *= FLT_SCALE;
b2 *= FLT_SCALE;
Point2f delta( (float)((A12*b2 - A22*b1) * D),
(float)((A12*b1 - A11*b2) * D));
nextPt += delta;
nextPts[ptidx] = nextPt - halfWin;
if( delta.ddot(delta) <= criteria.epsilon)
break;
if(j > 0 && std::abs(delta.x - prevDelta.x) < 0.01 &&
std::abs(delta.y - prevDelta.y) < 0.01)
{
nextPts[ptidx] -= delta*0.5f;
break;
}
prevDelta = delta;
}
}
}
const Mat* prevImg;
const Mat* nextImg;
const Mat* prevDeriv;
const Mat* rgbPrevImg;
const Mat* rgbNextImg;
const Point2f* prevPts;
Point2f* nextPts;
uchar* status;
float* err;
int maxWinSize;
int minWinSize;
TermCriteria criteria;
int level;
int maxLevel;
int windowType;
float minEigThreshold;
bool useInitialFlow;
int crossSegmentationThreshold;
};
}}}} // namespace
#endif
modules/optflow/src/rlof/rlof_invoker.hpp
+1125
View File
File diff suppressed because it is too large Load Diff
modules/optflow/src/rlof/rlof_invokerbase.hpp
+422
View File
@@ -0,0 +1,422 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef _RLOF_INVOKERBASE_HPP_
#define _RLOF_INVOKERBASE_HPP_
#ifndef CV_DESCALE
#define CV_DESCALE(x, n) (((x) + (1 << ((n)-1))) >> (n))
#endif
#define FLT_RESCALE 1
#include "rlof_localflow.h"
#include <unordered_map>
#include "opencv2/core/hal/intrin.hpp"
using namespace std;
using namespace cv;
namespace cv {
namespace optflow {
typedef short deriv_type;
#if CV_SIMD128
static inline void getVBitMask(const int & width, v_int32x4 & mask0, v_int32x4 & mask1)
{
int noBits = width - static_cast<int>(floor(width / 8.f) * 8.f);
unsigned int val[8];
for (int n = 0; n < 8; n++)
{
val[n] = (noBits > n) ? (std::numeric_limits<unsigned int>::max()) : 0;
}
mask0 = v_int32x4(val[0], val[1], val[2], val[3]);
mask1 = v_int32x4(val[4], val[5], val[6], val[7]);
}
#endif
typedef uchar tMaskType;
#define tCVMaskType CV_8UC1
#define MaskSet 0xffffffff
static inline void copyWinBuffers(int iw00, int iw01, int iw10, int iw11,
Size winSize,
const Mat & I, const Mat & derivI, const Mat & winMaskMat,
Mat & IWinBuf, Mat & derivIWinBuf,
Point iprevPt)
{
int cn = I.channels(), cn2 = cn * 2;
const int W_BITS = 14;
#if CV_SIMD128
v_int16x8 vqw0((short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01));
v_int16x8 vqw1((short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11));
v_int32x4 vdelta_d = v_setall_s32(1 << (W_BITS - 1));
v_int32x4 vdelta = v_setall_s32(1 << (W_BITS - 5 - 1));
v_int32x4 vmax_val_32 = v_setall_s32(std::numeric_limits<unsigned int>::max());
v_int32x4 vmask_border_0, vmask_border_1;
getVBitMask(winSize.width, vmask_border_0, vmask_border_1);
#endif
// extract the patch from the first image, compute covariation matrix of derivatives
int x, y;
for (y = 0; y < winSize.height; y++)
{
const uchar* src = I.ptr<uchar>(y + iprevPt.y, 0) + iprevPt.x*cn;
const uchar* src1 = I.ptr<uchar>(y + iprevPt.y + 1, 0) + iprevPt.x*cn;
const short* dsrc = derivI.ptr<short>(y + iprevPt.y, 0) + iprevPt.x*cn2;
const short* dsrc1 = derivI.ptr<short>(y + iprevPt.y + 1, 0) + iprevPt.x*cn2;
short* Iptr = IWinBuf.ptr<short>(y, 0);
short* dIptr = derivIWinBuf.ptr<short>(y, 0);
const tMaskType* maskPtr = winMaskMat.ptr<tMaskType>(y, 0);
x = 0;
#if CV_SIMD128
for (; x <= winSize.width*cn; x += 8, dsrc += 8 * 2, dsrc1 += 8 * 2, dIptr += 8 * 2)
{
v_int32x4 vmask0 = v_mul(v_reinterpret_as_s32(v_load_expand_q(maskPtr + x)), vmax_val_32);
v_int32x4 vmask1 = v_mul(v_reinterpret_as_s32(v_load_expand_q(maskPtr + x + 4)), vmax_val_32);
if (x + 4 > winSize.width)
{
vmask0 = v_and(vmask0, vmask_border_0);
}
if (x + 8 > winSize.width)
{
vmask1 = v_and(vmask1, vmask_border_1);
}
v_int32x4 t0, t1;
v_int16x8 v00, v01, v10, v11, t00, t01, t10, t11;
v00 = v_reinterpret_as_s16(v_load_expand(src + x));
v01 = v_reinterpret_as_s16(v_load_expand(src + x + cn));
v10 = v_reinterpret_as_s16(v_load_expand(src1 + x));
v11 = v_reinterpret_as_s16(v_load_expand(src1 + x + cn));
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_add(v_dotprod(t00, vqw0, vdelta), v_dotprod(t10, vqw1));
t1 = v_add(v_dotprod(t01, vqw0, vdelta), v_dotprod(t11, vqw1));
t0 = v_and(v_shr(t0, W_BITS - 5), vmask0);
t1 = v_and(v_shr(t1, W_BITS - 5), vmask1);
v_store(Iptr + x, v_pack(t0, t1));
v00 = v_reinterpret_as_s16(v_load(dsrc));
v01 = v_reinterpret_as_s16(v_load(dsrc + cn2));
v10 = v_reinterpret_as_s16(v_load(dsrc1));
v11 = v_reinterpret_as_s16(v_load(dsrc1 + cn2));
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_add(v_dotprod(t00, vqw0, vdelta_d), v_dotprod(t10, vqw1));
t1 = v_add(v_dotprod(t01, vqw0, vdelta_d), v_dotprod(t11, vqw1));
t0 = v_shr(t0, W_BITS);
t1 = v_shr(t1, W_BITS);
v00 = v_pack(t0, t1); // Ix0 Iy0 Ix1 Iy1 ...
v00 = v_and(v00, v_reinterpret_as_s16(vmask0));
v_store(dIptr, v00);
v00 = v_reinterpret_as_s16(v_load(dsrc + 4 * 2));
v01 = v_reinterpret_as_s16(v_load(dsrc + 4 * 2 + cn2));
v10 = v_reinterpret_as_s16(v_load(dsrc1 + 4 * 2));
v11 = v_reinterpret_as_s16(v_load(dsrc1 + 4 * 2 + cn2));
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_add(v_dotprod(t00, vqw0, vdelta_d), v_dotprod(t10, vqw1));
t1 = v_add(v_dotprod(t01, vqw0, vdelta_d), v_dotprod(t11, vqw1));
t0 = v_shr(t0, W_BITS);
t1 = v_shr(t1, W_BITS);
v00 = v_pack(t0, t1); // Ix0 Iy0 Ix1 Iy1 ...
v00 = v_and(v00, v_reinterpret_as_s16(vmask1));
v_store(dIptr + 4 * 2, v00);
}
#else
for (; x < winSize.width*cn; x++, dsrc += 2, dsrc1 += 2, dIptr += 2)
{
if (maskPtr[x] == 0)
{
dIptr[0] = 0;
dIptr[1] = 0;
continue;
}
int ival = CV_DESCALE(src[x] * iw00 + src[x + cn] * iw01 +
src1[x] * iw10 + src1[x + cn] * iw11, W_BITS - 5);
int ixval = CV_DESCALE(dsrc[0] * iw00 + dsrc[cn2] * iw01 +
dsrc1[0] * iw10 + dsrc1[cn2] * iw11, W_BITS);
int iyval = CV_DESCALE(dsrc[1] * iw00 + dsrc[cn2 + 1] * iw01 + dsrc1[1] * iw10 +
dsrc1[cn2 + 1] * iw11, W_BITS);
Iptr[x] = (short)ival;
dIptr[0] = (short)ixval;
dIptr[1] = (short)iyval;
}
#endif
}
}
static inline void copyWinBuffers(int iw00, int iw01, int iw10, int iw11,
Size winSize,
const Mat & I, const Mat & derivI, const Mat & winMaskMat,
Mat & IWinBuf, Mat & derivIWinBuf,
float & A11, float & A22, float & A12,
Point iprevPt)
{
const float FLT_SCALE = (1.f / (1 << 20));
int cn = I.channels(), cn2 = cn * 2;
const int W_BITS = 14;
#if CV_SIMD128
v_int16x8 vqw0((short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01), (short)(iw00), (short)(iw01));
v_int16x8 vqw1((short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11), (short)(iw10), (short)(iw11));
v_int32x4 vdelta_d = v_setall_s32(1 << (W_BITS - 1));
v_int32x4 vdelta = v_setall_s32(1 << (W_BITS - 5 - 1));
v_int32x4 vmax_val_32 = v_setall_s32(std::numeric_limits<unsigned int>::max());
v_int32x4 vmask_border0, vmask_border1;
v_float32x4 vA11 = v_setzero_f32(), vA22 = v_setzero_f32(), vA12 = v_setzero_f32();
getVBitMask(winSize.width, vmask_border0, vmask_border1);
#endif
// extract the patch from the first image, compute covariation matrix of derivatives
for (int y = 0; y < winSize.height; y++)
{
const uchar* src = I.ptr<uchar>(y + iprevPt.y, 0) + iprevPt.x*cn;
const uchar* src1 = I.ptr<uchar>(y + iprevPt.y + 1, 0) + iprevPt.x*cn;
const short* dsrc = derivI.ptr<short>(y + iprevPt.y, 0) + iprevPt.x*cn2;
const short* dsrc1 = derivI.ptr<short>(y + iprevPt.y + 1, 0) + iprevPt.x*cn2;
short* Iptr = IWinBuf.ptr<short>(y, 0);
short* dIptr = derivIWinBuf.ptr<short>(y, 0);
const tMaskType* maskPtr = winMaskMat.ptr<tMaskType>(y, 0);
#if CV_SIMD128
for (int x = 0; x <= winSize.width*cn; x += 8, dsrc += 8 * 2, dsrc1 += 8 * 2, dIptr += 8 * 2)
{
v_int32x4 vmask0 = v_mul(v_reinterpret_as_s32(v_load_expand_q(maskPtr + x)), vmax_val_32);
v_int32x4 vmask1 = v_mul(v_reinterpret_as_s32(v_load_expand_q(maskPtr + x + 4)), vmax_val_32);
if (x + 4 > winSize.width)
{
vmask0 = v_and(vmask0, vmask_border0);
}
if (x + 8 > winSize.width)
{
vmask1 = v_and(vmask1, vmask_border1);
}
v_int32x4 t0, t1;
v_int16x8 v00, v01, v10, v11, t00, t01, t10, t11;
v00 = v_reinterpret_as_s16(v_load_expand(src + x));
v01 = v_reinterpret_as_s16(v_load_expand(src + x + cn));
v10 = v_reinterpret_as_s16(v_load_expand(src1 + x));
v11 = v_reinterpret_as_s16(v_load_expand(src1 + x + cn));
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_add(v_dotprod(t00, vqw0, vdelta), v_dotprod(t10, vqw1));
t1 = v_add(v_dotprod(t01, vqw0, vdelta), v_dotprod(t11, vqw1));
t0 = v_shr(t0, W_BITS - 5);
t1 = v_shr(t1, W_BITS - 5);
t0 = v_and(t0, vmask0);
t1 = v_and(t1, vmask1);
v_store(Iptr + x, v_pack(t0, t1));
v00 = v_reinterpret_as_s16(v_load(dsrc));
v01 = v_reinterpret_as_s16(v_load(dsrc + cn2));
v10 = v_reinterpret_as_s16(v_load(dsrc1));
v11 = v_reinterpret_as_s16(v_load(dsrc1 + cn2));
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_add(v_dotprod(t00, vqw0, vdelta_d), v_dotprod(t10, vqw1));
t1 = v_add(v_dotprod(t01, vqw0, vdelta_d), v_dotprod(t11, vqw1));
t0 = v_shr(t0, W_BITS);
t1 = v_shr(t1, W_BITS);
v00 = v_pack(t0, t1); // Ix0 Iy0 Ix1 Iy1 ...
v00 = v_and(v00, v_reinterpret_as_s16(vmask0));
v_store(dIptr, v00);
v00 = v_reinterpret_as_s16(v_interleave_pairs(v_reinterpret_as_s32(v_interleave_pairs(v00))));
v_expand(v00, t1, t0);
v_float32x4 fy = v_cvt_f32(t0);
v_float32x4 fx = v_cvt_f32(t1);
vA22 = v_muladd(fy, fy, vA22);
vA12 = v_muladd(fx, fy, vA12);
vA11 = v_muladd(fx, fx, vA11);
v00 = v_reinterpret_as_s16(v_load(dsrc + 4 * 2));
v01 = v_reinterpret_as_s16(v_load(dsrc + 4 * 2 + cn2));
v10 = v_reinterpret_as_s16(v_load(dsrc1 + 4 * 2));
v11 = v_reinterpret_as_s16(v_load(dsrc1 + 4 * 2 + cn2));
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_add(v_dotprod(t00, vqw0, vdelta_d), v_dotprod(t10, vqw1));
t1 = v_add(v_dotprod(t01, vqw0, vdelta_d), v_dotprod(t11, vqw1));
t0 = v_shr(t0, W_BITS);
t1 = v_shr(t1, W_BITS);
v00 = v_pack(t0, t1); // Ix0 Iy0 Ix1 Iy1 ...
v00 = v_and(v00, v_reinterpret_as_s16(vmask1));
v_store(dIptr + 4 * 2, v00);
v00 = v_reinterpret_as_s16(v_interleave_pairs(v_reinterpret_as_s32(v_interleave_pairs(v00))));
v_expand(v00, t1, t0);
fy = v_cvt_f32(t0);
fx = v_cvt_f32(t1);
vA22 = v_muladd(fy, fy, vA22);
vA12 = v_muladd(fx, fy, vA12);
vA11 = v_muladd(fx, fx, vA11);
}
#else
for (int x = 0; x < winSize.width*cn; x++, dsrc += 2, dsrc1 += 2, dIptr += 2)
{
if (maskPtr[x] == 0)
{
dIptr[0] = 0;
dIptr[1] = 0;
continue;
}
int ival = CV_DESCALE(src[x] * iw00 + src[x + cn] * iw01 +
src1[x] * iw10 + src1[x + cn] * iw11, W_BITS - 5);
int ixval = CV_DESCALE(dsrc[0] * iw00 + dsrc[cn2] * iw01 +
dsrc1[0] * iw10 + dsrc1[cn2] * iw11, W_BITS);
int iyval = CV_DESCALE(dsrc[1] * iw00 + dsrc[cn2 + 1] * iw01 + dsrc1[1] * iw10 +
dsrc1[cn2 + 1] * iw11, W_BITS);
Iptr[x] = (short)ival;
dIptr[0] = (short)ixval;
dIptr[1] = (short)iyval;
A11 += (float)(ixval*ixval);
A12 += (float)(ixval*iyval);
A22 += (float)(iyval*iyval);
}
#endif
}
#if CV_SIMD128
A11 += v_reduce_sum(vA11);
A12 += v_reduce_sum(vA12);
A22 += v_reduce_sum(vA22);
#endif
A11 *= FLT_SCALE;
A12 *= FLT_SCALE;
A22 *= FLT_SCALE;
}
static void getLocalPatch(
const cv::Mat & src,
const cv::Point2i & prevPoint, // feature points
cv::Mat & winPointMask,
int & noPoints,
cv::Rect & winRoi,
int minWinSize)
{
int maxWinSizeH = (winPointMask.cols - 1) / 2;
winRoi.x = prevPoint.x;// - maxWinSizeH;
winRoi.y = prevPoint.y;// - maxWinSizeH;
winRoi.width = winPointMask.cols;
winRoi.height = winPointMask.rows;
if( minWinSize == winPointMask.cols || prevPoint.x < 0 || prevPoint.y < 0
|| prevPoint.x + 2*maxWinSizeH >= src.cols || prevPoint.y + 2*maxWinSizeH >= src.rows)
{
winRoi.x = prevPoint.x - maxWinSizeH;
winRoi.y = prevPoint.y - maxWinSizeH;
winPointMask.setTo(1);
noPoints = winPointMask.size().area();
return;
}
winPointMask.setTo(0);
noPoints = 0;
int c = prevPoint.x + maxWinSizeH;
int r = prevPoint.y + maxWinSizeH;
int min_c = c;
int max_c = c;
int border_left = c - maxWinSizeH;
int border_top = r - maxWinSizeH;
cv::Vec4i bounds = src.at<cv::Vec4i>(r,c);
int min_r = bounds.val[2];
int max_r = bounds.val[3];
for( int _r = min_r; _r <= max_r; _r++)
{
cv::Rect roi(maxWinSizeH, _r - border_top, winPointMask.cols, 1);
if( _r >= 0 && _r < src.cols)
{
bounds = src.at<cv::Vec4i>(_r,c);
roi.x = bounds.val[0] - border_left;
roi.width = bounds.val[1] - bounds.val[0];
cv::Mat(winPointMask, roi).setTo(1);
}
else
{
bounds.val[0] = border_left;
bounds.val[1] = border_left + roi.width;
}
min_c = MIN(min_c, bounds.val[0]);
max_c = MAX(max_c, bounds.val[1]);
noPoints += roi.width;
}
if( noPoints < minWinSize * minWinSize)
{
cv::Rect roi( winPointMask.cols / 2 - (minWinSize-1)/2,
winPointMask.rows / 2 - (minWinSize-1)/2,
minWinSize, minWinSize);
cv::Mat(winPointMask, roi).setTo(1);
roi.x += border_left;
roi.y += border_top;
min_c = MIN(MIN(min_c, roi.tl().x),roi.br().x);
max_c = MAX(MAX(max_c, roi.tl().x),roi.br().x);
min_r = MIN(MIN(min_r, roi.tl().y),roi.br().y);
max_r = MAX(MAX(max_r, roi.tl().y),roi.br().y);
noPoints += minWinSize * minWinSize;
}
winRoi.x = min_c - maxWinSizeH;
winRoi.y = min_r - maxWinSizeH;
winRoi.width = max_c - min_c;
winRoi.height = max_r - min_r;
winPointMask = winPointMask(cv::Rect(min_c - border_left, min_r - border_top, winRoi.width, winRoi.height));
}
static inline
bool calcWinMaskMat(
const cv::Mat & BI,
const int windowType,
const cv::Point2i & iprevPt,
cv::Mat & winMaskMat,
cv::Size & winSize,
cv::Point2f & halfWin,
int & winArea,
const int minWinSize,
const int maxWinSize)
{
if (windowType == SR_CROSS && maxWinSize != minWinSize)
{
// patch generation
cv::Rect winRoi;
getLocalPatch(BI, iprevPt, winMaskMat, winArea, winRoi, minWinSize);
if (winArea == 0)
return false;
winSize = winRoi.size();
halfWin = Point2f(static_cast<float>(iprevPt.x - winRoi.tl().x),
static_cast<float>(iprevPt.y - winRoi.tl().y));
}
else
{
winSize = cv::Size(maxWinSize, maxWinSize);
halfWin = Point2f((winSize.width - 1) / 2.f, (winSize.height - 1) / 2.f);
winMaskMat.setTo(1);
}
return true;
}
static inline
short estimateScale(cv::Mat & residuals)
{
cv::Mat absMat = cv::abs(residuals);
return quickselect<short>(absMat, absMat.rows / 2);
}
}} // namespace
#endif
modules/optflow/src/rlof/rlof_localflow.cpp
+711
View File
@@ -0,0 +1,711 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "../precomp.hpp"
#include "opencv2/geometry.hpp" // findHomography
#include "rlof_localflow.h"
#include "berlof_invoker.hpp"
#include "rlof_invoker.hpp"
#include "plk_invoker.hpp"
using namespace std;
using namespace cv;
namespace cv {
namespace detail {
typedef short deriv_type;
} // namespace
namespace {
static void calcSharrDeriv(const cv::Mat& src, cv::Mat& dst)
{
using namespace cv;
using cv::detail::deriv_type;
int rows = src.rows, cols = src.cols, cn = src.channels(), colsn = cols * cn, depth = src.depth();
CV_Assert(depth == CV_8U);
dst.create(rows, cols, CV_MAKETYPE(DataType<deriv_type>::depth, cn * 2));
int x, y, delta = (int)alignSize((cols + 2)*cn, 16);
AutoBuffer<deriv_type> _tempBuf(delta * 2 + 64);
deriv_type *trow0 = alignPtr(_tempBuf.data() + cn, 16), *trow1 = alignPtr(trow0 + delta, 16);
#if CV_SIMD128
v_int16x8 c3 = v_setall_s16(3), c10 = v_setall_s16(10);
bool haveSIMD = checkHardwareSupport(CV_CPU_SSE2) || checkHardwareSupport(CV_CPU_NEON);
#endif
for (y = 0; y < rows; y++)
{
const uchar* srow0 = src.ptr<uchar>(y > 0 ? y - 1 : rows > 1 ? 1 : 0);
const uchar* srow1 = src.ptr<uchar>(y);
const uchar* srow2 = src.ptr<uchar>(y < rows - 1 ? y + 1 : rows > 1 ? rows - 2 : 0);
deriv_type* drow = dst.ptr<deriv_type>(y);
// do vertical convolution
x = 0;
#if CV_SIMD128
if (haveSIMD)
{
for (; x <= colsn - 8; x += 8)
{
v_int16x8 s0 = v_reinterpret_as_s16(v_load_expand(srow0 + x));
v_int16x8 s1 = v_reinterpret_as_s16(v_load_expand(srow1 + x));
v_int16x8 s2 = v_reinterpret_as_s16(v_load_expand(srow2 + x));
v_int16x8 t1 = v_sub(s2, s0);
v_int16x8 t0 = v_add(v_mul_wrap(v_add(s0, s2), c3), v_mul_wrap(s1, c10));
v_store(trow0 + x, t0);
v_store(trow1 + x, t1);
}
}
#endif
for (; x < colsn; x++)
{
int t0 = (srow0[x] + srow2[x]) * 3 + srow1[x] * 10;
int t1 = srow2[x] - srow0[x];
trow0[x] = (deriv_type)t0;
trow1[x] = (deriv_type)t1;
}
// make border
int x0 = (cols > 1 ? 1 : 0)*cn, x1 = (cols > 1 ? cols - 2 : 0)*cn;
for (int k = 0; k < cn; k++)
{
trow0[-cn + k] = trow0[x0 + k]; trow0[colsn + k] = trow0[x1 + k];
trow1[-cn + k] = trow1[x0 + k]; trow1[colsn + k] = trow1[x1 + k];
}
// do horizontal convolution, interleave the results and store them to dst
x = 0;
#if CV_SIMD128
if (haveSIMD)
{
for (; x <= colsn - 8; x += 8)
{
v_int16x8 s0 = v_load(trow0 + x - cn);
v_int16x8 s1 = v_load(trow0 + x + cn);
v_int16x8 s2 = v_load(trow1 + x - cn);
v_int16x8 s3 = v_load(trow1 + x);
v_int16x8 s4 = v_load(trow1 + x + cn);
v_int16x8 t0 = v_sub(s1, s0);
v_int16x8 t1 = v_add(v_mul_wrap(v_add(s2, s4), c3), v_mul_wrap(s3, c10));
v_store_interleave((drow + x * 2), t0, t1);
}
}
#endif
for (; x < colsn; x++)
{
deriv_type t0 = (deriv_type)(trow0[x + cn] - trow0[x - cn]);
deriv_type t1 = (deriv_type)((trow1[x + cn] + trow1[x - cn]) * 3 + trow1[x] * 10);
drow[x * 2] = t0; drow[x * 2 + 1] = t1;
}
}
}
} // namespace
namespace optflow {
/*! Helper function for preCalcCrossSegmentation. Everything is performed on the large
*\param data CV_8UC3 image ( use extended image mit winSize)
*\param winSize
*\param dst CV_32SC1 bounding map
*\param threshold
*\param stride if true store into first two bounding maps
*/
class HorizontalCrossSegmentation : public cv::ParallelLoopBody
{
public:
HorizontalCrossSegmentation(
const cv::Point2f * ptList,
int npoints,
float pointScale,
const cv::Mat * data,
const int winSize,
cv::Mat * dst,
int threshold,
bool stride,
const cv::Mat * mask
)
{
m_ptList = ptList;
m_npoints = npoints;
m_pointScale = pointScale;
m_data = data;
m_winSize = winSize;
m_dst = dst;
m_threshold = threshold;
m_stride = stride;
m_mask = mask;
}
void operator()(const cv::Range& range) const CV_OVERRIDE
{
uchar channel[2];
channel[0] = m_stride ? 2 : 0;
channel[1] = m_stride ? 3 : 1;
int hWinSize = (m_winSize - 1) / 2;
std::vector<int> differenz(m_winSize);
for( int r = range.start; r < range.end; r++ )
{
for(int c = hWinSize; c < m_data->cols - hWinSize; c++)
{
if( m_mask->at<uchar>(r,c) == 0)
continue;
const Point3_<uchar> & ucval = m_data->at<Point3_<uchar>>(r,c);
Point3i val(static_cast<int>(ucval.x), static_cast<int>(ucval.y), static_cast<int>(ucval.z));
int x = c - hWinSize;
Point dstPos = m_stride ? Point(r,c) : Point(c,r);
for(int ix = 0; ix < m_winSize; ix++, x++)
{
const Point3_<uchar> & valref = m_data->at<Point3_<uchar>>(r,x);
differenz[ix] = MAX(std::abs(static_cast<int>(valref.x) - val.x),
MAX(std::abs(static_cast<int>(valref.y) - val.y),
(std::abs(static_cast<int>(valref.z) - val.z))));
}
cv::Vec4i & bounds = m_dst->at<cv::Vec4i>(dstPos);
bounds.val[channel[0]] = c - hWinSize;
bounds.val[channel[1]] = c + hWinSize;
int * diffPtr = &differenz[hWinSize];
bool useUpperBound = false;
bool useLowerBound = false;
for(int ix = 1; ix <= hWinSize; ix++)
{
if( !useUpperBound && diffPtr[-ix] > m_threshold)
{
useUpperBound = true;
bounds.val[channel[0]] = c - ix;
}
if( !useLowerBound && diffPtr[ix-1] > m_threshold)
{
useLowerBound = true;
bounds.val[channel[1]] = c + ix - 1;
}
if( useUpperBound && useLowerBound)
break;
}
}
}
}
const cv::Point2f * m_ptList;
int m_npoints;
float m_pointScale;
const cv::Mat * m_data;
int m_winSize;
cv::Mat * m_dst;
int m_threshold;
bool m_stride;
const cv::Mat * m_mask;
};
static
void preCalcCrossSegmentation(
const cv::Point2f * ptList,
int npoints,
float pointScale,
const cv::Mat & img,
const int winSize,
cv::Mat & dst,
int threshold
)
{
int hWinSize = (winSize - 1) / 2;
cv::Mat data = img;
data.adjustROI(hWinSize, hWinSize, hWinSize, hWinSize);
if( dst.size() != dst.size() || dst.type() != CV_32SC4)
{
dst.release();
dst.create(data.size(), CV_32SC4);
}
cv::Mat mask(data.cols, data.rows, CV_8UC1);
mask.setTo(0);
for( unsigned int n = 0; n < static_cast<unsigned int>(npoints); n++)
{
cv::Point ipos( static_cast<int>(floor(ptList[n].y * pointScale)),
static_cast<int>(floor(ptList[n].x * pointScale) + hWinSize));
ipos.x = MAX( MIN(ipos.x, mask.cols - 1), 0);
int to = MIN( mask.cols - 1, ipos.x + winSize );
int ypos = MAX( MIN(ipos.y, mask.rows - 1), 0);
for(int x = ipos.x; x <= to ; x++)
{
mask.at<uchar>(ypos, x) = 255;
}
}
cv::Mat datat = data.t();
cv::Mat maskt = mask.t();
parallel_for_(cv::Range(0, datat.rows), HorizontalCrossSegmentation(ptList, npoints, pointScale, &datat, winSize, &dst, threshold, true, &mask));
parallel_for_(cv::Range(0, data.rows), HorizontalCrossSegmentation(ptList, npoints, pointScale, &data, winSize, &dst, threshold, false, &maskt));
}
static inline
bool isrobust(const RLOFOpticalFlowParameter & param)
{
return (param.normSigma0 < 255 && param.normSigma1 < 255);
}
static inline
std::vector<float> get_norm(float sigma0, float sigma1)
{
std::vector<float> result = { sigma0, sigma1, sigma0 / (sigma0 - sigma1), sigma0 * sigma1 };
return result;
}
static
int buildOpticalFlowPyramidScale(InputArray _img, OutputArrayOfArrays pyramid, Size winSize, int maxLevel, bool withDerivatives,
int pyrBorder, int derivBorder, bool tryReuseInputImage, float levelScale[2])
{
Mat img = _img.getMat();
CV_Assert(img.depth() == CV_8U && winSize.width > 2 && winSize.height > 2);
int pyrstep = withDerivatives ? 2 : 1;
pyramid.create(1, (maxLevel + 1) * pyrstep, 0 /*type*/, -1, true);
int derivType = CV_MAKETYPE(DataType<short>::depth, img.channels() * 2);
//level 0
bool lvl0IsSet = false;
if (tryReuseInputImage && img.isSubmatrix() && (pyrBorder & BORDER_ISOLATED) == 0)
{
Size wholeSize;
Point ofs;
img.locateROI(wholeSize, ofs);
if (ofs.x >= winSize.width && ofs.y >= winSize.height
&& ofs.x + img.cols + winSize.width <= wholeSize.width
&& ofs.y + img.rows + winSize.height <= wholeSize.height)
{
pyramid.getMatRef(0) = img;
lvl0IsSet = true;
}
}
if (!lvl0IsSet)
{
Mat& temp = pyramid.getMatRef(0);
if (!temp.empty())
temp.adjustROI(winSize.height, winSize.height, winSize.width, winSize.width);
if (temp.type() != img.type() || temp.cols != winSize.width * 2 + img.cols || temp.rows != winSize.height * 2 + img.rows)
temp.create(img.rows + winSize.height * 2, img.cols + winSize.width * 2, img.type());
if (pyrBorder == BORDER_TRANSPARENT)
img.copyTo(temp(Rect(winSize.width, winSize.height, img.cols, img.rows)));
else
copyMakeBorder(img, temp, winSize.height, winSize.height, winSize.width, winSize.width, pyrBorder);
temp.adjustROI(-winSize.height, -winSize.height, -winSize.width, -winSize.width);
}
Size sz = img.size();
Mat prevLevel = pyramid.getMatRef(0);
Mat thisLevel = prevLevel;
for (int level = 0; level <= maxLevel; ++level)
{
if (level != 0)
{
Mat& temp = pyramid.getMatRef(level * pyrstep);
if (!temp.empty())
temp.adjustROI(winSize.height, winSize.height, winSize.width, winSize.width);
if (temp.type() != img.type() || temp.cols != winSize.width * 2 + sz.width || temp.rows != winSize.height * 2 + sz.height)
temp.create(sz.height + winSize.height * 2, sz.width + winSize.width * 2, img.type());
thisLevel = temp(Rect(winSize.width, winSize.height, sz.width, sz.height));
pyrDown(prevLevel, thisLevel, sz);
if (pyrBorder != BORDER_TRANSPARENT)
copyMakeBorder(thisLevel, temp, winSize.height, winSize.height, winSize.width, winSize.width, pyrBorder | BORDER_ISOLATED);
temp.adjustROI(-winSize.height, -winSize.height, -winSize.width, -winSize.width);
}
if (withDerivatives)
{
Mat& deriv = pyramid.getMatRef(level * pyrstep + 1);
if (!deriv.empty())
deriv.adjustROI(winSize.height, winSize.height, winSize.width, winSize.width);
if (deriv.type() != derivType || deriv.cols != winSize.width * 2 + sz.width || deriv.rows != winSize.height * 2 + sz.height)
deriv.create(sz.height + winSize.height * 2, sz.width + winSize.width * 2, derivType);
Mat derivI = deriv(Rect(winSize.width, winSize.height, sz.width, sz.height));
calcSharrDeriv(thisLevel, derivI);
if (derivBorder != BORDER_TRANSPARENT)
copyMakeBorder(derivI, deriv, winSize.height, winSize.height, winSize.width, winSize.width, derivBorder | BORDER_ISOLATED);
deriv.adjustROI(-winSize.height, -winSize.height, -winSize.width, -winSize.width);
}
sz = Size(static_cast<int>((sz.width + 1) / levelScale[0]),
static_cast<int>((sz.height + 1) / levelScale[1]));
if (sz.width <= winSize.width || sz.height <= winSize.height)
{
pyramid.create(1, (level + 1) * pyrstep, 0 /*type*/, -1, true);//check this
return level;
}
prevLevel = thisLevel;
}
return maxLevel;
}
int CImageBuffer::buildPyramid(cv::Size winSize, int maxLevel, float levelScale[2],bool withBlurredImage )
{
if (! m_Overwrite)
return m_maxLevel;
if (withBlurredImage)
m_maxLevel = buildOpticalFlowPyramidScale(m_BlurredImage, m_ImagePyramid, winSize, maxLevel, false, 4, 0, true, levelScale);
else
m_maxLevel = buildOpticalFlowPyramidScale(m_Image, m_ImagePyramid, winSize, maxLevel, false, 4, 0, true, levelScale);
return m_maxLevel;
}
static
void calcLocalOpticalFlowCore(
Ptr<CImageBuffer> prevPyramids[2],
Ptr<CImageBuffer> currPyramids[2],
InputArray _prevPts,
InputOutputArray _nextPts,
const RLOFOpticalFlowParameter & param)
{
bool useAdditionalRGB = param.supportRegionType == SR_CROSS;
int iWinSize = param.largeWinSize;
int winSizes[2] = { iWinSize, iWinSize };
if (param.supportRegionType != SR_FIXED)
{
winSizes[0] = param.smallWinSize;
}
//cv::Size winSize(iWinSize, iWinSize);
cv::TermCriteria criteria(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, param.maxIteration, 0.01);
std::vector<float> rlofNorm = get_norm(param.normSigma0, param.normSigma1);
CV_Assert(winSizes[0] <= winSizes[1]);
bool usePreComputedCross = winSizes[0] != winSizes[1];
Mat prevPtsMat = _prevPts.getMat();
const int derivDepth = DataType<detail::deriv_type>::depth;
CV_Assert(param.maxLevel >= 0 && iWinSize > 2);
int level = 0, npoints;
CV_Assert((npoints = prevPtsMat.checkVector(2, CV_32F, true)) >= 0);
if (!(param.useInitialFlow))
_nextPts.create(prevPtsMat.size(), prevPtsMat.type(), -1, true);
Mat nextPtsMat = _nextPts.getMat();
CV_Assert(nextPtsMat.checkVector(2, CV_32F, true) == npoints);
const Point2f* prevPts = (const Point2f*)prevPtsMat.data;
Point2f* nextPts = (Point2f*)nextPtsMat.data;
std::vector<uchar> status(npoints);
std::vector<float> err(npoints);
std::vector<Point2f> gainPts(npoints);
float levelScale[2] = { 2.f,2.f };
int maxLevel = prevPyramids[0]->buildPyramid(cv::Size(iWinSize, iWinSize), param.maxLevel, levelScale);
maxLevel = currPyramids[0]->buildPyramid(cv::Size(iWinSize, iWinSize), maxLevel, levelScale);
if (useAdditionalRGB)
{
prevPyramids[1]->buildPyramid(cv::Size(iWinSize, iWinSize), maxLevel, levelScale, true);
currPyramids[1]->buildPyramid(cv::Size(iWinSize, iWinSize), maxLevel, levelScale, true);
}
if ((criteria.type & TermCriteria::COUNT) == 0)
criteria.maxCount = 30;
else
criteria.maxCount = std::min(std::max(criteria.maxCount, 0), 100);
if ((criteria.type & TermCriteria::EPS) == 0)
criteria.epsilon = 0.001;
else
criteria.epsilon = std::min(std::max(criteria.epsilon, 0.), 10.);
criteria.epsilon *= criteria.epsilon;
// dI/dx ~ Ix, dI/dy ~ Iy
Mat derivIBuf;
derivIBuf.create(prevPyramids[0]->m_ImagePyramid[0].rows + iWinSize * 2, prevPyramids[0]->m_ImagePyramid[0].cols + iWinSize * 2, CV_MAKETYPE(derivDepth, prevPyramids[0]->m_ImagePyramid[0].channels() * 2));
for (level = maxLevel; level >= 0; level--)
{
Mat derivI;
Size imgSize = prevPyramids[0]->getImage(level).size();
Mat _derivI(imgSize.height + iWinSize * 2, imgSize.width + iWinSize * 2, derivIBuf.type(), derivIBuf.data);
derivI = _derivI(Rect(iWinSize, iWinSize, imgSize.width, imgSize.height));
calcSharrDeriv(prevPyramids[0]->getImage(level), derivI);
copyMakeBorder(derivI, _derivI, iWinSize, iWinSize, iWinSize, iWinSize, BORDER_CONSTANT | BORDER_ISOLATED);
cv::Mat tRGBPrevPyr;
cv::Mat tRGBNextPyr;
if (useAdditionalRGB)
{
tRGBPrevPyr = prevPyramids[1]->getImage(level);
tRGBNextPyr = prevPyramids[1]->getImage(level);
prevPyramids[1]->m_Overwrite = false;
currPyramids[1]->m_Overwrite = false;
}
cv::Mat prevImage = prevPyramids[0]->getImage(level);
cv::Mat currImage = currPyramids[0]->getImage(level);
cv::Mat preCrossMap;
if( usePreComputedCross )
{
preCalcCrossSegmentation(prevPts, npoints, (float)(1./(1 << level)), tRGBPrevPyr, winSizes[1], preCrossMap, param.crossSegmentationThreshold);
tRGBNextPyr = cv::Mat();
tRGBPrevPyr = preCrossMap;
}
// apply plk like tracker
prevImage.adjustROI(iWinSize, iWinSize, iWinSize, iWinSize);
currImage.adjustROI(iWinSize, iWinSize, iWinSize, iWinSize);
derivI.adjustROI(iWinSize, iWinSize, iWinSize, iWinSize);
if (isrobust(param) == false)
{
if (param.useIlluminationModel)
{
if (param.solverType == SolverType::ST_STANDART)
{
cv::parallel_for_(cv::Range(0, npoints),
plk::radial::TrackerInvoker(
prevImage, derivI, currImage, tRGBPrevPyr, tRGBNextPyr,
prevPts, nextPts, &status[0], &err[0], &gainPts[0],
level, maxLevel, winSizes,
param.maxIteration,
param.useInitialFlow,
param.supportRegionType,
param.minEigenValue,
param.crossSegmentationThreshold));
}
else
{
cv::parallel_for_(cv::Range(0, npoints),
beplk::radial::TrackerInvoker(
prevImage, derivI, currImage, tRGBPrevPyr, tRGBNextPyr,
prevPts, nextPts, &status[0], &err[0], &gainPts[0],
level, maxLevel, winSizes,
param.maxIteration,
param.useInitialFlow,
param.supportRegionType,
param.crossSegmentationThreshold,
param.minEigenValue));
}
}
else
{
if (param.solverType == SolverType::ST_STANDART)
{
cv::parallel_for_(cv::Range(0, npoints),
plk::ica::TrackerInvoker(
prevImage, derivI, currImage, tRGBPrevPyr, tRGBNextPyr,
prevPts, nextPts, &status[0], &err[0],
level, maxLevel, winSizes,
param.maxIteration,
param.useInitialFlow,
param.supportRegionType,
param.crossSegmentationThreshold,
param.minEigenValue));
}
else
{
cv::parallel_for_(cv::Range(0, npoints),
beplk::ica::TrackerInvoker(prevImage, derivI, currImage, tRGBPrevPyr, tRGBNextPyr,
prevPts, nextPts, &status[0], &err[0],
level, maxLevel, winSizes,
param.maxIteration,
param.useInitialFlow,
param.supportRegionType,
param.crossSegmentationThreshold,
param.minEigenValue));
}
}
}
// for robust models
else
{
if (param.useIlluminationModel)
{
if (param.solverType == SolverType::ST_STANDART)
{
cv::parallel_for_(cv::Range(0, npoints),
rlof::radial::TrackerInvoker(
prevImage, derivI, currImage, tRGBPrevPyr, tRGBNextPyr,
prevPts, nextPts, &status[0], &err[0], &gainPts[0],
level, maxLevel, winSizes,
param.maxIteration,
param.useInitialFlow,
param.supportRegionType,
rlofNorm,
param.minEigenValue,
param.crossSegmentationThreshold));
}
else
{
cv::parallel_for_(cv::Range(0, npoints),
berlof::radial::TrackerInvoker(prevImage, derivI, currImage, tRGBPrevPyr, tRGBNextPyr,
prevPts, nextPts, &status[0], &err[0], &gainPts[0],
level, maxLevel, winSizes,
param.maxIteration,
param.useInitialFlow,
param.supportRegionType,
param.crossSegmentationThreshold,
rlofNorm,
param.minEigenValue));
}
}
else
{
if (param.solverType == SolverType::ST_STANDART)
{
cv::parallel_for_(cv::Range(0, npoints),
rlof::ica::TrackerInvoker(
prevImage, derivI, currImage, tRGBPrevPyr, tRGBNextPyr,
prevPts, nextPts, &status[0], &err[0],
level, maxLevel, winSizes,
param.maxIteration,
param.useInitialFlow,
param.supportRegionType,
rlofNorm,
param.minEigenValue,
param.crossSegmentationThreshold));
}
else
{
cv::parallel_for_(cv::Range(0, npoints),
berlof::ica::TrackerInvoker(prevImage, derivI, currImage, tRGBPrevPyr, tRGBNextPyr,
prevPts, nextPts, &status[0], &err[0],
level, maxLevel, winSizes,
param.maxIteration,
param.useInitialFlow,
param.supportRegionType,
param.crossSegmentationThreshold,
rlofNorm,
param.minEigenValue));
}
}
}
prevPyramids[0]->m_Overwrite = true;
currPyramids[0]->m_Overwrite = true;
}
}
static
void preprocess(Ptr<CImageBuffer> prevPyramids[2],
Ptr<CImageBuffer> currPyramids[2],
const std::vector<cv::Point2f> & prevPoints,
std::vector<cv::Point2f> & currPoints,
const RLOFOpticalFlowParameter & param)
{
cv::Mat mask, homography;
if (param.useGlobalMotionPrior == false)
return;
currPoints.resize(prevPoints.size());
RLOFOpticalFlowParameter gmeTrackerParam = param;
gmeTrackerParam.useGlobalMotionPrior = false;
gmeTrackerParam.largeWinSize = 17;
// use none robust tracker for global motion estimation since it is faster
gmeTrackerParam.normSigma0 = std::numeric_limits<float>::max();
gmeTrackerParam.maxIteration = MAX(15, param.maxIteration);
gmeTrackerParam.minEigenValue = 0.000001f;
std::vector<cv::Point2f> gmPrevPoints, gmCurrPoints;
// Initialize point grid
int stepr = prevPyramids[0]->m_Image.rows / 30;
int stepc = prevPyramids[0]->m_Image.cols / 40;
for (int r = stepr / 2; r < prevPyramids[0]->m_Image.rows; r += stepr)
{
for (int c = stepc / 2; c < prevPyramids[0]->m_Image.cols; c += stepc)
{
gmPrevPoints.push_back(cv::Point2f(static_cast<float>(c), static_cast<float>(r)));
}
}
// perform motion estimation
calcLocalOpticalFlowCore(prevPyramids, currPyramids, gmPrevPoints, gmCurrPoints, gmeTrackerParam);
cv::Mat prevPointsMat(static_cast<int>(gmPrevPoints.size()), 1, CV_32FC2);
cv::Mat currPointsMat(static_cast<int>(gmPrevPoints.size()), 1, CV_32FC2);
cv::Mat distMat(static_cast<int>(gmPrevPoints.size()), 1, CV_32FC1);
// Forward backward confidence to estimate optimal ransac reprojection error
int noPoints = 0;
for (unsigned int n = 0; n < gmPrevPoints.size(); n++)
{
cv::Point2f flow = gmCurrPoints[n] - gmPrevPoints[n];
prevPointsMat.at<cv::Point2f>(noPoints) = gmPrevPoints[n];
currPointsMat.at<cv::Point2f>(noPoints) = gmCurrPoints[n];
distMat.at<float>(noPoints) = flow.x * flow.x + flow.y* flow.y;
if (isnan(distMat.at<float>(noPoints)) == false)
noPoints++;
}
float medianDist = (param.globalMotionRansacThreshold == 0) ? 1.f :
quickselect<float>(distMat, static_cast<int>(noPoints * static_cast<float>(param.globalMotionRansacThreshold) / 100.f));
medianDist = sqrt(medianDist);
if (noPoints < 8)
return;
cv::findHomography(prevPointsMat(cv::Rect(0, 0, 1, noPoints)), currPointsMat(cv::Rect(0, 0, 1, noPoints)), cv::RANSAC, medianDist, mask).convertTo(homography, CV_32FC1);
if (homography.empty())
return;
cv::perspectiveTransform(prevPoints, currPoints, homography);
}
void calcLocalOpticalFlow(
const Mat prevImage,
const Mat currImage,
Ptr<CImageBuffer> prevPyramids[2],
Ptr<CImageBuffer> currPyramids[2],
const std::vector<Point2f> & prevPoints,
std::vector<Point2f> & currPoints,
const RLOFOpticalFlowParameter & param)
{
if (prevImage.empty() == false && currImage.empty()== false)
{
prevPyramids[0]->m_Overwrite = true;
currPyramids[0]->m_Overwrite = true;
prevPyramids[1]->m_Overwrite = true;
// perform blurring and build blur pyramid only for the prev image
currPyramids[1]->m_Overwrite = false;
if (prevImage.type() == CV_8UC3)
{
prevPyramids[0]->setGrayFromRGB(prevImage);
currPyramids[0]->setGrayFromRGB(currImage);
prevPyramids[1]->setImage(prevImage);
currPyramids[1]->setImage(currImage);
if (param.supportRegionType == SR_CROSS)
{
prevPyramids[1]->setBlurFromRGB(prevImage);
currPyramids[1]->setBlurFromRGB(currImage);
}
}
else
{
prevPyramids[0]->setImage(prevImage);
currPyramids[0]->setImage(currImage);
}
}
preprocess(prevPyramids, currPyramids, prevPoints, currPoints, param);
RLOFOpticalFlowParameter internParam = param;
if (param.useGlobalMotionPrior == true)
internParam.useInitialFlow = true;
calcLocalOpticalFlowCore(prevPyramids, currPyramids, prevPoints, currPoints, internParam);
}
}} // namespace
modules/optflow/src/rlof/rlof_localflow.h
+117
View File
@@ -0,0 +1,117 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef _RLOF_LOCALFLOW_H_
#define _RLOF_LOCALFLOW_H_
#include <limits>
#include <math.h>
#include <float.h>
#include <stdio.h>
#include "opencv2/imgproc.hpp"
#include "opencv2/optflow/rlofflow.hpp"
//! Fast median estimation method based on @cite Tibshirani2008. This implementation relates to http://www.stat.cmu.edu/~ryantibs/median/
using namespace cv;
template<typename T>
T quickselect(const Mat & inp, int k)
{
unsigned long i;
unsigned long ir;
unsigned long j;
unsigned long l;
unsigned long mid;
Mat values = inp.clone();
T a;
l = 0;
ir = MAX(values.rows, values.cols) - 1;
while(true)
{
if (ir <= l + 1)
{
if (ir == l + 1 && values.at<T>(ir) < values.at<T>(l))
std::swap(values.at<T>(l), values.at<T>(ir));
return values.at<T>(k);
}
else
{
mid = (l + ir) >> 1;
std::swap(values.at<T>(mid), values.at<T>(l+1));
if (values.at<T>(l) > values.at<T>(ir))
std::swap(values.at<T>(l), values.at<T>(ir));
if (values.at<T>(l+1) > values.at<T>(ir))
std::swap(values.at<T>(l+1), values.at<T>(ir));
if (values.at<T>(l) > values.at<T>(l+1))
std::swap(values.at<T>(l), values.at<T>(l+1));
i = l + 1;
j = ir;
a = values.at<T>(l+1);
while (true)
{
do
{
i++;
}
while (values.at<T>(i) < a);
do
{
j--;
}
while (values.at<T>(j) > a);
if (j < i) break;
std::swap(values.at<T>(i), values.at<T>(j));
}
values.at<T>(l+1) = values.at<T>(j);
values.at<T>(j) = a;
if (j >= static_cast<unsigned long>(k)) ir = j - 1;
if (j <= static_cast<unsigned long>(k)) l = i;
}
}
}
namespace cv {
namespace optflow {
class CImageBuffer
{
public:
CImageBuffer()
: m_Overwrite(true)
{}
void setGrayFromRGB(const cv::Mat & inp)
{
if(m_Overwrite)
cv::cvtColor(inp, m_Image, cv::COLOR_BGR2GRAY);
}
void setImage(const cv::Mat & inp)
{
if(m_Overwrite)
inp.copyTo(m_Image);
}
void setBlurFromRGB(const cv::Mat & inp)
{
if(m_Overwrite)
cv::GaussianBlur(inp, m_BlurredImage, cv::Size(7,7), -1);
}
int buildPyramid(cv::Size winSize, int maxLevel, float levelScale[2], bool withBlurredImage = false);
cv::Mat & getImage(int level) {return m_ImagePyramid[level];}
std::vector<cv::Mat> m_ImagePyramid;
cv::Mat m_BlurredImage;
cv::Mat m_Image;
std::vector<cv::Mat> m_CrossPyramid;
int m_maxLevel;
bool m_Overwrite;
};
void calcLocalOpticalFlow(
const Mat prevImage,
const Mat currImage,
Ptr<CImageBuffer> prevPyramids[2],
Ptr<CImageBuffer> currPyramids[2],
const std::vector<Point2f> & prevPoints,
std::vector<Point2f> & currPoints,
const RLOFOpticalFlowParameter & param);
}} // namespace
#endif
modules/optflow/src/rlofflow.cpp
+482
View File
@@ -0,0 +1,482 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "precomp.hpp"
#include "rlof/rlof_localflow.h"
#include "rlof/geo_interpolation.hpp"
#include "opencv2/ximgproc.hpp"
namespace cv {
namespace optflow {
Ptr<RLOFOpticalFlowParameter> RLOFOpticalFlowParameter::create()
{
return Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter);
}
void RLOFOpticalFlowParameter::setUseMEstimator(bool val)
{
if (val)
{
normSigma0 = 3.2f;
normSigma1 = 7.f;
}
else
{
normSigma0 = std::numeric_limits<float>::max();
normSigma1 = std::numeric_limits<float>::max();
}
}
void RLOFOpticalFlowParameter::setSolverType(SolverType val){ solverType = val;}
SolverType RLOFOpticalFlowParameter::getSolverType() const { return solverType;}
void RLOFOpticalFlowParameter::setSupportRegionType(SupportRegionType val){ supportRegionType = val;}
SupportRegionType RLOFOpticalFlowParameter::getSupportRegionType() const { return supportRegionType;}
void RLOFOpticalFlowParameter::setNormSigma0(float val){ normSigma0 = val;}
float RLOFOpticalFlowParameter::getNormSigma0() const { return normSigma0;}
void RLOFOpticalFlowParameter::setNormSigma1(float val){ normSigma1 = val;}
float RLOFOpticalFlowParameter::getNormSigma1() const { return normSigma1;}
void RLOFOpticalFlowParameter::setSmallWinSize(int val){ smallWinSize = val;}
int RLOFOpticalFlowParameter::getSmallWinSize() const { return smallWinSize;}
void RLOFOpticalFlowParameter::setLargeWinSize(int val){ largeWinSize = val;}
int RLOFOpticalFlowParameter::getLargeWinSize() const { return largeWinSize;}
void RLOFOpticalFlowParameter::setCrossSegmentationThreshold(int val){ crossSegmentationThreshold = val;}
int RLOFOpticalFlowParameter::getCrossSegmentationThreshold() const { return crossSegmentationThreshold;}
void RLOFOpticalFlowParameter::setMaxLevel(int val){ maxLevel = val;}
int RLOFOpticalFlowParameter::getMaxLevel() const { return maxLevel;}
void RLOFOpticalFlowParameter::setUseInitialFlow(bool val){ useInitialFlow = val;}
bool RLOFOpticalFlowParameter::getUseInitialFlow() const { return useInitialFlow;}
void RLOFOpticalFlowParameter::setUseIlluminationModel(bool val){ useIlluminationModel = val;}
bool RLOFOpticalFlowParameter::getUseIlluminationModel() const { return useIlluminationModel;}
void RLOFOpticalFlowParameter::setUseGlobalMotionPrior(bool val){ useGlobalMotionPrior = val;}
bool RLOFOpticalFlowParameter::getUseGlobalMotionPrior() const { return useGlobalMotionPrior;}
void RLOFOpticalFlowParameter::setMaxIteration(int val){ maxIteration = val;}
int RLOFOpticalFlowParameter::getMaxIteration() const { return maxIteration;}
void RLOFOpticalFlowParameter::setMinEigenValue(float val){ minEigenValue = val;}
float RLOFOpticalFlowParameter::getMinEigenValue() const { return minEigenValue;}
void RLOFOpticalFlowParameter::setGlobalMotionRansacThreshold(float val){ globalMotionRansacThreshold = val;}
float RLOFOpticalFlowParameter::getGlobalMotionRansacThreshold() const { return globalMotionRansacThreshold;}
class DenseOpticalFlowRLOFImpl : public DenseRLOFOpticalFlow
{
public:
DenseOpticalFlowRLOFImpl()
: param(Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter))
, forwardBackwardThreshold(1.f)
, gridStep(6, 6)
, interp_type(InterpolationType::INTERP_GEO)
, k(128)
, sigma(0.05f)
, lambda(999.f)
, fgs_lambda(500.0f)
, fgs_sigma(1.5f)
, use_post_proc(true)
, use_variational_refinement(false)
, sp_size(15)
, slic_type(ximgproc::SLIC)
{
prevPyramid[0] = cv::Ptr<CImageBuffer>(new CImageBuffer);
prevPyramid[1] = cv::Ptr<CImageBuffer>(new CImageBuffer);
currPyramid[0] = cv::Ptr<CImageBuffer>(new CImageBuffer);
currPyramid[1] = cv::Ptr<CImageBuffer>(new CImageBuffer);
}
virtual void setRLOFOpticalFlowParameter(Ptr<RLOFOpticalFlowParameter> val) CV_OVERRIDE { param = val; }
virtual Ptr<RLOFOpticalFlowParameter> getRLOFOpticalFlowParameter() const CV_OVERRIDE { return param; }
virtual float getForwardBackward() const CV_OVERRIDE { return forwardBackwardThreshold; }
virtual void setForwardBackward(float val) CV_OVERRIDE { forwardBackwardThreshold = val; }
virtual void setInterpolation(InterpolationType val) CV_OVERRIDE { interp_type = val; }
virtual InterpolationType getInterpolation() const CV_OVERRIDE { return interp_type; }
virtual Size getGridStep() const CV_OVERRIDE { return gridStep; }
virtual void setGridStep(Size val) CV_OVERRIDE { gridStep = val; }
virtual int getEPICK() const CV_OVERRIDE { return k; }
virtual void setEPICK(int val) CV_OVERRIDE { k = val; }
virtual float getEPICSigma() const CV_OVERRIDE { return sigma; }
virtual void setEPICSigma(float val) CV_OVERRIDE { sigma = val; }
virtual float getEPICLambda() const CV_OVERRIDE { return lambda; }
virtual void setEPICLambda(float val) CV_OVERRIDE { lambda = val; }
virtual float getFgsLambda() const CV_OVERRIDE { return fgs_lambda; }
virtual void setFgsLambda(float val) CV_OVERRIDE { fgs_lambda = val; }
virtual float getFgsSigma() const CV_OVERRIDE { return fgs_sigma; }
virtual void setFgsSigma(float val) CV_OVERRIDE { fgs_sigma = val; }
virtual bool getUsePostProc() const CV_OVERRIDE { return use_post_proc; }
virtual void setUsePostProc(bool val) CV_OVERRIDE { use_post_proc = val; }
virtual void setUseVariationalRefinement(bool val) CV_OVERRIDE { use_variational_refinement = val; }
virtual bool getUseVariationalRefinement() const CV_OVERRIDE { return use_variational_refinement; }
virtual void setRICSPSize(int val) CV_OVERRIDE { sp_size = val; }
virtual int getRICSPSize() const CV_OVERRIDE { return sp_size; }
virtual void setRICSLICType(int val) CV_OVERRIDE { slic_type = static_cast<ximgproc::SLICType>(val); }
virtual int getRICSLICType() const CV_OVERRIDE { return slic_type; }
virtual void calc(InputArray I0, InputArray I1, InputOutputArray flow) CV_OVERRIDE
{
CV_Assert(!I0.empty() && I0.depth() == CV_8U && (I0.channels() == 3 || I0.channels() == 1));
CV_Assert(!I1.empty() && I1.depth() == CV_8U && (I1.channels() == 3 || I1.channels() == 1));
CV_Assert(I0.sameSize(I1));
if (param.empty())
param = Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter());
if (param->supportRegionType == SR_CROSS)
CV_Assert( I0.channels() == 3 && I1.channels() == 3);
CV_Assert(interp_type == InterpolationType::INTERP_EPIC || interp_type == InterpolationType::INTERP_GEO || interp_type == InterpolationType::INTERP_RIC);
// if no parameter is used use the default parameter
Mat prevImage = I0.getMat();
Mat currImage = I1.getMat();
int noPoints = prevImage.cols * prevImage.rows;
std::vector<cv::Point2f> prevPoints(noPoints);
std::vector<cv::Point2f> currPoints, refPoints;
noPoints = 0;
cv::Size grid_h = gridStep / 2;
for (int r = grid_h.height; r < prevImage.rows - grid_h.height; r += gridStep.height)
{
for (int c = grid_h.width; c < prevImage.cols - grid_h.width; c += gridStep.width)
{
prevPoints[noPoints++] = cv::Point2f(static_cast<float>(c), static_cast<float>(r));
}
}
prevPoints.erase(prevPoints.begin() + noPoints, prevPoints.end());
currPoints.resize(prevPoints.size());
calcLocalOpticalFlow(prevImage, currImage, prevPyramid, currPyramid, prevPoints, currPoints, *(param.get()));
flow.create(prevImage.size(), CV_32FC2);
Mat dense_flow = flow.getMat();
std::vector<Point2f> filtered_prevPoints;
std::vector<Point2f> filtered_currPoints;
if (gridStep == cv::Size(1, 1) && forwardBackwardThreshold <= 0)
{
for (unsigned int n = 0; n < prevPoints.size(); n++)
{
dense_flow.at<Point2f>(prevPoints[n]) = currPoints[n] - prevPoints[n];
}
return;
}
if (forwardBackwardThreshold > 0)
{
// reuse image pyramids
calcLocalOpticalFlow(currImage, prevImage, currPyramid, prevPyramid, currPoints, refPoints, *(param.get()));
filtered_prevPoints.resize(prevPoints.size());
filtered_currPoints.resize(prevPoints.size());
float sqrForwardBackwardThreshold = forwardBackwardThreshold * forwardBackwardThreshold;
noPoints = 0;
for (unsigned int r = 0; r < refPoints.size(); r++)
{
Point2f diff = refPoints[r] - prevPoints[r];
if (diff.x * diff.x + diff.y * diff.y < sqrForwardBackwardThreshold)
{
filtered_prevPoints[noPoints] = prevPoints[r];
filtered_currPoints[noPoints++] = currPoints[r];
}
}
filtered_prevPoints.erase(filtered_prevPoints.begin() + noPoints, filtered_prevPoints.end());
filtered_currPoints.erase(filtered_currPoints.begin() + noPoints, filtered_currPoints.end());
}
else
{
filtered_prevPoints = prevPoints;
filtered_currPoints = currPoints;
}
// Interpolators below expect non empty matches
if (filtered_prevPoints.empty()) {
flow.setTo(0);
return;
}
if (interp_type == InterpolationType::INTERP_EPIC)
{
Ptr<ximgproc::EdgeAwareInterpolator> gd = ximgproc::createEdgeAwareInterpolator();
gd->setK(k);
gd->setSigma(sigma);
gd->setLambda(lambda);
gd->setFGSLambda(fgs_lambda);
gd->setFGSSigma(fgs_sigma);
gd->setUsePostProcessing(use_post_proc);
gd->interpolate(prevImage, filtered_prevPoints, currImage, filtered_currPoints, dense_flow);
}
else if (interp_type == InterpolationType::INTERP_RIC)
{
Ptr<ximgproc::RICInterpolator> gd = ximgproc::createRICInterpolator();
gd->setK(k);
gd->setFGSLambda(fgs_lambda);
gd->setFGSSigma(fgs_sigma);
gd->setSuperpixelSize(sp_size);
gd->setSuperpixelMode(slic_type);
gd->setUseGlobalSmootherFilter(use_post_proc);
gd->setUseVariationalRefinement(false);
gd->interpolate(prevImage, filtered_prevPoints, currImage, filtered_currPoints, dense_flow);
}
else
{
Mat blurredPrevImage, blurredCurrImage;
GaussianBlur(prevImage, blurredPrevImage, cv::Size(5, 5), -1);
std::vector<uchar> status(filtered_currPoints.size(), 1);
interpolate_irregular_nn_raster(filtered_prevPoints, filtered_currPoints, status, blurredPrevImage).copyTo(dense_flow);
std::vector<Mat> vecMats;
std::vector<Mat> vecMats2(2);
cv::split(dense_flow, vecMats);
cv::bilateralFilter(vecMats[0], vecMats2[0], 5, 2, 20);
cv::bilateralFilter(vecMats[1], vecMats2[1], 5, 2, 20);
cv::merge(vecMats2, dense_flow);
if (use_post_proc)
{
ximgproc::fastGlobalSmootherFilter(prevImage, flow, flow, fgs_lambda, fgs_sigma);
}
}
if (use_variational_refinement)
{
Mat prevGrey, currGrey;
Ptr<VariationalRefinement > variationalrefine = VariationalRefinement::create();
cvtColor(prevImage, prevGrey, COLOR_BGR2GRAY);
cvtColor(currImage, currGrey, COLOR_BGR2GRAY);
variationalrefine->setOmega(1.9f);
variationalrefine->calc(prevGrey, currGrey, flow);
}
}
virtual void collectGarbage() CV_OVERRIDE
{
prevPyramid[0].release();
prevPyramid[1].release();
currPyramid[0].release();
currPyramid[1].release();
}
protected:
Ptr<RLOFOpticalFlowParameter> param;
float forwardBackwardThreshold;
Ptr<CImageBuffer> prevPyramid[2];
Ptr<CImageBuffer> currPyramid[2];
cv::Size gridStep;
InterpolationType interp_type;
int k;
float sigma;
float lambda;
float fgs_lambda;
float fgs_sigma;
bool use_post_proc;
bool use_variational_refinement;
int sp_size;
ximgproc::SLICType slic_type;
};
Ptr<DenseRLOFOpticalFlow> DenseRLOFOpticalFlow::create(
Ptr<RLOFOpticalFlowParameter> rlofParam,
float forwardBackwardThreshold,
cv::Size gridStep,
InterpolationType interp_type,
int epicK,
float epicSigma,
float epicLambda,
int ricSPSize,
int ricSLICType,
bool use_post_proc,
float fgs_lambda,
float fgs_sigma,
bool use_variational_refinement)
{
Ptr<DenseRLOFOpticalFlow> algo = makePtr<DenseOpticalFlowRLOFImpl>();
algo->setRLOFOpticalFlowParameter(rlofParam);
algo->setForwardBackward(forwardBackwardThreshold);
algo->setGridStep(gridStep);
algo->setInterpolation(interp_type);
algo->setEPICK(epicK);
algo->setEPICSigma(epicSigma);
algo->setEPICLambda(epicLambda);
algo->setUsePostProc(use_post_proc);
algo->setFgsLambda(fgs_lambda);
algo->setFgsSigma(fgs_sigma);
algo->setRICSLICType(ricSLICType);
algo->setRICSPSize(ricSPSize);
algo->setUseVariationalRefinement(use_variational_refinement);
return algo;
}
class SparseRLOFOpticalFlowImpl : public SparseRLOFOpticalFlow
{
public:
SparseRLOFOpticalFlowImpl()
: param(Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter))
, forwardBackwardThreshold(1.f)
{
prevPyramid[0] = cv::Ptr< CImageBuffer>(new CImageBuffer);
prevPyramid[1] = cv::Ptr< CImageBuffer>(new CImageBuffer);
currPyramid[0] = cv::Ptr< CImageBuffer>(new CImageBuffer);
currPyramid[1] = cv::Ptr< CImageBuffer>(new CImageBuffer);
}
virtual void setRLOFOpticalFlowParameter(Ptr<RLOFOpticalFlowParameter> val) CV_OVERRIDE { param = val; }
virtual Ptr<RLOFOpticalFlowParameter> getRLOFOpticalFlowParameter() const CV_OVERRIDE { return param; }
virtual float getForwardBackward() const CV_OVERRIDE { return forwardBackwardThreshold; }
virtual void setForwardBackward(float val) CV_OVERRIDE { forwardBackwardThreshold = val; }
virtual void calc(InputArray prevImg, InputArray nextImg,
InputArray prevPts, InputOutputArray nextPts,
OutputArray status,
OutputArray err) CV_OVERRIDE
{
CV_Assert(!prevImg.empty() && prevImg.depth() == CV_8U && (prevImg.channels() == 3 || prevImg.channels() == 1));
CV_Assert(!nextImg.empty() && nextImg.depth() == CV_8U && (nextImg.channels() == 3 || nextImg.channels() == 1));
CV_Assert(prevImg.sameSize(nextImg));
if (param.empty())
{
param = makePtr<RLOFOpticalFlowParameter>();
}
CV_DbgAssert(!param.empty());
if (param->supportRegionType == SR_CROSS)
{
CV_CheckChannelsEQ(prevImg.channels(), 3, "SR_CROSS mode requires images with 3 channels");
CV_CheckChannelsEQ(nextImg.channels(), 3, "SR_CROSS mode requires images with 3 channels");
}
Mat prevImage = prevImg.getMat();
Mat nextImage = nextImg.getMat();
Mat prevPtsMat = prevPts.getMat();
if (param->useInitialFlow == false)
nextPts.create(prevPtsMat.size(), prevPtsMat.type(), -1, true);
int npoints = 0;
CV_Assert((npoints = prevPtsMat.checkVector(2, CV_32F, true)) >= 0);
if (npoints == 0)
{
nextPts.release();
status.release();
err.release();
return;
}
Mat nextPtsMat = nextPts.getMat();
CV_Assert(nextPtsMat.checkVector(2, CV_32F, true) == npoints);
std::vector<cv::Point2f> prevPoints(npoints), nextPoints(npoints), refPoints;
if (prevPtsMat.channels() != 2)
prevPtsMat = prevPtsMat.reshape(2, npoints);
prevPtsMat.copyTo(prevPoints);
if (param->useInitialFlow )
{
if (nextPtsMat.channels() != 2)
nextPtsMat = nextPtsMat.reshape(2, npoints);
nextPtsMat.copyTo(nextPoints);
}
cv::Mat statusMat;
cv::Mat errorMat;
if (status.needed() || forwardBackwardThreshold > 0)
{
status.create((int)npoints, 1, CV_8U, -1, true);
statusMat = status.getMat();
statusMat.setTo(1);
}
if (err.needed() || forwardBackwardThreshold > 0)
{
err.create((int)npoints, 1, CV_32F, -1, true);
errorMat = err.getMat();
errorMat.setTo(0);
}
calcLocalOpticalFlow(prevImage, nextImage, prevPyramid, currPyramid, prevPoints, nextPoints, *(param.get()));
cv::Mat(1,npoints , CV_32FC2, &nextPoints[0]).copyTo(nextPtsMat);
if (forwardBackwardThreshold > 0)
{
// use temp variable to properly initialize refPoints
// inside 'calcLocalOpticalFlow' when 'use_init_flow' and 'fwd_bwd_thresh' parameters are used
bool temp_param = param->getUseInitialFlow();
param->setUseInitialFlow(false);
// reuse image pyramids
calcLocalOpticalFlow(nextImage, prevImage, currPyramid, prevPyramid, nextPoints, refPoints, *(param.get()));
param->setUseInitialFlow(temp_param);
}
for (unsigned int r = 0; r < refPoints.size(); r++)
{
Point2f diff = refPoints[r] - prevPoints[r];
errorMat.at<float>(r) = sqrt(diff.x * diff.x + diff.y * diff.y);
if (errorMat.at<float>(r) > forwardBackwardThreshold)
statusMat.at<uchar>(r) = 0;
}
}
protected:
Ptr<RLOFOpticalFlowParameter> param;
float forwardBackwardThreshold;
Ptr<CImageBuffer> prevPyramid[2];
Ptr<CImageBuffer> currPyramid[2];
};
Ptr<SparseRLOFOpticalFlow> SparseRLOFOpticalFlow::create(
Ptr<RLOFOpticalFlowParameter> rlofParam,
float forwardBackwardThreshold)
{
Ptr<SparseRLOFOpticalFlow> algo = makePtr<SparseRLOFOpticalFlowImpl>();
algo->setRLOFOpticalFlowParameter(rlofParam);
algo->setForwardBackward(forwardBackwardThreshold);
return algo;
}
void calcOpticalFlowDenseRLOF(InputArray I0, InputArray I1, InputOutputArray flow,
Ptr<RLOFOpticalFlowParameter> rlofParam ,
float forewardBackwardThreshold, Size gridStep,
InterpolationType interp_type,
int epicK, float epicSigma, float epicLambda,
int superpixelSize, int superpixelType,
bool use_post_proc, float fgsLambda, float fgsSigma, bool use_variational_refinement)
{
Ptr<DenseRLOFOpticalFlow> algo = DenseRLOFOpticalFlow::create(
rlofParam, forewardBackwardThreshold, gridStep, interp_type,
epicK, epicSigma, epicLambda, superpixelSize, superpixelType,
use_post_proc, fgsLambda, fgsSigma, use_variational_refinement);
algo->calc(I0, I1, flow);
algo->collectGarbage();
}
void calcOpticalFlowSparseRLOF(InputArray prevImg, InputArray nextImg,
InputArray prevPts, InputOutputArray nextPts,
OutputArray status, OutputArray err,
Ptr<RLOFOpticalFlowParameter> rlofParam,
float forewardBackwardThreshold)
{
Ptr<SparseRLOFOpticalFlow> algo = SparseRLOFOpticalFlow::create(
rlofParam, forewardBackwardThreshold);
algo->calc(prevImg, nextImg, prevPts, nextPts, status, err);
}
Ptr<DenseOpticalFlow> createOptFlow_DenseRLOF()
{
return DenseRLOFOpticalFlow::create();
}
Ptr<SparseOpticalFlow> createOptFlow_SparseRLOF()
{
return SparseRLOFOpticalFlow::create();
}
}} // namespace
modules/optflow/src/simpleflow.cpp
+779
View File
@@ -0,0 +1,779 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
#ifdef _MSC_VER
# pragma warning(disable: 4512)
#endif
//
// 2D dense optical flow algorithm from the following paper:
// Michael Tao, Jiamin Bai, Pushmeet Kohli, and Sylvain Paris.
// "SimpleFlow: A Non-iterative, Sublinear Optical Flow Algorithm"
// Computer Graphics Forum (Eurographics 2012)
// http://graphics.berkeley.edu/papers/Tao-SAN-2012-05/
//
namespace cv
{
namespace optflow
{
static const uchar MASK_TRUE_VALUE = (uchar)255;
inline static int dist(const Vec3b &p1, const Vec3b &p2) {
int a = p1[0] - p2[0];
int b = p1[1] - p2[1];
int c = p1[2] - p2[2];
return a*a+b*b+c*c;
}
inline static float dist(const Vec2f &p1, const Vec2f &p2) {
float a = p1[0] - p2[0];
float b = p1[1] - p2[1];
return a*a+b*b;
}
template<class T>
inline static T min(T t1, T t2, T t3) {
return (t1 <= t2 && t1 <= t3) ? t1 : min(t2, t3);
}
static void removeOcclusions(const Mat& flow,
const Mat& flow_inv,
float occ_thr,
Mat& confidence) {
const int rows = flow.rows;
const int cols = flow.cols;
if (!confidence.data) {
confidence = Mat::zeros(rows, cols, CV_32F);
}
for (int r = 0; r < rows; ++r) {
for (int c = 0; c < cols; ++c) {
if (dist(flow.at<Vec2f>(r, c), -flow_inv.at<Vec2f>(r, c)) > occ_thr) {
confidence.at<float>(r, c) = 0;
} else {
confidence.at<float>(r, c) = 1;
}
}
}
}
static void wd(Mat& d, int top_shift, int bottom_shift, int left_shift, int right_shift, double sigma) {
for (int dr = -top_shift, r = 0; dr <= bottom_shift; ++dr, ++r) {
for (int dc = -left_shift, c = 0; dc <= right_shift; ++dc, ++c) {
d.at<float>(r, c) = (float)-(dr*dr + dc*dc);
}
}
d *= 1.0 / (2.0 * sigma * sigma);
exp(d, d);
}
template<typename JointVec, typename SrcVec>
class CrossBilateralFilter : public ParallelLoopBody {
Mat &joint, &confidence, &src;
Mat &dst;
int radius;
bool flag;
Mat &spaceWeights;
std::vector<double> &expLut;
public:
CrossBilateralFilter(Mat &joint_, Mat &confidence_, Mat &src_, Mat &dst_, int radius_, bool flag_, Mat &spaceWeights_, std::vector<double> &expLut_)
:
joint(joint_),
confidence(confidence_),
src(src_),
dst(dst_),
radius(radius_),
flag(flag_),
spaceWeights(spaceWeights_),
expLut(expLut_) {
CV_DbgAssert(joint.type() == traits::Type<JointVec>::value && confidence.type() == CV_32F && src.type() == dst.type() && src.type() == traits::Type<SrcVec>::value);
CV_DbgAssert(joint.rows == src.rows && confidence.rows == src.rows && src.rows == dst.rows + 2 * radius);
CV_DbgAssert(joint.cols == src.cols && confidence.cols == src.cols && src.cols == dst.cols + 2 * radius);
}
void operator()(const Range &range) const CV_OVERRIDE {
const int d = 2*radius +1;
for (int i = range.start; i < range.end; i++) {
SrcVec* dstRow = dst.ptr<SrcVec>(i);
for (int j = 0; j < dst.cols; j++) {
const JointVec& centeralPoint = joint.at<JointVec>(i+radius, j+radius);
Scalar totalSum = Scalar::all(0);
double weightsSum = 0;
for (int dr = i, r = 0; dr < i + d; ++dr, ++r) {
const JointVec *jointRow = joint.ptr<JointVec>(dr);
const SrcVec *srcRow = src.ptr<SrcVec>(dr);
const float *confidenceRow = confidence.ptr<float>(dr);
const float *spaceWeightsRow = spaceWeights.ptr<float>(r);
for (int dc = j, c = 0; dc < j + d; ++dc, ++c) {
double weight = spaceWeightsRow[c]*confidenceRow[dc];
for (int cn = 0; cn < JointVec::channels; cn++) {
weight *= expLut[std::abs(centeralPoint[cn] - jointRow[dc][cn])];
}
for (int cn = 0; cn < SrcVec::channels; cn++) {
totalSum[cn] += weight * srcRow[dc][cn];
}
weightsSum += weight;
}
}
SrcVec& srcSum = dstRow[j];
for (int cn = 0; cn < SrcVec::channels; cn++) {
srcSum[cn] = (flag && fabs(weightsSum) < 1e-9)
? src.at<SrcVec>(i+radius, j+radius)[cn]
: static_cast<float>(totalSum[cn] / weightsSum);
}
}
}
}
};
static void crossBilateralFilter(InputArray joint_,
InputArray confidence_,
InputOutputArray src_,
int radius,
double sigmaColor, double sigmaSpace,
bool flag = false) {
CV_Assert(!src_.empty());
CV_Assert(!confidence_.empty());
CV_Assert(!joint_.empty());
Mat src = src_.getMat();
Mat joint = joint_.getMat();
Mat confidence = confidence_.getMat();
CV_Assert(src.size() == joint.size() && confidence.size() == src.size());
CV_Assert(joint.depth() == CV_8U && confidence.type() == CV_32F);
if (sigmaColor <= 0)
sigmaColor = 1;
if (sigmaSpace <= 0)
sigmaSpace = 1;
if (radius <= 0)
radius = cvRound(sigmaSpace * 1.5);
radius = std::max(radius, 1);
if (src.data == joint.data)
joint = joint.clone();
int d = 2 * radius + 1;
Mat jointTemp, confidenceTemp, srcTemp;
copyMakeBorder(joint, jointTemp, radius, radius, radius, radius, BORDER_DEFAULT);
copyMakeBorder(confidence, confidenceTemp, radius, radius, radius, radius, BORDER_CONSTANT, Scalar(0));
copyMakeBorder(src, srcTemp, radius, radius, radius, radius, BORDER_DEFAULT);
Mat spaceWeights(d, d, CV_32F);
wd(spaceWeights, radius, radius, radius, radius, sigmaSpace);
double gaussColorCoeff = -0.5 / (sigmaColor * sigmaColor);
std::vector<double> expLut (256);
for (size_t i = 0; i < expLut.size(); i++) {
expLut[i] = std::exp(i * i * gaussColorCoeff);
}
Range range(0, src.rows);
parallel_for_(range, CrossBilateralFilter<Vec3b, Vec2f>(jointTemp, confidenceTemp, srcTemp, src, radius, flag, spaceWeights, expLut));
}
static void calcConfidence(const Mat& prev,
const Mat& next,
const Mat& flow,
Mat& confidence,
int max_flow) {
const int rows = prev.rows;
const int cols = prev.cols;
confidence = Mat::zeros(rows, cols, CV_32F);
for (int r0 = 0; r0 < rows; ++r0) {
for (int c0 = 0; c0 < cols; ++c0) {
Vec2f flow_at_point = flow.at<Vec2f>(r0, c0);
int u0 = cvRound(flow_at_point[0]);
if (r0 + u0 < 0) { u0 = -r0; }
if (r0 + u0 >= rows) { u0 = rows - 1 - r0; }
int v0 = cvRound(flow_at_point[1]);
if (c0 + v0 < 0) { v0 = -c0; }
if (c0 + v0 >= cols) { v0 = cols - 1 - c0; }
const int top_row_shift = -std::min(r0 + u0, max_flow);
const int bottom_row_shift = std::min(rows - 1 - (r0 + u0), max_flow);
const int left_col_shift = -std::min(c0 + v0, max_flow);
const int right_col_shift = std::min(cols - 1 - (c0 + v0), max_flow);
bool first_flow_iteration = true;
int sum_e = 0, min_e = 0;
for (int u = top_row_shift; u <= bottom_row_shift; ++u) {
for (int v = left_col_shift; v <= right_col_shift; ++v) {
int e = dist(prev.at<Vec3b>(r0, c0), next.at<Vec3b>(r0 + u0 + u, c0 + v0 + v));
if (first_flow_iteration) {
sum_e = e;
min_e = e;
first_flow_iteration = false;
} else {
sum_e += e;
min_e = std::min(min_e, e);
}
}
}
int windows_square = (bottom_row_shift - top_row_shift + 1) *
(right_col_shift - left_col_shift + 1);
confidence.at<float>(r0, c0) = (windows_square == 0) ? 0
: static_cast<float>(sum_e) / windows_square - min_e;
CV_Assert(confidence.at<float>(r0, c0) >= 0);
}
}
}
template<typename SrcVec, typename DstVec>
class CalcOpticalFlowSingleScaleSF : public ParallelLoopBody {
Mat &prev, &next;
Mat &mask;
Mat &dst;
int radius, maxFlow;
Mat &spaceWeights;
std::vector<double> &expLut;
public:
CalcOpticalFlowSingleScaleSF(Mat &prev_, Mat &next_, Mat &mask_, Mat &dst_, int radius_, int maxFlow_, Mat &spaceWeights_, std::vector<double> &expLut_)
:
prev(prev_),
next(next_),
mask(mask_),
dst(dst_),
radius(radius_),
maxFlow(maxFlow_),
spaceWeights(spaceWeights_),
expLut(expLut_){
CV_DbgAssert(prev.type() == next.type());
CV_DbgAssert(prev.rows == next.rows && prev.rows == dst.rows + 2 * radius);
CV_DbgAssert(prev.cols == next.cols && next.cols == dst.cols + 2 * radius);
}
void operator()(const Range &range) const CV_OVERRIDE {
int d = 2 * radius + 1;
Mat weights(d, d, CV_32F);
for (int i = range.start; i < range.end; i++) {
const uchar *maskRow = mask.ptr<uchar>(i);
const DstVec *dstRow = dst.ptr<DstVec>(i);
for (int j = 0; j < dst.cols; j++) {
if (!maskRow[j]) {
continue;
}
// TODO: do smth with this creepy staff
const DstVec &flowAtPoint = dstRow[j];
int u0 = cvRound(flowAtPoint[0]);
if (i + u0 < 0) {u0 = -i;}
if (i + u0 >= dst.rows) {u0 = dst.rows - 1 - i;}
int v0 = cvRound(flowAtPoint[1]);
if (j + v0 < 0) {v0 = -j;}
if (j + v0 >= dst.cols) {v0 = dst.cols - 1 - j;}
const int topRowShift = -std::min(i + u0, maxFlow);
const int bottomRowShift = std::min(dst.rows - 1 - (i + u0), maxFlow);
const int leftColShift = -std::min(j + v0, maxFlow);
const int rightColShift = std::min(dst.cols - 1 - (j + v0), maxFlow);
float minCost = FLT_MAX, bestU = (float) u0, bestV = (float) v0;
const SrcVec& centeralPoint = prev.at<SrcVec>(i+radius, j+radius);
int left_border = j, right_border = j + d;
for (int dr = i, r = 0; dr < i + d; ++dr, ++r) {
const SrcVec *prevRow = prev.ptr<SrcVec>(dr);
const float* spaceWeightsRow = spaceWeights.ptr<float>(r);
float *weightsRow = weights.ptr<float>(r);
for (int dc = left_border, c = 0; dc < right_border; ++dc, ++c) {
double weight = spaceWeightsRow[c];
for(int cn=0;cn<SrcVec::channels;cn++){
weight *= expLut[std::abs(centeralPoint[cn]-prevRow[dc][cn])];
}
weightsRow[c] = static_cast<float>(weight);
}
}
for (int u = topRowShift; u <= bottomRowShift; ++u) {
const int next_extended_top_window_row = i + u0 + u;
for (int v = leftColShift; v <= rightColShift; ++v) {
const int next_extended_left_window_col = j + v0 + v;
float cost = 0;
for (int r = 0; r < d; ++r) {
const SrcVec *prev_extended_window_row = prev.ptr<SrcVec>(i + r);
const SrcVec *next_extended_window_row = next.ptr<SrcVec>(next_extended_top_window_row + r);
const float *weight_window_row = weights.ptr<float>(r);
for (int c = 0; c < d; ++c) {
cost += weight_window_row[c] *
dist(prev_extended_window_row[j + c],
next_extended_window_row[next_extended_left_window_col + c]);
}
}
// cost should be divided by sum(weight_window), but because
// we interested only in min(cost) and sum(weight_window) is constant
// for every point - we remove it
if (cost < minCost) {
minCost = cost;
bestU = (float) (u + u0);
bestV = (float) (v + v0);
}
}
}
dst.at<DstVec>(i, j) = DstVec(bestU, bestV);
}
}
}
};
static void calcOpticalFlowSingleScaleSF(InputArray prev_,
InputArray next_,
InputArray mask_,
InputOutputArray dst_,
int radius,
int max_flow,
float sigmaSpace,
float sigmaColor) {
Mat prev = prev_.getMat();
Mat next = next_.getMat();
Mat mask = mask_.getMat();
Mat dst = dst_.getMat();
Mat prevTemp, nextTemp;
copyMakeBorder(prev, prevTemp, radius, radius, radius, radius, BORDER_DEFAULT);
copyMakeBorder(next, nextTemp, radius, radius, radius, radius, BORDER_DEFAULT);
int d = 2 * radius + 1;
Mat spaceWeights(d, d, CV_32F);
wd(spaceWeights, radius, radius, radius, radius, sigmaSpace);
double gaussColorCoeff = -0.5 / (sigmaColor * sigmaColor);
std::vector<double> expLut (256);
for (size_t i = 0; i < expLut.size(); i++) {
expLut[i] = std::exp(i * i * gaussColorCoeff);
}
Range range(0, dst.rows);
parallel_for_(range, CalcOpticalFlowSingleScaleSF<Vec3b, Vec2f>(prevTemp, nextTemp, mask, dst, radius, max_flow, spaceWeights, expLut));
}
static Mat upscaleOpticalFlow(int new_rows,
int new_cols,
const Mat& image,
const Mat& confidence,
Mat& flow,
int averaging_radius,
float sigma_dist,
float sigma_color) {
crossBilateralFilter(image, confidence, flow, averaging_radius, sigma_color, sigma_dist, true);
Mat new_flow;
resize(flow, new_flow, Size(new_cols, new_rows), 0, 0, INTER_NEAREST);
new_flow *= 2;
return new_flow;
}
static Mat calcIrregularityMat(const Mat& flow, int radius) {
const int rows = flow.rows;
const int cols = flow.cols;
Mat irregularity = Mat::zeros(rows, cols, CV_32F);
for (int r = 0; r < rows; ++r) {
const int start_row = std::max(0, r - radius);
const int end_row = std::min(rows - 1, r + radius);
for (int c = 0; c < cols; ++c) {
const int start_col = std::max(0, c - radius);
const int end_col = std::min(cols - 1, c + radius);
for (int dr = start_row; dr <= end_row; ++dr) {
for (int dc = start_col; dc <= end_col; ++dc) {
const float diff = dist(flow.at<Vec2f>(r, c), flow.at<Vec2f>(dr, dc));
if (diff > irregularity.at<float>(r, c)) {
irregularity.at<float>(r, c) = diff;
}
}
}
}
}
return irregularity;
}
static void selectPointsToRecalcFlow(const Mat& flow,
int irregularity_metric_radius,
float speed_up_thr,
int curr_rows,
int curr_cols,
const Mat& prev_speed_up,
Mat& speed_up,
Mat& mask) {
const int prev_rows = flow.rows;
const int prev_cols = flow.cols;
Mat is_flow_regular = calcIrregularityMat(flow, irregularity_metric_radius)
< speed_up_thr;
Mat done = Mat::zeros(prev_rows, prev_cols, CV_8U);
speed_up = Mat::zeros(curr_rows, curr_cols, CV_8U);
mask = Mat::zeros(curr_rows, curr_cols, CV_8U);
for (int r = 0; r < is_flow_regular.rows; ++r) {
for (int c = 0; c < is_flow_regular.cols; ++c) {
if (!done.at<uchar>(r, c)) {
if (is_flow_regular.at<uchar>(r, c) &&
2*r + 1 < curr_rows && 2*c + 1< curr_cols) {
bool all_flow_in_region_regular = true;
int speed_up_at_this_point = prev_speed_up.at<uchar>(r, c);
int step = (1 << speed_up_at_this_point) - 1;
int prev_top = r;
int prev_bottom = std::min(r + step, prev_rows - 1);
int prev_left = c;
int prev_right = std::min(c + step, prev_cols - 1);
for (int rr = prev_top; rr <= prev_bottom; ++rr) {
for (int cc = prev_left; cc <= prev_right; ++cc) {
done.at<uchar>(rr, cc) = 1;
if (!is_flow_regular.at<uchar>(rr, cc)) {
all_flow_in_region_regular = false;
}
}
}
int curr_top = std::min(2 * r, curr_rows - 1);
int curr_bottom = std::min(2*(r + step) + 1, curr_rows - 1);
int curr_left = std::min(2 * c, curr_cols - 1);
int curr_right = std::min(2*(c + step) + 1, curr_cols - 1);
if (all_flow_in_region_regular &&
curr_top != curr_bottom &&
curr_left != curr_right) {
mask.at<uchar>(curr_top, curr_left) = MASK_TRUE_VALUE;
mask.at<uchar>(curr_bottom, curr_left) = MASK_TRUE_VALUE;
mask.at<uchar>(curr_top, curr_right) = MASK_TRUE_VALUE;
mask.at<uchar>(curr_bottom, curr_right) = MASK_TRUE_VALUE;
for (int rr = curr_top; rr <= curr_bottom; ++rr) {
for (int cc = curr_left; cc <= curr_right; ++cc) {
speed_up.at<uchar>(rr, cc) = (uchar)(speed_up_at_this_point + 1);
}
}
} else {
for (int rr = curr_top; rr <= curr_bottom; ++rr) {
for (int cc = curr_left; cc <= curr_right; ++cc) {
mask.at<uchar>(rr, cc) = MASK_TRUE_VALUE;
}
}
}
} else {
done.at<uchar>(r, c) = 1;
for (int dr = 0; dr <= 1; ++dr) {
int nr = 2*r + dr;
for (int dc = 0; dc <= 1; ++dc) {
int nc = 2*c + dc;
if (nr < curr_rows && nc < curr_cols) {
mask.at<uchar>(nr, nc) = MASK_TRUE_VALUE;
}
}
}
}
}
}
}
}
static inline float extrapolateValueInRect(int height, int width,
float v11, float v12,
float v21, float v22,
int r, int c) {
if (r == 0 && c == 0) { return v11;}
if (r == 0 && c == width) { return v12;}
if (r == height && c == 0) { return v21;}
if (r == height && c == width) { return v22;}
CV_Assert(height > 0 && width > 0);
float qr = float(r) / height;
float pr = 1.0f - qr;
float qc = float(c) / width;
float pc = 1.0f - qc;
return v11*pr*pc + v12*pr*qc + v21*qr*pc + v22*qc*qr;
}
static void extrapolateFlow(Mat& flow,
const Mat& speed_up) {
const int rows = flow.rows;
const int cols = flow.cols;
Mat done = Mat::zeros(rows, cols, CV_8U);
for (int r = 0; r < rows; ++r) {
for (int c = 0; c < cols; ++c) {
if (!done.at<uchar>(r, c) && speed_up.at<uchar>(r, c) > 1) {
int step = (1 << speed_up.at<uchar>(r, c)) - 1;
int top = r;
int bottom = std::min(r + step, rows - 1);
int left = c;
int right = std::min(c + step, cols - 1);
int height = bottom - top;
int width = right - left;
for (int rr = top; rr <= bottom; ++rr) {
for (int cc = left; cc <= right; ++cc) {
done.at<uchar>(rr, cc) = 1;
Vec2f flow_at_point;
Vec2f top_left = flow.at<Vec2f>(top, left);
Vec2f top_right = flow.at<Vec2f>(top, right);
Vec2f bottom_left = flow.at<Vec2f>(bottom, left);
Vec2f bottom_right = flow.at<Vec2f>(bottom, right);
flow_at_point[0] = extrapolateValueInRect(height, width,
top_left[0], top_right[0],
bottom_left[0], bottom_right[0],
rr-top, cc-left);
flow_at_point[1] = extrapolateValueInRect(height, width,
top_left[1], top_right[1],
bottom_left[1], bottom_right[1],
rr-top, cc-left);
flow.at<Vec2f>(rr, cc) = flow_at_point;
}
}
}
}
}
}
static void buildPyramidWithResizeMethod(const Mat& src,
std::vector<Mat>& pyramid,
int layers,
int interpolation_type) {
pyramid.push_back(src);
for (int i = 1; i <= layers; ++i) {
Mat prev = pyramid[i - 1];
if (prev.rows <= 1 || prev.cols <= 1) {
break;
}
Mat next;
resize(prev, next, Size((prev.cols + 1) / 2, (prev.rows + 1) / 2), 0, 0, interpolation_type);
pyramid.push_back(next);
}
}
CV_EXPORTS_W void calcOpticalFlowSF(InputArray _from,
InputArray _to,
OutputArray _resulted_flow,
int layers,
int averaging_radius,
int max_flow,
double sigma_dist,
double sigma_color,
int postprocess_window,
double sigma_dist_fix,
double sigma_color_fix,
double occ_thr,
int upscale_averaging_radius,
double upscale_sigma_dist,
double upscale_sigma_color,
double speed_up_thr)
{
Mat from = _from.getMat();
Mat to = _to.getMat();
std::vector<Mat> pyr_from_images;
std::vector<Mat> pyr_to_images;
buildPyramidWithResizeMethod(from, pyr_from_images, layers - 1, INTER_CUBIC);
buildPyramidWithResizeMethod(to, pyr_to_images, layers - 1, INTER_CUBIC);
CV_Assert((int)pyr_from_images.size() == layers && (int)pyr_to_images.size() == layers);
Mat curr_from, curr_to, prev_from, prev_to;
curr_from = pyr_from_images[layers - 1];
curr_to = pyr_to_images[layers - 1];
Mat mask = Mat::ones(curr_from.size(), CV_8U);
Mat mask_inv = Mat::ones(curr_from.size(), CV_8U);
Mat flow = Mat::zeros(curr_from.size(), CV_32FC2);
Mat flow_inv = Mat::zeros(curr_to.size(), CV_32FC2);
Mat confidence;
Mat confidence_inv;
calcOpticalFlowSingleScaleSF(curr_from,
curr_to,
mask,
flow,
averaging_radius,
max_flow,
(float)sigma_dist,
(float)sigma_color);
calcOpticalFlowSingleScaleSF(curr_to,
curr_from,
mask_inv,
flow_inv,
averaging_radius,
max_flow,
(float)sigma_dist,
(float)sigma_color);
removeOcclusions(flow,
flow_inv,
(float)occ_thr,
confidence);
removeOcclusions(flow_inv,
flow,
(float)occ_thr,
confidence_inv);
Mat speed_up = Mat::zeros(curr_from.size(), CV_8U);
Mat speed_up_inv = Mat::zeros(curr_from.size(), CV_8U);
for (int curr_layer = layers - 2; curr_layer >= 0; --curr_layer) {
curr_from = pyr_from_images[curr_layer];
curr_to = pyr_to_images[curr_layer];
prev_from = pyr_from_images[curr_layer + 1];
prev_to = pyr_to_images[curr_layer + 1];
const int curr_rows = curr_from.rows;
const int curr_cols = curr_from.cols;
Mat new_speed_up, new_speed_up_inv;
selectPointsToRecalcFlow(flow,
averaging_radius,
(float)speed_up_thr,
curr_rows,
curr_cols,
speed_up,
new_speed_up,
mask);
selectPointsToRecalcFlow(flow_inv,
averaging_radius,
(float)speed_up_thr,
curr_rows,
curr_cols,
speed_up_inv,
new_speed_up_inv,
mask_inv);
speed_up = new_speed_up;
speed_up_inv = new_speed_up_inv;
flow = upscaleOpticalFlow(curr_rows,
curr_cols,
prev_from,
confidence,
flow,
upscale_averaging_radius,
(float)upscale_sigma_dist,
(float)upscale_sigma_color);
flow_inv = upscaleOpticalFlow(curr_rows,
curr_cols,
prev_to,
confidence_inv,
flow_inv,
upscale_averaging_radius,
(float)upscale_sigma_dist,
(float)upscale_sigma_color);
calcConfidence(curr_from, curr_to, flow, confidence, max_flow);
calcOpticalFlowSingleScaleSF(curr_from,
curr_to,
mask,
flow,
averaging_radius,
max_flow,
(float)sigma_dist,
(float)sigma_color);
calcConfidence(curr_to, curr_from, flow_inv, confidence_inv, max_flow);
calcOpticalFlowSingleScaleSF(curr_to,
curr_from,
mask_inv,
flow_inv,
averaging_radius,
max_flow,
(float)sigma_dist,
(float)sigma_color);
extrapolateFlow(flow, speed_up);
extrapolateFlow(flow_inv, speed_up_inv);
//TODO: should we remove occlusions for the last stage?
removeOcclusions(flow, flow_inv, (float)occ_thr, confidence);
removeOcclusions(flow_inv, flow, (float)occ_thr, confidence_inv);
}
crossBilateralFilter(curr_from, confidence, flow,
postprocess_window, (float)sigma_color_fix, (float)sigma_dist_fix);
GaussianBlur(flow, flow, Size(3, 3), 5);
_resulted_flow.create(flow.size(), CV_32FC2);
Mat resulted_flow = _resulted_flow.getMat();
int from_to[] = {0,1 , 1,0};
mixChannels(&flow, 1, &resulted_flow, 1, from_to, 2);
}
CV_EXPORTS_W void calcOpticalFlowSF(InputArray from,
InputArray to,
OutputArray flow,
int layers,
int averaging_block_size,
int max_flow) {
calcOpticalFlowSF(from, to, flow, layers, averaging_block_size, max_flow,
4.1, 25.5, 18, 55.0, 25.5, 0.35, 18, 55.0, 25.5, 10);
}
}
}
modules/optflow/src/sparse_matching_gpc.cpp
+788
View File
@@ -0,0 +1,788 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
#undef CV_FORCE_SIMD128_CPP // mixed HAL SIMD/SSE code
#include "opencv2/core/private.hpp"
#include "opencv2/flann/miniflann.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencl_kernels_optflow.hpp"
#include "opencv2/core/hal/intrin.hpp"
#ifdef CV_CXX11
#include <random> // std::mt19937
#endif
/* Disable "from double to float" and "from size_t to int" warnings.
* Fixing these would make the code look ugly by introducing explicit cast all around.
* Here these warning are pointless anyway.
*/
#ifdef _MSC_VER
#pragma warning( disable : 4244 4267 4838 )
#endif
#ifdef __clang__
#pragma clang diagnostic ignored "-Wshorten-64-to-32"
#endif
namespace cv
{
namespace optflow
{
namespace
{
#define PATCH_RADIUS 10
#define PATCH_RADIUS_DOUBLED 20
#define SQRT2_INV 0.7071067811865475
const int patchRadius = PATCH_RADIUS;
const int globalIters = 3;
const int localIters = 500;
const double thresholdOutliers = 0.98;
const double thresholdMagnitudeFrac = 0.8;
const double epsTolerance = 1e-12;
const unsigned scoreGainPos = 5;
const unsigned scoreGainNeg = 1;
const unsigned negSearchKNN = 5;
const double simulatedAnnealingTemperatureCoef = 200.0;
const double sigmaGrowthRate = 0.2;
RNG rng;
struct Magnitude
{
float val;
int i;
int j;
Magnitude( float _val, int _i, int _j ) : val( _val ), i( _i ), j( _j ) {}
Magnitude() {}
bool operator<( const Magnitude &m ) const { return val > m.val; }
};
struct PartitionPredicate1
{
Vec< double, GPCPatchDescriptor::nFeatures > coef;
double rhs;
PartitionPredicate1( const Vec< double, GPCPatchDescriptor::nFeatures > &_coef, double _rhs ) : coef( _coef ), rhs( _rhs ) {}
bool operator()( const GPCPatchSample &sample ) const
{
bool refdir, posdir, negdir;
sample.getDirections( refdir, posdir, negdir, coef, rhs );
return refdir == false && ( posdir == false || negdir == true );
}
};
struct PartitionPredicate2
{
Vec< double, GPCPatchDescriptor::nFeatures > coef;
double rhs;
PartitionPredicate2( const Vec< double, GPCPatchDescriptor::nFeatures > &_coef, double _rhs ) : coef( _coef ), rhs( _rhs ) {}
bool operator()( const GPCPatchSample &sample ) const
{
bool refdir, posdir, negdir;
sample.getDirections( refdir, posdir, negdir, coef, rhs );
return refdir != posdir && refdir == negdir;
}
};
struct CompareWithTolerance
{
double val;
CompareWithTolerance( double _val ) : val( _val ) {};
bool operator()( const double &elem ) const
{
const double diff = ( val + elem == 0 ) ? std::abs( val - elem ) : std::abs( ( val - elem ) / ( val + elem ) );
return diff <= epsTolerance;
}
};
float normL2Sqr( const Vec2f &v ) { return v[0] * v[0] + v[1] * v[1]; }
int normL2Sqr( const Point2i &v ) { return v.x * v.x + v.y * v.y; }
bool checkBounds( int i, int j, Size sz )
{
return i >= patchRadius && j >= patchRadius && i + patchRadius < sz.height && j + patchRadius < sz.width;
}
void getDCTPatchDescriptor( GPCPatchDescriptor &patchDescr, const Mat *imgCh, int i, int j )
{
Rect roi( j - patchRadius, i - patchRadius, 2 * patchRadius, 2 * patchRadius );
Mat freqDomain;
dct( imgCh[0]( roi ), freqDomain );
double *feature = patchDescr.feature.val;
feature[0] = freqDomain.at< float >( 0, 0 );
feature[1] = freqDomain.at< float >( 0, 1 );
feature[2] = freqDomain.at< float >( 0, 2 );
feature[3] = freqDomain.at< float >( 0, 3 );
feature[4] = freqDomain.at< float >( 1, 0 );
feature[5] = freqDomain.at< float >( 1, 1 );
feature[6] = freqDomain.at< float >( 1, 2 );
feature[7] = freqDomain.at< float >( 1, 3 );
feature[8] = freqDomain.at< float >( 2, 0 );
feature[9] = freqDomain.at< float >( 2, 1 );
feature[10] = freqDomain.at< float >( 2, 2 );
feature[11] = freqDomain.at< float >( 2, 3 );
feature[12] = freqDomain.at< float >( 3, 0 );
feature[13] = freqDomain.at< float >( 3, 1 );
feature[14] = freqDomain.at< float >( 3, 2 );
feature[15] = freqDomain.at< float >( 3, 3 );
feature[16] = cv::sum( imgCh[1]( roi ) )[0] / ( 2 * patchRadius );
feature[17] = cv::sum( imgCh[2]( roi ) )[0] / ( 2 * patchRadius );
}
double sumInt( const Mat &integ, int i, int j, int h, int w )
{
return integ.at< double >( i + h, j + w ) - integ.at< double >( i + h, j ) - integ.at< double >( i, j + w ) + integ.at< double >( i, j );
}
void getWHTPatchDescriptor( GPCPatchDescriptor &patchDescr, const Mat *imgCh, int i, int j )
{
i -= patchRadius;
j -= patchRadius;
const int k = 2 * patchRadius;
const double s = sumInt( imgCh[0], i, j, k, k );
double *feature = patchDescr.feature.val;
feature[0] = s;
feature[1] = s - 2 * sumInt( imgCh[0], i, j + k / 2, k, k / 2 );
feature[2] = s - 2 * sumInt( imgCh[0], i, j + k / 4, k, k / 2 );
feature[3] = s - 2 * sumInt( imgCh[0], i, j + k / 4, k, k / 4 ) - 2 * sumInt( imgCh[0], i, j + 3 * k / 4, k, k / 4 );
feature[4] = s - 2 * sumInt( imgCh[0], i + k / 2, j, k / 2, k );
feature[5] = s - 2 * sumInt( imgCh[0], i, j + k / 2, k / 2, k / 2 ) - 2 * sumInt( imgCh[0], i + k / 2, j, k / 2, k / 2 );
feature[6] = s - 2 * sumInt( imgCh[0], i, j + k / 4, k / 2, k / 2 ) - 2 * sumInt( imgCh[0], i + k / 2, j, k / 2, k / 4 ) -
2 * sumInt( imgCh[0], i + k / 2, j + 3 * k / 4, k / 2, k / 4 );
feature[7] = s - 2 * sumInt( imgCh[0], i, j + k / 4, k / 2, k / 4 ) - 2 * sumInt( imgCh[0], i, j + 3 * k / 4, k / 2, k / 4 ) -
2 * sumInt( imgCh[0], i + k / 2, j, k / 2, k / 4 ) - 2 * sumInt( imgCh[0], i + k / 2, j + k / 2, k / 2, k / 4 );
feature[8] = s - 2 * sumInt( imgCh[0], i + k / 4, j, k / 2, k );
feature[9] = s - 2 * sumInt( imgCh[0], i + k / 4, j, k / 2, k / 2 ) - 2 * sumInt( imgCh[0], i, j + k / 2, k / 4, k / 2 ) -
2 * sumInt( imgCh[0], i + 3 * k / 4, j + k / 2, k / 4, k / 2 );
feature[10] = s - 2 * sumInt( imgCh[0], i + k / 4, j, k / 2, k / 4 ) - 2 * sumInt( imgCh[0], i + k / 4, j + 3 * k / 4, k / 2, k / 4 ) -
2 * sumInt( imgCh[0], i, j + k / 4, k / 4, k / 2 ) - 2 * sumInt( imgCh[0], i + 3 * k / 4, j + k / 4, k / 4, k / 2 );
feature[11] = s - 2 * sumInt( imgCh[0], i, j + k / 4, k / 4, k / 4 ) - 2 * sumInt( imgCh[0], i, j + 3 * k / 4, k / 4, k / 4 ) -
2 * sumInt( imgCh[0], i + k / 4, j, k / 2, k / 4 ) - 2 * sumInt( imgCh[0], i + k / 4, j + k / 2, k / 2, k / 4 ) -
2 * sumInt( imgCh[0], i + 3 * k / 4, j + k / 4, k / 4, k / 4 ) -
2 * sumInt( imgCh[0], i + 3 * k / 4, j + 3 * k / 4, k / 4, k / 4 );
feature[12] = s - 2 * sumInt( imgCh[0], i + k / 4, j, k / 4, k ) - 2 * sumInt( imgCh[0], i + 3 * k / 4, j, k / 4, k );
feature[13] = s - 2 * sumInt( imgCh[0], i + k / 4, j, k / 4, k / 2 ) - 2 * sumInt( imgCh[0], i + 3 * k / 4, j, k / 4, k / 2 ) -
2 * sumInt( imgCh[0], i, j + k / 2, k / 4, k / 2 ) - 2 * sumInt( imgCh[0], i + k / 2, j + k / 2, k / 4, k / 2 );
feature[14] = s - 2 * sumInt( imgCh[0], i + k / 4, j, k / 4, k / 4 ) - 2 * sumInt( imgCh[0], i + 3 * k / 4, j, k / 4, k / 4 ) -
2 * sumInt( imgCh[0], i, j + k / 4, k / 4, k / 2 ) - 2 * sumInt( imgCh[0], i + k / 2, j + k / 4, k / 4, k / 2 ) -
2 * sumInt( imgCh[0], i + k / 4, j + 3 * k / 4, k / 4, k / 4 ) -
2 * sumInt( imgCh[0], i + 3 * k / 4, j + 3 * k / 4, k / 4, k / 4 );
feature[15] = s - 2 * sumInt( imgCh[0], i, j + k / 4, k / 4, k / 4 ) - 2 * sumInt( imgCh[0], i, j + 3 * k / 4, k / 4, k / 4 ) -
2 * sumInt( imgCh[0], i + k / 4, j, k / 4, k / 4 ) - 2 * sumInt( imgCh[0], i + k / 4, j + k / 2, k / 4, k / 4 ) -
2 * sumInt( imgCh[0], i + k / 2, j + k / 4, k / 4, k / 4 ) -
2 * sumInt( imgCh[0], i + k / 2, j + 3 * k / 4, k / 4, k / 4 ) - 2 * sumInt( imgCh[0], i + 3 * k / 4, j, k / 4, k / 4 ) -
2 * sumInt( imgCh[0], i + 3 * k / 4, j + k / 2, k / 4, k / 4 );
feature[16] = sumInt( imgCh[1], i, j, k, k );
feature[17] = sumInt( imgCh[2], i, j, k, k );
patchDescr.feature /= patchRadius;
}
class ParallelDCTFiller : public ParallelLoopBody
{
private:
const Size sz;
const Mat *imgCh;
std::vector< GPCPatchDescriptor > *descr;
ParallelDCTFiller &operator=( const ParallelDCTFiller & );
public:
ParallelDCTFiller( const Size &_sz, const Mat *_imgCh, std::vector< GPCPatchDescriptor > *_descr )
: sz( _sz ), imgCh( _imgCh ), descr( _descr ){};
void operator()( const Range &range ) const CV_OVERRIDE
{
for ( int i = range.start; i < range.end; ++i )
{
int x, y;
GPCDetails::getCoordinatesFromIndex( i, sz, x, y );
getDCTPatchDescriptor( descr->at( i ), imgCh, y, x );
}
}
};
#ifdef HAVE_OPENCL
bool ocl_getAllDCTDescriptorsForImage( const Mat *imgCh, std::vector< GPCPatchDescriptor > &descr )
{
const Size sz = imgCh[0].size();
ocl::Kernel kernel( "getPatchDescriptor", ocl::optflow::sparse_matching_gpc_oclsrc,
format( "-DPATCH_RADIUS_DOUBLED=%d -DCV_PI=%f -DSQRT2_INV=%f", PATCH_RADIUS_DOUBLED, CV_PI, SQRT2_INV ) );
CV_Assert(sz.height - 2 * patchRadius > 0);
CV_Assert(sz.width - 2 * patchRadius > 0);
size_t globSize[] = {(size_t)(sz.height - 2 * patchRadius), (size_t)(sz.width - 2 * patchRadius)};
UMat out( globSize[0] * globSize[1], GPCPatchDescriptor::nFeatures, CV_64F );
if (
kernel
.args( cv::ocl::KernelArg::ReadOnlyNoSize( imgCh[0].getUMat( ACCESS_READ ) ),
cv::ocl::KernelArg::ReadOnlyNoSize( imgCh[1].getUMat( ACCESS_READ ) ),
cv::ocl::KernelArg::ReadOnlyNoSize( imgCh[2].getUMat( ACCESS_READ ) ),
cv::ocl::KernelArg::WriteOnlyNoSize( out ),
(int)globSize[0], (int)globSize[1], (int)patchRadius )
.run( 2, globSize, 0, true ) == false )
return false;
Mat cpuOut = out.getMat( ACCESS_READ );
for ( int i = 0; i + 2 * patchRadius < sz.height; ++i )
for ( int j = 0; j + 2 * patchRadius < sz.width; ++j )
descr.push_back( *cpuOut.ptr< GPCPatchDescriptor >( i * globSize[1] + j ) );
return true;
}
#endif
void getAllDCTDescriptorsForImage( const Mat *imgCh, std::vector< GPCPatchDescriptor > &descr, const GPCMatchingParams &mp )
{
const Size sz = imgCh[0].size();
descr.reserve( ( sz.height - 2 * patchRadius ) * ( sz.width - 2 * patchRadius ) );
CV_UNUSED(mp); // Fix unused parameter warning in case OpenCL is not available
CV_OCL_RUN( mp.useOpenCL, ocl_getAllDCTDescriptorsForImage( imgCh, descr ) )
descr.resize( ( sz.height - 2 * patchRadius ) * ( sz.width - 2 * patchRadius ) );
parallel_for_( Range( 0, descr.size() ), ParallelDCTFiller( sz, imgCh, &descr ) );
}
class ParallelWHTFiller : public ParallelLoopBody
{
private:
const Size sz;
const Mat *imgChInt;
std::vector< GPCPatchDescriptor > *descr;
ParallelWHTFiller &operator=( const ParallelWHTFiller & );
public:
ParallelWHTFiller( const Size &_sz, const Mat *_imgChInt, std::vector< GPCPatchDescriptor > *_descr )
: sz( _sz ), imgChInt( _imgChInt ), descr( _descr ){};
void operator()( const Range &range ) const CV_OVERRIDE
{
for ( int i = range.start; i < range.end; ++i )
{
int x, y;
GPCDetails::getCoordinatesFromIndex( i, sz, x, y );
getWHTPatchDescriptor( descr->at( i ), imgChInt, y, x );
}
}
};
void getAllWHTDescriptorsForImage( const Mat *imgCh, std::vector< GPCPatchDescriptor > &descr, const GPCMatchingParams & )
{
const Size sz = imgCh[0].size();
descr.resize( ( sz.height - 2 * patchRadius ) * ( sz.width - 2 * patchRadius ) );
Mat imgChInt[3];
integral( imgCh[0], imgChInt[0], CV_64F );
integral( imgCh[1], imgChInt[1], CV_64F );
integral( imgCh[2], imgChInt[2], CV_64F );
parallel_for_( Range( 0, descr.size() ), ParallelWHTFiller( sz, imgChInt, &descr ) );
}
void buildIndex( OutputArray featuresOut, flann::Index &index, const Mat *imgCh,
void ( *getAllDescrFn )( const Mat *, std::vector< GPCPatchDescriptor > &, const GPCMatchingParams & ) )
{
std::vector< GPCPatchDescriptor > descriptors;
getAllDescrFn( imgCh, descriptors, GPCMatchingParams() );
featuresOut.create( descriptors.size(), GPCPatchDescriptor::nFeatures, CV_32F );
Mat features = featuresOut.getMat();
for ( size_t i = 0; i < descriptors.size(); ++i )
*features.ptr< Vec< float, GPCPatchDescriptor::nFeatures > >( i ) = descriptors[i].feature;
cv::flann::KDTreeIndexParams indexParams;
index.build( features, indexParams, cvflann::FLANN_DIST_L2 );
}
void getTriplet( const Magnitude &mag, const Mat &gt, const Mat *fromCh, const Mat *toCh, GPCSamplesVector &samples, flann::Index &index,
void ( *getDescFn )( GPCPatchDescriptor &, const Mat *, int, int ) )
{
const Size sz = gt.size();
const int i0 = mag.i;
const int j0 = mag.j;
const int i1 = i0 + cvRound( gt.at< Vec2f >( i0, j0 )[1] );
const int j1 = j0 + cvRound( gt.at< Vec2f >( i0, j0 )[0] );
if ( checkBounds( i1, j1, sz ) )
{
GPCPatchSample ps;
getDescFn( ps.ref, fromCh, i0, j0 );
getDescFn( ps.pos, toCh, i1, j1 );
ps.neg.markAsSeparated();
Matx< float, 1, GPCPatchDescriptor::nFeatures > ref32;
Matx< int, 1, negSearchKNN > indices;
int maxDist = 0;
for ( unsigned i = 0; i < GPCPatchDescriptor::nFeatures; ++i )
ref32( 0, i ) = ps.ref.feature[i];
index.knnSearch( ref32, indices, noArray(), negSearchKNN );
for ( unsigned i = 0; i < negSearchKNN; ++i )
{
int i2, j2;
GPCDetails::getCoordinatesFromIndex( indices( 0, i ), sz, j2, i2 );
const int dist = ( i2 - i1 ) * ( i2 - i1 ) + ( j2 - j1 ) * ( j2 - j1 );
if ( maxDist < dist )
{
maxDist = dist;
getDescFn( ps.neg, toCh, i2, j2 );
}
}
samples.push_back( ps );
}
}
void getTrainingSamples( const Mat &from, const Mat &to, const Mat &gt, GPCSamplesVector &samples, const int type )
{
const Size sz = gt.size();
std::vector< Magnitude > mag;
for ( int i = patchRadius; i + patchRadius < sz.height; ++i )
for ( int j = patchRadius; j + patchRadius < sz.width; ++j )
mag.push_back( Magnitude( normL2Sqr( gt.at< Vec2f >( i, j ) ), i, j ) );
size_t n = size_t( mag.size() * thresholdMagnitudeFrac ); // As suggested in the paper, we discard part of the training samples
// with a small displacement and train to better distinguish hard pairs.
std::nth_element( mag.begin(), mag.begin() + n, mag.end() );
mag.resize( n );
#ifdef CV_CXX11
std::mt19937 std_rng(cv::theRNG()());
std::shuffle(mag.begin(), mag.end(), std_rng);
#else
std::random_shuffle( mag.begin(), mag.end() );
#endif
n /= patchRadius;
mag.resize( n );
if ( type == GPC_DESCRIPTOR_DCT )
{
Mat fromCh[3], toCh[3];
split( from, fromCh );
split( to, toCh );
Mat allDescriptors;
flann::Index index;
buildIndex( allDescriptors, index, toCh, getAllDCTDescriptorsForImage );
for ( size_t k = 0; k < n; ++k )
getTriplet( mag[k], gt, fromCh, toCh, samples, index, getDCTPatchDescriptor );
}
else if ( type == GPC_DESCRIPTOR_WHT )
{
Mat fromCh[3], toCh[3], fromChInt[3], toChInt[3];
split( from, fromCh );
split( to, toCh );
integral( fromCh[0], fromChInt[0], CV_64F );
integral( fromCh[1], fromChInt[1], CV_64F );
integral( fromCh[2], fromChInt[2], CV_64F );
integral( toCh[0], toChInt[0], CV_64F );
integral( toCh[1], toChInt[1], CV_64F );
integral( toCh[2], toChInt[2], CV_64F );
Mat allDescriptors;
flann::Index index;
buildIndex( allDescriptors, index, toCh, getAllWHTDescriptorsForImage );
for ( size_t k = 0; k < n; ++k )
getTriplet( mag[k], gt, fromChInt, toChInt, samples, index, getWHTPatchDescriptor );
}
else
CV_Error( Error::StsBadArg, "Unknown descriptor type" );
}
/* Sample random number from Cauchy distribution. */
double getRandomCauchyScalar()
{
return tan( rng.uniform( -1.54, 1.54 ) ); // I intentionally used the value slightly less than PI/2 to enforce strictly
// zero probability for large numbers. Resulting PDF for Cauchy has
// truncated "tails".
}
/* Sample random vector from Cauchy distribution (pointwise, i.e. vector whose components are independent random
* variables from Cauchy distribution) */
void getRandomCauchyVector( Vec< double, GPCPatchDescriptor::nFeatures > &v )
{
for ( unsigned i = 0; i < GPCPatchDescriptor::nFeatures; ++i )
v[i] = getRandomCauchyScalar();
}
double getRobustMedian( double m ) { return m < 0 ? m * ( 1.0 + epsTolerance ) : m * ( 1.0 - epsTolerance ); }
}
double GPCPatchDescriptor::dot( const Vec< double, nFeatures > &coef ) const
{
#if CV_SIMD128_64F
v_float64x2 sum = v_setzero_f64();
for ( unsigned i = 0; i < nFeatures; i += 2 )
{
v_float64x2 x = v_load( &feature.val[i] );
v_float64x2 y = v_load( &coef.val[i] );
sum = v_muladd( x, y, sum );
}
#if CV_SSE2
__m128d sumrev = _mm_shuffle_pd( sum.val, sum.val, _MM_SHUFFLE2( 0, 1 ) );
return _mm_cvtsd_f64( _mm_add_pd( sum.val, sumrev ) );
#else
double CV_DECL_ALIGNED( 16 ) buf[2];
v_store_aligned( buf, sum );
return OPENCV_HAL_ADD( buf[0], buf[1] );
#endif
#else
return feature.dot( coef );
#endif
}
void GPCPatchSample::getDirections( bool &refdir, bool &posdir, bool &negdir, const Vec< double, GPCPatchDescriptor::nFeatures > &coef, double rhs ) const
{
refdir = ( ref.dot( coef ) < rhs );
posdir = pos.isSeparated() ? ( !refdir ) : ( pos.dot( coef ) < rhs );
negdir = neg.isSeparated() ? ( !refdir ) : ( neg.dot( coef ) < rhs );
}
void GPCDetails::getAllDescriptorsForImage( const Mat *imgCh, std::vector< GPCPatchDescriptor > &descr, const GPCMatchingParams &mp,
int type )
{
if ( type == GPC_DESCRIPTOR_DCT )
getAllDCTDescriptorsForImage( imgCh, descr, mp );
else if ( type == GPC_DESCRIPTOR_WHT )
getAllWHTDescriptorsForImage( imgCh, descr, mp );
else
CV_Error( Error::StsBadArg, "Unknown descriptor type" );
}
void GPCDetails::getCoordinatesFromIndex( size_t index, Size sz, int &x, int &y )
{
const size_t stride = sz.width - patchRadius * 2;
y = int( index / stride );
x = int( index - y * stride + patchRadius );
y += patchRadius;
}
bool GPCTree::trainNode( size_t nodeId, SIter begin, SIter end, unsigned depth )
{
const int nSamples = (int)std::distance( begin, end );
if ( nSamples < params.minNumberOfSamples || depth >= params.maxTreeDepth )
return false;
if ( nodeId >= nodes.size() )
nodes.resize( nodeId + 1 );
Node &node = nodes[nodeId];
// Select the best hyperplane
unsigned globalBestScore = 0;
std::vector< double > values;
values.reserve( nSamples * 2 );
for ( int j = 0; j < globalIters; ++j )
{ // Global search step
Vec< double, GPCPatchDescriptor::nFeatures > coef;
unsigned localBestScore = 0;
getRandomCauchyVector( coef );
for ( int i = 0; i < localIters; ++i )
{ // Local search step
double randomModification = getRandomCauchyScalar() * ( 1.0 + sigmaGrowthRate * int( i / GPCPatchDescriptor::nFeatures ) );
const int pos = i % GPCPatchDescriptor::nFeatures;
std::swap( coef[pos], randomModification );
values.clear();
for ( SIter iter = begin; iter != end; ++iter )
values.push_back( iter->ref.dot( coef ) );
std::nth_element( values.begin(), values.begin() + nSamples / 2, values.end() );
double median = values[nSamples / 2];
// Skip obviously malformed division. This may happen in case there are a large number of equal samples.
// Most likely this won't happen with samples collected from a good dataset.
// Happens in case dataset contains plain (or close to plain) images.
if ( std::count_if( values.begin(), values.end(), CompareWithTolerance( median ) ) > std::max( 1, nSamples / 4 ) )
continue;
median = getRobustMedian( median );
unsigned score = 0;
for ( SIter iter = begin; iter != end; ++iter )
{
bool refdir, posdir, negdir;
iter->getDirections( refdir, posdir, negdir, coef, median );
if ( refdir == posdir )
score += scoreGainPos;
if ( refdir != negdir )
score += scoreGainNeg;
}
if ( score > localBestScore )
localBestScore = score;
else
{
const double beta = simulatedAnnealingTemperatureCoef * std::sqrt( static_cast<float>(i) ) / ( nSamples * ( scoreGainPos + scoreGainNeg ) );
if ( rng.uniform( 0.0, 1.0 ) > std::exp( -beta * ( localBestScore - score) ) )
coef[pos] = randomModification;
}
if ( score > globalBestScore )
{
globalBestScore = score;
node.coef = coef;
node.rhs = median;
}
}
}
if ( globalBestScore == 0 )
return false;
if ( params.printProgress )
{
const int maxScore = nSamples * ( scoreGainPos + scoreGainNeg );
const double correctRatio = double( globalBestScore ) / maxScore;
printf( "[%u] Correct %.2f (%u/%d)\nWeights:", depth, correctRatio, globalBestScore, maxScore );
for ( unsigned k = 0; k < GPCPatchDescriptor::nFeatures; ++k )
printf( " %.3f", node.coef[k] );
printf( "\n" );
}
for ( SIter iter = begin; iter != end; ++iter )
{
bool refdir, posdir, negdir;
iter->getDirections( refdir, posdir, negdir, node.coef, node.rhs );
// We shouldn't account for positive sample in the scoring in case it was separated before. So mark it as separated.
// After all, we can't bring back samples which were separated from reference on early levels.
if ( refdir != posdir )
iter->pos.markAsSeparated();
// The same for negative sample.
if ( refdir != negdir )
iter->neg.markAsSeparated();
// If both positive and negative were separated before then such triplet doesn't make sense on deeper levels. We discard it.
}
// Partition vector with samples according to the hyperplane in QuickSort-like manner.
// Unlike QuickSort, we need to partition it into 3 parts (left subtree samples; undefined samples; right subtree
// samples), so we call it two times.
SIter leftEnd = std::partition( begin, end, PartitionPredicate1( node.coef, node.rhs ) ); // Separate left subtree samples from others.
SIter rightBegin =
std::partition( leftEnd, end, PartitionPredicate2( node.coef, node.rhs ) ); // Separate undefined samples from right subtree samples.
node.left = ( trainNode( nodeId * 2 + 1, begin, leftEnd, depth + 1 ) ) ? unsigned( nodeId * 2 + 1 ) : 0;
node.right = ( trainNode( nodeId * 2 + 2, rightBegin, end, depth + 1 ) ) ? unsigned( nodeId * 2 + 2 ) : 0;
return true;
}
void GPCTree::train( GPCTrainingSamples &samples, const GPCTrainingParams _params )
{
if ( _params.descriptorType != samples.type() )
CV_Error( Error::StsBadArg, "Descriptor type mismatch! Check that samples are collected with the same descriptor type." );
nodes.clear();
nodes.reserve( samples.size() * 2 - 1 ); // set upper bound for the possible number of nodes so all subsequent resize() will be no-op
params = _params;
GPCSamplesVector &sv = samples;
trainNode( 0, sv.begin(), sv.end(), 0 );
}
void GPCTree::write( FileStorage &fs ) const
{
if ( nodes.empty() )
CV_Error( Error::StsBadArg, "Tree have not been trained" );
fs << "nodes" << nodes;
fs << "dtype" << (int)params.descriptorType;
}
void GPCTree::read( const FileNode &fn )
{
fn["nodes"] >> nodes;
fn["dtype"] >> (int &)params.descriptorType;
}
unsigned GPCTree::findLeafForPatch( const GPCPatchDescriptor &descr ) const
{
unsigned id = 0, prevId;
do
{
prevId = id;
if ( descr.dot( nodes[id].coef ) < nodes[id].rhs )
id = nodes[id].right;
else
id = nodes[id].left;
} while ( id );
return prevId;
}
Ptr< GPCTrainingSamples > GPCTrainingSamples::create( const std::vector< String > &imagesFrom, const std::vector< String > &imagesTo,
const std::vector< String > &gt, int _descriptorType )
{
CV_Assert( imagesFrom.size() == imagesTo.size() );
CV_Assert( imagesFrom.size() == gt.size() );
Ptr< GPCTrainingSamples > ts = makePtr< GPCTrainingSamples >();
ts->descriptorType = _descriptorType;
for ( size_t i = 0; i < imagesFrom.size(); ++i )
{
Mat from = imread( imagesFrom[i] );
Mat to = imread( imagesTo[i] );
Mat gtFlow = readOpticalFlow( gt[i] );
CV_Assert( from.size == to.size );
CV_Assert( from.size == gtFlow.size );
CV_Assert( from.channels() == 3 );
CV_Assert( to.channels() == 3 );
from.convertTo( from, CV_32FC3 );
to.convertTo( to, CV_32FC3 );
cvtColor( from, from, COLOR_BGR2YCrCb );
cvtColor( to, to, COLOR_BGR2YCrCb );
getTrainingSamples( from, to, gtFlow, ts->samples, ts->descriptorType );
}
return ts;
}
Ptr< GPCTrainingSamples > GPCTrainingSamples::create( InputArrayOfArrays imagesFrom, InputArrayOfArrays imagesTo,
InputArrayOfArrays gt, int _descriptorType )
{
CV_Assert( imagesFrom.total() == imagesTo.total() );
CV_Assert( imagesFrom.total() == gt.total() );
Ptr< GPCTrainingSamples > ts = makePtr< GPCTrainingSamples >();
ts->descriptorType = _descriptorType;
for ( size_t i = 0; i < imagesFrom.total(); ++i )
{
Mat from = imagesFrom.getMat( static_cast<int>( i ) );
Mat to = imagesTo.getMat( static_cast<int>( i ) );
Mat gtFlow = gt.getMat( static_cast<int>( i ) );
CV_Assert( from.size == to.size );
CV_Assert( from.size == gtFlow.size );
CV_Assert( from.channels() == 3 );
CV_Assert( to.channels() == 3 );
from.convertTo( from, CV_32FC3 );
to.convertTo( to, CV_32FC3 );
cvtColor( from, from, COLOR_BGR2YCrCb );
cvtColor( to, to, COLOR_BGR2YCrCb );
getTrainingSamples( from, to, gtFlow, ts->samples, ts->descriptorType );
}
return ts;
}
void GPCDetails::dropOutliers( std::vector< std::pair< Point2i, Point2i > > &corr )
{
if ( corr.size() == 0 )
return;
std::vector< float > mag( corr.size() );
for ( size_t i = 0; i < corr.size(); ++i )
mag[i] = normL2Sqr( corr[i].first - corr[i].second );
const size_t threshold = size_t( mag.size() * thresholdOutliers );
std::nth_element( mag.begin(), mag.begin() + threshold, mag.end() );
const float percentile = mag[threshold];
size_t i = 0, j = 0;
while ( i < corr.size() )
{
if ( normL2Sqr( corr[i].first - corr[i].second ) <= percentile )
{
corr[j] = corr[i];
++j;
}
++i;
}
corr.resize( j );
}
} // namespace optflow
void write( FileStorage &fs, const String &name, const optflow::GPCTree::Node &node )
{
cv::internal::WriteStructContext ws( fs, name, FileNode::SEQ + FileNode::FLOW );
for ( unsigned i = 0; i < optflow::GPCPatchDescriptor::nFeatures; ++i )
write( fs, node.coef[i] );
write( fs, node.rhs );
write( fs, (int)node.left );
write( fs, (int)node.right );
}
void read( const FileNode &fn, optflow::GPCTree::Node &node, optflow::GPCTree::Node )
{
FileNodeIterator it = fn.begin();
for ( unsigned i = 0; i < optflow::GPCPatchDescriptor::nFeatures; ++i )
it >> node.coef[i];
it >> node.rhs >> (int &)node.left >> (int &)node.right;
}
} // namespace cv
modules/optflow/src/sparsetodenseflow.cpp
+112
View File
@@ -0,0 +1,112 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
#include "opencv2/ximgproc/sparse_match_interpolator.hpp"
using namespace std;
namespace cv {
namespace optflow {
CV_EXPORTS_W void calcOpticalFlowSparseToDense(InputArray from, InputArray to, OutputArray flow,
int grid_step, int k,
float sigma, bool use_post_proc,
float fgs_lambda, float fgs_sigma)
{
CV_Assert( grid_step>1 && k>3 && sigma>0.0001f && fgs_lambda>1.0f && fgs_sigma>0.01f );
CV_Assert( !from.empty() && from.depth() == CV_8U && (from.channels() == 3 || from.channels() == 1) );
CV_Assert( !to .empty() && to .depth() == CV_8U && (to .channels() == 3 || to .channels() == 1) );
CV_Assert( from.sameSize(to) );
Mat prev = from.getMat();
Mat cur = to.getMat();
Mat prev_grayscale, cur_grayscale;
while( (prev.cols/grid_step)*(prev.rows/grid_step) > SHRT_MAX ) //ensure that the number matches is not too big
grid_step*=2;
if(prev.channels()==3)
{
cvtColor(prev,prev_grayscale,COLOR_BGR2GRAY);
cvtColor(cur, cur_grayscale, COLOR_BGR2GRAY);
}
else
{
prev.copyTo(prev_grayscale);
cur .copyTo(cur_grayscale);
}
vector<Point2f> points;
vector<Point2f> dst_points;
vector<unsigned char> status;
vector<float> err;
vector<Point2f> points_filtered, dst_points_filtered;
for(int i=0;i<prev.rows;i+=grid_step)
for(int j=0;j<prev.cols;j+=grid_step)
points.push_back(Point2f((float)j,(float)i));
calcOpticalFlowPyrLK(prev_grayscale,cur_grayscale,points,dst_points,status,err,Size(21,21));
for(unsigned int i=0;i<points.size();i++)
{
if(status[i]!=0)
{
points_filtered.push_back(points[i]);
dst_points_filtered.push_back(dst_points[i]);
}
}
flow.create(from.size(),CV_32FC2);
Mat dense_flow = flow.getMat();
Ptr<ximgproc::EdgeAwareInterpolator> gd = ximgproc::createEdgeAwareInterpolator();
gd->setK(k);
gd->setSigma(sigma);
gd->setUsePostProcessing(use_post_proc);
gd->setFGSLambda(fgs_lambda);
gd->setFGSSigma (fgs_sigma);
gd->interpolate(prev,points_filtered,cur,dst_points_filtered,dense_flow);
}
}
}
modules/optflow/src/tvl1flow.cpp
+1490
View File
File diff suppressed because it is too large Load Diff
modules/optflow/test/ocl/test_motempl.cpp
+66
View File
@@ -0,0 +1,66 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2014, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../test_precomp.hpp"
#include "opencv2/ts/ocl_test.hpp"
#ifdef HAVE_OPENCL
namespace opencv_test { namespace {
PARAM_TEST_CASE(UpdateMotionHistory, bool)
{
double timestamp, duration;
bool use_roi;
TEST_DECLARE_INPUT_PARAMETER(silhouette);
TEST_DECLARE_OUTPUT_PARAMETER(mhi);
virtual void SetUp()
{
use_roi = GET_PARAM(0);
}
virtual void generateTestData()
{
Size roiSize = randomSize(1, MAX_VALUE);
Border silhouetteBorder = randomBorder(0, use_roi ? MAX_VALUE : 0);
randomSubMat(silhouette, silhouette_roi, roiSize, silhouetteBorder, CV_8UC1, -11, 11);
Border mhiBorder = randomBorder(0, use_roi ? MAX_VALUE : 0);
randomSubMat(mhi, mhi_roi, roiSize, mhiBorder, CV_32FC1, 0, 1);
timestamp = randomDouble(0, 1);
duration = randomDouble(0, 1);
if (timestamp < duration)
std::swap(timestamp, duration);
UMAT_UPLOAD_INPUT_PARAMETER(silhouette);
UMAT_UPLOAD_OUTPUT_PARAMETER(mhi);
}
};
OCL_TEST_P(UpdateMotionHistory, Mat)
{
for (int j = 0; j < cvtest::ocl::test_loop_times; j++)
{
generateTestData();
OCL_OFF(cv::motempl::updateMotionHistory(silhouette_roi, mhi_roi, timestamp, duration));
OCL_ON(cv::motempl::updateMotionHistory(usilhouette_roi, umhi_roi, timestamp, duration));
OCL_EXPECT_MATS_NEAR(mhi, 0);
}
}
//////////////////////////////////////// Instantiation /////////////////////////////////////////
OCL_INSTANTIATE_TEST_CASE_P(Video, UpdateMotionHistory, Values(false, true));
}} // namespace
#endif // HAVE_OPENCL
modules/optflow/test/ocl/test_optflow_tvl1flow.cpp
+117
View File
@@ -0,0 +1,117 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "../test_precomp.hpp"
#include "opencv2/ts/ocl_test.hpp"
#ifdef HAVE_OPENCL
namespace opencv_test {
namespace ocl {
/////////////////////////////////////////////////////////////////////////////////////////////////
// Optical_flow_tvl1
namespace
{
IMPLEMENT_PARAM_CLASS(UseInitFlow, bool)
IMPLEMENT_PARAM_CLASS(MedianFiltering, int)
IMPLEMENT_PARAM_CLASS(ScaleStep, double)
}
PARAM_TEST_CASE(OpticalFlowTVL1, UseInitFlow, MedianFiltering, ScaleStep)
{
bool useInitFlow;
int medianFiltering;
double scaleStep;
virtual void SetUp()
{
useInitFlow = GET_PARAM(0);
medianFiltering = GET_PARAM(1);
scaleStep = GET_PARAM(2);
}
};
OCL_TEST_P(OpticalFlowTVL1, Mat)
{
Mat frame0 = readImage("optflow/RubberWhale1.png", IMREAD_GRAYSCALE);
ASSERT_FALSE(frame0.empty());
Mat frame1 = readImage("optflow/RubberWhale2.png", IMREAD_GRAYSCALE);
ASSERT_FALSE(frame1.empty());
Mat flow; UMat uflow;
//create algorithm
Ptr<DualTVL1OpticalFlow> alg = createOptFlow_DualTVL1();
//set parameters
alg->setScaleStep(scaleStep);
alg->setMedianFiltering(medianFiltering);
//create initial flow as result of algorithm calculation
if (useInitFlow)
{
OCL_ON(alg->calc(frame0, frame1, uflow));
uflow.copyTo(flow);
}
//set flag to use initial flow as it is ready to use
alg->setUseInitialFlow(useInitFlow);
OCL_OFF(alg->calc(frame0, frame1, flow));
OCL_ON(alg->calc(frame0, frame1, uflow));
EXPECT_MAT_SIMILAR(flow, uflow, 1e-2);
}
OCL_INSTANTIATE_TEST_CASE_P(Contrib, OpticalFlowTVL1,
Combine(
Values(UseInitFlow(false), UseInitFlow(true)),
Values(MedianFiltering(3), MedianFiltering(-1)),
Values(ScaleStep(0.3),ScaleStep(0.5))
)
);
} } // namespace opencv_test::ocl
#endif // HAVE_OPENCL
modules/optflow/test/test_OF_accuracy.cpp
+353
View File
@@ -0,0 +1,353 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// Intel License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000, Intel Corporation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of Intel Corporation may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "test_precomp.hpp"
namespace opencv_test { namespace {
static string getDataDir() { return TS::ptr()->get_data_path(); }
static string getRubberWhaleFrame1() { return getDataDir() + "optflow/RubberWhale1.png"; }
static string getRubberWhaleFrame2() { return getDataDir() + "optflow/RubberWhale2.png"; }
static string getRubberWhaleGroundTruth() { return getDataDir() + "optflow/RubberWhale.flo"; }
static bool isFlowCorrect(float u) { return !cvIsNaN(u) && (fabs(u) < 1e9); }
static bool isFlowCorrect(double u) { return !cvIsNaN(u) && (fabs(u) < 1e9); }
static float calcRMSE(Mat flow1, Mat flow2)
{
float sum = 0;
int counter = 0;
const int rows = flow1.rows;
const int cols = flow1.cols;
for (int y = 0; y < rows; ++y)
{
for (int x = 0; x < cols; ++x)
{
Vec2f flow1_at_point = flow1.at<Vec2f>(y, x);
Vec2f flow2_at_point = flow2.at<Vec2f>(y, x);
float u1 = flow1_at_point[0];
float v1 = flow1_at_point[1];
float u2 = flow2_at_point[0];
float v2 = flow2_at_point[1];
if (isFlowCorrect(u1) && isFlowCorrect(u2) && isFlowCorrect(v1) && isFlowCorrect(v2))
{
sum += (u1 - u2) * (u1 - u2) + (v1 - v2) * (v1 - v2);
counter++;
}
}
}
return (float)sqrt(sum / (1e-9 + counter));
}
static float calcRMSE(vector<Point2f> prevPts, vector<Point2f> currPts, Mat flow)
{
vector<float> ee;
for (unsigned int n = 0; n < prevPts.size(); n++)
{
Point2f gtFlow = flow.at<Point2f>(prevPts[n]);
if (isFlowCorrect(gtFlow.x) && isFlowCorrect(gtFlow.y))
{
Point2f diffFlow = (currPts[n] - prevPts[n]) - gtFlow;
ee.push_back(sqrt(diffFlow.x * diffFlow.x + diffFlow.y * diffFlow.y));
}
}
return static_cast<float>(mean(ee).val[0]);
}
static float calcAvgEPE(vector< pair<Point2i, Point2i> > corr, Mat flow)
{
double sum = 0;
int counter = 0;
for (size_t i = 0; i < corr.size(); ++i)
{
Vec2f flow1_at_point = Point2f(corr[i].second - corr[i].first);
Vec2f flow2_at_point = flow.at<Vec2f>(corr[i].first.y, corr[i].first.x);
double u1 = (double)flow1_at_point[0];
double v1 = (double)flow1_at_point[1];
double u2 = (double)flow2_at_point[0];
double v2 = (double)flow2_at_point[1];
if (isFlowCorrect(u1) && isFlowCorrect(u2) && isFlowCorrect(v1) && isFlowCorrect(v2))
{
sum += sqrt((u1 - u2) * (u1 - u2) + (v1 - v2) * (v1 - v2));
counter++;
}
}
return (float)(sum / counter);
}
bool readRubberWhale(Mat &dst_frame_1, Mat &dst_frame_2, Mat &dst_GT)
{
string frame1_path = getRubberWhaleFrame1();
string frame2_path = getRubberWhaleFrame2();
string gt_flow_path = getRubberWhaleGroundTruth();
// removing space may be an issue on windows machines
frame1_path.erase(std::remove_if(frame1_path.begin(), frame1_path.end(), isspace), frame1_path.end());
frame2_path.erase(std::remove_if(frame2_path.begin(), frame2_path.end(), isspace), frame2_path.end());
gt_flow_path.erase(std::remove_if(gt_flow_path.begin(), gt_flow_path.end(), isspace), gt_flow_path.end());
dst_frame_1 = imread(frame1_path);
dst_frame_2 = imread(frame2_path);
dst_GT = readOpticalFlow(gt_flow_path);
if (dst_frame_1.empty() || dst_frame_2.empty() || dst_GT.empty())
return false;
else
return true;
}
TEST(DenseOpticalFlow_SimpleFlow, ReferenceAccuracy)
{
Mat frame1, frame2, GT;
ASSERT_TRUE(readRubberWhale(frame1, frame2, GT));
float target_RMSE = 0.37f;
Mat flow;
Ptr<DenseOpticalFlow> algo;
algo = createOptFlow_SimpleFlow();
algo->calc(frame1, frame2, flow);
ASSERT_EQ(GT.rows, flow.rows);
ASSERT_EQ(GT.cols, flow.cols);
EXPECT_LE(calcRMSE(GT, flow), target_RMSE);
}
TEST(DenseOpticalFlow_DeepFlow, ReferenceAccuracy)
{
Mat frame1, frame2, GT;
ASSERT_TRUE(readRubberWhale(frame1, frame2, GT));
float target_RMSE = 0.35f;
cvtColor(frame1, frame1, COLOR_BGR2GRAY);
cvtColor(frame2, frame2, COLOR_BGR2GRAY);
Mat flow;
Ptr<DenseOpticalFlow> algo;
algo = createOptFlow_DeepFlow();
algo->calc(frame1, frame2, flow);
ASSERT_EQ(GT.rows, flow.rows);
ASSERT_EQ(GT.cols, flow.cols);
EXPECT_LE(calcRMSE(GT, flow), target_RMSE);
}
TEST(SparseOpticalFlow, ReferenceAccuracy)
{
// with the following test each invoker class should be tested once
Mat frame1, frame2, GT;
ASSERT_TRUE(readRubberWhale(frame1, frame2, GT));
vector<Point2f> prevPts, currPts;
for (int r = 0; r < frame1.rows; r+=10)
{
for (int c = 0; c < frame1.cols; c+=10)
{
prevPts.push_back(Point2f(static_cast<float>(c), static_cast<float>(r)));
}
}
vector<uchar> status(prevPts.size());
vector<float> err(prevPts.size());
Ptr<SparseRLOFOpticalFlow> algo = SparseRLOFOpticalFlow::create();
algo->setForwardBackward(0.0f);
Ptr<RLOFOpticalFlowParameter> param = Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter);
param->supportRegionType = SR_CROSS;
param->useIlluminationModel = true;
param->solverType = ST_BILINEAR;
param->setUseMEstimator(true);
algo->setRLOFOpticalFlowParameter(param);
algo->calc(frame1, frame2, prevPts, currPts, status, err);
EXPECT_LE(calcRMSE(prevPts, currPts, GT), 0.3f);
param->solverType = ST_STANDART;
algo->setRLOFOpticalFlowParameter(param);
algo->calc(frame1, frame2, prevPts, currPts, status, err);
EXPECT_LE(calcRMSE(prevPts, currPts, GT), 0.34f);
param->useIlluminationModel = false;
param->solverType = ST_BILINEAR;
algo->setRLOFOpticalFlowParameter(param);
algo->calc(frame1, frame2, prevPts, currPts, status, err);
EXPECT_LE(calcRMSE(prevPts, currPts, GT), 0.27f);
param->solverType = ST_STANDART;
algo->setRLOFOpticalFlowParameter(param);
algo->calc(frame1, frame2, prevPts, currPts, status, err);
EXPECT_LE(calcRMSE(prevPts, currPts, GT), 0.27f);
param->setUseMEstimator(false);
param->useIlluminationModel = true;
param->solverType = ST_BILINEAR;
algo->setRLOFOpticalFlowParameter(param);
algo->calc(frame1, frame2, prevPts, currPts, status, err);
EXPECT_LE(calcRMSE(prevPts, currPts, GT), 0.28f);
param->solverType = ST_STANDART;
algo->setRLOFOpticalFlowParameter(param);
algo->calc(frame1, frame2, prevPts, currPts, status, err);
EXPECT_LE(calcRMSE(prevPts, currPts, GT), 0.28f);
param->useIlluminationModel = false;
param->solverType = ST_BILINEAR;
algo->setRLOFOpticalFlowParameter(param);
algo->calc(frame1, frame2, prevPts, currPts, status, err);
EXPECT_LE(calcRMSE(prevPts, currPts, GT), 0.80f);
param->solverType = ST_STANDART;
algo->setRLOFOpticalFlowParameter(param);
algo->calc(frame1, frame2, prevPts, currPts, status, err);
EXPECT_LE(calcRMSE(prevPts, currPts, GT), 0.28f);
}
TEST(DenseOpticalFlow_RLOF, ReferenceAccuracy)
{
Mat frame1, frame2, GT;
ASSERT_TRUE(readRubberWhale(frame1, frame2, GT));
Mat flow;
Ptr<DenseRLOFOpticalFlow> algo = DenseRLOFOpticalFlow::create();
Ptr<RLOFOpticalFlowParameter> param = Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter);
param->setUseMEstimator(true);
param->supportRegionType = SR_CROSS;
param->solverType = ST_BILINEAR;
algo->setRLOFOpticalFlowParameter(param);
algo->setForwardBackward(1.0f);
algo->setGridStep(cv::Size(4, 4));
algo->setInterpolation(INTERP_EPIC);
algo->calc(frame1, frame2, flow);
ASSERT_EQ(GT.rows, flow.rows);
ASSERT_EQ(GT.cols, flow.cols);
EXPECT_LE(calcRMSE(GT, flow), 0.46f);
algo->setInterpolation(INTERP_GEO);
algo->calc(frame1, frame2, flow);
ASSERT_EQ(GT.rows, flow.rows);
ASSERT_EQ(GT.cols, flow.cols);
EXPECT_LE(calcRMSE(GT, flow), 0.55f);
}
TEST(DenseOpticalFlow_SparseToDenseFlow, ReferenceAccuracy)
{
Mat frame1, frame2, GT;
ASSERT_TRUE(readRubberWhale(frame1, frame2, GT));
float target_RMSE = 0.52f;
Mat flow;
Ptr<DenseOpticalFlow> algo;
algo = createOptFlow_SparseToDense();
algo->calc(frame1, frame2, flow);
ASSERT_EQ(GT.rows, flow.rows);
ASSERT_EQ(GT.cols, flow.cols);
EXPECT_LE(calcRMSE(GT, flow), target_RMSE);
}
TEST(DenseOpticalFlow_PCAFlow, ReferenceAccuracy)
{
Mat frame1, frame2, GT;
ASSERT_TRUE(readRubberWhale(frame1, frame2, GT));
const float target_RMSE = 0.55f;
Mat flow;
Ptr<DenseOpticalFlow> algo = createOptFlow_PCAFlow();
algo->calc(frame1, frame2, flow);
ASSERT_EQ(GT.rows, flow.rows);
ASSERT_EQ(GT.cols, flow.cols);
EXPECT_LE(calcRMSE(GT, flow), target_RMSE);
}
TEST(DenseOpticalFlow_GlobalPatchColliderDCT, ReferenceAccuracy)
{
Mat frame1, frame2, GT;
ASSERT_TRUE(readRubberWhale(frame1, frame2, GT));
const Size sz = frame1.size() / 2;
frame1 = frame1(Rect(0, 0, sz.width, sz.height));
frame2 = frame2(Rect(0, 0, sz.width, sz.height));
GT = GT(Rect(0, 0, sz.width, sz.height));
vector<Mat> img1, img2, gt;
vector< pair<Point2i, Point2i> > corr;
img1.push_back(frame1);
img2.push_back(frame2);
gt.push_back(GT);
Ptr< GPCForest<5> > forest = GPCForest<5>::create();
forest->train(img1, img2, gt, GPCTrainingParams(8, 3, GPC_DESCRIPTOR_DCT, false));
forest->findCorrespondences(frame1, frame2, corr);
ASSERT_LE(7500U, corr.size());
ASSERT_LE(calcAvgEPE(corr, GT), 0.5f);
}
TEST(DenseOpticalFlow_GlobalPatchColliderWHT, ReferenceAccuracy)
{
Mat frame1, frame2, GT;
ASSERT_TRUE(readRubberWhale(frame1, frame2, GT));
const Size sz = frame1.size() / 2;
frame1 = frame1(Rect(0, 0, sz.width, sz.height));
frame2 = frame2(Rect(0, 0, sz.width, sz.height));
GT = GT(Rect(0, 0, sz.width, sz.height));
vector<Mat> img1, img2, gt;
vector< pair<Point2i, Point2i> > corr;
img1.push_back(frame1);
img2.push_back(frame2);
gt.push_back(GT);
Ptr< GPCForest<5> > forest = GPCForest<5>::create();
forest->train(img1, img2, gt, GPCTrainingParams(8, 3, GPC_DESCRIPTOR_WHT, false));
forest->findCorrespondences(frame1, frame2, corr);
ASSERT_LE(7000U, corr.size());
ASSERT_LE(calcAvgEPE(corr, GT), 0.5f);
}
}} // namespace
modules/optflow/test/test_main.cpp
+6
View File
@@ -0,0 +1,6 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "test_precomp.hpp"
CV_TEST_MAIN("cv")
modules/optflow/test/test_motiontemplates.cpp
+501
View File
@@ -0,0 +1,501 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// Intel License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000, Intel Corporation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of Intel Corporation may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "test_precomp.hpp"
namespace opencv_test { namespace {
///////////////////// base MHI class ///////////////////////
class CV_MHIBaseTest : public cvtest::ArrayTest
{
public:
CV_MHIBaseTest();
protected:
void get_test_array_types_and_sizes( int test_case_idx, vector<vector<Size> >& sizes, vector<vector<int> >& types );
void get_minmax_bounds( int i, int j, int type, Scalar& low, Scalar& high );
int prepare_test_case( int test_case_idx );
double timestamp, duration, max_log_duration;
int mhi_i, mhi_ref_i;
double silh_ratio;
};
CV_MHIBaseTest::CV_MHIBaseTest()
{
timestamp = duration = 0;
max_log_duration = 9;
mhi_i = mhi_ref_i = -1;
silh_ratio = 0.25;
}
void CV_MHIBaseTest::get_minmax_bounds( int i, int j, int type, Scalar& low, Scalar& high )
{
cvtest::ArrayTest::get_minmax_bounds( i, j, type, low, high );
if( i == INPUT && CV_MAT_DEPTH(type) == CV_8U )
{
low = Scalar::all(cvRound(-1./silh_ratio)+2.);
high = Scalar::all(2);
}
else if( i == mhi_i || i == mhi_ref_i )
{
low = Scalar::all(-exp(max_log_duration));
high = Scalar::all(0.);
}
}
void CV_MHIBaseTest::get_test_array_types_and_sizes( int test_case_idx,
vector<vector<Size> >& sizes, vector<vector<int> >& types )
{
RNG& rng = ts->get_rng();
cvtest::ArrayTest::get_test_array_types_and_sizes( test_case_idx, sizes, types );
types[INPUT][0] = CV_8UC1;
types[mhi_i][0] = types[mhi_ref_i][0] = CV_32FC1;
duration = exp(cvtest::randReal(rng)*max_log_duration);
timestamp = duration + cvtest::randReal(rng)*30.-10.;
}
int CV_MHIBaseTest::prepare_test_case( int test_case_idx )
{
int code = cvtest::ArrayTest::prepare_test_case( test_case_idx );
if( code > 0 )
{
Mat& mat = test_mat[mhi_i][0];
mat += Scalar::all(duration);
cv::max(mat, 0, mat);
if( mhi_i != mhi_ref_i )
{
Mat& mat0 = test_mat[mhi_ref_i][0];
cvtest::copy( mat, mat0 );
}
}
return code;
}
///////////////////// update motion history ////////////////////////////
static void test_updateMHI( const Mat& silh, Mat& mhi, double timestamp, double duration )
{
int i, j;
float delbound = (float)(timestamp - duration);
for( i = 0; i < mhi.rows; i++ )
{
const uchar* silh_row = silh.ptr(i);
float* mhi_row = mhi.ptr<float>(i);
for( j = 0; j < mhi.cols; j++ )
{
if( silh_row[j] )
mhi_row[j] = (float)timestamp;
else if( mhi_row[j] < delbound )
mhi_row[j] = 0.f;
}
}
}
class CV_UpdateMHITest : public CV_MHIBaseTest
{
public:
CV_UpdateMHITest();
protected:
double get_success_error_level( int test_case_idx, int i, int j );
void run_func();
void prepare_to_validation( int );
};
CV_UpdateMHITest::CV_UpdateMHITest()
{
test_array[INPUT].push_back(NULL);
test_array[INPUT_OUTPUT].push_back(NULL);
test_array[REF_INPUT_OUTPUT].push_back(NULL);
mhi_i = INPUT_OUTPUT; mhi_ref_i = REF_INPUT_OUTPUT;
}
double CV_UpdateMHITest::get_success_error_level( int /*test_case_idx*/, int /*i*/, int /*j*/ )
{
return 0;
}
void CV_UpdateMHITest::run_func()
{
cv::motempl::updateMotionHistory( test_mat[INPUT][0], test_mat[INPUT_OUTPUT][0], timestamp, duration);
}
void CV_UpdateMHITest::prepare_to_validation( int /*test_case_idx*/ )
{
//CvMat m0 = test_mat[REF_INPUT_OUTPUT][0];
test_updateMHI( test_mat[INPUT][0], test_mat[REF_INPUT_OUTPUT][0], timestamp, duration );
}
///////////////////// calc motion gradient ////////////////////////////
static void test_MHIGradient( const Mat& mhi, Mat& mask, Mat& orientation,
double delta1, double delta2, int aperture_size )
{
Point anchor( aperture_size/2, aperture_size/2 );
double limit = 1e-4*aperture_size*aperture_size;
Mat dx, dy, min_mhi, max_mhi;
Mat kernel = cvtest::calcSobelKernel2D( 1, 0, aperture_size );
cvtest::filter2D( mhi, dx, CV_32F, kernel, anchor, 0, BORDER_REPLICATE );
kernel = cvtest::calcSobelKernel2D( 0, 1, aperture_size );
cvtest::filter2D( mhi, dy, CV_32F, kernel, anchor, 0, BORDER_REPLICATE );
kernel = Mat::ones(aperture_size, aperture_size, CV_8U);
cvtest::erode(mhi, min_mhi, kernel, anchor, 0, BORDER_REPLICATE);
cvtest::dilate(mhi, max_mhi, kernel, anchor, 0, BORDER_REPLICATE);
if( delta1 > delta2 )
{
std::swap( delta1, delta2 );
}
for( int i = 0; i < mhi.rows; i++ )
{
uchar* mask_row = mask.ptr(i);
float* orient_row = orientation.ptr<float>(i);
const float* dx_row = dx.ptr<float>(i);
const float* dy_row = dy.ptr<float>(i);
const float* min_row = min_mhi.ptr<float>(i);
const float* max_row = max_mhi.ptr<float>(i);
for( int j = 0; j < mhi.cols; j++ )
{
double delta = max_row[j] - min_row[j];
double _dx = dx_row[j], _dy = dy_row[j];
if( delta1 <= delta && delta <= delta2 &&
(fabs(_dx) > limit || fabs(_dy) > limit) )
{
mask_row[j] = 1;
double angle = atan2( _dy, _dx ) * (180/CV_PI);
if( angle < 0 )
angle += 360.;
orient_row[j] = (float)angle;
}
else
{
mask_row[j] = 0;
orient_row[j] = 0.f;
}
}
}
}
class CV_MHIGradientTest : public CV_MHIBaseTest
{
public:
CV_MHIGradientTest();
protected:
void get_test_array_types_and_sizes( int test_case_idx, vector<vector<Size> >& sizes, vector<vector<int> >& types );
double get_success_error_level( int test_case_idx, int i, int j );
void run_func();
void prepare_to_validation( int );
double delta1, delta2, delta_range_log;
int aperture_size;
};
CV_MHIGradientTest::CV_MHIGradientTest()
{
mhi_i = mhi_ref_i = INPUT;
test_array[INPUT].push_back(NULL);
test_array[OUTPUT].push_back(NULL);
test_array[OUTPUT].push_back(NULL);
test_array[REF_OUTPUT].push_back(NULL);
test_array[REF_OUTPUT].push_back(NULL);
delta1 = delta2 = 0;
aperture_size = 0;
delta_range_log = 4;
}
void CV_MHIGradientTest::get_test_array_types_and_sizes( int test_case_idx, vector<vector<Size> >& sizes, vector<vector<int> >& types )
{
RNG& rng = ts->get_rng();
CV_MHIBaseTest::get_test_array_types_and_sizes( test_case_idx, sizes, types );
types[OUTPUT][0] = types[REF_OUTPUT][0] = CV_8UC1;
types[OUTPUT][1] = types[REF_OUTPUT][1] = CV_32FC1;
delta1 = exp(cvtest::randReal(rng)*delta_range_log + 1.);
delta2 = exp(cvtest::randReal(rng)*delta_range_log + 1.);
aperture_size = (cvtest::randInt(rng)%3)*2+3;
//duration = exp(cvtest::randReal(rng)*max_log_duration);
//timestamp = duration + cvtest::randReal(rng)*30.-10.;
}
double CV_MHIGradientTest::get_success_error_level( int /*test_case_idx*/, int /*i*/, int j )
{
return j == 0 ? 0 : 2e-1;
}
void CV_MHIGradientTest::run_func()
{
cv::motempl::calcMotionGradient(test_mat[INPUT][0], test_mat[OUTPUT][0],
test_mat[OUTPUT][1], delta1, delta2, aperture_size );
//cvCalcMotionGradient( test_array[INPUT][0], test_array[OUTPUT][0],
// test_array[OUTPUT][1], delta1, delta2, aperture_size );
}
void CV_MHIGradientTest::prepare_to_validation( int /*test_case_idx*/ )
{
test_MHIGradient( test_mat[INPUT][0], test_mat[REF_OUTPUT][0],
test_mat[REF_OUTPUT][1], delta1, delta2, aperture_size );
test_mat[REF_OUTPUT][0] += Scalar::all(1);
test_mat[OUTPUT][0] += Scalar::all(1);
}
////////////////////// calc global orientation /////////////////////////
static double test_calcGlobalOrientation( const Mat& orient, const Mat& mask,
const Mat& mhi, double timestamp, double duration )
{
const int HIST_SIZE = 12;
int y, x;
int histogram[HIST_SIZE];
int max_bin = 0;
double base_orientation = 0, delta_orientation = 0, weight = 0;
double low_time, global_orientation;
memset( histogram, 0, sizeof( histogram ));
timestamp = 0;
for( y = 0; y < orient.rows; y++ )
{
const float* orient_data = orient.ptr<float>(y);
const uchar* mask_data = mask.ptr(y);
const float* mhi_data = mhi.ptr<float>(y);
for( x = 0; x < orient.cols; x++ )
if( mask_data[x] )
{
int bin = cvFloor( (orient_data[x]*HIST_SIZE)/360 );
histogram[bin < 0 ? 0 : bin >= HIST_SIZE ? HIST_SIZE-1 : bin]++;
if( mhi_data[x] > timestamp )
timestamp = mhi_data[x];
}
}
low_time = timestamp - duration;
for( x = 1; x < HIST_SIZE; x++ )
{
if( histogram[x] > histogram[max_bin] )
max_bin = x;
}
base_orientation = ((double)max_bin*360)/HIST_SIZE;
for( y = 0; y < orient.rows; y++ )
{
const float* orient_data = orient.ptr<float>(y);
const float* mhi_data = mhi.ptr<float>(y);
const uchar* mask_data = mask.ptr(y);
for( x = 0; x < orient.cols; x++ )
{
if( mask_data[x] && mhi_data[x] > low_time )
{
double diff = orient_data[x] - base_orientation;
double delta_weight = (((mhi_data[x] - low_time)/duration)*254 + 1)/255;
if( diff < -180 ) diff += 360;
if( diff > 180 ) diff -= 360;
if( delta_weight > 0 && fabs(diff) < 45 )
{
delta_orientation += diff*delta_weight;
weight += delta_weight;
}
}
}
}
if( weight == 0 )
global_orientation = base_orientation;
else
{
global_orientation = base_orientation + delta_orientation/weight;
if( global_orientation < 0 ) global_orientation += 360;
if( global_orientation > 360 ) global_orientation -= 360;
}
return global_orientation;
}
class CV_MHIGlobalOrientTest : public CV_MHIBaseTest
{
public:
CV_MHIGlobalOrientTest();
protected:
void get_test_array_types_and_sizes( int test_case_idx, vector<vector<Size> >& sizes, vector<vector<int> >& types );
void get_minmax_bounds( int i, int j, int type, Scalar& low, Scalar& high );
double get_success_error_level( int test_case_idx, int i, int j );
int validate_test_results( int test_case_idx );
void run_func();
double angle, min_angle, max_angle;
};
CV_MHIGlobalOrientTest::CV_MHIGlobalOrientTest()
{
mhi_i = mhi_ref_i = INPUT;
test_array[INPUT].push_back(NULL);
test_array[INPUT].push_back(NULL);
test_array[INPUT].push_back(NULL);
min_angle = max_angle = 0;
}
void CV_MHIGlobalOrientTest::get_test_array_types_and_sizes( int test_case_idx, vector<vector<Size> >& sizes, vector<vector<int> >& types )
{
RNG& rng = ts->get_rng();
CV_MHIBaseTest::get_test_array_types_and_sizes( test_case_idx, sizes, types );
Size size = sizes[INPUT][0];
size.width = MAX( size.width, 16 );
size.height = MAX( size.height, 16 );
sizes[INPUT][0] = sizes[INPUT][1] = sizes[INPUT][2] = size;
types[INPUT][1] = CV_8UC1; // mask
types[INPUT][2] = CV_32FC1; // orientation
min_angle = cvtest::randReal(rng)*359.9;
max_angle = cvtest::randReal(rng)*359.9;
if( min_angle >= max_angle )
{
std::swap( min_angle, max_angle);
}
max_angle += 0.1;
duration = exp(cvtest::randReal(rng)*max_log_duration);
timestamp = duration + cvtest::randReal(rng)*30.-10.;
}
void CV_MHIGlobalOrientTest::get_minmax_bounds( int i, int j, int type, Scalar& low, Scalar& high )
{
CV_MHIBaseTest::get_minmax_bounds( i, j, type, low, high );
if( i == INPUT && j == 2 )
{
low = Scalar::all(min_angle);
high = Scalar::all(max_angle);
}
}
double CV_MHIGlobalOrientTest::get_success_error_level( int /*test_case_idx*/, int /*i*/, int /*j*/ )
{
return 15;
}
void CV_MHIGlobalOrientTest::run_func()
{
//angle = cvCalcGlobalOrientation( test_array[INPUT][2], test_array[INPUT][1],
// test_array[INPUT][0], timestamp, duration );
angle = cv::motempl::calcGlobalOrientation(test_mat[INPUT][2], test_mat[INPUT][1],
test_mat[INPUT][0], timestamp, duration );
}
int CV_MHIGlobalOrientTest::validate_test_results( int test_case_idx )
{
//printf("%d. rows=%d, cols=%d, nzmask=%d\n", test_case_idx, test_mat[INPUT][1].rows, test_mat[INPUT][1].cols,
// cvCountNonZero(test_array[INPUT][1]));
double ref_angle = test_calcGlobalOrientation( test_mat[INPUT][2], test_mat[INPUT][1],
test_mat[INPUT][0], timestamp, duration );
double err_level = get_success_error_level( test_case_idx, 0, 0 );
int code = cvtest::TS::OK;
int nz = countNonZero( test_mat[INPUT][1] );
if( nz > 32 && !(min_angle - err_level <= angle &&
max_angle + err_level >= angle) &&
!(min_angle - err_level <= angle+360 &&
max_angle + err_level >= angle+360) )
{
ts->printf( cvtest::TS::LOG, "The angle=%g is outside (%g,%g) range\n",
angle, min_angle - err_level, max_angle + err_level );
code = cvtest::TS::FAIL_BAD_ACCURACY;
}
else if( fabs(angle - ref_angle) > err_level &&
fabs(360 - fabs(angle - ref_angle)) > err_level )
{
ts->printf( cvtest::TS::LOG, "The angle=%g differs too much from reference value=%g\n",
angle, ref_angle );
code = cvtest::TS::FAIL_BAD_ACCURACY;
}
if( code < 0 )
ts->set_failed_test_info( code );
return code;
}
TEST(Video_MHIUpdate, accuracy) { CV_UpdateMHITest test; test.safe_run(); }
TEST(Video_MHIGradient, accuracy) { CV_MHIGradientTest test; test.safe_run(); }
TEST(Video_MHIGlobalOrient, accuracy) { CV_MHIGlobalOrientTest test; test.safe_run(); }
}} // namespace
modules/optflow/test/test_precomp.hpp
+15
View File
@@ -0,0 +1,15 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef __OPENCV_TEST_PRECOMP_HPP__
#define __OPENCV_TEST_PRECOMP_HPP__
#include "opencv2/ts.hpp"
#include "opencv2/optflow.hpp"
namespace opencv_test {
using namespace cv::optflow;
using namespace perf;
}
#endif
modules/optflow/test/test_tvl1optflow.cpp
+173
View File
@@ -0,0 +1,173 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// Intel License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000, Intel Corporation, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of Intel Corporation may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "test_precomp.hpp"
namespace opencv_test { namespace {
//#define DUMP
// first four bytes, should be the same in little endian
const float FLO_TAG_FLOAT = 202021.25f; // check for this when READING the file
#ifdef DUMP
// binary file format for flow data specified here:
// http://vision.middlebury.edu/flow/data/
void writeOpticalFlowToFile(const Mat_<Point2f>& flow, const string& fileName)
{
const char FLO_TAG_STRING[] = "PIEH"; // use this when WRITING the file
ofstream file(fileName.c_str(), ios_base::binary);
file << FLO_TAG_STRING;
file.write((const char*) &flow.cols, sizeof(int));
file.write((const char*) &flow.rows, sizeof(int));
for (int i = 0; i < flow.rows; ++i)
{
for (int j = 0; j < flow.cols; ++j)
{
const Point2f u = flow(i, j);
file.write((const char*) &u.x, sizeof(float));
file.write((const char*) &u.y, sizeof(float));
}
}
}
#endif
// binary file format for flow data specified here:
// http://vision.middlebury.edu/flow/data/
void readOpticalFlowFromFile(Mat_<Point2f>& flow, const string& fileName)
{
std::ifstream file(fileName.c_str(), std::ios_base::binary);
float tag;
file.read((char*) &tag, sizeof(float));
CV_Assert( tag == FLO_TAG_FLOAT );
Size size;
file.read((char*) &size.width, sizeof(int));
file.read((char*) &size.height, sizeof(int));
flow.create(size);
for (int i = 0; i < flow.rows; ++i)
{
for (int j = 0; j < flow.cols; ++j)
{
Point2f u;
file.read((char*) &u.x, sizeof(float));
file.read((char*) &u.y, sizeof(float));
flow(i, j) = u;
}
}
file.close();
}
bool isFlowCorrect(Point2f u)
{
return !cvIsNaN(u.x) && !cvIsNaN(u.y) && (fabs(u.x) < 1e9) && (fabs(u.y) < 1e9);
}
void check(const Mat_<Point2f>& gold, const Mat_<Point2f>& flow, double threshold = 0.1, double expectedAccuracy = 0.95)
{
threshold = threshold*threshold;
size_t gold_counter = 0;
size_t valid_counter = 0;
for (int i = 0; i < gold.rows; ++i)
{
for (int j = 0; j < gold.cols; ++j)
{
const Point2f u1 = gold(i, j);
const Point2f u2 = flow(i, j);
if (isFlowCorrect(u1))
{
gold_counter++;
if (isFlowCorrect(u2))
{
const Point2f diff = u1 - u2;
double err = diff.ddot(diff);
if (err <= threshold)
valid_counter++;
}
}
}
}
EXPECT_GE(valid_counter, expectedAccuracy * gold_counter);
}
TEST(Contrib_calcOpticalFlowDual_TVL1, Regression)
{
const string frame1_path = TS::ptr()->get_data_path() + "optflow/RubberWhale1.png";
const string frame2_path = TS::ptr()->get_data_path() + "optflow/RubberWhale2.png";
const string gold_flow_path = TS::ptr()->get_data_path() + "optflow/tvl1_flow.flo";
Mat frame1 = imread(frame1_path, IMREAD_GRAYSCALE);
Mat frame2 = imread(frame2_path, IMREAD_GRAYSCALE);
ASSERT_FALSE(frame1.empty());
ASSERT_FALSE(frame2.empty());
Mat_<Point2f> flow;
Ptr<DualTVL1OpticalFlow> tvl1 = cv::optflow::DualTVL1OpticalFlow::create();
tvl1->calc(frame1, frame2, flow);
#ifdef DUMP
writeOpticalFlowToFile(flow, gold_flow_path);
#else
Mat_<Point2f> gold;
readOpticalFlowFromFile(gold, gold_flow_path);
ASSERT_EQ(gold.rows, flow.rows);
ASSERT_EQ(gold.cols, flow.cols);
check(gold, flow);
#endif
}
}} // namespace