MRPT  2.0.1
tracking.h
Go to the documentation of this file.
1 /* +------------------------------------------------------------------------+
2  | Mobile Robot Programming Toolkit (MRPT) |
3  | https://www.mrpt.org/ |
4  | |
5  | Copyright (c) 2005-2020, Individual contributors, see AUTHORS file |
6  | See: https://www.mrpt.org/Authors - All rights reserved. |
7  | Released under BSD License. See: https://www.mrpt.org/License |
8  +------------------------------------------------------------------------+ */
9 
10 #pragma once
11 
12 #include <mrpt/vision/types.h>
13 
14 #include <mrpt/img/CImage.h>
17 #include <mrpt/vision/TKeyPoint.h>
18 #include <memory> // for unique_ptr
19 
20 namespace mrpt::vision
21 {
22 /** \addtogroup vision_tracking Feature detection and tracking
23  * \ingroup mrpt_vision_grp
24  * @{ */
25 
26 /** A virtual interface for all feature trackers, implementing the part of
27  * feature tracking that is common to any specific tracker implementation.
28  * This class provides a quite robust tracking of features, avoiding as many
29  * outliers as possible but not all of them:
30  * more robust tracking would require application-specific information and
31  * could be done in a number of very different approaches,
32  * so this class will not try to do any kind of RANSAC or any other advanced
33  * outlier rejection; instead, it should
34  * be done by the users or the classes that employ this class.
35  *
36  * The basic usage of this class is as follows:
37  * \code
38  * CFeatureTracker_KL tracker; // Note: CFeatureTracker_KL is the
39  * most robust implementation for now.
40  * tracker.extra_params["add_new_features"] = 1; // Enable detection of
41  * new features, not only tracking
42  * tracker.extra_params[...] = ...
43  * // ....
44  * TKeyPointList theFeats; // The list of features
45  * mrpt::img::CImage previous_img, current_img;
46  *
47  * while (true) {
48  * current_img = ... // Grab new image.
49  * if ( previous_img_is_ok )
50  * tracker.trackFeatures(previous_img, current_img, theFeats);
51  * previous_img = current_img;
52  * }
53  * \endcode
54  *
55  * Below follows the list of optional parameters for "extra_params" which can
56  * be set
57  * and will be understood by this base class for any specific tracker
58  * implementation.
59  * Note that all parameters are double's, but boolean flags are emulated by
60  * the values 0.0 (false) and 1.0 (true).
61  *
62  * List of parameters:
63  * <table border="1" >
64  * <tr><td align="center" > <b>Parameter name</b> </td> <td align="center"
65  * > <b>Default value</b> </td> <td align="center" > <b>Comments</b> </td> </tr>
66  * <tr><td align="center" > add_new_features </td> <td align="center" > 0
67  * </td>
68  * <td> If set to "1", the class will not only track existing features,
69  * but will also perform (after doing the actual tracking) an efficient
70  * search for new features with the FAST detector, and will add them
71  * to the passed feature list if they fulfill a set of restrictions,
72  * as stablished by the other parameters (see
73  * <i>add_new_feat_min_separation</i>,<i>add_new_feat_max_features</i>,<i>minimum_KLT_response_to_add</i>).
74  * </td> </tr>
75  * <tr><td align="center" > add_new_feat_min_separation </td> <td
76  * align="center" > 15 </td>
77  * <td> If <i>add_new_features</i>==1, this is the minimum separation (in
78  * pixels) to any other (old, or new) feature for it
79  * being considered a candidate to be added.
80  * </td> </tr>
81  * <tr><td align="center" > desired_num_features_adapt </td> <td
82  * align="center" > (img_width*img_height)/512 </td>
83  * <td> If <i>add_new_features</i>==1, the threshold of the FAST(ER)
84  * feature detector is dynamically adapted such as the number of
85  * raw FAST keypoints is around this number. This number should be much
86  * higher than the real desired numbre of features, since this
87  * one includes many features concentrated in space which are later
88  * discarded for the minimum distance.
89  * </td> </tr>
90  * <tr><td align="center" > desired_num_features </td> <td align="center" >
91  * 100 </td>
92  * <td> If <i>add_new_features</i>==1, the target number of the patch
93  * associated to each feature will be updated with every N'th frame. </td> </tr>
94  * <tr><td align="center" > add_new_feat_patch_size </td> <td
95  * align="center" > 11 </td>
96  * <td> If <i>add_new_features</i>==1, for each new added feature, this
97  * is the size of the patch to be extracted around the keypoint (set to 0 if
98  * patches are not required at all).
99  * </td> </tr>
100  * <tr><td align="center" > minimum_KLT_response_to_add </td> <td
101  * align="center" > 10 </td>
102  * <td> If <i>add_new_features</i>==1, this sets the minimum KLT response
103  * of candidate FAST features to be added in each frame, if they also fulfil the
104  * other restrictions (e.g. min.distance).
105  * </td> </tr>
106  * <tr><td align="center" > check_KLT_response_every </td> <td
107  * align="center" > 0 </td>
108  * <td> If >0, it will compute the KLT response at each feature point
109  * every <i>N</i> frames
110  * and those below <i>minimum_KLT_response</i> will be marked as
111  * "lost" in their "track_status" field.
112  * </td> </tr>
113  * <tr><td align="center" > minimum_KLT_response </td> <td align="center" >
114  * 5 </td>
115  * <td> See explanation of <i>check_KLT_response_every</i>.
116  * </td> </tr>
117  * <tr><td align="center" > KLT_response_half_win </td> <td align="center"
118  * > 4 </td>
119  * <td> When computing the KLT response of features (see
120  * <i>minimum_KLT_response</i> and <i>minimum_KLT_response_to_add</i>),
121  * the window centered at the point for its estimation will be of
122  * size (2*W+1)x(2*W+1), with <i>W</i> being this parameter value.
123  * </td> </tr>
124  * <tr><td align="center" > update_patches_every </td> <td align="center" >
125  * 0 </td>
126  * <td> If !=0, the patch associated to each feature will be updated with
127  * every N'th frame. </td> </tr>
128  * <tr><td align="center" > remove_lost_features </td> <td align="center" >
129  * 0 </td>
130  * <td> If !=0, out-of-bound features or those lost while tracking, will
131  * be automatically removed from the list of features.
132  * Otherwise, the user will have to manually remove them by checking
133  * the track_status field. </td> </tr>
134  * </table>
135  *
136  * This class also offers a time profiler, disabled by default (see
137  * getProfiler and enableTimeLogger).
138  *
139  * \sa CFeatureTracker_KL, the example application "track-video-features".
140  */
142 {
143  /** Optional list of extra parameters to the algorithm. */
145 
146  /** Default ctor */
147  inline CGenericFeatureTracker() : m_timlog(false) {}
148  /** Ctor with extra parameters */
150  : extra_params(extraParams),
151  m_timlog(false),
155  {
156  }
157  /** Dtor */
158  virtual ~CGenericFeatureTracker() = default;
159  /** Perform feature tracking from "old_img" to "new_img", with a (possibly
160  *empty) list of previously tracked features "inout_featureList".
161  * This is a list of parameters (in "extraParams") accepted by ALL
162  *implementations of feature tracker (see each derived class for more
163  *specific parameters).
164  * - "add_new_features" (Default=0). If set to "1", new features will
165  *be
166  *also added to the existing ones in areas of the image poor of features.
167  * This method does:
168  * - Convert old and new images to grayscale, if they're in color.
169  * - Call the pure virtual "trackFeatures_impl" method.
170  * - Implement the optional detection of new features if
171  *"add_new_features"!=0.
172  */
173  void trackFeatures(
174  const mrpt::img::CImage& old_img, const mrpt::img::CImage& new_img,
175  TKeyPointList& inout_featureList);
176 
177  /** overload with subpixel precision */
178  void trackFeatures(
179  const mrpt::img::CImage& old_img, const mrpt::img::CImage& new_img,
180  TKeyPointfList& inout_featureList);
181 
182  /** Returns a read-only reference to the internal time logger */
184  {
185  return m_timlog;
186  }
187  /** Returns a reference to the internal time logger */
189  /** Returns a read-only reference to the internal time logger */
190  inline void enableTimeLogger(bool enable = true)
191  {
192  m_timlog.enable(enable);
193  }
194 
195  /** Returns the current adaptive threshold used by the FAST(ER) detector to
196  * find out new features in empty areas */
197  inline int getDetectorAdaptiveThreshold() const
198  {
200  }
201 
203  {
204  /** In the new_img with the last adaptive threshold */
206  /** The number of features which were deleted due to OOB, bad tracking,
207  * etc... (only if "remove_lost_features" is enabled) */
209  };
210 
211  /** Updated with each call to trackFeatures() */
213 
214  protected:
215  /** The tracking method implementation, to be implemented in children
216  * classes. */
217  virtual void trackFeatures_impl(
218  const mrpt::img::CImage& old_img, const mrpt::img::CImage& new_img,
219  TKeyPointfList& inout_featureList);
220 
221  /** The tracking method implementation, to be implemented in children
222  * classes. */
223  virtual void trackFeatures_impl(
224  const mrpt::img::CImage& old_img, const mrpt::img::CImage& new_img,
225  TKeyPointList& inout_featureList) = 0;
226 
227  /** the internal time logger, disabled by default. */
229 
230  /** This field is clared by \a trackFeatures() before calling \a
231  * trackFeatures_impl(), and
232  * can be filled out with newly defected FAST(ER) features in the latter.
233  * If it's not the case, feats will be computed anyway if the user enabled
234  * the "add_new_features" option.
235  */
237 
238  /** Adapts the threshold \a m_detector_adaptive_thres according to the real
239  * and desired number of features just detected */
241  const size_t nNewlyDetectedFeats, const size_t desired_num_features);
242 
243  private:
244  /** for use when "update_patches_every">=1 */
246  /** For use when "check_KLT_response_every">=1 */
248  /** For use in "add_new_features" == true */
250 
251  template <typename FEATLIST>
253  const mrpt::img::CImage& old_img, const mrpt::img::CImage& new_img,
254  FEATLIST& inout_featureList);
255 };
256 
257 using CGenericFeatureTrackerAutoPtr = std::unique_ptr<CGenericFeatureTracker>;
258 
259 /** Track a set of features from old_img -> new_img using sparse optimal flow
260  *(classic KL method).
261  *
262  * See CGenericFeatureTracker for a more detailed explanation on how to use
263  *this class.
264  *
265  * List of additional parameters in "extra_params" (apart from those in
266  *CGenericFeatureTracker) accepted by this class:
267  * - "window_width" (Default=15)
268  * - "window_height" (Default=15)
269  * - "LK_levels" (Default=3) Number of pyramids to build for LK tracking
270  *(this
271  *parameter only has effects when tracking with CImage's, not with
272  *CImagePyramid's).
273  * - "LK_max_iters" (Default=10) Max. number of iterations in LK tracking.
274  * - "LK_epsilon" (Default=0.1) Minimum epsilon step in interations of
275  *LK_tracking.
276  * - "LK_max_tracking_error" (Default=150.0) The maximum "tracking error"
277  *of
278  *LK tracking such as a feature is marked as "lost".
279  *
280  * \sa OpenCV's method cvCalcOpticalFlowPyrLK
281  */
283 {
284  /** Default ctor */
285  inline CFeatureTracker_KL() = default;
286  /** Ctor with extra parameters */
288  : CGenericFeatureTracker(extraParams)
289  {
290  }
291 
292  protected:
293  void trackFeatures_impl(
294  const mrpt::img::CImage& old_img, const mrpt::img::CImage& new_img,
295  TKeyPointList& inout_featureList) override;
296  void trackFeatures_impl(
297  const mrpt::img::CImage& old_img, const mrpt::img::CImage& new_img,
298  TKeyPointfList& inout_featureList) override;
299 
300  private:
301  template <typename FEATLIST>
303  const mrpt::img::CImage& old_img, const mrpt::img::CImage& new_img,
304  FEATLIST& inout_featureList);
305 };
306 
307 /** @} */ // end of grouping
308 } // namespace mrpt::vision
void trackFeatures(const mrpt::img::CImage &old_img, const mrpt::img::CImage &new_img, TKeyPointList &inout_featureList)
Perform feature tracking from "old_img" to "new_img", with a (possibly empty) list of previously trac...
Definition: tracking.cpp:647
mrpt::system::TParametersDouble extra_params
Optional list of extra parameters to the algorithm.
Definition: tracking.h:144
void internal_trackFeatures(const mrpt::img::CImage &old_img, const mrpt::img::CImage &new_img, FEATLIST &inout_featureList)
Perform feature tracking from "old_img" to "new_img", with a (possibly empty) list of previously trac...
Definition: tracking.cpp:444
mrpt::system::CTimeLogger m_timlog
the internal time logger, disabled by default.
Definition: tracking.h:228
int m_detector_adaptive_thres
For use in "add_new_features" == true.
Definition: tracking.h:249
virtual void trackFeatures_impl(const mrpt::img::CImage &old_img, const mrpt::img::CImage &new_img, TKeyPointfList &inout_featureList)
The tracking method implementation, to be implemented in children classes.
CFeatureTracker_KL(mrpt::system::TParametersDouble extraParams)
Ctor with extra parameters.
Definition: tracking.h:287
void updateAdaptiveNewFeatsThreshold(const size_t nNewlyDetectedFeats, const size_t desired_num_features)
Adapts the threshold m_detector_adaptive_thres according to the real and desired number of features j...
Definition: tracking.cpp:659
Classes for computer vision, detectors, features, etc.
Definition: CDifodo.h:17
mrpt::vision::TKeyPointList m_newly_detected_feats
This field is clared by trackFeatures() before calling trackFeatures_impl(), and can be filled out wi...
Definition: tracking.h:236
size_t raw_FAST_feats_detected
In the new_img with the last adaptive threshold.
Definition: tracking.h:205
void trackFeatures_impl_templ(const mrpt::img::CImage &old_img, const mrpt::img::CImage &new_img, FEATLIST &inout_featureList)
Track a set of features from old_img -> new_img using sparse optimal flow (classic KL method) Optiona...
Definition: tracking_KL.cpp:37
size_t num_deleted_feats
The number of features which were deleted due to OOB, bad tracking, etc...
Definition: tracking.h:208
std::unique_ptr< CGenericFeatureTracker > CGenericFeatureTrackerAutoPtr
Definition: tracking.h:257
CFeatureTracker_KL()=default
Default ctor.
void enable(bool enabled=true)
void enableTimeLogger(bool enable=true)
Returns a read-only reference to the internal time logger.
Definition: tracking.h:190
Track a set of features from old_img -> new_img using sparse optimal flow (classic KL method)...
Definition: tracking.h:282
mrpt::system::CTimeLogger & getProfiler()
Returns a reference to the internal time logger.
Definition: tracking.h:188
A versatile "profiler" that logs the time spent within each pair of calls to enter(X)-leave(X), among other stats.
CGenericFeatureTracker(mrpt::system::TParametersDouble extraParams)
Ctor with extra parameters.
Definition: tracking.h:149
void trackFeatures_impl(const mrpt::img::CImage &old_img, const mrpt::img::CImage &new_img, TKeyPointList &inout_featureList) override
The tracking method implementation, to be implemented in children classes.
size_t m_check_KLT_counter
For use when "check_KLT_response_every">=1.
Definition: tracking.h:247
virtual ~CGenericFeatureTracker()=default
Dtor.
size_t m_update_patches_counter
for use when "update_patches_every">=1
Definition: tracking.h:245
TExtraOutputInfo last_execution_extra_info
Updated with each call to trackFeatures()
Definition: tracking.h:212
CGenericFeatureTracker()
Default ctor.
Definition: tracking.h:147
A virtual interface for all feature trackers, implementing the part of feature tracking that is commo...
Definition: tracking.h:141
int getDetectorAdaptiveThreshold() const
Returns the current adaptive threshold used by the FAST(ER) detector to find out new features in empt...
Definition: tracking.h:197
const mrpt::system::CTimeLogger & getProfiler() const
Returns a read-only reference to the internal time logger.
Definition: tracking.h:183
A class for storing images as grayscale or RGB bitmaps.
Definition: img/CImage.h:148



Page generated by Doxygen 1.8.14 for MRPT 2.0.1 Git: 0fef1a6d7 Fri Apr 3 23:00:21 2020 +0200 at vie abr 3 23:20:28 CEST 2020