~charon-developers/hekate/trunk

« back to all changes in this revision

Viewing changes to src/TrackAnnotationBoxesKLT.cpp

  • Committer: Moritz Becker
  • Date: 2014-06-17 13:35:07 UTC
  • Revision ID: moritz.becker@pallas-ludens.com-20140617133507-z6dm5bd3rsukj05d
tracker

Show diffs side-by-side

added added

removed removed

Lines of Context:
 
1
/*  Copyright (C) 2014 Moritz Becker
 
2
 
 
3
        This file is part of Charon.
 
4
 
 
5
        Charon is free software: you can redistribute it and/or modify
 
6
        it under the terms of the GNU Lesser General Public License as published by
 
7
        the Free Software Foundation, either version 3 of the License, or
 
8
        (at your option) any later version.
 
9
 
 
10
        Charon is distributed in the hope that it will be useful,
 
11
        but WITHOUT ANY WARRANTY; without even the implied warranty of
 
12
        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
13
        GNU Lesser General Public License for more details.
 
14
 
 
15
        You should have received a copy of the GNU Lesser General Public License
 
16
        along with Charon.  If not, see <http://www.gnu.org/licenses/>.
 
17
*/
 
18
/** \file TrackAnnotationBoxesKLT.cpp
 
19
 *  Implementation of parameter class TrackAnnotationBoxesKLT.
 
20
 *  \author <a href="mailto:moritz.becker@pallas-ludens.com">
 
21
 *      Moritz Becker</a>
 
22
 *  \date 03.06.2014
 
23
 */
 
24
 
 
25
#include <charon-core/ParameteredObject.hxx>
 
26
#include <hekate/TrackAnnotationBoxesKLT.h>
 
27
#include <regex>
 
28
 
 
29
using namespace cv;
 
30
 
 
31
TrackAnnotationBoxesKLT::TrackAnnotationBoxesKLT(const std::string& name) :
 
32
                ParameteredObject(
 
33
                        "TrackAnnotationBoxesKLT", name,
 
34
                        "<h2>Track annotation boxes using KLT from OpenCV</h2><br>"
 
35
                        "Track annotation boxes using KLT from OpenCV"
 
36
                )
 
37
{
 
38
 
 
39
        ParameteredObject::_addInputSlot(
 
40
                annotations, "annotations",
 
41
                "Box objects",
 
42
                "AnnotationList");
 
43
 
 
44
        ParameteredObject::_addInputSlot(
 
45
                filepaths, "filepaths",
 
46
                "filepaths",
 
47
                "vector<string>");
 
48
 
 
49
        ParameteredObject::_addOutputSlot(
 
50
                boxesTracked, "boxesTracked",
 
51
                "Tracked Boxes",
 
52
                "AnnotationList");
 
53
 
 
54
        ParameteredObject::_addOutputSlot(
 
55
                debug, "debug",
 
56
                "debug",
 
57
                "CImgList<double>");
 
58
 
 
59
        ParameteredObject::_addParameter< int >(
 
60
                trackLength, "trackLength",
 
61
                "Number of frames to track boxes",
 
62
                1, "int");
 
63
 
 
64
        ParameteredObject::_addParameter< int >(
 
65
                minBoxSize, "minBoxSize",
 
66
                "Minimum width of height of annotation box",
 
67
                10, "int");
 
68
 
 
69
        ParameteredObject::_addParameter< int >(
 
70
                maxBoxSize, "maxBoxSize",
 
71
                "Maximum width of height of annotation box",
 
72
                500, "int");
 
73
 
 
74
}
 
75
 
 
76
std::string getFilepath(std::string filename, const std::vector<std::string> &list, int &_id){
 
77
 
 
78
        std::string path = "";
 
79
        std::string filename2 = "";
 
80
        int id = 0;
 
81
        for (auto it = list.begin(); it != list.end(); it++, id++){
 
82
                filename2 = (*it).substr((*it).find_last_of(FileTool::slash) + 1, (*it).length());
 
83
                if (filename2 == filename){
 
84
                        path = (*it);
 
85
                        _id = id;
 
86
                }
 
87
        }
 
88
 
 
89
        //sout << "filename2 = " << filename2 << std::endl;
 
90
 
 
91
        return path;
 
92
 
 
93
}
 
94
 
 
95
std::string getFilename(std::string filepath){
 
96
 
 
97
        return filepath.substr(filepath.find_last_of(FileTool::slash) + 1, filepath.length());
 
98
 
 
99
}
 
100
 
 
101
int getFilenameId(std::string filename){
 
102
 
 
103
        std::regex e("(\\d+)(\\D*)\\.");
 
104
        std::smatch sm;
 
105
        
 
106
        if (std::regex_search(filename, sm, e)) {
 
107
                std::stringstream s;
 
108
                s << sm[1];
 
109
                return stoi(s.str());
 
110
        }
 
111
        else
 
112
                return -1;
 
113
 
 
114
}
 
115
 
 
116
float weight(int xN, int yN, int x1, int y1){
 
117
 
 
118
        float weight = std::sqrt( (float) (xN - x1)*(xN - x1) - (yN - y1)*(yN - y1));
 
119
 
 
120
        return weight;
 
121
 
 
122
}
 
123
 
 
124
/**
 
125
* Function to perform fast template matching with image pyramid
 
126
*/
 
127
void fastMatchTemplate(cv::Mat& srca,  // The reference image
 
128
        cv::Mat& srcb,  // The template image
 
129
        cv::Mat& dst,   // Template matching result
 
130
        int maxlevel)   // Number of levels
 
131
{
 
132
        std::vector<cv::Mat> refs, tpls, results;
 
133
 
 
134
        // Build Gaussian pyramid
 
135
        cv::buildPyramid(srca, refs, maxlevel);
 
136
        cv::buildPyramid(srcb, tpls, maxlevel);
 
137
 
 
138
        cv::Mat ref, tpl, res;
 
139
 
 
140
        // Process each level
 
141
        for (int level = maxlevel; level >= 0; level--)
 
142
        {
 
143
                ref = refs[level];
 
144
                tpl = tpls[level];
 
145
                res = cv::Mat::zeros(ref.size() + cv::Size(1, 1) - tpl.size(), CV_32FC1);
 
146
 
 
147
                if (level == maxlevel)
 
148
                {
 
149
                        // On the smallest level, just perform regular template matching
 
150
                        cv::matchTemplate(ref, tpl, res, CV_TM_CCORR_NORMED);
 
151
                }
 
152
                else
 
153
                {
 
154
                        // On the next layers, template matching is performed on pre-defined 
 
155
                        // ROI areas.  We define the ROI using the template matching result 
 
156
                        // from the previous layer.
 
157
 
 
158
                        cv::Mat mask;
 
159
                        cv::pyrUp(results.back(), mask);
 
160
 
 
161
                        cv::Mat mask8u;
 
162
                        mask.convertTo(mask8u, CV_8U);
 
163
 
 
164
                        // Find matches from previous layer
 
165
                        std::vector<std::vector<cv::Point> > contours;
 
166
                        cv::findContours(mask8u, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
 
167
 
 
168
                        // Use the contours to define region of interest and 
 
169
                        // perform template matching on the areas
 
170
                        for (int i = 0; i < contours.size(); i++)
 
171
                        {
 
172
                                cv::Rect r = cv::boundingRect(contours[i]);
 
173
                                cv::matchTemplate(
 
174
                                        ref(r + (tpl.size() - cv::Size(1, 1))),
 
175
                                        tpl,
 
176
                                        res(r),
 
177
                                        CV_TM_CCORR_NORMED
 
178
                                        );
 
179
                        }
 
180
                }
 
181
 
 
182
                // Only keep good matches
 
183
                cv::threshold(res, res, 0.94, 1., CV_THRESH_TOZERO);
 
184
                results.push_back(res);
 
185
        }
 
186
 
 
187
        res.copyTo(dst);
 
188
}
 
189
 
 
190
void TrackAnnotationBoxesKLT::execute() {
 
191
 
 
192
//      cv::Mat ref = cv::imread(this->filepaths()[0].c_str());
 
193
//      cv::Mat refPrint = cv::imread(this->filepaths()[0].c_str());
 
194
//      Rect rect(378, 125, 30, 30);
 
195
//      cv::Mat tpl = ref(rect); // cv::imread(this->filepaths()[1].c_str());
 
196
//
 
197
//
 
198
//      cv::Mat ref_gray, tpl_gray;
 
199
//      cv::cvtColor(ref, ref_gray, CV_BGR2GRAY);
 
200
//      cv::cvtColor(tpl, tpl_gray, CV_BGR2GRAY);
 
201
//
 
202
//      cv::Mat dst;
 
203
//
 
204
//      double maxvalAll = 0;
 
205
//      cv::Point  maxlocAll(0,0);
 
206
//      int patchHeight = 0;
 
207
//      int patchWidth = 0;
 
208
//
 
209
//      for (int i = 0; i < 100; i++){
 
210
//
 
211
//              Mat tplScaled = tpl_gray;
 
212
//              Size sz = tpl_gray.size();
 
213
//              sz.height = sz.height * ( 0.25 + i * 0.02 );
 
214
//              sz.width = sz.width * ( 0.25 + i * 0.02 );
 
215
//
 
216
//              resize(tpl_gray /* input image*/, tplScaled /*result image */, sz/*new dimensions*/, 0, 0, INTER_CUBIC/* interpolation method*/);
 
217
//
 
218
//              fastMatchTemplate(ref_gray, tplScaled, dst, 2);
 
219
//
 
220
//              double minval, maxval;
 
221
//              cv::Point minloc, maxloc;
 
222
//              cv::minMaxLoc(dst, &minval, &maxval, &minloc, &maxloc);
 
223
//
 
224
//              if (maxval > maxvalAll){
 
225
//                      maxvalAll = maxval;
 
226
//                      maxlocAll = maxloc;
 
227
//                      patchHeight = sz.height;
 
228
//                      patchWidth = sz.width;
 
229
//              }
 
230
//
 
231
//      }
 
232
//
 
233
//      cv::rectangle(
 
234
//              refPrint, maxlocAll,
 
235
//              cv::Point(maxlocAll.x + patchWidth, maxlocAll.y + patchHeight),
 
236
//              CV_RGB(0, std::min(float(1), float((maxvalAll - 0.94) * 10)) * 255, 0), 2
 
237
//              );
 
238
//
 
239
//      cv::imshow("result", refPrint);
 
240
//      cv::waitKey();
 
241
 
 
242
 
 
243
 
 
244
        const int minSize = 5;
 
245
        
 
246
        const annotation::AnnotationList &in = this->annotations();
 
247
 
 
248
        this->boxesTracked() = in;
 
249
 
 
250
        const std::vector< const annotation::Frame* > frames = in.getFrames();
 
251
 
 
252
        sout << "Read " << frames.size() << " annotations" << std::endl;
 
253
 
 
254
        TermCriteria termcrit(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03);
 
255
 
 
256
        // iterate over list of all available frames
 
257
        auto it = frames.begin();
 
258
        for (; it != frames.end(); it++){
 
259
 
 
260
                if ( !(*it)->annotationIds.size() )
 
261
                        continue;
 
262
 
 
263
                sout << "\tProcess frame " << (*it)->frameId << std::endl;
 
264
 
 
265
                // get filename saved in annoation list
 
266
                std::string filename = (*it)->filename;
 
267
 
 
268
                // id of filepath entry inside of this->filepaths()
 
269
                int filepathId;
 
270
 
 
271
                // find corresponding filepath from filelist
 
272
                std::string filepath = getFilepath(filename, this->filepaths(), filepathId);
 
273
 
 
274
                // get numbers of frames to track +1
 
275
                const int length = std::min( (int) this->trackLength() + 1, (int) this->filepaths().size() - filepathId);
 
276
 
 
277
                if (length < 1)
 
278
                        continue;
 
279
                
 
280
                //GFTTDetector detector;
 
281
 
 
282
 
 
283
                //-- Step 1: Detect the keypoints using SURF Detector
 
284
                //SurfFeatureDetector detectorSURF(400);
 
285
                //SurfFeatureDetector detector( minHessian );   // detector
 
286
                //SurfDescriptorExtractor extractor;            // descriptor generator
 
287
 
 
288
                // Load images
 
289
                for (int i = filepathId; i < filepathId + length -1; i++){
 
290
 
 
291
                        //cv::Mat ref = cv::imread(this->filepaths()[0].c_str());
 
292
                        //cv::Mat refPrint = cv::imread(this->filepaths()[0].c_str());
 
293
                        //Rect rect(378, 125, 30, 30);
 
294
                        //cv::Mat tpl = ref(rect); // cv::imread(this->filepaths()[1].c_str());
 
295
                        //
 
296
                        //
 
297
                        //cv::Mat ref_gray, tpl_gray;
 
298
                        //cv::cvtColor(ref, ref_gray, CV_BGR2GRAY);
 
299
                        //cv::cvtColor(tpl, tpl_gray, CV_BGR2GRAY);
 
300
                        //
 
301
                        //cv::Mat dst;
 
302
 
 
303
                        /////////////////////////////////////////////////
 
304
 
 
305
                        // frames
 
306
                        cv::Mat ref = cv::imread(this->filepaths()[i + 1].c_str());
 
307
                        cv::Mat tpl = cv::imread(this->filepaths()[i].c_str());
 
308
                        cv::Mat ref_gray;
 
309
                        cv::cvtColor(ref, ref_gray, CV_BGR2GRAY);
 
310
 
 
311
                        cv::Mat dst;
 
312
 
 
313
                        //Mat img = imread(this->filepaths()[i].c_str(), CV_LOAD_IMAGE_GRAYSCALE);
 
314
                        //Mat img2 = imread(this->filepaths()[i + 1].c_str(), CV_LOAD_IMAGE_GRAYSCALE);
 
315
 
 
316
                        cv::Size s = tpl.size();
 
317
                        int h = s.height;
 
318
                        int w = s.width;
 
319
 
 
320
                        // masks for boxes
 
321
                        //Mat mask(s, CV_8UC1);
 
322
                        //mask = Scalar(0);
 
323
 
 
324
                        std::string currentFilename = getFilename(this->filepaths()[i]);
 
325
                        std::string refFilename = getFilename(this->filepaths()[i+1]);
 
326
                        int currentId = getFilenameId(this->filepaths()[i]);
 
327
 
 
328
                        sout << "\tTrack boxes of frame " << currentId << ": " << currentFilename << " in frame " << refFilename << std::endl;
 
329
 
 
330
                        try{
 
331
 
 
332
                                const vector< const annotation::AnnotationBox* > boxes = this->boxesTracked().getAnnotationBoxes(currentFilename );
 
333
 
 
334
                                auto it2 = boxes.begin();
 
335
                                const auto it2End = boxes.end();
 
336
                                for (; it2 != it2End; it2++){
 
337
 
 
338
                                        // get coordinates
 
339
                                        int x1 = std::max(0, (int)(*it2)->x1);
 
340
                                        int y1 = std::max(0, (int)(*it2)->y1);
 
341
                                        int x2 = std::min(w-1, (int)(*it2)->x2);
 
342
                                        int y2 = std::min(h-1, (int)(*it2)->y2);
 
343
                                        int _w = x2 - x1 +1;
 
344
                                        int _h = y2 - y1 + 1;
 
345
 
 
346
                                        sout << "\t\ttrack annotation box " << (*it2)->annotationId << ": x1=" << x1 << ": y1=" << y1 << ": x2=" << x2 << ": y2=" << y2 << ": w=" << _w << ", h=" << _h << std::endl;
 
347
 
 
348
                                        if (_h < this->minBoxSize() || _h > this->maxBoxSize() || _w < this->minBoxSize() || _w > this->maxBoxSize() )
 
349
                                                continue;
 
350
                                        
 
351
                                        Rect r(x1, y1, _w, _h);
 
352
                                        Mat patch = tpl(r);
 
353
                                        Mat patch_gray;
 
354
                                        cv::cvtColor(patch, patch_gray, CV_BGR2GRAY);
 
355
 
 
356
                                        double maxvalAll = 0;
 
357
                                        cv::Point  maxlocAll(0, 0);
 
358
                                        int patchHeightAll = 0;
 
359
                                        int patchWidthAll = 0;
 
360
 
 
361
                                        float threshold = 0.598;
 
362
                                        float minSize = 0.25; // minimal size of comparison template (relative to src)
 
363
                                        float maxSize = 4; // maximal size of comparison template (relative to src)
 
364
                                        int steps = 100; // number of steps between min and max size
 
365
                                        int pyramids = 3; // number of pyramids for matchTemplate
 
366
 
 
367
                                        for (int i = 0; i < steps; i++){
 
368
 
 
369
                                                Mat tplScaled = patch_gray;
 
370
                                                Size sz = patch_gray.size();
 
371
 
 
372
                                                int patchHeight = sz.height * (minSize + i * 0.01 * maxSize);
 
373
                                                int patchWidth  = sz.width  * (minSize + i * 0.01 * maxSize);
 
374
 
 
375
                                                if (patchHeight < this->minBoxSize() || patchHeight > this->maxBoxSize() || patchWidth < this->minBoxSize() || patchWidth > this->maxBoxSize() || patchHeight >= h || patchWidth >= w || patchHeight < minSize || patchWidth < minSize)
 
376
                                                        continue;
 
377
 
 
378
                                                sz.height = patchHeight;
 
379
                                                sz.width = patchWidth;
 
380
 
 
381
                                                resize(patch_gray /* input image*/, tplScaled /*result image */, sz/*new dimensions*/, 0, 0, INTER_CUBIC/* interpolation method*/);
 
382
                                                //cv::cvtColor(tplScaled, tplScaled, CV_BGR2GRAY);
 
383
 
 
384
                                                cimg_library::CImg<double> dbg = openCvUtil::Mat2CImg<double>(tplScaled);
 
385
                                                this->debug().push_back(dbg);
 
386
 
 
387
                                                fastMatchTemplate(ref_gray, tplScaled, dst, pyramids);
 
388
 
 
389
                                                double minval, maxval;
 
390
                                                cv::Point minloc, maxloc;
 
391
                                                cv::minMaxLoc(dst, &minval, &maxval, &minloc, &maxloc);
 
392
                                                
 
393
                                                if (maxval >threshold && maxval > maxvalAll){
 
394
                                                        maxvalAll = maxval;
 
395
                                                        maxlocAll = maxloc;
 
396
                                                        patchHeightAll = sz.height;
 
397
                                                        patchWidthAll = sz.width;
 
398
                                                }
 
399
 
 
400
                                                //while (true)
 
401
                                                //{
 
402
                                                //
 
403
                                                //      double minval, maxval;
 
404
                                                //      cv::Point minloc, maxloc;
 
405
                                                //      cv::minMaxLoc(dst, &minval, &maxval, &minloc, &maxloc);
 
406
                                                //
 
407
                                                //      if (maxval >= 0.97)
 
408
                                                //      {
 
409
                                                //              cv::rectangle(
 
410
                                                //                      ref, maxloc,
 
411
                                                //                      cv::Point(maxloc.x + tplScaled.cols, maxloc.y + tplScaled.rows),
 
412
                                                //                      CV_RGB(0, std::min(float(1), float((maxval - 0.94) * 10)) * 255, 0), 2
 
413
                                                //                      );
 
414
                                                //              cv::floodFill(
 
415
                                                //                      dst, maxloc,
 
416
                                                //                      cv::Scalar(0), 0,
 
417
                                                //                      cv::Scalar(.1),
 
418
                                                //                      cv::Scalar(1.)
 
419
                                                //                      );
 
420
                                                //      }
 
421
                                                //      else
 
422
                                                //              break;
 
423
                                                //}
 
424
 
 
425
                                        }
 
426
 
 
427
                                        if (maxvalAll > threshold)
 
428
                                                this->boxesTracked().addAnnotationBox(annotation::AnnotationBox(refFilename, true, false, maxlocAll.x, maxlocAll.y, maxlocAll.x + patchWidthAll, maxlocAll.y + patchHeightAll));
 
429
 
 
430
                                        //cv::rectangle(
 
431
                                        //      refPrint, maxlocAll,
 
432
                                        //      cv::Point(maxlocAll.x + patchWidth, maxlocAll.y + patchHeight),
 
433
                                        //      CV_RGB(0, std::min(float(1), float((maxvalAll - 0.94) * 10)) * 255, 0), 2
 
434
                                        //      );
 
435
 
 
436
                                        //cimg_library::CImg<double> dbg = openCvUtil::Mat2CImg<double>(img);
 
437
                                        //this->debug().push_back(dbg);
 
438
                                        //
 
439
                                        //dbg = openCvUtil::Mat2CImg<double>(patch);
 
440
                                        //this->debug().push_back(dbg);
 
441
                                        //
 
442
                                        //cv::Size s3(_w, _h);
 
443
                                        //sout << s3.width << " // " << s3.height << std::endl;
 
444
                                        //Mat patch3(s3, CV_LOAD_IMAGE_GRAYSCALE);
 
445
                                        //patch = Scalar(0);
 
446
                                        //
 
447
                                        //sout << "img.type() = " << img.type() << std::endl;
 
448
                                        //sout << "patch3.type() = " << patch3.type() << std::endl;
 
449
                                        //sout << "patch.type() = " << patch.type() << std::endl;
 
450
                                        //
 
451
                                        //for (int x = x1; x <= x2; x++){
 
452
                                        //      for (int y = y1; y <= y2; y++){
 
453
                                        //              sout << "x = " << x << " / y = " << y << std::endl;
 
454
                                        //              //patch3.at<uchar>(y - y1, x - x1) = 0;
 
455
                                        //              patch3.at<uchar>(y - y1, x - x1) = img.at<uchar>(y, x);
 
456
                                        //              //unsigned char px = img.at<uchar>(y, x);
 
457
                                        //              //sout << "img.at<uchar>(y, x) = " <<  px << std::endl;
 
458
                                        //              //patch3.at<float>(y - y1, x - x1) = img.at<float>(y, x);
 
459
                                        //      //      img.at<uchar>(y, x) = 128;
 
460
                                        //
 
461
                                        //      }
 
462
                                        //}
 
463
                                        //
 
464
                                        //dbg = openCvUtil::Mat2CImg<double>(patch3);
 
465
                                        //this->debug().push_back(dbg);
 
466
                                        //
 
467
                                        //sout << "\t\tdraw box" << std::endl;
 
468
                                        //
 
469
                                        //Mat result;
 
470
                                        //int method = CV_TM_SQDIFF_NORMED;
 
471
                                        ////matchTemplate(img2, patch, result, method);
 
472
                                        //matchTemplate(img, patch, result, method);    // MATCH WITH SELF
 
473
                                        //
 
474
                                        //sout << "\t\tRESULT DIMENSION: " << result.cols << " ::: " << result.rows << " ::: " << result.size << std::endl;
 
475
                                        //
 
476
                                        //dbg = openCvUtil::Mat2CImg<double>(result);
 
477
                                        //this->debug().push_back(dbg);
 
478
                                        //
 
479
                                        //dbg = openCvUtil::Mat2CImg<double>(img);
 
480
                                        //this->debug().push_back(dbg);
 
481
                                        //
 
482
                                        //cv::Size s2 = result.size();
 
483
                                        //int h2 = s2.height;
 
484
                                        //int w2 = s2.width;
 
485
                                        //
 
486
                                        //sout << "\t\t result width = " << w2 << std::endl;
 
487
                                        //sout << "\t\t result height = " << h2 << std::endl;
 
488
                                        //
 
489
                                        //int xN = 0, yN = 0;
 
490
                                        //float cost = result.at<uchar>(xN, yN) * weight(xN, yN, x1, y1);
 
491
                                        //sout << cost << std::endl;
 
492
                                        //// get best result
 
493
                                        //for (int x = 0; x < w2; x++){
 
494
                                        //      for (int y = 0; y < h2; y++){
 
495
                                        //              //mask.at<uchar>(y, x) = 1;
 
496
                                        //
 
497
                                        //              float costN = result.at<float>(y, x) *weight(x, y, x1, y1);
 
498
                                        //              
 
499
                                        //              if (costN < cost){
 
500
                                        //                      cost = costN;
 
501
                                        //                      xN = x;
 
502
                                        //                      yN = y;
 
503
                                        //              }
 
504
                                        //              
 
505
                                        //      }
 
506
                                        //}
 
507
                                        //
 
508
                                        //Rect r2(xN, yN, _w, _h);
 
509
                                        //Mat patch2 = img(r2);
 
510
                                        //dbg = openCvUtil::Mat2CImg<double>(patch2);
 
511
                                        //this->debug().push_back(dbg);
 
512
 
 
513
                                }
 
514
 
 
515
                                //cv::imshow("result", ref);
 
516
                                //cv::waitKey();
 
517
 
 
518
                        }
 
519
                        catch (std::range_error& e) {
 
520
                                sout << e.what() << std::endl;
 
521
                                //sout << "catched... " << currentFilename << std::endl;
 
522
                                continue;
 
523
                        }
 
524
 
 
525
                        /*
 
526
                        std::vector< KeyPoint > keypoints;
 
527
                        std::vector< KeyPoint > keypoints2;
 
528
 
 
529
                        cimg_library::CImg<double> dbg = openCvUtil::Mat2CImg<double>(mask);
 
530
                        this->debug().push_back(dbg);
 
531
 
 
532
                        vector<Point2f> points;
 
533
                        vector<Point2f> points2;
 
534
                        //int MAX_COUNT = 100;
 
535
                        //Mat gray;
 
536
                        //cvtColor(img, gray, CV_BGR2GRAY);
 
537
                        //goodFeaturesToTrack(gray, points, MAX_COUNT, 0.01, 10, mask, 3, 0, 0.04);
 
538
                        //sout << points.size() << std::endl;
 
539
                        //cornerSubPix(img, points, Size(10, 10), Size(-1, -1), termcrit);
 
540
 
 
541
                        // get feature points inside of masked areas
 
542
                        //detector.detect(img, keypoints, mask);
 
543
 
 
544
                        detectorSURF.detect(img, keypoints, mask);
 
545
                        
 
546
                        for (auto itP = keypoints.begin(); itP != keypoints.end(); itP++){
 
547
                                points.push_back((*itP).pt);
 
548
                                //sout << (*itP).pt.x << " / " << (*itP).pt.y << std::endl;
 
549
                        }
 
550
 
 
551
                        points2 = points;
 
552
                        //for ()
 
553
                        //vector<Point2f> points = keypoints;
 
554
 
 
555
                        vector<uchar> status;
 
556
                        vector<float> err;
 
557
 
 
558
                        if (!points.size())
 
559
                                continue;
 
560
 
 
561
                        //calcOpticalFlowPyrLK(img, img2, points, points2, status, err, Size winSize = Size(21, 21), int maxLevel = 3, TermCriteria criteria = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 30, 0.01), int flags = 0, double minEigThreshold = 1e-4);
 
562
                        calcOpticalFlowPyrLK(img, img2, points, points2, status, err, Size(32, 32), 0);//, TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 30, 0.01), 0, 1e-4);
 
563
 
 
564
                        sout << points2.size() << " --> " << points.size() << std::endl;
 
565
                        sout << status.size() << std::endl;
 
566
 
 
567
                        //Mat imgShow = img;
 
568
                        //Mat imgShow2 = img2;
 
569
 
 
570
                        cimg_library::CImg<double> dbg2(this->filepaths()[i].c_str());
 
571
                        dbg2.resize(dbg2.width(), dbg2.height(), 1, 3);
 
572
 
 
573
                        const char color[] = { 255, 0, 0 };
 
574
 
 
575
                        for (int k = 0; k < points2.size(); k++){
 
576
                                if (status[k] != 0) {
 
577
                                        sout << points[k].x << " / " << points[k].y << " --> " << points2[k].x << " / " << points2[k].y << ": status " << (status[k] == 0 ? 0 : 1) << ": err " << err[k] << std::endl;
 
578
                                        //circle(imgShow, points[k], 3, Scalar(0, 255, 0), -1, 8);
 
579
                                        //circle(imgShow2, points2[k], 3, Scalar(0, 255, 0), -1, 8);
 
580
                                        dbg2.draw_arrow(points[k].x, points[k].y, points2[k].x, points2[k].y, color);
 
581
                                }
 
582
                        }
 
583
 
 
584
                        this->debug().push_back(dbg2);
 
585
 
 
586
 
 
587
                        // DO WARP OF BOXES
 
588
 
 
589
                        // save new boxes
 
590
                        const vector< const annotation::AnnotationBox* > boxes = this->boxesTracked().getAnnotationBoxes(currentFilename);
 
591
 
 
592
                        std::string currentFilename2 = getFilename(this->filepaths()[i + 1]);
 
593
                        this->boxesTracked().addFrame(annotation::Frame(currentFilename2, i + 1));
 
594
 
 
595
                        auto it2 = boxes.begin();
 
596
                        const auto it2End = boxes.end();
 
597
                        for (; it2 != it2End; it2++){
 
598
 
 
599
                                // get coordinates
 
600
                                int x1 = std::max(0, std::min(w - 1, (int)(*it2)->x1));
 
601
                                int y1 = std::max(0, std::min(h - 1, (int)(*it2)->y1));
 
602
                                int x2 = std::max(0, std::min(w - 1, (int)(*it2)->x2));
 
603
                                int y2 = std::max(0, std::min(h - 1, (int)(*it2)->y2));
 
604
 
 
605
                                int count = 0;
 
606
                                float u = 0, v = 0;
 
607
 
 
608
                                float maxError = 1000;
 
609
 
 
610
                                // get flow of points inside of this mask
 
611
                                for (int ptIt = 0; ptIt < points.size(); ptIt++){
 
612
                                        if (err[ptIt]< maxError)
 
613
                                                if (points[ptIt].x >= x1 && points[ptIt].x <= x2 && points[ptIt].y >= y1 && points[ptIt].y <= y2){
 
614
                                                        count++;
 
615
                                                        u += points2[ptIt].x - points[ptIt].x;
 
616
                                                        v += points2[ptIt].y - points[ptIt].y;
 
617
                                                }
 
618
                                }
 
619
 
 
620
                                // do not add warped box without given flow
 
621
                                if (!count)
 
622
                                        continue;
 
623
 
 
624
                                u /= count;
 
625
                                v /= count;
 
626
 
 
627
                                x1 += u;
 
628
                                x2 += u;
 
629
                                y1 += v;
 
630
                                y2 += v;
 
631
 
 
632
                                this->boxesTracked().addAnnotationBox(annotation::AnnotationBox(currentFilename2, true, false, x1, y1, x2, y2));
 
633
                                
 
634
                        }
 
635
 
 
636
                        */
 
637
 
 
638
                }
 
639
 
 
640
                //cvCalcOpticalFlowPyrLK()
 
641
 
 
642
                //sout << "\t\tkeypoints: " << keypoints[0].size() << std::endl;
 
643
 
 
644
        }
 
645
 
 
646
}
 
647
 
 
648
// the following functions are needed
 
649
// for class TrackAnnotationBoxesKLT to work as a charon plugin.
 
650
extern "C" trackannotationboxesklt_DECLDIR ParameteredObject*
 
651
                create(const std::string& name, ParameteredObject::template_type) {
 
652
        return new TrackAnnotationBoxesKLT(name);
 
653
}
 
654
 
 
655
extern "C" trackannotationboxesklt_DECLDIR void destroy(ParameteredObject* b) {
 
656
        delete b;
 
657
}
 
658
 
 
659
/// Report build configuration to prevent linking of incompatibel runtime libs
 
660
extern "C" trackannotationboxesklt_DECLDIR ParameteredObject::build_type getBuildType() {
 
661
#ifdef _DEBUG
 
662
        return ParameteredObject::DEBUG_BUILD;
 
663
#else
 
664
        return ParameteredObject::RELEASE_BUILD;
 
665
#endif
 
666
}