1
/* Copyright (C) 2014 Moritz Becker
3
This file is part of Charon.
5
Charon is free software: you can redistribute it and/or modify
6
it under the terms of the GNU Lesser General Public License as published by
7
the Free Software Foundation, either version 3 of the License, or
8
(at your option) any later version.
10
Charon is distributed in the hope that it will be useful,
11
but WITHOUT ANY WARRANTY; without even the implied warranty of
12
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
GNU Lesser General Public License for more details.
15
You should have received a copy of the GNU Lesser General Public License
16
along with Charon. If not, see <http://www.gnu.org/licenses/>.
18
/** \file TrackAnnotationBoxesKLT.cpp
19
* Implementation of parameter class TrackAnnotationBoxesKLT.
20
* \author <a href="mailto:moritz.becker@pallas-ludens.com">
25
#include <charon-core/ParameteredObject.hxx>
26
#include <hekate/TrackAnnotationBoxesKLT.h>
31
TrackAnnotationBoxesKLT::TrackAnnotationBoxesKLT(const std::string& name) :
33
"TrackAnnotationBoxesKLT", name,
34
"<h2>Track annotation boxes using KLT from OpenCV</h2><br>"
35
"Track annotation boxes using KLT from OpenCV"
39
ParameteredObject::_addInputSlot(
40
annotations, "annotations",
44
ParameteredObject::_addInputSlot(
45
filepaths, "filepaths",
49
ParameteredObject::_addOutputSlot(
50
boxesTracked, "boxesTracked",
54
ParameteredObject::_addOutputSlot(
59
ParameteredObject::_addParameter< int >(
60
trackLength, "trackLength",
61
"Number of frames to track boxes",
64
ParameteredObject::_addParameter< int >(
65
minBoxSize, "minBoxSize",
66
"Minimum width of height of annotation box",
69
ParameteredObject::_addParameter< int >(
70
maxBoxSize, "maxBoxSize",
71
"Maximum width of height of annotation box",
76
std::string getFilepath(std::string filename, const std::vector<std::string> &list, int &_id){
78
std::string path = "";
79
std::string filename2 = "";
81
for (auto it = list.begin(); it != list.end(); it++, id++){
82
filename2 = (*it).substr((*it).find_last_of(FileTool::slash) + 1, (*it).length());
83
if (filename2 == filename){
89
//sout << "filename2 = " << filename2 << std::endl;
95
std::string getFilename(std::string filepath){
97
return filepath.substr(filepath.find_last_of(FileTool::slash) + 1, filepath.length());
101
int getFilenameId(std::string filename){
103
std::regex e("(\\d+)(\\D*)\\.");
106
if (std::regex_search(filename, sm, e)) {
109
return stoi(s.str());
116
float weight(int xN, int yN, int x1, int y1){
118
float weight = std::sqrt( (float) (xN - x1)*(xN - x1) - (yN - y1)*(yN - y1));
125
* Function to perform fast template matching with image pyramid
127
void fastMatchTemplate(cv::Mat& srca, // The reference image
128
cv::Mat& srcb, // The template image
129
cv::Mat& dst, // Template matching result
130
int maxlevel) // Number of levels
132
std::vector<cv::Mat> refs, tpls, results;
134
// Build Gaussian pyramid
135
cv::buildPyramid(srca, refs, maxlevel);
136
cv::buildPyramid(srcb, tpls, maxlevel);
138
cv::Mat ref, tpl, res;
140
// Process each level
141
for (int level = maxlevel; level >= 0; level--)
145
res = cv::Mat::zeros(ref.size() + cv::Size(1, 1) - tpl.size(), CV_32FC1);
147
if (level == maxlevel)
149
// On the smallest level, just perform regular template matching
150
cv::matchTemplate(ref, tpl, res, CV_TM_CCORR_NORMED);
154
// On the next layers, template matching is performed on pre-defined
155
// ROI areas. We define the ROI using the template matching result
156
// from the previous layer.
159
cv::pyrUp(results.back(), mask);
162
mask.convertTo(mask8u, CV_8U);
164
// Find matches from previous layer
165
std::vector<std::vector<cv::Point> > contours;
166
cv::findContours(mask8u, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
168
// Use the contours to define region of interest and
169
// perform template matching on the areas
170
for (int i = 0; i < contours.size(); i++)
172
cv::Rect r = cv::boundingRect(contours[i]);
174
ref(r + (tpl.size() - cv::Size(1, 1))),
182
// Only keep good matches
183
cv::threshold(res, res, 0.94, 1., CV_THRESH_TOZERO);
184
results.push_back(res);
190
void TrackAnnotationBoxesKLT::execute() {
192
// cv::Mat ref = cv::imread(this->filepaths()[0].c_str());
193
// cv::Mat refPrint = cv::imread(this->filepaths()[0].c_str());
194
// Rect rect(378, 125, 30, 30);
195
// cv::Mat tpl = ref(rect); // cv::imread(this->filepaths()[1].c_str());
198
// cv::Mat ref_gray, tpl_gray;
199
// cv::cvtColor(ref, ref_gray, CV_BGR2GRAY);
200
// cv::cvtColor(tpl, tpl_gray, CV_BGR2GRAY);
204
// double maxvalAll = 0;
205
// cv::Point maxlocAll(0,0);
206
// int patchHeight = 0;
207
// int patchWidth = 0;
209
// for (int i = 0; i < 100; i++){
211
// Mat tplScaled = tpl_gray;
212
// Size sz = tpl_gray.size();
213
// sz.height = sz.height * ( 0.25 + i * 0.02 );
214
// sz.width = sz.width * ( 0.25 + i * 0.02 );
216
// resize(tpl_gray /* input image*/, tplScaled /*result image */, sz/*new dimensions*/, 0, 0, INTER_CUBIC/* interpolation method*/);
218
// fastMatchTemplate(ref_gray, tplScaled, dst, 2);
220
// double minval, maxval;
221
// cv::Point minloc, maxloc;
222
// cv::minMaxLoc(dst, &minval, &maxval, &minloc, &maxloc);
224
// if (maxval > maxvalAll){
225
// maxvalAll = maxval;
226
// maxlocAll = maxloc;
227
// patchHeight = sz.height;
228
// patchWidth = sz.width;
234
// refPrint, maxlocAll,
235
// cv::Point(maxlocAll.x + patchWidth, maxlocAll.y + patchHeight),
236
// CV_RGB(0, std::min(float(1), float((maxvalAll - 0.94) * 10)) * 255, 0), 2
239
// cv::imshow("result", refPrint);
244
const int minSize = 5;
246
const annotation::AnnotationList &in = this->annotations();
248
this->boxesTracked() = in;
250
const std::vector< const annotation::Frame* > frames = in.getFrames();
252
sout << "Read " << frames.size() << " annotations" << std::endl;
254
TermCriteria termcrit(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03);
256
// iterate over list of all available frames
257
auto it = frames.begin();
258
for (; it != frames.end(); it++){
260
if ( !(*it)->annotationIds.size() )
263
sout << "\tProcess frame " << (*it)->frameId << std::endl;
265
// get filename saved in annoation list
266
std::string filename = (*it)->filename;
268
// id of filepath entry inside of this->filepaths()
271
// find corresponding filepath from filelist
272
std::string filepath = getFilepath(filename, this->filepaths(), filepathId);
274
// get numbers of frames to track +1
275
const int length = std::min( (int) this->trackLength() + 1, (int) this->filepaths().size() - filepathId);
280
//GFTTDetector detector;
283
//-- Step 1: Detect the keypoints using SURF Detector
284
//SurfFeatureDetector detectorSURF(400);
285
//SurfFeatureDetector detector( minHessian ); // detector
286
//SurfDescriptorExtractor extractor; // descriptor generator
289
for (int i = filepathId; i < filepathId + length -1; i++){
291
//cv::Mat ref = cv::imread(this->filepaths()[0].c_str());
292
//cv::Mat refPrint = cv::imread(this->filepaths()[0].c_str());
293
//Rect rect(378, 125, 30, 30);
294
//cv::Mat tpl = ref(rect); // cv::imread(this->filepaths()[1].c_str());
297
//cv::Mat ref_gray, tpl_gray;
298
//cv::cvtColor(ref, ref_gray, CV_BGR2GRAY);
299
//cv::cvtColor(tpl, tpl_gray, CV_BGR2GRAY);
303
/////////////////////////////////////////////////
306
cv::Mat ref = cv::imread(this->filepaths()[i + 1].c_str());
307
cv::Mat tpl = cv::imread(this->filepaths()[i].c_str());
309
cv::cvtColor(ref, ref_gray, CV_BGR2GRAY);
313
//Mat img = imread(this->filepaths()[i].c_str(), CV_LOAD_IMAGE_GRAYSCALE);
314
//Mat img2 = imread(this->filepaths()[i + 1].c_str(), CV_LOAD_IMAGE_GRAYSCALE);
316
cv::Size s = tpl.size();
321
//Mat mask(s, CV_8UC1);
324
std::string currentFilename = getFilename(this->filepaths()[i]);
325
std::string refFilename = getFilename(this->filepaths()[i+1]);
326
int currentId = getFilenameId(this->filepaths()[i]);
328
sout << "\tTrack boxes of frame " << currentId << ": " << currentFilename << " in frame " << refFilename << std::endl;
332
const vector< const annotation::AnnotationBox* > boxes = this->boxesTracked().getAnnotationBoxes(currentFilename );
334
auto it2 = boxes.begin();
335
const auto it2End = boxes.end();
336
for (; it2 != it2End; it2++){
339
int x1 = std::max(0, (int)(*it2)->x1);
340
int y1 = std::max(0, (int)(*it2)->y1);
341
int x2 = std::min(w-1, (int)(*it2)->x2);
342
int y2 = std::min(h-1, (int)(*it2)->y2);
344
int _h = y2 - y1 + 1;
346
sout << "\t\ttrack annotation box " << (*it2)->annotationId << ": x1=" << x1 << ": y1=" << y1 << ": x2=" << x2 << ": y2=" << y2 << ": w=" << _w << ", h=" << _h << std::endl;
348
if (_h < this->minBoxSize() || _h > this->maxBoxSize() || _w < this->minBoxSize() || _w > this->maxBoxSize() )
351
Rect r(x1, y1, _w, _h);
354
cv::cvtColor(patch, patch_gray, CV_BGR2GRAY);
356
double maxvalAll = 0;
357
cv::Point maxlocAll(0, 0);
358
int patchHeightAll = 0;
359
int patchWidthAll = 0;
361
float threshold = 0.598;
362
float minSize = 0.25; // minimal size of comparison template (relative to src)
363
float maxSize = 4; // maximal size of comparison template (relative to src)
364
int steps = 100; // number of steps between min and max size
365
int pyramids = 3; // number of pyramids for matchTemplate
367
for (int i = 0; i < steps; i++){
369
Mat tplScaled = patch_gray;
370
Size sz = patch_gray.size();
372
int patchHeight = sz.height * (minSize + i * 0.01 * maxSize);
373
int patchWidth = sz.width * (minSize + i * 0.01 * maxSize);
375
if (patchHeight < this->minBoxSize() || patchHeight > this->maxBoxSize() || patchWidth < this->minBoxSize() || patchWidth > this->maxBoxSize() || patchHeight >= h || patchWidth >= w || patchHeight < minSize || patchWidth < minSize)
378
sz.height = patchHeight;
379
sz.width = patchWidth;
381
resize(patch_gray /* input image*/, tplScaled /*result image */, sz/*new dimensions*/, 0, 0, INTER_CUBIC/* interpolation method*/);
382
//cv::cvtColor(tplScaled, tplScaled, CV_BGR2GRAY);
384
cimg_library::CImg<double> dbg = openCvUtil::Mat2CImg<double>(tplScaled);
385
this->debug().push_back(dbg);
387
fastMatchTemplate(ref_gray, tplScaled, dst, pyramids);
389
double minval, maxval;
390
cv::Point minloc, maxloc;
391
cv::minMaxLoc(dst, &minval, &maxval, &minloc, &maxloc);
393
if (maxval >threshold && maxval > maxvalAll){
396
patchHeightAll = sz.height;
397
patchWidthAll = sz.width;
403
// double minval, maxval;
404
// cv::Point minloc, maxloc;
405
// cv::minMaxLoc(dst, &minval, &maxval, &minloc, &maxloc);
407
// if (maxval >= 0.97)
411
// cv::Point(maxloc.x + tplScaled.cols, maxloc.y + tplScaled.rows),
412
// CV_RGB(0, std::min(float(1), float((maxval - 0.94) * 10)) * 255, 0), 2
427
if (maxvalAll > threshold)
428
this->boxesTracked().addAnnotationBox(annotation::AnnotationBox(refFilename, true, false, maxlocAll.x, maxlocAll.y, maxlocAll.x + patchWidthAll, maxlocAll.y + patchHeightAll));
431
// refPrint, maxlocAll,
432
// cv::Point(maxlocAll.x + patchWidth, maxlocAll.y + patchHeight),
433
// CV_RGB(0, std::min(float(1), float((maxvalAll - 0.94) * 10)) * 255, 0), 2
436
//cimg_library::CImg<double> dbg = openCvUtil::Mat2CImg<double>(img);
437
//this->debug().push_back(dbg);
439
//dbg = openCvUtil::Mat2CImg<double>(patch);
440
//this->debug().push_back(dbg);
442
//cv::Size s3(_w, _h);
443
//sout << s3.width << " // " << s3.height << std::endl;
444
//Mat patch3(s3, CV_LOAD_IMAGE_GRAYSCALE);
447
//sout << "img.type() = " << img.type() << std::endl;
448
//sout << "patch3.type() = " << patch3.type() << std::endl;
449
//sout << "patch.type() = " << patch.type() << std::endl;
451
//for (int x = x1; x <= x2; x++){
452
// for (int y = y1; y <= y2; y++){
453
// sout << "x = " << x << " / y = " << y << std::endl;
454
// //patch3.at<uchar>(y - y1, x - x1) = 0;
455
// patch3.at<uchar>(y - y1, x - x1) = img.at<uchar>(y, x);
456
// //unsigned char px = img.at<uchar>(y, x);
457
// //sout << "img.at<uchar>(y, x) = " << px << std::endl;
458
// //patch3.at<float>(y - y1, x - x1) = img.at<float>(y, x);
459
// // img.at<uchar>(y, x) = 128;
464
//dbg = openCvUtil::Mat2CImg<double>(patch3);
465
//this->debug().push_back(dbg);
467
//sout << "\t\tdraw box" << std::endl;
470
//int method = CV_TM_SQDIFF_NORMED;
471
////matchTemplate(img2, patch, result, method);
472
//matchTemplate(img, patch, result, method); // MATCH WITH SELF
474
//sout << "\t\tRESULT DIMENSION: " << result.cols << " ::: " << result.rows << " ::: " << result.size << std::endl;
476
//dbg = openCvUtil::Mat2CImg<double>(result);
477
//this->debug().push_back(dbg);
479
//dbg = openCvUtil::Mat2CImg<double>(img);
480
//this->debug().push_back(dbg);
482
//cv::Size s2 = result.size();
483
//int h2 = s2.height;
486
//sout << "\t\t result width = " << w2 << std::endl;
487
//sout << "\t\t result height = " << h2 << std::endl;
489
//int xN = 0, yN = 0;
490
//float cost = result.at<uchar>(xN, yN) * weight(xN, yN, x1, y1);
491
//sout << cost << std::endl;
493
//for (int x = 0; x < w2; x++){
494
// for (int y = 0; y < h2; y++){
495
// //mask.at<uchar>(y, x) = 1;
497
// float costN = result.at<float>(y, x) *weight(x, y, x1, y1);
499
// if (costN < cost){
508
//Rect r2(xN, yN, _w, _h);
509
//Mat patch2 = img(r2);
510
//dbg = openCvUtil::Mat2CImg<double>(patch2);
511
//this->debug().push_back(dbg);
515
//cv::imshow("result", ref);
519
catch (std::range_error& e) {
520
sout << e.what() << std::endl;
521
//sout << "catched... " << currentFilename << std::endl;
526
std::vector< KeyPoint > keypoints;
527
std::vector< KeyPoint > keypoints2;
529
cimg_library::CImg<double> dbg = openCvUtil::Mat2CImg<double>(mask);
530
this->debug().push_back(dbg);
532
vector<Point2f> points;
533
vector<Point2f> points2;
534
//int MAX_COUNT = 100;
536
//cvtColor(img, gray, CV_BGR2GRAY);
537
//goodFeaturesToTrack(gray, points, MAX_COUNT, 0.01, 10, mask, 3, 0, 0.04);
538
//sout << points.size() << std::endl;
539
//cornerSubPix(img, points, Size(10, 10), Size(-1, -1), termcrit);
541
// get feature points inside of masked areas
542
//detector.detect(img, keypoints, mask);
544
detectorSURF.detect(img, keypoints, mask);
546
for (auto itP = keypoints.begin(); itP != keypoints.end(); itP++){
547
points.push_back((*itP).pt);
548
//sout << (*itP).pt.x << " / " << (*itP).pt.y << std::endl;
553
//vector<Point2f> points = keypoints;
555
vector<uchar> status;
561
//calcOpticalFlowPyrLK(img, img2, points, points2, status, err, Size winSize = Size(21, 21), int maxLevel = 3, TermCriteria criteria = TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 30, 0.01), int flags = 0, double minEigThreshold = 1e-4);
562
calcOpticalFlowPyrLK(img, img2, points, points2, status, err, Size(32, 32), 0);//, TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 30, 0.01), 0, 1e-4);
564
sout << points2.size() << " --> " << points.size() << std::endl;
565
sout << status.size() << std::endl;
568
//Mat imgShow2 = img2;
570
cimg_library::CImg<double> dbg2(this->filepaths()[i].c_str());
571
dbg2.resize(dbg2.width(), dbg2.height(), 1, 3);
573
const char color[] = { 255, 0, 0 };
575
for (int k = 0; k < points2.size(); k++){
576
if (status[k] != 0) {
577
sout << points[k].x << " / " << points[k].y << " --> " << points2[k].x << " / " << points2[k].y << ": status " << (status[k] == 0 ? 0 : 1) << ": err " << err[k] << std::endl;
578
//circle(imgShow, points[k], 3, Scalar(0, 255, 0), -1, 8);
579
//circle(imgShow2, points2[k], 3, Scalar(0, 255, 0), -1, 8);
580
dbg2.draw_arrow(points[k].x, points[k].y, points2[k].x, points2[k].y, color);
584
this->debug().push_back(dbg2);
590
const vector< const annotation::AnnotationBox* > boxes = this->boxesTracked().getAnnotationBoxes(currentFilename);
592
std::string currentFilename2 = getFilename(this->filepaths()[i + 1]);
593
this->boxesTracked().addFrame(annotation::Frame(currentFilename2, i + 1));
595
auto it2 = boxes.begin();
596
const auto it2End = boxes.end();
597
for (; it2 != it2End; it2++){
600
int x1 = std::max(0, std::min(w - 1, (int)(*it2)->x1));
601
int y1 = std::max(0, std::min(h - 1, (int)(*it2)->y1));
602
int x2 = std::max(0, std::min(w - 1, (int)(*it2)->x2));
603
int y2 = std::max(0, std::min(h - 1, (int)(*it2)->y2));
608
float maxError = 1000;
610
// get flow of points inside of this mask
611
for (int ptIt = 0; ptIt < points.size(); ptIt++){
612
if (err[ptIt]< maxError)
613
if (points[ptIt].x >= x1 && points[ptIt].x <= x2 && points[ptIt].y >= y1 && points[ptIt].y <= y2){
615
u += points2[ptIt].x - points[ptIt].x;
616
v += points2[ptIt].y - points[ptIt].y;
620
// do not add warped box without given flow
632
this->boxesTracked().addAnnotationBox(annotation::AnnotationBox(currentFilename2, true, false, x1, y1, x2, y2));
640
//cvCalcOpticalFlowPyrLK()
642
//sout << "\t\tkeypoints: " << keypoints[0].size() << std::endl;
648
// the following functions are needed
649
// for class TrackAnnotationBoxesKLT to work as a charon plugin.
650
extern "C" trackannotationboxesklt_DECLDIR ParameteredObject*
651
create(const std::string& name, ParameteredObject::template_type) {
652
return new TrackAnnotationBoxesKLT(name);
655
extern "C" trackannotationboxesklt_DECLDIR void destroy(ParameteredObject* b) {
659
/// Report build configuration to prevent linking of incompatibel runtime libs
660
extern "C" trackannotationboxesklt_DECLDIR ParameteredObject::build_type getBuildType() {
662
return ParameteredObject::DEBUG_BUILD;
664
return ParameteredObject::RELEASE_BUILD;