~charon-developers/charon-utils/trunk : revision 474

1

2

3

This file is part of Charon.

4

5

Charon is free software: you can redistribute it and/or modify

6

it under the terms of the GNU Lesser General Public License as published by

7

the Free Software Foundation, either version 3 of the License, or

8

(at your option) any later version.

9

10

Charon is distributed in the hope that it will be useful,

11

but WITHOUT ANY WARRANTY; without even the implied warranty of

12

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

13

GNU Lesser General Public License for more details.

14

15

You should have received a copy of the GNU Lesser General Public License

16

along with Charon. If not, see <http://www.gnu.org/licenses/>.

17

*/

18

/** \file AnnotationFinalizer.cpp

19

* Implementation of parameter class AnnotationFinalizer.

20

* \author <a href="mailto:moritz.becker@pallas-ludens.com">

21

* Moritz Becker</a>

22

* \date 22.07.2014

23

*/

24

25

#include <charon-core/ParameteredObject.hxx>

26

#include <charon-utils/AnnotationFinalizer.h>

27

28

AnnotationFinalizer::AnnotationFinalizer(const std::string& name) :

29

ParameteredObject(

30

"AnnotationFinalizer", name,

31

"<h2>Postprocessing for annotation</h2><br>"

32

"Rules to decide which annotation can be flagged as final due to "

33

"enough matches etc."

34

)

35

{

36

37

ParameteredObject::_addInputSlot(

38

annotations, "annotations",

39

"Annotations",

40

"AnnotationList");

41

42

ParameteredObject::_addInputSlot(

43

filepaths, "filepaths",

44

"filepaths",

45

"vector<string>");

46

47

ParameteredObject::_addOutputSlot(

48

annotationsFinal, "annotationsFinal",

49

"Final annotations",

50

"AnnotationList");

51

52

ParameteredObject::_addParameter<float>(

53

minMatchDistance, "minMatchDistance",

54

"Minimum match distance. Annotations with a distance within "

55

"this threshold do match.", 3.0f, "float");

56

57

ParameteredObject::_addParameter<float>(

58

minMatchDistanceOutlier, "minMatchDistanceOutlier",

59

"This is a seperate threshold for the forth coordinate which is allowed "

60

"to have a taller distance than the other three coords.", 5.0f, "float");

61

62

ParameteredObject::_addParameter<float>(

63

maxMatchDistance, "maxMatchDistance",

64

"Maximum match distance. Annotations with a distance within "

65

"this threshold do match.", 10.0f, "float");

66

67

ParameteredObject::_addParameter<float>(

68

relativeMatchDistance, "relativeMatchDistance",

69

"Minimum match distance. Annotations with a distance within "

70

"this threshold do match. Threshold is given in percentage", 0.1f, "float");

71

72

ParameteredObject::_addParameter<int>(

73

thresholdMatchCount, "thresholdMatchCount",

74

"Threshold for match count. Annotations with this number of "

75

"matches are inliers.", 3, "int");

76

77

ParameteredObject::_addParameter<float>(

78

maxSize, "maxSize",

79

"Maximal size of boxes given in percentage.", 1.0f, "float");

80

81

ParameteredObject::_addParameter<bool>(

82

propagateNeighbours, "propagateNeighbours",

83

"Propagate similar annotations.", false, "bool");

84

85

ParameteredObject::_addParameter<bool>(

86

saveDeleted, "saveDeleted",

87

"Also save deleted annotations (marked as deleted)", false, "bool");

88

89

}

90

91

/// Check if box is an outlier by position and returns true if so+

92

/**

93

coordinates represent an outlier box. The function will

94

return true for all annotations lying inside that given box .

95

*/

96

bool outlierPosition(const annotation::AnnotationPoints* box, float x1, float y1, float x2, float y2){

97

98

for (int i = 0; i < box->points.size(); i++){

99

100

// first coord inside outlier box

101

if (box->points[i].first >= x1 &&

102

box->points[i].second >= y1 &&

103

box->points[i].first <= x2 &&

104

box->points[i].second <= y2){

105

return true;

106

}

107

}

108

109

return false;

110

111

}

112

113

/// check if size of box extends a given size

114

bool outlierSize(const annotation::AnnotationPoints* box, int _width, int _height, float _size){

115

116

if (box->points.size() != 2)

117

return false;

118

119

float x1 = box->points[0].first;

120

float y1 = box->points[0].second;

121

float x2 = box->points[1].first;

122

float y2 = box->points[1].second;

123

124

int w = x2 - x1;

125

int h = y2 - y1;

126

float n1 = w*h;

127

float n2 = _width * _height;

128

129

float size = std::min(float(1), std::max(float(0), _size));

130

131

if (n1 >= size * n2){

132

return true;

133

}

134

135

return false;

136

137

}

138

139

void AnnotationFinalizer::getMedians(std::vector<std::pair<float, float>> &medianPoints, const std::vector< const annotation::AnnotationPoints* > &matchList){

140

141

// iterate over all coordinates

142

for (int j = 0; j < matchList[0]->points.size(); j++){

143

144

float medianX, medianY;

145

146

std::list<float> listX, listY; // list of all x and y coordinates

147

148

// iterate over all matches

149

for (int i = 0; i < matchList.size(); i++){

150

listX.push_back(matchList[i]->points[j].first);

151

listY.push_back(matchList[i]->points[j].second);

152

}

153

154

// sort by value

155

listX.sort();

156

listY.sort();

157

158

std::vector<float> vecX( std::begin(listX), std::end(listX) );

159

std::vector<float> vecY( std::begin(listY), std::end(listY) );

160

161

// get the median index

162

int medianIndex = listX.size() / 2;

163

164

// if number of coords in list is odd, everything is good

165

if (listX.size() % 2 != 0){

166

medianX = vecX[medianIndex];

167

medianY = vecY[medianIndex];

168

}

169

// else, choose both middle coords for mean value

170

else{

171

medianX = 0.5 * (vecX[medianIndex] + vecX[medianIndex - 1]);

172

medianY = 0.5 * (vecY[medianIndex] + vecY[medianIndex - 1]);

173

}

174

175

medianPoints.push_back(std::pair<float, float>(medianX, medianY));

176

177

}

178

}

179

180

/// Get filepath for given filename

181

std::string getFilepath(std::string filename, const std::vector<std::string> &list){

182

183

std::string path = "";

184

std::string filename2 = "";

185

for (auto it = list.begin(); it != list.end(); it++){

186

filename2 = (*it).substr((*it).find_last_of(FileTool::slash) + 1, (*it).length());

187

if (filename2 == filename)

188

path = (*it);

189

}

190

191

//sout << "filename2 = " << filename2 << std::endl;

192

193

return path;

194

195

}

196

197

/// check if there are enough similar boxes to take the median of all of them and set it final

198

bool AnnotationFinalizer::outlierMatchCount(const annotation::Frame* frame,

199

const annotation::AnnotationPoints* box,

200

std::vector<std::pair<float, float>> &medianPoints,

201

int w, int h){

202

203

const annotation::AnnotationList &in = this->annotations();

204

205

// fill match list with all similar annotations of this frame

206

std::vector< const annotation::AnnotationPoints* > matchList;

207

matchList.push_back(box);

208

209

std::vector<int> annotationIdsOld;

210

211

int oldSize = 0;

212

int newSize = 1;

213

214

//sout << "---" << std::endl;

215

216

int itera = 0;

217

218

// iterate until no new similar points are shown

219

// Iteration because similar points can have further similar points

220

// which are not similar to the first point.

221

while (oldSize != newSize){

222

223

if (!this->propagateNeighbours() && itera > 0)

224

break;

225

226

itera++;

227

228

//sout << "Oldsize = " << oldSize << " / " << "newSize = " << newSize << std::endl;

229

230

oldSize = matchList.size();

231

232

// search for every point in matchList (in case of several runs,

233

// the loop runs serveral times for the same point.

234

// OPTIMIZE THAT WHEN YOU HAVE TIME, MR. BECKER!

235

for (int i = 0; i < matchList.size(); i++){

236

237

// comparison box 1

238

const annotation::AnnotationPoints* box1 = matchList[i];

239

240

// -- check if box already was used for search

241

bool processed = false;

242

for (int j = 0; j < annotationIdsOld.size(); j++){

243

if (box1->annotationId == annotationIdsOld[j]){

244

processed = true;

245

break;

246

}

247

}

248

if (processed)

249

continue;

250

251

// flag box as "already used for search"

252

annotationIdsOld.push_back(box1->annotationId);

253

254

// second iteration over list of annotations

255

auto it3 = frame->annotationIds.begin();

256

auto it3End = frame->annotationIds.end();

257

for (; it3 != it3End; it3++){

258

259

// comparison box 2

260

const annotation::AnnotationPoints* box2 = in.getAnnotationPoints(*it3);

261

262

// check if box 2 is already in match list

263

bool processed2 = false;

264

for (int j = 0; j < matchList.size(); j++){

265

if (box2->annotationId == matchList[j]->annotationId){

266

processed2 = true;

267

break;

268

}

269

}

270

if (processed2)

271

continue;

272

273

int matchCount = 0;

274

int matchCountOutlier = 0;

275

276

// iterate over all coordinates of this annotation

277

for (int i = 0; i < box1->points.size(); i++){

278

279

// get distance between both coordinates

280

float dist1 = std::abs(box1->points[i].first - box2->points[i].first);

281

float dist2 = std::abs(box1->points[i].second - box2->points[i].second);

282

283

float distanceX = this->minMatchDistance();

284

float distanceY = this->minMatchDistance();

285

float distanceOutlierX = this->minMatchDistanceOutlier();

286

float distanceOutlierY = this->minMatchDistanceOutlier();

287

288

// choose relative distance according to box size

289

if (box1->points.size() == 2){

290

int w1 = box1->points[1].first - box1->points[0].first;

291

int h1 = box1->points[1].second - box1->points[0].second;

292

int w2 = box2->points[1].first - box2->points[0].first;

293

int h2 = box2->points[1].second - box2->points[0].second;

294

int w = std::max(w1, w2);

295

int h = std::max(h1, h2);

296

distanceX = std::min( this->maxMatchDistance(), std::max(this->minMatchDistance(), this->relativeMatchDistance() * w) );

297

distanceY = std::max(this->minMatchDistance(), this->relativeMatchDistance() * h);

298

}

299

300

// x coord

301

if (dist1 <= distanceX){

302

matchCount++;

303

}

304

else if (dist1 <= distanceOutlierX){

305

matchCountOutlier++;

306

}

307

308

// y coord

309

if (dist2 <= distanceY){

310

matchCount++;

311

}

312

else if (dist2 <= distanceOutlierY){

313

matchCountOutlier++;

314

}

315

316

/*

317

Check if the shape of the current box compared to all others indicates an outlier:

318

In this case, the indicator for an outlier is the case, whether a box has a similar height

319

as another box but is wider. This mostly happens for vehicles where the whole object was boxed

320

and not only its rear.

321

*/

322

//else if (dist1 > _distance &&

323

// dist2 <= _distance){

324

// return true;

325

//}

326

327

328

}

329

330

// if all coordinates ly within the match distance threshold, flag as match

331

if (

332

(matchCount >= box1->points.size() * 2) ||

333

(matchCount >= box1->points.size() * 2 - 1 && matchCountOutlier == 1)

334

){

335

336

// check if annotation is marked as deleted

337

if (!box2->deleted &&

338

!outlierPosition(box2, 455, 345, 573, 438) &&

339

!outlierSize(box2, w, h, this->maxSize() )

340

)

341

matchList.push_back(box2);

342

}

343

344

}

345

346

}

347

348

newSize = matchList.size();

349

350

}

351

352

// remove this point due to less matches with other annotations

353

if (matchList.size() < this->thresholdMatchCount() ){

354

return true;

355

}

356

357

if ( itera > 2 )

358

sout << itera << std::endl;

359

360

if (matchList.size() <= 2){

361

sout << "Only two or less matches.." << std::endl;

362

}

363

364

// get medians of every coordinate and save it to median points

365

getMedians(medianPoints, matchList);

366

367

return false;

368

369

}

370

371

372

void AnnotationFinalizer::execute() {

373

374

const annotation::AnnotationList &in = this->annotations();

375

annotation::AnnotationList &out = this->annotationsFinal();

376

377

const std::vector< const annotation::Frame* > frames = in.getFrames();

378

379

// iterate over list of all available frames

380

auto it = frames.begin();

381

for (; it != frames.end(); it++){

382

383

// get filename saved in annoation list

384

std::string filename = (*it)->filename;

385

386

// --- get image object of current annotation

387

388

// find corresponding filepath from filelist

389

std::string filepath = getFilepath(filename, this->filepaths());

390

391

if (filepath == "")

392

continue;

393

394

cimg_library::CImg<float> img(filepath.c_str());

395

const int w = img.width();

396

const int h = img.height();

397

398

// iterate over list of annotations

399

auto it2 = (*it)->annotationIds.begin();

400

const auto it2End = (*it)->annotationIds.end();

401

for (; it2 != it2End; it2++){

402

403

bool isDeleted = false;

404

405

const annotation::AnnotationPoints* box = in.getAnnotationPoints(*it2);

406

407

//sout << "check box " << box->annotationId << std::endl;

408

409

// check if annotation is marked as deleted

410

if (box->deleted)

411

isDeleted = true; // continue;

412

413

// check for outlier by positions

414

if (outlierPosition(box, 455, 345, 573, 438))

415

isDeleted = true; // continue;

416

417

if (outlierSize(box, w, h, this->maxSize()))

418

isDeleted = true; // continue;

419

420

// list of median coordinates between all similar boxes matched in the 'outlierMatchCount' method

421

std::vector<std::pair<float, float>> medianPoints;

422

423

//sout << "vor" << std::endl;

424

// check if there are enough similar boxes to take the median of all of them and set it final

425

if (outlierMatchCount(*it, box, medianPoints, w, h))

426

isDeleted = true; // continue;

427

428

//sout << "nach" << std::endl;

429

430

annotation::AnnotationPoints newBox(filename, box->annotationId, isDeleted, !isDeleted, box->taskResultId, box->taskResultState, medianPoints);

431

432

if (!isDeleted ||this->saveDeleted())

433

out.addAnnotationPoints(newBox);

434

435

}

436

437

}

438

439

440

}

441

442

// the following functions are needed

443

// for class AnnotationFinalizer to work as a charon plugin.

444

extern "C" annotationfinalizer_DECLDIR ParameteredObject*

445

create(const std::string& name, ParameteredObject::template_type) {

446

return new AnnotationFinalizer(name);

447

}

448

449

extern "C" annotationfinalizer_DECLDIR void destroy(ParameteredObject* b) {

450

delete b;

451

}

452

453

/// Report build configuration to prevent linking of incompatibel runtime libs

454

extern "C" annotationfinalizer_DECLDIR ParameteredObject::build_type getBuildType() {

455

#ifdef _DEBUG

456

return ParameteredObject::DEBUG_BUILD;

457

#else

458

return ParameteredObject::RELEASE_BUILD;

459

#endif

460

}