1
#include "../APTClasses.h"
2
#include "../xmlHelper.h"
4
#include "../translation.h"
6
#include "ionDownsample.h"
10
KEY_IONDOWNSAMPLE_FRACTION=1,
11
KEY_IONDOWNSAMPLE_FIXEDOUT,
12
KEY_IONDOWNSAMPLE_COUNT,
13
KEY_IONDOWNSAMPLE_PERSPECIES,
14
KEY_IONDOWNSAMPLE_ENABLE,
15
//Dynamic area for this filter class. May validly use any index after this value
16
KEY_IONDOWNSAMPLE_DYNAMIC,
19
//!Downsampling filter
22
IONDOWNSAMPLE_ABORT_ERR=1,
23
IONDOWNSAMPLE_BAD_ALLOC,
27
// == Ion Downsampling filter ==
29
IonDownsampleFilter::IonDownsampleFilter()
39
cache=true; //By default, we should cache, but decision is made higher up
43
void IonDownsampleFilter::initFilter(const std::vector<const FilterStreamData *> &dataIn,
44
std::vector<const FilterStreamData *> &dataOut)
46
const RangeStreamData *c=0;
47
//Determine if we have an incoming range
48
for (size_t i = 0; i < dataIn.size(); i++)
50
if(dataIn[i]->getStreamType() == STREAM_TYPE_RANGE)
52
c=(const RangeStreamData *)dataIn[i];
58
//we no longer (or never did) have any incoming ranges. Not much to do
61
//delete the old incoming range pointer
66
//Well, don't use per-species info anymore
73
//If we didn't have a previously incoming rsd, then make one up!
74
// - we can't use a reference, as the rangestreams are technically transient,
75
// so we have to copy.
78
rsdIncoming = new RangeStreamData;
81
if(ionFractions.size() != c->rangeFile->getNumIons())
83
//set up some defaults; seeded from normal
84
ionFractions.resize(c->rangeFile->getNumIons(),fraction);
85
ionLimits.resize(c->rangeFile->getNumIons(),maxAfterFilter);
91
//OK, so we have a range incoming already (from last time)
92
//-- the question is, is it the same one we had before ?
94
//Do a pointer comparison (its a hack, yes, but it should work)
95
if(rsdIncoming->rangeFile != c->rangeFile)
97
//hmm, it is different. well, trash the old incoming rng
100
rsdIncoming = new RangeStreamData;
103
ionFractions.resize(c->rangeFile->getNumIons(),fraction);
104
ionLimits.resize(c->rangeFile->getNumIons(),maxAfterFilter);
106
else if(ionFractions.size() !=c->rangeFile->getNumIons())
108
//well its the same range, but somehow the number of ions
109
//have changed. Could be range was reloaded.
110
ionFractions.resize(rsdIncoming->rangeFile->getNumIons(),fraction);
111
ionLimits.resize(rsdIncoming->rangeFile->getNumIons(),maxAfterFilter);
114
//Ensure what is enabled and is disabled is up-to-date
115
for(unsigned int ui=0;ui<rsdIncoming->enabledRanges.size();ui++)
116
rsdIncoming->enabledRanges[ui] = c->enabledRanges[ui];
117
for(unsigned int ui=0;ui<rsdIncoming->enabledIons.size();ui++)
118
rsdIncoming->enabledIons[ui] = c->enabledIons[ui];
125
ASSERT(ionLimits.size() == ionFractions.size());
128
Filter *IonDownsampleFilter::cloneUncached() const
130
IonDownsampleFilter *p=new IonDownsampleFilter();
132
p->maxAfterFilter=maxAfterFilter;
133
p->fraction=fraction;
134
p->perSpecies=perSpecies;
135
p->rsdIncoming=rsdIncoming;
137
p->ionFractions.resize(ionFractions.size());
138
std::copy(ionFractions.begin(),ionFractions.end(),p->ionFractions.begin());
139
p->ionLimits.resize(ionLimits.size());
140
std::copy(ionLimits.begin(),ionLimits.end(),p->ionLimits.begin());
143
//We are copying wether to cache or not,
144
//not the cache itself
147
p->userString=userString;
148
p->fixedNumOut=fixedNumOut;
152
size_t IonDownsampleFilter::numBytesForCache(size_t nObjects) const
156
if(nObjects > maxAfterFilter)
157
return maxAfterFilter*IONDATA_SIZE;
159
return nObjects*IONDATA_SIZE;
163
return (size_t)((float)(nObjects*IONDATA_SIZE)*fraction);
167
unsigned int IonDownsampleFilter::refresh(const std::vector<const FilterStreamData *> &dataIn,
168
std::vector<const FilterStreamData *> &getOut, ProgressData &progress, bool (*callback)(bool))
170
//use the cached copy if we have it.
173
for(size_t ui=0;ui<dataIn.size();ui++)
175
if(dataIn[ui]->getStreamType() != STREAM_TYPE_IONS)
176
getOut.push_back(dataIn[ui]);
178
for(size_t ui=0;ui<filterOutputs.size();ui++)
179
getOut.push_back(filterOutputs[ui]);
184
size_t totalSize = numElements(dataIn,STREAM_TYPE_IONS);
187
for(size_t ui=0;ui<dataIn.size() ;ui++)
189
switch(dataIn[ui]->getStreamType())
191
case STREAM_TYPE_IONS:
204
frac = (float)(((const IonStreamData*)dataIn[ui])->data.size())/(float)totalSize;
206
randomSelect(d->data,((const IonStreamData *)dataIn[ui])->data,
207
rng,maxAfterFilter*frac,progress.filterProgress,callback,strongRandom);
214
unsigned int curProg=NUM_CALLBACK;
216
//Reserve 90% of storage needed.
217
//highly likely with even modest numbers of ions
218
//that this will be exceeeded
219
d->data.reserve(fraction*0.9*totalSize);
221
ASSERT(dataIn[ui]->getStreamType() == STREAM_TYPE_IONS);
223
for(vector<IonHit>::const_iterator it=((const IonStreamData *)dataIn[ui])->data.begin();
224
it!=((const IonStreamData *)dataIn[ui])->data.end(); ++it)
226
if(rng.genUniformDev() < fraction)
227
d->data.push_back(*it);
229
//update progress every CALLBACK ions
232
curProg=NUM_CALLBACK;
234
progress.filterProgress= (unsigned int)((float)(n)/((float)totalSize)*100.0f);
235
if(!(*callback)(false))
238
return IONDOWNSAMPLE_ABORT_ERR;
244
catch(std::bad_alloc)
247
return IONDOWNSAMPLE_BAD_ALLOC;
251
//Copy over other attributes
252
d->r = ((IonStreamData *)dataIn[ui])->r;
253
d->g = ((IonStreamData *)dataIn[ui])->g;
254
d->b =((IonStreamData *)dataIn[ui])->b;
255
d->a =((IonStreamData *)dataIn[ui])->a;
256
d->ionSize =((IonStreamData *)dataIn[ui])->ionSize;
257
d->representationType=((IonStreamData *)dataIn[ui])->representationType;
258
d->valueType=((IonStreamData *)dataIn[ui])->valueType;
260
//getOut is const, so shouldn't be modified
264
filterOutputs.push_back(d);
276
getOut.push_back(dataIn[ui]);
285
const IonStreamData *input;
287
//Construct two vectors. One with the ion IDs for each input
288
//ion stream. the other with the total number of ions in the input
290
vector<size_t> numIons,ionIDVec;
291
numIons.resize(rsdIncoming->rangeFile->getNumIons(),0);
293
for(unsigned int uj=0;uj<dataIn.size() ;uj++)
295
if(dataIn[uj]->getStreamType() == STREAM_TYPE_IONS)
297
input=(const IonStreamData*)dataIn[uj];
298
if(input->data.size())
301
ionID=rsdIncoming->rangeFile->getIonID(
302
input->data[0].getMassToCharge());
304
if(ionID != (unsigned int)-1)
305
numIons[ionID]+=input->data.size();
307
ionIDVec.push_back(ionID);
313
unsigned int idPos=0;
314
for(size_t ui=0;ui<dataIn.size() ;ui++)
316
switch(dataIn[ui]->getStreamType())
318
case STREAM_TYPE_IONS:
320
input=(const IonStreamData*)dataIn[ui];
322
//Don't process ionstreams that are empty
323
if(!input->data.size())
326
//FIXME: Allow processing of unranged data
327
//Don't process streams that are not ranged, as we cannot get their desired fractions
329
if(ionIDVec[idPos]==(unsigned int)-1)
339
//if we are building the fixed number for output,
340
//then compute the relative fraction for this ion set
342
frac = (float)(input->data.size())/(float)(numIons[ionIDVec[idPos]]);
344
//The total number of ions is the specified value for this ionID, multiplied by
345
//this stream's fraction of the total incoming data
346
randomSelect(d->data,input->data, rng,frac*ionLimits[ionIDVec[idPos]],
347
progress.filterProgress,callback,strongRandom);
351
//Use the direct fractions as entered in by user.
353
unsigned int curProg=NUM_CALLBACK;
355
float thisFraction = ionFractions[ionIDVec[idPos]];
357
//Reserve 90% of storage needed.
358
//highly likely (poisson) with even modest numbers of ions
359
//that this will be exceeeded, and thus we won't over-allocate
360
d->data.reserve(thisFraction*0.9*numIons[ionIDVec[idPos]]);
364
for(vector<IonHit>::const_iterator it=input->data.begin();
365
it!=input->data.end(); ++it)
367
if(rng.genUniformDev() < thisFraction)
368
d->data.push_back(*it);
370
//update progress every CALLBACK ions
374
progress.filterProgress=
375
(unsigned int)((float)(n)/((float)totalSize)*100.0f);
376
if(!(*callback)(false))
379
return IONDOWNSAMPLE_ABORT_ERR;
387
catch(std::bad_alloc)
390
return IONDOWNSAMPLE_BAD_ALLOC;
396
//Copy over other attributes
401
d->ionSize =input->ionSize;
402
d->representationType=input->representationType;
403
d->valueType=input->valueType;
406
//getOut is const, so shouldn't be modified
410
filterOutputs.push_back(d);
428
getOut.push_back(dataIn[ui]);
441
void IonDownsampleFilter::getProperties(FilterProperties &propertyList) const
443
propertyList.data.clear();
444
propertyList.keys.clear();
445
propertyList.types.clear();
447
vector<unsigned int> type,keys;
448
vector<pair<string,string> > s;
451
stream_cast(tmpStr,fixedNumOut);
452
s.push_back(std::make_pair(TRANS("By Count"), tmpStr));
453
keys.push_back(KEY_IONDOWNSAMPLE_FIXEDOUT);
454
type.push_back(PROPERTY_TYPE_BOOL);
458
stream_cast(tmpStr,perSpecies);
459
s.push_back(std::make_pair(TRANS("Per Species"), tmpStr));
460
keys.push_back(KEY_IONDOWNSAMPLE_PERSPECIES);
461
type.push_back(PROPERTY_TYPE_BOOL);
465
propertyList.data.push_back(s);
466
propertyList.types.push_back(type);
467
propertyList.keys.push_back(keys);
469
//Start a new section
474
if(rsdIncoming && perSpecies)
476
unsigned int typeVal;
478
typeVal=PROPERTY_TYPE_INTEGER;
480
typeVal=PROPERTY_TYPE_REAL;
482
//create a single line for each
483
for(unsigned int ui=0; ui<rsdIncoming->enabledIons.size(); ui++)
485
if(rsdIncoming->enabledIons[ui])
488
stream_cast(tmpStr,ionLimits[ui]);
490
stream_cast(tmpStr,ionFractions[ui]);
492
s.push_back(make_pair(
493
rsdIncoming->rangeFile->getName(ui), tmpStr));
494
type.push_back(typeVal);
495
keys.push_back(KEY_IONDOWNSAMPLE_DYNAMIC+ui);
503
stream_cast(tmpStr,maxAfterFilter);
504
keys.push_back(KEY_IONDOWNSAMPLE_COUNT);
505
s.push_back(make_pair(TRANS("Output Count"), tmpStr));
506
type.push_back(PROPERTY_TYPE_INTEGER);
510
stream_cast(tmpStr,fraction);
511
s.push_back(make_pair(TRANS("Out Fraction"), tmpStr));
512
keys.push_back(KEY_IONDOWNSAMPLE_FRACTION);
513
type.push_back(PROPERTY_TYPE_REAL);
518
propertyList.data.push_back(s);
519
propertyList.types.push_back(type);
520
propertyList.keys.push_back(keys);
523
bool IonDownsampleFilter::setProperty( unsigned int set, unsigned int key,
524
const std::string &value, bool &needUpdate)
529
case KEY_IONDOWNSAMPLE_FIXEDOUT:
531
string stripped=stripWhite(value);
533
if(!(stripped == "1"|| stripped == "0"))
536
bool lastVal=fixedNumOut;
542
//if the result is different, the
543
//cache should be invalidated
544
if(lastVal!=fixedNumOut)
552
case KEY_IONDOWNSAMPLE_FRACTION:
555
if(stream_cast(newFraction,value))
558
if(newFraction < 0.0f || newFraction > 1.0f)
561
//In the case of fixed number output,
562
//our cache is invalidated
569
fraction=newFraction;
574
case KEY_IONDOWNSAMPLE_COUNT:
578
if(stream_cast(count,value))
581
maxAfterFilter=count;
582
//In the case of fixed number output,
583
//our cache is invalidated
592
case KEY_IONDOWNSAMPLE_PERSPECIES:
594
string stripped=stripWhite(value);
596
if(!(stripped == "1"|| stripped == "0"))
599
bool lastVal=perSpecies;
605
//if the result is different, the
606
//cache should be invalidated
607
if(lastVal!=perSpecies)
618
ASSERT(key >=KEY_IONDOWNSAMPLE_DYNAMIC);
619
ASSERT(key < KEY_IONDOWNSAMPLE_DYNAMIC+ionLimits.size());
620
ASSERT(ionLimits.size() == ionFractions.size());
623
offset=key-KEY_IONDOWNSAMPLE_DYNAMIC;
625
//Dynamically generated list of downsamples
630
if(stream_cast(v,value))
638
if(stream_cast(v,value))
641
if(v < 0.0f || v> 1.0f)
644
ionFractions[offset]=v;
657
std::string IonDownsampleFilter::getErrString(unsigned int code) const
661
case IONDOWNSAMPLE_ABORT_ERR:
662
return std::string(TRANS("Downsample Aborted"));
663
case IONDOWNSAMPLE_BAD_ALLOC:
664
return std::string(TRANS("Insuffient memory for downsample"));
667
return std::string("BUG! Should not see this (IonDownsample)");
670
bool IonDownsampleFilter::writeState(std::ofstream &f,unsigned int format, unsigned int depth) const
675
case STATE_FORMAT_XML:
677
f << tabs(depth) << "<" << trueName() << ">" << endl;
678
f << tabs(depth+1) << "<userstring value=\""<< escapeXML(userString) << "\"/>" << endl;
680
f << tabs(depth+1) << "<fixednumout value=\""<<fixedNumOut<< "\"/>" << endl;
681
f << tabs(depth+1) << "<fraction value=\""<<fraction<< "\"/>" << endl;
682
f << tabs(depth+1) << "<maxafterfilter value=\"" << maxAfterFilter << "\"/>" << endl;
683
f << tabs(depth+1) << "<perspecies value=\""<<perSpecies<< "\"/>" << endl;
684
f << tabs(depth+1) << "<fractions>" << endl;
685
for(unsigned int ui=0;ui<ionFractions.size(); ui++)
686
f << tabs(depth+2) << "<scalar value=\"" << ionFractions[ui] << "\"/>" << endl;
687
f << tabs(depth+1) << "</fractions>" << endl;
688
f << tabs(depth+1) << "<limits>" << endl;
689
for(unsigned int ui=0;ui<ionLimits.size(); ui++)
690
f << tabs(depth+2) << "<scalar value=\"" << ionLimits[ui] << "\"/>" << endl;
691
f << tabs(depth+1) << "</limits>" << endl;
692
f << tabs(depth) << "</" <<trueName()<< ">" << endl;
703
bool IonDownsampleFilter::readState(xmlNodePtr &nodePtr, const std::string &stateFileDir)
709
//Retrieve user string
710
if(XMLHelpFwdToElem(nodePtr,"userstring"))
713
xmlString=xmlGetProp(nodePtr,(const xmlChar *)"value");
716
userString=(char *)xmlString;
719
//Retrieve number out (yes/no) mode
720
if(!XMLGetNextElemAttrib(nodePtr,tmpStr,"fixednumout","value"))
725
else if(tmpStr== "0")
733
if(!XMLGetNextElemAttrib(nodePtr,fraction,"fraction","value"))
735
//disallow negative or values gt 1.
736
if(fraction < 0.0f || fraction > 1.0f)
741
//Retrieve "perspecies" attrib
742
if(!XMLGetNextElemAttrib(nodePtr,tmpStr,"perspecies","value"))
747
else if(tmpStr== "0")
754
//Retrieve the ion per-species fractions
755
if(XMLHelpFwdToElem(nodePtr,"fractions"))
758
nodePtr=nodePtr->xmlChildrenNode;
760
//Populate the ion fraction vector
762
while(XMLGetNextElemAttrib(nodePtr,fracThis,"scalar","value"))
763
ionFractions.push_back(fracThis);
768
//Retrieve the ion per-species fractions
769
if(XMLHelpFwdToElem(nodePtr,"limits"))
772
nodePtr=nodePtr->xmlChildrenNode;
774
while(XMLGetNextElemAttrib(nodePtr,limitThis,"scalar","value"))
775
ionLimits.push_back(limitThis);
777
if(ionLimits.size()!=ionFractions.size())
784
unsigned int IonDownsampleFilter::getRefreshBlockMask() const
786
return STREAM_TYPE_IONS ;
789
unsigned int IonDownsampleFilter::getRefreshEmitMask() const
791
return STREAM_TYPE_IONS;
797
//Unit testing for this class
801
//Create a synthetic dataset of points
802
// returned pointer *must* be deleted. Span must have 3 elements, and for best results sould be co-prime with one another; eg all prime numbers
803
IonStreamData *synthDataPts(unsigned int span[],unsigned int numPts);
805
//Test for fixed number of output ions
806
bool fixedSampleTest();
808
//Test for variable number of output ions
809
bool variableSampleTest();
812
bool IonDownsampleFilter::runUnitTests()
814
if(!fixedSampleTest())
817
if(!variableSampleTest())
823
bool fixedSampleTest()
825
//Simulate some data to send to the filter
826
vector<const FilterStreamData*> streamIn,streamOut;
827
IonStreamData *d= new IonStreamData;
829
const unsigned int NUM_PTS=10000;
830
for(unsigned int ui=0;ui<NUM_PTS;ui++)
833
h.setPos(Point3D(ui,ui,ui));
834
h.setMassToCharge(ui);
835
d->data.push_back(h);
839
streamIn.push_back(d);
840
//Set up the filter itself
841
IonDownsampleFilter *f=new IonDownsampleFilter;
842
f->setCaching(false);
846
unsigned int numOutput=NUM_PTS/10;
848
f->setProperty(0,KEY_IONDOWNSAMPLE_FIXEDOUT,"1",needUp);
849
stream_cast(s,numOutput);
850
f->setProperty(0,KEY_IONDOWNSAMPLE_COUNT,s,needUp);
854
f->refresh(streamIn,streamOut,p,dummyCallback);
860
TEST(streamOut.size() == 1, "Stream count");
861
TEST(streamOut[0]->getStreamType() == STREAM_TYPE_IONS, "stream type");
862
TEST(streamOut[0]->getNumBasicObjects() == numOutput, "output ions (basicobject)");
863
TEST( ((IonStreamData*)streamOut[0])->data.size() == numOutput, "output ions (direct)")
870
bool variableSampleTest()
872
//Build some points to pass to the filter
873
vector<const FilterStreamData*> streamIn,streamOut;
875
unsigned int span[]={
878
const unsigned int NUM_PTS=10000;
879
IonStreamData *d=synthDataPts(span,NUM_PTS);
881
streamIn.push_back(d);
882
IonDownsampleFilter *f=new IonDownsampleFilter;
883
f->setCaching(false);
886
f->setProperty(0,KEY_IONDOWNSAMPLE_FIXEDOUT,"0",needUp);
887
f->setProperty(0,KEY_IONDOWNSAMPLE_FRACTION,"0.1",needUp);
891
TEST(!(f->refresh(streamIn,streamOut,p,dummyCallback)),"refresh error code");
897
TEST(streamOut.size() == 1,"stream count");
898
TEST(streamOut[0]->getStreamType() == STREAM_TYPE_IONS,"stream type");
900
//It is HIGHLY improbable that it will be <1/10th of the requested number
901
TEST(streamOut[0]->getNumBasicObjects() > 0.01*NUM_PTS
902
&& streamOut[0]->getNumBasicObjects() <= NUM_PTS,"ion fraction");
910
IonStreamData *synthDataPts(unsigned int span[], unsigned int numPts)
912
IonStreamData *d = new IonStreamData;
914
for(unsigned int ui=0;ui<numPts;ui++)
917
h.setPos(Point3D(ui%span[0],
918
ui%span[1],ui%span[2]));
919
h.setMassToCharge(ui);
920
d->data.push_back(h);
added
removed
src/filters/ionDownsample.cpp
