2
* This program is free software; you can redistribute it and/or modify
3
* it under the terms of the GNU General Public License as published by
4
* the Free Software Foundation; either version 2 of the License, or
5
* (at your option) any later version.
7
* This program is distributed in the hope that it will be useful,
8
* but WITHOUT ANY WARRANTY; without even the implied warranty of
9
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10
* GNU General Public License for more details.
12
* You should have received a copy of the GNU General Public License
13
* along with this program; if not, write to the Free Software
14
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18
* ClusterMembership.java
19
* Copyright (C) 2004 University of Waikato, Hamilton, New Zealand
23
package weka.filters.unsupervised.attribute;
25
import weka.clusterers.DensityBasedClusterer;
26
import weka.core.Attribute;
27
import weka.core.Capabilities;
28
import weka.core.FastVector;
29
import weka.core.Instance;
30
import weka.core.Instances;
31
import weka.core.Option;
32
import weka.core.OptionHandler;
33
import weka.core.Range;
34
import weka.core.Utils;
35
import weka.filters.Filter;
36
import weka.filters.UnsupervisedFilter;
38
import java.util.Enumeration;
39
import java.util.Vector;
42
<!-- globalinfo-start -->
43
* A filter that uses a density-based clusterer to generate cluster membership values; filtered instances are composed of these values plus the class attribute (if set in the input data). If a (nominal) class attribute is set, the clusterer is run separately for each class. The class attribute (if set) and any user-specified attributes are ignored during the clustering operation
45
<!-- globalinfo-end -->
47
<!-- options-start -->
48
* Valid options are: <p/>
50
* <pre> -W <clusterer name>
51
* Full name of clusterer to use. eg:
53
* Additional options after the '--'.
54
* (default: weka.clusterers.EM)</pre>
56
* <pre> -I <att1,att2-att4,...>
57
* The range of attributes the clusterer should ignore.
58
* (the class attribute is automatically ignored)</pre>
62
* Options after the -- are passed on to the clusterer.
64
* @author Mark Hall (mhall@cs.waikato.ac.nz)
66
* @version $Revision: 1.14 $
68
public class ClusterMembership
70
implements UnsupervisedFilter, OptionHandler {
72
/** for serialization */
73
static final long serialVersionUID = 6675702504667714026L;
76
protected DensityBasedClusterer m_clusterer = new weka.clusterers.EM();
78
/** Array for storing the clusterers */
79
protected DensityBasedClusterer[] m_clusterers;
81
/** Range of attributes to ignore */
82
protected Range m_ignoreAttributesRange;
84
/** Filter for removing attributes */
85
protected Filter m_removeAttributes;
87
/** The prior probability for each class */
88
protected double[] m_priors;
91
* Returns the Capabilities of this filter.
93
* @return the capabilities of this object
96
public Capabilities getCapabilities() {
97
Capabilities result = m_clusterer.getCapabilities();
99
result.setMinimumNumberInstances(0);
105
* Returns the Capabilities of this filter, makes sure that the class is
106
* never set (for the clusterer).
108
* @param data the data to use for customization
109
* @return the capabilities of this object, based on the data
110
* @see #getCapabilities()
112
public Capabilities getCapabilities(Instances data) {
115
newData = new Instances(data, 0);
116
newData.setClassIndex(-1);
118
return super.getCapabilities(newData);
122
* tests the data whether the filter can actually handle it
124
* @param instanceInfo the data to test
125
* @throws Exception if the test fails
127
protected void testInputFormat(Instances instanceInfo) throws Exception {
128
getCapabilities(instanceInfo).testWithFail(removeIgnored(instanceInfo));
132
* Sets the format of the input instances.
134
* @param instanceInfo an Instances object containing the input instance
135
* structure (any instances contained in the object are ignored - only the
136
* structure is required).
137
* @return true if the outputFormat may be collected immediately
138
* @throws Exception if the inputFormat can't be set successfully
140
public boolean setInputFormat(Instances instanceInfo) throws Exception {
142
super.setInputFormat(instanceInfo);
143
m_removeAttributes = null;
150
* filters all attributes that should be ignored
152
* @param data the data to filter
153
* @return the filtered data
154
* @throws Exception if filtering fails
156
protected Instances removeIgnored(Instances data) throws Exception {
157
Instances result = data;
159
if (m_ignoreAttributesRange != null || data.classIndex() >= 0) {
160
result = new Instances(data);
161
m_removeAttributes = new Remove();
162
String rangeString = "";
163
if (m_ignoreAttributesRange != null) {
164
rangeString += m_ignoreAttributesRange.getRanges();
166
if (data.classIndex() >= 0) {
167
if (rangeString.length() > 0) {
168
rangeString += "," + (data.classIndex() + 1);
170
rangeString = "" + (data.classIndex() + 1);
173
((Remove) m_removeAttributes).setAttributeIndices(rangeString);
174
((Remove) m_removeAttributes).setInvertSelection(false);
175
m_removeAttributes.setInputFormat(data);
176
result = Filter.useFilter(data, m_removeAttributes);
183
* Signify that this batch of input to the filter is finished.
185
* @return true if there are instances pending output
186
* @throws IllegalStateException if no input structure has been defined
188
public boolean batchFinished() throws Exception {
190
if (getInputFormat() == null) {
191
throw new IllegalStateException("No input instance format defined");
194
if (outputFormatPeek() == null) {
195
Instances toFilter = getInputFormat();
196
Instances[] toFilterIgnoringAttributes;
198
// Make subsets if class is nominal
199
if ((toFilter.classIndex() >= 0) && toFilter.classAttribute().isNominal()) {
200
toFilterIgnoringAttributes = new Instances[toFilter.numClasses()];
201
for (int i = 0; i < toFilter.numClasses(); i++) {
202
toFilterIgnoringAttributes[i] = new Instances(toFilter, toFilter.numInstances());
204
for (int i = 0; i < toFilter.numInstances(); i++) {
205
toFilterIgnoringAttributes[(int)toFilter.instance(i).classValue()].add(toFilter.instance(i));
207
m_priors = new double[toFilter.numClasses()];
208
for (int i = 0; i < toFilter.numClasses(); i++) {
209
toFilterIgnoringAttributes[i].compactify();
210
m_priors[i] = toFilterIgnoringAttributes[i].sumOfWeights();
212
Utils.normalize(m_priors);
214
toFilterIgnoringAttributes = new Instances[1];
215
toFilterIgnoringAttributes[0] = toFilter;
216
m_priors = new double[1];
220
// filter out attributes if necessary
221
for (int i = 0; i < toFilterIgnoringAttributes.length; i++)
222
toFilterIgnoringAttributes[i] = removeIgnored(toFilterIgnoringAttributes[i]);
224
// build the clusterers
225
if ((toFilter.classIndex() <= 0) || !toFilter.classAttribute().isNominal()) {
226
m_clusterers = DensityBasedClusterer.makeCopies(m_clusterer, 1);
227
m_clusterers[0].buildClusterer(toFilterIgnoringAttributes[0]);
229
m_clusterers = DensityBasedClusterer.makeCopies(m_clusterer, toFilter.numClasses());
230
for (int i = 0; i < m_clusterers.length; i++) {
231
if (toFilterIgnoringAttributes[i].numInstances() == 0) {
232
m_clusterers[i] = null;
234
m_clusterers[i].buildClusterer(toFilterIgnoringAttributes[i]);
239
// create output dataset
240
FastVector attInfo = new FastVector();
241
for (int j = 0; j < m_clusterers.length; j++) {
242
if (m_clusterers[j] != null) {
243
for (int i = 0; i < m_clusterers[j].numberOfClusters(); i++) {
244
attInfo.addElement(new Attribute("pCluster_" + j + "_" + i));
248
if (toFilter.classIndex() >= 0) {
249
attInfo.addElement(toFilter.classAttribute().copy());
251
attInfo.trimToSize();
252
Instances filtered = new Instances(toFilter.relationName()+"_clusterMembership",
254
if (toFilter.classIndex() >= 0) {
255
filtered.setClassIndex(filtered.numAttributes() - 1);
257
setOutputFormat(filtered);
260
for (int i = 0; i < toFilter.numInstances(); i++) {
261
convertInstance(toFilter.instance(i));
267
return (numPendingOutput() != 0);
271
* Input an instance for filtering. Ordinarily the instance is processed
272
* and made available for output immediately. Some filters require all
273
* instances be read before producing output.
275
* @param instance the input instance
276
* @return true if the filtered instance may now be
277
* collected with output().
278
* @throws IllegalStateException if no input format has been defined.
280
public boolean input(Instance instance) throws Exception {
282
if (getInputFormat() == null) {
283
throw new IllegalStateException("No input instance format defined");
290
if (outputFormatPeek() != null) {
291
convertInstance(instance);
295
bufferInput(instance);
300
* Converts logs back to density values.
302
* @param j the index of the clusterer
303
* @param in the instance to convert the logs back
304
* @return the densities
305
* @throws Exception if something goes wrong
307
protected double[] logs2densities(int j, Instance in) throws Exception {
309
double[] logs = m_clusterers[j].logJointDensitiesForInstance(in);
311
for (int i = 0; i < logs.length; i++) {
312
logs[i] += Math.log(m_priors[j]);
318
* Convert a single instance over. The converted instance is added to
319
* the end of the output queue.
321
* @param instance the instance to convert
322
* @throws Exception if something goes wrong
324
protected void convertInstance(Instance instance) throws Exception {
327
double [] instanceVals = new double[outputFormatPeek().numAttributes()];
329
if (instance.classIndex() >= 0) {
330
tempvals = new double[outputFormatPeek().numAttributes() - 1];
332
tempvals = new double[outputFormatPeek().numAttributes()];
335
for (int j = 0; j < m_clusterers.length; j++) {
336
if (m_clusterers[j] != null) {
338
if (m_removeAttributes != null) {
339
m_removeAttributes.input(instance);
340
probs = logs2densities(j, m_removeAttributes.output());
342
probs = logs2densities(j, instance);
344
System.arraycopy(probs, 0, tempvals, pos, probs.length);
348
tempvals = Utils.logs2probs(tempvals);
349
System.arraycopy(tempvals, 0, instanceVals, 0, tempvals.length);
350
if (instance.classIndex() >= 0) {
351
instanceVals[instanceVals.length - 1] = instance.classValue();
354
push(new Instance(instance.weight(), instanceVals));
358
* Returns an enumeration describing the available options.
360
* @return an enumeration of all the available options.
362
public Enumeration listOptions() {
364
Vector newVector = new Vector(2);
367
addElement(new Option("\tFull name of clusterer to use. eg:\n"
368
+ "\t\tweka.clusterers.EM\n"
369
+ "\tAdditional options after the '--'.\n"
370
+ "\t(default: weka.clusterers.EM)",
371
"W", 1, "-W <clusterer name>"));
374
addElement(new Option("\tThe range of attributes the clusterer should ignore."
375
+"\n\t(the class attribute is automatically ignored)",
376
"I", 1,"-I <att1,att2-att4,...>"));
378
return newVector.elements();
382
* Parses a given list of options. <p/>
384
<!-- options-start -->
385
* Valid options are: <p/>
387
* <pre> -W <clusterer name>
388
* Full name of clusterer to use. eg:
390
* Additional options after the '--'.
391
* (default: weka.clusterers.EM)</pre>
393
* <pre> -I <att1,att2-att4,...>
394
* The range of attributes the clusterer should ignore.
395
* (the class attribute is automatically ignored)</pre>
399
* Options after the -- are passed on to the clusterer.
401
* @param options the list of options as an array of strings
402
* @throws Exception if an option is not supported
404
public void setOptions(String[] options) throws Exception {
406
String clustererString = Utils.getOption('W', options);
407
if (clustererString.length() == 0)
408
clustererString = weka.clusterers.EM.class.getName();
409
setDensityBasedClusterer((DensityBasedClusterer)Utils.
410
forName(DensityBasedClusterer.class, clustererString,
411
Utils.partitionOptions(options)));
413
setIgnoredAttributeIndices(Utils.getOption('I', options));
414
Utils.checkForRemainingOptions(options);
418
* Gets the current settings of the filter.
420
* @return an array of strings suitable for passing to setOptions
422
public String [] getOptions() {
424
String [] clustererOptions = new String [0];
425
if ((m_clusterer != null) &&
426
(m_clusterer instanceof OptionHandler)) {
427
clustererOptions = ((OptionHandler)m_clusterer).getOptions();
429
String [] options = new String [clustererOptions.length + 5];
432
if (!getIgnoredAttributeIndices().equals("")) {
433
options[current++] = "-I";
434
options[current++] = getIgnoredAttributeIndices();
437
if (m_clusterer != null) {
438
options[current++] = "-W";
439
options[current++] = getDensityBasedClusterer().getClass().getName();
442
options[current++] = "--";
443
System.arraycopy(clustererOptions, 0, options, current,
444
clustererOptions.length);
445
current += clustererOptions.length;
447
while (current < options.length) {
448
options[current++] = "";
454
* Returns a string describing this filter
456
* @return a description of the filter suitable for
457
* displaying in the explorer/experimenter gui
459
public String globalInfo() {
461
return "A filter that uses a density-based clusterer to generate cluster "
462
+ "membership values; filtered instances are composed of these values "
463
+ "plus the class attribute (if set in the input data). If a (nominal) "
464
+ "class attribute is set, the clusterer is run separately for each "
465
+ "class. The class attribute (if set) and any user-specified "
466
+ "attributes are ignored during the clustering operation";
470
* Returns a description of this option suitable for display
471
* as a tip text in the gui.
473
* @return description of this option
475
public String densityBasedClustererTipText() {
476
return "The clusterer that will generate membership values for the instances.";
480
* Set the clusterer for use in filtering
482
* @param newClusterer the clusterer to use
484
public void setDensityBasedClusterer(DensityBasedClusterer newClusterer) {
485
m_clusterer = newClusterer;
489
* Get the clusterer used by this filter
491
* @return the clusterer used
493
public DensityBasedClusterer getDensityBasedClusterer() {
498
* Returns the tip text for this property
500
* @return tip text for this property suitable for
501
* displaying in the explorer/experimenter gui
503
public String ignoredAttributeIndicesTipText() {
505
return "The range of attributes to be ignored by the clusterer. eg: first-3,5,9-last";
509
* Gets ranges of attributes to be ignored.
511
* @return a string containing a comma-separated list of ranges
513
public String getIgnoredAttributeIndices() {
515
if (m_ignoreAttributesRange == null) {
518
return m_ignoreAttributesRange.getRanges();
523
* Sets the ranges of attributes to be ignored. If provided string
524
* is null, no attributes will be ignored.
526
* @param rangeList a string representing the list of attributes.
527
* eg: first-3,5,6-last
528
* @throws IllegalArgumentException if an invalid range list is supplied
530
public void setIgnoredAttributeIndices(String rangeList) {
532
if ((rangeList == null) || (rangeList.length() == 0)) {
533
m_ignoreAttributesRange = null;
535
m_ignoreAttributesRange = new Range();
536
m_ignoreAttributesRange.setRanges(rangeList);
541
* Main method for testing this class.
543
* @param argv should contain arguments to the filter: use -h for help
545
public static void main(String [] argv) {
546
runFilter(new ClusterMembership(), argv);
added
removed
weka/filters/unsupervised/attribute/ClusterMembership.java
