2
* This program is free software; you can redistribute it and/or modify
3
* it under the terms of the GNU General Public License as published by
4
* the Free Software Foundation; either version 2 of the License, or
5
* (at your option) any later version.
7
* This program is distributed in the hope that it will be useful,
8
* but WITHOUT ANY WARRANTY; without even the implied warranty of
9
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10
* GNU General Public License for more details.
12
* You should have received a copy of the GNU General Public License
13
* along with this program; if not, write to the Free Software
14
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18
* PairedCorrectedTTester.java
19
* Copyright (C) 2003 University of Waikato, Hamilton, New Zealand
24
package weka.experiment;
26
import weka.core.Attribute;
27
import weka.core.FastVector;
28
import weka.core.Instance;
29
import weka.core.Instances;
30
import weka.core.Option;
31
import weka.core.Utils;
32
import weka.core.TechnicalInformation;
33
import weka.core.TechnicalInformation.Type;
34
import weka.core.TechnicalInformation.Field;
35
import weka.core.TechnicalInformationHandler;
37
import java.io.BufferedReader;
38
import java.io.FileReader;
39
import java.util.Enumeration;
42
* Behaves the same as PairedTTester, only it uses the corrected
43
* resampled t-test statistic.<p/>
45
* For more information see:<p/>
47
<!-- technical-plaintext-start -->
48
* Claude Nadeau, Yoshua Bengio (2001). Inference for the Generalization Error. Machine Learning..
49
<!-- technical-plaintext-end -->
53
<!-- technical-bibtex-start -->
56
* @article{Nadeau2001,
57
* author = {Claude Nadeau and Yoshua Bengio},
58
* journal = {Machine Learning},
59
* title = {Inference for the Generalization Error},
61
* PDF = {http://www.iro.umontreal.ca/\~lisa/bib/pub_subject/comparative/pointeurs/nadeau_MLJ1597.pdf}
65
<!-- technical-bibtex-end -->
67
<!-- options-start -->
68
* Valid options are: <p/>
70
* <pre> -D <index,index2-index4,...>
71
* Specify list of columns that specify a unique
73
* First and last are valid indexes. (default none)</pre>
75
* <pre> -R <index>
76
* Set the index of the column containing the run number</pre>
78
* <pre> -F <index>
79
* Set the index of the column containing the fold number</pre>
81
* <pre> -G <index1,index2-index4,...>
82
* Specify list of columns that specify a unique
83
* 'result generator' (eg: classifier name and options).
84
* First and last are valid indexes. (default none)</pre>
86
* <pre> -S <significance level>
87
* Set the significance level for comparisons (default 0.05)</pre>
90
* Show standard deviations</pre>
93
* Produce table comparisons in Latex table format</pre>
96
* Produce table comparisons in CSV table format</pre>
99
* Produce table comparisons in HTML table format</pre>
101
* <pre> -significance
102
* Produce table comparisons with only the significance values</pre>
105
* Produce table comparisons output suitable for GNUPlot</pre>
109
* @author Richard Kirkby (rkirkby@cs.waikato.ac.nz)
110
* @version $Revision: 1.12 $
112
public class PairedCorrectedTTester
113
extends PairedTTester
114
implements TechnicalInformationHandler {
116
/** for serialization */
117
static final long serialVersionUID = -3105268939845653323L;
120
* Returns an instance of a TechnicalInformation object, containing
121
* detailed information about the technical background of this class,
122
* e.g., paper reference or book this class is based on.
124
* @return the technical information about this class
126
public TechnicalInformation getTechnicalInformation() {
127
TechnicalInformation result;
129
result = new TechnicalInformation(Type.ARTICLE);
130
result.setValue(Field.AUTHOR, "Claude Nadeau and Yoshua Bengio");
131
result.setValue(Field.YEAR, "2001");
132
result.setValue(Field.TITLE, "Inference for the Generalization Error");
133
result.setValue(Field.JOURNAL, "Machine Learning");
134
result.setValue(Field.PDF, "http://www.iro.umontreal.ca/~lisa/bib/pub_subject/comparative/pointeurs/nadeau_MLJ1597.pdf");
140
* Computes a paired t-test comparison for a specified dataset between
143
* @param datasetSpecifier the dataset specifier
144
* @param resultset1Index the index of the first resultset
145
* @param resultset2Index the index of the second resultset
146
* @param comparisonColumn the column containing values to compare
147
* @return the results of the paired comparison
148
* @throws Exception if an error occurs
150
public PairedStats calculateStatistics(Instance datasetSpecifier,
153
int comparisonColumn) throws Exception {
155
if (m_Instances.attribute(comparisonColumn).type()
156
!= Attribute.NUMERIC) {
157
throw new Exception("Comparison column " + (comparisonColumn + 1)
159
+ m_Instances.attribute(comparisonColumn).name()
160
+ ") is not numeric");
162
if (!m_ResultsetsValid) {
166
Resultset resultset1 = (Resultset) m_Resultsets.elementAt(resultset1Index);
167
Resultset resultset2 = (Resultset) m_Resultsets.elementAt(resultset2Index);
168
FastVector dataset1 = resultset1.dataset(datasetSpecifier);
169
FastVector dataset2 = resultset2.dataset(datasetSpecifier);
170
String datasetName = templateString(datasetSpecifier);
171
if (dataset1 == null) {
172
throw new Exception("No results for dataset=" + datasetName
173
+ " for resultset=" + resultset1.templateString());
174
} else if (dataset2 == null) {
175
throw new Exception("No results for dataset=" + datasetName
176
+ " for resultset=" + resultset2.templateString());
177
} else if (dataset1.size() != dataset2.size()) {
178
throw new Exception("Results for dataset=" + datasetName
179
+ " differ in size for resultset="
180
+ resultset1.templateString()
182
+ resultset2.templateString()
186
// calculate the test/train ratio
187
double testTrainRatio = 0.0;
188
int trainSizeIndex = -1;
189
int testSizeIndex = -1;
190
// find the columns with the train/test sizes
191
for (int i=0; i<m_Instances.numAttributes(); i++) {
192
if (m_Instances.attribute(i).name().toLowerCase().equals("number_of_training_instances")) {
194
} else if (m_Instances.attribute(i).name().toLowerCase().equals("number_of_testing_instances")) {
198
if (trainSizeIndex >= 0 && testSizeIndex >= 0) {
199
double totalTrainSize = 0.0;
200
double totalTestSize = 0.0;
201
for (int k = 0; k < dataset1.size(); k ++) {
202
Instance current = (Instance) dataset1.elementAt(k);
203
totalTrainSize += current.value(trainSizeIndex);
204
totalTestSize += current.value(testSizeIndex);
206
testTrainRatio = totalTestSize / totalTrainSize;
208
PairedStats pairedStats =
209
new PairedStatsCorrected(m_SignificanceLevel, testTrainRatio);
211
for (int k = 0; k < dataset1.size(); k ++) {
212
Instance current1 = (Instance) dataset1.elementAt(k);
213
Instance current2 = (Instance) dataset2.elementAt(k);
214
if (current1.isMissing(comparisonColumn)) {
215
System.err.println("Instance has missing value in comparison "
216
+ "column!\n" + current1);
219
if (current2.isMissing(comparisonColumn)) {
220
System.err.println("Instance has missing value in comparison "
221
+ "column!\n" + current2);
224
if (current1.value(m_RunColumn) != current2.value(m_RunColumn)) {
225
System.err.println("Run numbers do not match!\n"
226
+ current1 + current2);
228
if (m_FoldColumn != -1) {
229
if (current1.value(m_FoldColumn) != current2.value(m_FoldColumn)) {
230
System.err.println("Fold numbers do not match!\n"
231
+ current1 + current2);
235
double value1 = current1.value(comparisonColumn);
236
double value2 = current2.value(comparisonColumn);
237
pairedStats.add(value1, value2);
239
pairedStats.calculateDerived();
244
* Test the class from the command line.
246
* @param args contains options for the instance ttests
248
public static void main(String args[]) {
251
PairedCorrectedTTester tt = new PairedCorrectedTTester();
252
String datasetName = Utils.getOption('t', args);
253
String compareColStr = Utils.getOption('c', args);
254
String baseColStr = Utils.getOption('b', args);
255
boolean summaryOnly = Utils.getFlag('s', args);
256
boolean rankingOnly = Utils.getFlag('r', args);
258
if ((datasetName.length() == 0)
259
|| (compareColStr.length() == 0)) {
260
throw new Exception("-t and -c options are required");
263
Utils.checkForRemainingOptions(args);
264
} catch (Exception ex) {
266
Enumeration enu = tt.listOptions();
267
while (enu.hasMoreElements()) {
268
Option option = (Option) enu.nextElement();
269
result += option.synopsis() + '\n'
270
+ option.description() + '\n';
275
+ "\tSet the dataset containing data to evaluate\n"
277
+ "\tSet the resultset to base comparisons against (optional)\n"
279
+ "\tSet the column to perform a comparison on\n"
281
+ "\tSummarize wins over all resultset pairs\n\n"
283
+ "\tGenerate a resultset ranking\n\n"
286
Instances data = new Instances(new BufferedReader(
287
new FileReader(datasetName)));
288
tt.setInstances(data);
290
int compareCol = Integer.parseInt(compareColStr) - 1;
291
System.out.println(tt.header(compareCol));
293
System.out.println(tt.multiResultsetRanking(compareCol));
294
} else if (summaryOnly) {
295
System.out.println(tt.multiResultsetSummary(compareCol));
297
System.out.println(tt.resultsetKey());
298
if (baseColStr.length() == 0) {
299
for (int i = 0; i < tt.getNumResultsets(); i++) {
300
System.out.println(tt.multiResultsetFull(i, compareCol));
303
int baseCol = Integer.parseInt(baseColStr) - 1;
304
System.out.println(tt.multiResultsetFull(baseCol, compareCol));
307
} catch(Exception e) {
309
System.err.println(e.getMessage());
314
* returns the name of the tester
316
* @return the display name
318
public String getDisplayName() {
319
return "Paired T-Tester (corrected)";
323
* returns a string that is displayed as tooltip on the "perform test"
324
* button in the experimenter
326
* @return the string for the tool tip
328
public String getToolTipText() {
329
return "Performs test using corrected resampled t-test statistic (Nadeau and Bengio)";