2
* This program is free software; you can redistribute it and/or modify
3
* it under the terms of the GNU General Public License as published by
4
* the Free Software Foundation; either version 2 of the License, or
5
* (at your option) any later version.
7
* This program is distributed in the hope that it will be useful,
8
* but WITHOUT ANY WARRANTY; without even the implied warranty of
9
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10
* GNU General Public License for more details.
12
* You should have received a copy of the GNU General Public License
13
* along with this program; if not, write to the Free Software
14
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18
* NaiveBayesMultinomial.java
19
* Copyright (C) 2003 University of Waikato, Hamilton, New Zealand
22
package weka.classifiers.bayes;
24
import weka.classifiers.Classifier;
25
import weka.core.Capabilities;
26
import weka.core.Instance;
27
import weka.core.Instances;
28
import weka.core.TechnicalInformation;
29
import weka.core.TechnicalInformationHandler;
30
import weka.core.Utils;
31
import weka.core.WeightedInstancesHandler;
32
import weka.core.Capabilities.Capability;
33
import weka.core.TechnicalInformation.Field;
34
import weka.core.TechnicalInformation.Type;
37
<!-- globalinfo-start -->
38
* Class for building and using a multinomial Naive Bayes classifier. For more information see,<br/>
40
* Andrew Mccallum, Kamal Nigam: A Comparison of Event Models for Naive Bayes Text Classification. In: AAAI-98 Workshop on 'Learning for Text Categorization', 1998.<br/>
42
* The core equation for this classifier:<br/>
44
* P[Ci|D] = (P[D|Ci] x P[Ci]) / P[D] (Bayes rule)<br/>
46
* where Ci is class i and D is a document.
48
<!-- globalinfo-end -->
50
<!-- technical-bibtex-start -->
53
* @inproceedings{Mccallum1998,
54
* author = {Andrew Mccallum and Kamal Nigam},
55
* booktitle = {AAAI-98 Workshop on 'Learning for Text Categorization'},
56
* title = {A Comparison of Event Models for Naive Bayes Text Classification},
61
<!-- technical-bibtex-end -->
63
<!-- options-start -->
64
* Valid options are: <p/>
67
* If set, classifier is run in debug mode and
68
* may output additional info to the console</pre>
72
* @author Andrew Golightly (acg4@cs.waikato.ac.nz)
73
* @author Bernhard Pfahringer (bernhard@cs.waikato.ac.nz)
74
* @version $Revision: 1.15 $
76
public class NaiveBayesMultinomial
78
implements WeightedInstancesHandler,TechnicalInformationHandler {
80
/** for serialization */
81
static final long serialVersionUID = 5932177440181257085L;
84
* probability that a word (w) exists in a class (H) (i.e. Pr[w|H])
85
* The matrix is in the this format: probOfWordGivenClass[class][wordAttribute]
86
* NOTE: the values are actually the log of Pr[w|H]
88
protected double[][] m_probOfWordGivenClass;
90
/** the probability of a class (i.e. Pr[H]) */
91
protected double[] m_probOfClass;
93
/** number of unique words */
94
protected int m_numAttributes;
96
/** number of class values */
97
protected int m_numClasses;
99
/** cache lnFactorial computations */
100
protected double[] m_lnFactorialCache = new double[]{0.0,0.0};
102
/** copy of header information for use in toString method */
103
protected Instances m_headerInfo;
106
* Returns a string describing this classifier
107
* @return a description of the classifier suitable for
108
* displaying in the explorer/experimenter gui
110
public String globalInfo() {
112
"Class for building and using a multinomial Naive Bayes classifier. "
113
+ "For more information see,\n\n"
114
+ getTechnicalInformation().toString() + "\n\n"
115
+ "The core equation for this classifier:\n\n"
116
+ "P[Ci|D] = (P[D|Ci] x P[Ci]) / P[D] (Bayes rule)\n\n"
117
+ "where Ci is class i and D is a document.";
121
* Returns an instance of a TechnicalInformation object, containing
122
* detailed information about the technical background of this class,
123
* e.g., paper reference or book this class is based on.
125
* @return the technical information about this class
127
public TechnicalInformation getTechnicalInformation() {
128
TechnicalInformation result;
130
result = new TechnicalInformation(Type.INPROCEEDINGS);
131
result.setValue(Field.AUTHOR, "Andrew Mccallum and Kamal Nigam");
132
result.setValue(Field.YEAR, "1998");
133
result.setValue(Field.TITLE, "A Comparison of Event Models for Naive Bayes Text Classification");
134
result.setValue(Field.BOOKTITLE, "AAAI-98 Workshop on 'Learning for Text Categorization'");
140
* Returns default capabilities of the classifier.
142
* @return the capabilities of this classifier
144
public Capabilities getCapabilities() {
145
Capabilities result = super.getCapabilities();
148
result.enable(Capability.NUMERIC_ATTRIBUTES);
151
result.enable(Capability.NOMINAL_CLASS);
152
result.enable(Capability.MISSING_CLASS_VALUES);
158
* Generates the classifier.
160
* @param instances set of instances serving as training data
161
* @throws Exception if the classifier has not been generated successfully
163
public void buildClassifier(Instances instances) throws Exception
165
// can classifier handle the data?
166
getCapabilities().testWithFail(instances);
168
// remove instances with missing class
169
instances = new Instances(instances);
170
instances.deleteWithMissingClass();
172
m_headerInfo = new Instances(instances, 0);
173
m_numClasses = instances.numClasses();
174
m_numAttributes = instances.numAttributes();
175
m_probOfWordGivenClass = new double[m_numClasses][];
178
initialising the matrix of word counts
179
NOTE: Laplace estimator introduced in case a word that does not appear for a class in the
180
training set does so for the test set
182
for(int c = 0; c<m_numClasses; c++)
184
m_probOfWordGivenClass[c] = new double[m_numAttributes];
185
for(int att = 0; att<m_numAttributes; att++)
187
m_probOfWordGivenClass[c][att] = 1;
191
//enumerate through the instances
194
double numOccurences;
195
double[] docsPerClass = new double[m_numClasses];
196
double[] wordsPerClass = new double[m_numClasses];
198
java.util.Enumeration enumInsts = instances.enumerateInstances();
199
while (enumInsts.hasMoreElements())
201
instance = (Instance) enumInsts.nextElement();
202
classIndex = (int)instance.value(instance.classIndex());
203
docsPerClass[classIndex] += instance.weight();
205
for(int a = 0; a<instance.numValues(); a++)
206
if(instance.index(a) != instance.classIndex())
208
if(!instance.isMissing(a))
210
numOccurences = instance.valueSparse(a) * instance.weight();
211
if(numOccurences < 0)
212
throw new Exception("Numeric attribute values must all be greater or equal to zero.");
213
wordsPerClass[classIndex] += numOccurences;
214
m_probOfWordGivenClass[classIndex][instance.index(a)] += numOccurences;
220
normalising probOfWordGivenClass values
221
and saving each value as the log of each value
223
for(int c = 0; c<m_numClasses; c++)
224
for(int v = 0; v<m_numAttributes; v++)
225
m_probOfWordGivenClass[c][v] = Math.log(m_probOfWordGivenClass[c][v] / (wordsPerClass[c] + m_numAttributes - 1));
229
NOTE: Laplace estimator introduced in case a class does not get mentioned in the set of
232
final double numDocs = instances.sumOfWeights() + m_numClasses;
233
m_probOfClass = new double[m_numClasses];
234
for(int h=0; h<m_numClasses; h++)
235
m_probOfClass[h] = (double)(docsPerClass[h] + 1)/numDocs;
239
* Calculates the class membership probabilities for the given test
242
* @param instance the instance to be classified
243
* @return predicted class probability distribution
244
* @throws Exception if there is a problem generating the prediction
246
public double [] distributionForInstance(Instance instance) throws Exception
248
double[] probOfClassGivenDoc = new double[m_numClasses];
250
//calculate the array of log(Pr[D|C])
251
double[] logDocGivenClass = new double[m_numClasses];
252
for(int h = 0; h<m_numClasses; h++)
253
logDocGivenClass[h] = probOfDocGivenClass(instance, h);
255
double max = logDocGivenClass[Utils.maxIndex(logDocGivenClass)];
256
double probOfDoc = 0.0;
258
for(int i = 0; i<m_numClasses; i++)
260
probOfClassGivenDoc[i] = Math.exp(logDocGivenClass[i] - max) * m_probOfClass[i];
261
probOfDoc += probOfClassGivenDoc[i];
264
Utils.normalize(probOfClassGivenDoc,probOfDoc);
266
return probOfClassGivenDoc;
270
* log(N!) + (for all the words)(log(Pi^ni) - log(ni!))
273
* N is the total number of words
274
* Pi is the probability of obtaining word i
275
* ni is the number of times the word at index i occurs in the document
277
* @param inst The instance to be classified
278
* @param classIndex The index of the class we are calculating the probability with respect to
280
* @return The log of the probability of the document occuring given the class
283
private double probOfDocGivenClass(Instance inst, int classIndex)
286
//double totalWords = 0; //no need as we are not calculating the factorial at all.
288
double freqOfWordInDoc; //should be double
289
for(int i = 0; i<inst.numValues(); i++)
290
if(inst.index(i) != inst.classIndex())
292
freqOfWordInDoc = inst.valueSparse(i);
293
//totalWords += freqOfWordInDoc;
294
answer += (freqOfWordInDoc * m_probOfWordGivenClass[classIndex][inst.index(i)]
295
); //- lnFactorial(freqOfWordInDoc));
298
//answer += lnFactorial(totalWords);//The factorial terms don't make
299
//any difference to the classifier's
300
//accuracy, so not needed.
306
* Fast computation of ln(n!) for non-negative ints
308
* negative ints are passed on to the general gamma-function
309
* based version in weka.core.SpecialFunctions
311
* if the current n value is higher than any previous one,
312
* the cache is extended and filled to cover it
314
* the common case is reduced to a simple array lookup
316
* @param n the integer
320
public double lnFactorial(int n)
322
if (n < 0) return weka.core.SpecialFunctions.lnFactorial(n);
324
if (m_lnFactorialCache.length <= n) {
325
double[] tmp = new double[n+1];
326
System.arraycopy(m_lnFactorialCache,0,tmp,0,m_lnFactorialCache.length);
327
for(int i = m_lnFactorialCache.length; i < tmp.length; i++)
328
tmp[i] = tmp[i-1] + Math.log(i);
329
m_lnFactorialCache = tmp;
332
return m_lnFactorialCache[n];
336
* Returns a string representation of the classifier.
338
* @return a string representation of the classifier
340
public String toString()
342
StringBuffer result = new StringBuffer("The independent probability of a class\n--------------------------------------\n");
344
for(int c = 0; c<m_numClasses; c++)
345
result.append(m_headerInfo.classAttribute().value(c)).append("\t").append(Double.toString(m_probOfClass[c])).append("\n");
347
result.append("\nThe probability of a word given the class\n-----------------------------------------\n\t");
349
for(int c = 0; c<m_numClasses; c++)
350
result.append(m_headerInfo.classAttribute().value(c)).append("\t");
354
for(int w = 0; w<m_numAttributes; w++)
356
result.append(m_headerInfo.attribute(w).name()).append("\t");
357
for(int c = 0; c<m_numClasses; c++)
358
result.append(Double.toString(Math.exp(m_probOfWordGivenClass[c][w]))).append("\t");
362
return result.toString();
366
* Main method for testing this class.
368
* @param argv the options
370
public static void main(String [] argv) {
371
runClassifier(new NaiveBayesMultinomial(), argv);
added
removed
weka/classifiers/bayes/NaiveBayesMultinomial.java
