2
* Licensed to the Apache Software Foundation (ASF) under one or more
3
* contributor license agreements. See the NOTICE file distributed with
4
* this work for additional information regarding copyright ownership.
5
* The ASF licenses this file to You under the Apache License, Version 2.0
6
* (the "License"); you may not use this file except in compliance with
7
* the License. You may obtain a copy of the License at
9
* http://www.apache.org/licenses/LICENSE-2.0
11
* Unless required by applicable law or agreed to in writing, software
12
* distributed under the License is distributed on an "AS IS" BASIS,
13
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
* See the License for the specific language governing permissions and
15
* limitations under the License.
17
package org.apache.commons.io.input.compatibility;
19
import java.io.BufferedInputStream;
20
import java.io.BufferedReader;
22
import java.io.FileInputStream;
23
import java.io.IOException;
24
import java.io.InputStream;
25
import java.io.InputStreamReader;
26
import java.io.Reader;
27
import java.io.StringReader;
28
import java.net.HttpURLConnection;
30
import java.net.URLConnection;
31
import java.text.MessageFormat;
32
import java.util.regex.Matcher;
33
import java.util.regex.Pattern;
35
import org.apache.commons.io.output.XmlStreamWriter;
38
* Character stream that handles all the necessary Voodo to figure out the
39
* charset encoding of the XML document within the stream.
41
* IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader.
42
* This one IS a character stream.
44
* All this has to be done without consuming characters from the stream, if not
45
* the XML parser will not recognized the document as a valid XML. This is not
46
* 100% true, but it's close enough (UTF-8 BOM is not handled by all parsers
47
* right now, XmlStreamReader handles it and things work in all parsers).
49
* The XmlStreamReader class handles the charset encoding of XML documents in
50
* Files, raw streams and HTTP streams by offering a wide set of constructors.
52
* By default the charset encoding detection is lenient, the constructor with
53
* the lenient flag can be used for an script (following HTTP MIME and XML
54
* specifications). All this is nicely explained by Mark Pilgrim in his blog, <a
55
* href="http://diveintomark.org/archives/2004/02/13/xml-media-types">
56
* Determining the character encoding of a feed</a>.
58
* Originally developed for <a href="http://rome.dev.java.net">ROME</a> under
61
* @version $Id: XmlStreamReader.java 1346400 2012-06-05 14:48:01Z ggregory $
62
* @see XmlStreamWriter
64
public class XmlStreamReader extends Reader {
65
private static final int BUFFER_SIZE = 4096;
67
private static final String UTF_8 = "UTF-8";
69
private static final String US_ASCII = "US-ASCII";
71
private static final String UTF_16BE = "UTF-16BE";
73
private static final String UTF_16LE = "UTF-16LE";
75
private static final String UTF_16 = "UTF-16";
77
private static final String UTF_32BE = "UTF-32BE";
79
private static final String UTF_32LE = "UTF-32LE";
81
private static final String UTF_32 = "UTF-32";
83
private static final String EBCDIC = "CP1047";
85
private static String staticDefaultEncoding = null;
87
private Reader reader;
89
private String encoding;
91
private String defaultEncoding;
94
* Sets the default encoding to use if none is set in HTTP content-type, XML
95
* prolog and the rules based on content-type are not adequate.
97
* If it is set to NULL the content-type based rules are used.
99
* By default it is NULL.
101
* @param encoding charset encoding to default to.
103
public static void setDefaultEncoding(String encoding) {
104
staticDefaultEncoding = encoding;
108
* Returns the default encoding to use if none is set in HTTP content-type,
109
* XML prolog and the rules based on content-type are not adequate.
111
* If it is NULL the content-type based rules are used.
113
* @return the default encoding to use.
115
public static String getDefaultEncoding() {
116
return staticDefaultEncoding;
120
* Creates a Reader for a File.
122
* It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset,
123
* if this is also missing defaults to UTF-8.
125
* It does a lenient charset encoding detection, check the constructor with
126
* the lenient parameter for details.
128
* @param file File to create a Reader from.
129
* @throws IOException thrown if there is a problem reading the file.
131
public XmlStreamReader(File file) throws IOException {
132
this(new FileInputStream(file));
136
* Creates a Reader for a raw InputStream.
138
* It follows the same logic used for files.
140
* It does a lenient charset encoding detection, check the constructor with
141
* the lenient parameter for details.
143
* @param is InputStream to create a Reader from.
144
* @throws IOException thrown if there is a problem reading the stream.
146
public XmlStreamReader(InputStream is) throws IOException {
151
* Creates a Reader for a raw InputStream.
153
* It follows the same logic used for files.
155
* If lenient detection is indicated and the detection above fails as per
156
* specifications it then attempts the following:
158
* If the content type was 'text/html' it replaces it with 'text/xml' and
159
* tries the detection again.
161
* Else if the XML prolog had a charset encoding that encoding is used.
163
* Else if the content type had a charset encoding that encoding is used.
165
* Else 'UTF-8' is used.
167
* If lenient detection is indicated an XmlStreamReaderException is never
170
* @param is InputStream to create a Reader from.
171
* @param lenient indicates if the charset encoding detection should be
173
* @throws IOException thrown if there is a problem reading the stream.
174
* @throws XmlStreamReaderException thrown if the charset encoding could not
175
* be determined according to the specs.
177
public XmlStreamReader(InputStream is, boolean lenient) throws IOException,
178
XmlStreamReaderException {
179
defaultEncoding = staticDefaultEncoding;
181
doRawStream(is, lenient);
182
} catch (XmlStreamReaderException ex) {
186
doLenientDetection(null, ex);
192
* Creates a Reader using the InputStream of a URL.
194
* If the URL is not of type HTTP and there is not 'content-type' header in
195
* the fetched data it uses the same logic used for Files.
197
* If the URL is a HTTP Url or there is a 'content-type' header in the
198
* fetched data it uses the same logic used for an InputStream with
201
* It does a lenient charset encoding detection, check the constructor with
202
* the lenient parameter for details.
204
* @param url URL to create a Reader from.
205
* @throws IOException thrown if there is a problem reading the stream of
208
public XmlStreamReader(URL url) throws IOException {
209
this(url.openConnection());
213
* Creates a Reader using the InputStream of a URLConnection.
215
* If the URLConnection is not of type HttpURLConnection and there is not
216
* 'content-type' header in the fetched data it uses the same logic used for
219
* If the URLConnection is a HTTP Url or there is a 'content-type' header in
220
* the fetched data it uses the same logic used for an InputStream with
223
* It does a lenient charset encoding detection, check the constructor with
224
* the lenient parameter for details.
226
* @param conn URLConnection to create a Reader from.
227
* @throws IOException thrown if there is a problem reading the stream of
230
public XmlStreamReader(URLConnection conn) throws IOException {
231
defaultEncoding = staticDefaultEncoding;
232
boolean lenient = true;
233
if (conn instanceof HttpURLConnection) {
235
doHttpStream(conn.getInputStream(), conn.getContentType(),
237
} catch (XmlStreamReaderException ex) {
238
doLenientDetection(conn.getContentType(), ex);
240
} else if (conn.getContentType() != null) {
242
doHttpStream(conn.getInputStream(), conn.getContentType(),
244
} catch (XmlStreamReaderException ex) {
245
doLenientDetection(conn.getContentType(), ex);
249
doRawStream(conn.getInputStream(), lenient);
250
} catch (XmlStreamReaderException ex) {
251
doLenientDetection(null, ex);
257
* Creates a Reader using an InputStream an the associated content-type
260
* First it checks if the stream has BOM. If there is not BOM checks the
261
* content-type encoding. If there is not content-type encoding checks the
262
* XML prolog encoding. If there is not XML prolog encoding uses the default
263
* encoding mandated by the content-type MIME type.
265
* It does a lenient charset encoding detection, check the constructor with
266
* the lenient parameter for details.
268
* @param is InputStream to create the reader from.
269
* @param httpContentType content-type header to use for the resolution of
270
* the charset encoding.
271
* @throws IOException thrown if there is a problem reading the file.
273
public XmlStreamReader(InputStream is, String httpContentType)
275
this(is, httpContentType, true);
279
* Creates a Reader using an InputStream an the associated content-type
280
* header. This constructor is lenient regarding the encoding detection.
282
* First it checks if the stream has BOM. If there is not BOM checks the
283
* content-type encoding. If there is not content-type encoding checks the
284
* XML prolog encoding. If there is not XML prolog encoding uses the default
285
* encoding mandated by the content-type MIME type.
287
* If lenient detection is indicated and the detection above fails as per
288
* specifications it then attempts the following:
290
* If the content type was 'text/html' it replaces it with 'text/xml' and
291
* tries the detection again.
293
* Else if the XML prolog had a charset encoding that encoding is used.
295
* Else if the content type had a charset encoding that encoding is used.
297
* Else 'UTF-8' is used.
299
* If lenient detection is indicated an XmlStreamReaderException is never
302
* @param is InputStream to create the reader from.
303
* @param httpContentType content-type header to use for the resolution of
304
* the charset encoding.
305
* @param lenient indicates if the charset encoding detection should be
307
* @throws IOException thrown if there is a problem reading the file.
308
* @throws XmlStreamReaderException thrown if the charset encoding could not
309
* be determined according to the specs.
311
public XmlStreamReader(InputStream is, String httpContentType,
312
boolean lenient, String defaultEncoding) throws IOException,
313
XmlStreamReaderException {
314
this.defaultEncoding = defaultEncoding == null ? staticDefaultEncoding
317
doHttpStream(is, httpContentType, lenient);
318
} catch (XmlStreamReaderException ex) {
322
doLenientDetection(httpContentType, ex);
328
* Creates a Reader using an InputStream an the associated content-type
329
* header. This constructor is lenient regarding the encoding detection.
331
* First it checks if the stream has BOM. If there is not BOM checks the
332
* content-type encoding. If there is not content-type encoding checks the
333
* XML prolog encoding. If there is not XML prolog encoding uses the default
334
* encoding mandated by the content-type MIME type.
336
* If lenient detection is indicated and the detection above fails as per
337
* specifications it then attempts the following:
339
* If the content type was 'text/html' it replaces it with 'text/xml' and
340
* tries the detection again.
342
* Else if the XML prolog had a charset encoding that encoding is used.
344
* Else if the content type had a charset encoding that encoding is used.
346
* Else 'UTF-8' is used.
348
* If lenient detection is indicated an XmlStreamReaderException is never
351
* @param is InputStream to create the reader from.
352
* @param httpContentType content-type header to use for the resolution of
353
* the charset encoding.
354
* @param lenient indicates if the charset encoding detection should be
356
* @throws IOException thrown if there is a problem reading the file.
357
* @throws XmlStreamReaderException thrown if the charset encoding could not
358
* be determined according to the specs.
360
public XmlStreamReader(InputStream is, String httpContentType,
361
boolean lenient) throws IOException, XmlStreamReaderException {
362
this(is, httpContentType, lenient, null);
365
private void doLenientDetection(String httpContentType,
366
XmlStreamReaderException ex) throws IOException {
367
if (httpContentType != null) {
368
if (httpContentType.startsWith("text/html")) {
369
httpContentType = httpContentType.substring("text/html"
371
httpContentType = "text/xml" + httpContentType;
373
doHttpStream(ex.getInputStream(), httpContentType, true);
375
} catch (XmlStreamReaderException ex2) {
381
String encoding = ex.getXmlEncoding();
382
if (encoding == null) {
383
encoding = ex.getContentTypeEncoding();
385
if (encoding == null) {
386
encoding = defaultEncoding == null ? UTF_8 : defaultEncoding;
388
prepareReader(ex.getInputStream(), encoding);
393
* Returns the charset encoding of the XmlStreamReader.
395
* @return charset encoding.
397
public String getEncoding() {
402
public int read(char[] buf, int offset, int len) throws IOException {
403
return reader.read(buf, offset, len);
407
* Closes the XmlStreamReader stream.
409
* @throws IOException thrown if there was a problem closing the stream.
412
public void close() throws IOException {
416
private void doRawStream(InputStream is, boolean lenient)
418
BufferedInputStream pis = new BufferedInputStream(is, BUFFER_SIZE);
419
String bomEnc = getBOMEncoding(pis);
420
String xmlGuessEnc = getXMLGuessEncoding(pis);
421
String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
422
String encoding = calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc, pis);
423
prepareReader(pis, encoding);
426
private void doHttpStream(InputStream is, String httpContentType,
427
boolean lenient) throws IOException {
428
BufferedInputStream pis = new BufferedInputStream(is, BUFFER_SIZE);
429
String cTMime = getContentTypeMime(httpContentType);
430
String cTEnc = getContentTypeEncoding(httpContentType);
431
String bomEnc = getBOMEncoding(pis);
432
String xmlGuessEnc = getXMLGuessEncoding(pis);
433
String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
434
String encoding = calculateHttpEncoding(cTMime, cTEnc, bomEnc,
435
xmlGuessEnc, xmlEnc, pis, lenient);
436
prepareReader(pis, encoding);
439
private void prepareReader(InputStream is, String encoding)
441
reader = new InputStreamReader(is, encoding);
442
this.encoding = encoding;
445
// InputStream is passed for XmlStreamReaderException creation only
446
String calculateRawEncoding(String bomEnc, String xmlGuessEnc,
447
String xmlEnc, InputStream is) throws IOException {
449
if (bomEnc == null) {
450
if (xmlGuessEnc == null || xmlEnc == null) {
451
encoding = defaultEncoding == null ? UTF_8 : defaultEncoding;
452
} else if (xmlEnc.equals(UTF_16)
453
&& (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc
454
.equals(UTF_16LE))) {
455
encoding = xmlGuessEnc;
456
} else if (xmlEnc.equals(UTF_32)
457
&& (xmlGuessEnc.equals(UTF_32BE) || xmlGuessEnc
458
.equals(UTF_32LE))) {
459
encoding = xmlGuessEnc;
463
} else if (bomEnc.equals(UTF_8)) {
464
if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) {
465
throw new XmlStreamReaderException(RAW_EX_1
466
.format(new Object[] { bomEnc, xmlGuessEnc, xmlEnc }),
467
bomEnc, xmlGuessEnc, xmlEnc, is);
469
if (xmlEnc != null && !xmlEnc.equals(UTF_8)) {
470
throw new XmlStreamReaderException(RAW_EX_1
471
.format(new Object[] { bomEnc, xmlGuessEnc, xmlEnc }),
472
bomEnc, xmlGuessEnc, xmlEnc, is);
475
} else if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) {
476
if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
477
throw new XmlStreamReaderException(RAW_EX_1.format(new Object[] { bomEnc,
478
xmlGuessEnc, xmlEnc }), bomEnc, xmlGuessEnc, xmlEnc, is);
480
if (xmlEnc != null && !xmlEnc.equals(UTF_16)
481
&& !xmlEnc.equals(bomEnc)) {
482
throw new XmlStreamReaderException(RAW_EX_1
483
.format(new Object[] { bomEnc, xmlGuessEnc, xmlEnc }),
484
bomEnc, xmlGuessEnc, xmlEnc, is);
487
} else if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) {
488
if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
489
throw new XmlStreamReaderException(RAW_EX_1.format(new Object[] { bomEnc,
490
xmlGuessEnc, xmlEnc }), bomEnc, xmlGuessEnc, xmlEnc, is);
492
if (xmlEnc != null && !xmlEnc.equals(UTF_32)
493
&& !xmlEnc.equals(bomEnc)) {
494
throw new XmlStreamReaderException(RAW_EX_1
495
.format(new Object[] { bomEnc, xmlGuessEnc, xmlEnc }),
496
bomEnc, xmlGuessEnc, xmlEnc, is);
500
throw new XmlStreamReaderException(RAW_EX_2.format(new Object[] {
501
bomEnc, xmlGuessEnc, xmlEnc }), bomEnc, xmlGuessEnc,
507
// InputStream is passed for XmlStreamReaderException creation only
508
String calculateHttpEncoding(String cTMime, String cTEnc,
509
String bomEnc, String xmlGuessEnc, String xmlEnc, InputStream is,
510
boolean lenient) throws IOException {
512
if (lenient & xmlEnc != null) {
515
boolean appXml = isAppXml(cTMime);
516
boolean textXml = isTextXml(cTMime);
517
if (appXml || textXml) {
520
encoding = calculateRawEncoding(bomEnc, xmlGuessEnc,
523
encoding = defaultEncoding == null ? US_ASCII
526
} else if (bomEnc != null
527
&& (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE))) {
528
throw new XmlStreamReaderException(HTTP_EX_1
529
.format(new Object[] { cTMime, cTEnc, bomEnc,
530
xmlGuessEnc, xmlEnc }), cTMime, cTEnc,
531
bomEnc, xmlGuessEnc, xmlEnc, is);
532
} else if (cTEnc.equals(UTF_16)) {
533
if (bomEnc != null && bomEnc.startsWith(UTF_16)) {
536
throw new XmlStreamReaderException(HTTP_EX_2
537
.format(new Object[] { cTMime, cTEnc, bomEnc,
538
xmlGuessEnc, xmlEnc }), cTMime, cTEnc,
539
bomEnc, xmlGuessEnc, xmlEnc, is);
541
} else if (bomEnc != null
542
&& (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE))) {
543
throw new XmlStreamReaderException(HTTP_EX_1
544
.format(new Object[] { cTMime, cTEnc, bomEnc,
545
xmlGuessEnc, xmlEnc }), cTMime, cTEnc,
546
bomEnc, xmlGuessEnc, xmlEnc, is);
547
} else if (cTEnc.equals(UTF_32)) {
548
if (bomEnc != null && bomEnc.startsWith(UTF_32)) {
551
throw new XmlStreamReaderException(HTTP_EX_2
552
.format(new Object[] { cTMime, cTEnc, bomEnc,
553
xmlGuessEnc, xmlEnc }), cTMime, cTEnc,
554
bomEnc, xmlGuessEnc, xmlEnc, is);
560
throw new XmlStreamReaderException(HTTP_EX_3
561
.format(new Object[] { cTMime, cTEnc, bomEnc,
562
xmlGuessEnc, xmlEnc }), cTMime, cTEnc, bomEnc,
563
xmlGuessEnc, xmlEnc, is);
569
// returns MIME type or NULL if httpContentType is NULL
570
static String getContentTypeMime(String httpContentType) {
572
if (httpContentType != null) {
573
int i = httpContentType.indexOf(";");
574
mime = (i == -1 ? httpContentType : httpContentType.substring(0,
580
private static final Pattern CHARSET_PATTERN = Pattern
581
.compile("charset=[\"']?([.[^; \"']]*)[\"']?");
583
// returns charset parameter value, NULL if not present, NULL if
584
// httpContentType is NULL
585
static String getContentTypeEncoding(String httpContentType) {
586
String encoding = null;
587
if (httpContentType != null) {
588
int i = httpContentType.indexOf(";");
590
String postMime = httpContentType.substring(i + 1);
591
Matcher m = CHARSET_PATTERN.matcher(postMime);
592
encoding = m.find() ? m.group(1) : null;
593
encoding = encoding != null ? encoding.toUpperCase() : null;
599
// returns the BOM in the stream, NULL if not present,
600
// if there was BOM the in the stream it is consumed
601
static String getBOMEncoding(BufferedInputStream is)
603
String encoding = null;
604
int[] bytes = new int[3];
606
bytes[0] = is.read();
607
bytes[1] = is.read();
608
bytes[2] = is.read();
610
if (bytes[0] == 0xFE && bytes[1] == 0xFF) {
615
} else if (bytes[0] == 0xFF && bytes[1] == 0xFE) {
620
} else if (bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) {
628
// returns the best guess for the encoding by looking the first bytes of the
630
private static String getXMLGuessEncoding(BufferedInputStream is)
632
String encoding = null;
633
int[] bytes = new int[4];
635
bytes[0] = is.read();
636
bytes[1] = is.read();
637
bytes[2] = is.read();
638
bytes[3] = is.read();
641
if (bytes[0] == 0x00 && bytes[1] == 0x3C && bytes[2] == 0x00
642
&& bytes[3] == 0x3F) {
644
} else if (bytes[0] == 0x3C && bytes[1] == 0x00 && bytes[2] == 0x3F
645
&& bytes[3] == 0x00) {
647
} else if (bytes[0] == 0x3C && bytes[1] == 0x3F && bytes[2] == 0x78
648
&& bytes[3] == 0x6D) {
650
} else if (bytes[0] == 0x4C && bytes[1] == 0x6F && bytes[2] == 0xA7
651
&& bytes[3] == 0x94) {
657
public static final Pattern ENCODING_PATTERN = Pattern.compile(
658
"<\\?xml.*encoding[\\s]*=[\\s]*((?:\".[^\"]*\")|(?:'.[^']*'))",
661
// returns the encoding declared in the <?xml encoding=...?>, NULL if none
662
private static String getXmlProlog(BufferedInputStream is, String guessedEnc)
664
String encoding = null;
665
if (guessedEnc != null) {
666
byte[] bytes = new byte[BUFFER_SIZE];
667
is.mark(BUFFER_SIZE);
669
int max = BUFFER_SIZE;
670
int c = is.read(bytes, offset, max);
672
String xmlProlog = null;
673
while (c != -1 && firstGT == -1 && offset < BUFFER_SIZE) {
676
c = is.read(bytes, offset, max);
677
xmlProlog = new String(bytes, 0, offset, guessedEnc);
678
firstGT = xmlProlog.indexOf('>');
682
throw new IOException("Unexpected end of XML stream");
684
throw new IOException(
685
"XML prolog or ROOT element not found on first "
686
+ offset + " bytes");
689
int bytesRead = offset;
692
BufferedReader bReader = new BufferedReader(new StringReader(
693
xmlProlog.substring(0, firstGT + 1)));
694
StringBuffer prolog = new StringBuffer();
695
String line = bReader.readLine();
696
while (line != null) {
698
line = bReader.readLine();
700
Matcher m = ENCODING_PATTERN.matcher(prolog);
702
encoding = m.group(1).toUpperCase();
703
encoding = encoding.substring(1, encoding.length() - 1);
710
// indicates if the MIME type belongs to the APPLICATION XML family
711
static boolean isAppXml(String mime) {
713
&& (mime.equals("application/xml")
714
|| mime.equals("application/xml-dtd")
716
.equals("application/xml-external-parsed-entity") || mime
717
.startsWith("application/") && mime.endsWith("+xml"));
720
// indicates if the MIME type belongs to the TEXT XML family
721
static boolean isTextXml(String mime) {
723
&& (mime.equals("text/xml")
724
|| mime.equals("text/xml-external-parsed-entity") || mime
725
.startsWith("text/") && mime.endsWith("+xml"));
728
private static final MessageFormat RAW_EX_1 = new MessageFormat(
729
"Invalid encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch");
731
private static final MessageFormat RAW_EX_2 = new MessageFormat(
732
"Invalid encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM");
734
private static final MessageFormat HTTP_EX_1 = new MessageFormat(
735
"Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be NULL");
737
private static final MessageFormat HTTP_EX_2 = new MessageFormat(
738
"Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch");
740
private static final MessageFormat HTTP_EX_3 = new MessageFormat(
741
"Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Invalid MIME");