1
######################## BEGIN LICENSE BLOCK ########################
2
# The Original Code is mozilla.org code.
4
# The Initial Developer of the Original Code is
5
# Netscape Communications Corporation.
6
# Portions created by the Initial Developer are Copyright (C) 1998
7
# the Initial Developer. All Rights Reserved.
10
# Mark Pilgrim - port to Python
12
# This library is free software; you can redistribute it and/or
13
# modify it under the terms of the GNU Lesser General Public
14
# License as published by the Free Software Foundation; either
15
# version 2.1 of the License, or (at your option) any later version.
17
# This library is distributed in the hope that it will be useful,
18
# but WITHOUT ANY WARRANTY; without even the implied warranty of
19
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20
# Lesser General Public License for more details.
22
# You should have received a copy of the GNU Lesser General Public
23
# License along with this library; if not, write to the Free Software
24
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
26
######################### END LICENSE BLOCK #########################
29
from .mbcharsetprober import MultiByteCharSetProber
30
from .codingstatemachine import CodingStateMachine
31
from .chardistribution import SJISDistributionAnalysis
32
from .jpcntx import SJISContextAnalysis
33
from .mbcssm import SJISSMModel
34
from . import constants
37
class SJISProber(MultiByteCharSetProber):
39
MultiByteCharSetProber.__init__(self)
40
self._mCodingSM = CodingStateMachine(SJISSMModel)
41
self._mDistributionAnalyzer = SJISDistributionAnalysis()
42
self._mContextAnalyzer = SJISContextAnalysis()
46
MultiByteCharSetProber.reset(self)
47
self._mContextAnalyzer.reset()
49
def get_charset_name(self):
54
for i in range(0, aLen):
55
codingState = self._mCodingSM.next_state(aBuf[i])
56
if codingState == constants.eError:
58
sys.stderr.write(self.get_charset_name()
59
+ ' prober hit error at byte ' + str(i)
61
self._mState = constants.eNotMe
63
elif codingState == constants.eItsMe:
64
self._mState = constants.eFoundIt
66
elif codingState == constants.eStart:
67
charLen = self._mCodingSM.get_current_charlen()
69
self._mLastChar[1] = aBuf[0]
70
self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:],
72
self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
74
self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3
76
self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
79
self._mLastChar[0] = aBuf[aLen - 1]
81
if self.get_state() == constants.eDetecting:
82
if (self._mContextAnalyzer.got_enough_data() and
83
(self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
84
self._mState = constants.eFoundIt
86
return self.get_state()
88
def get_confidence(self):
89
contxtCf = self._mContextAnalyzer.get_confidence()
90
distribCf = self._mDistributionAnalyzer.get_confidence()
91
return max(contxtCf, distribCf)