Index: third_party/requests/packages/charade/eucjpprober.py |
diff --git a/third_party/requests/packages/charade/eucjpprober.py b/third_party/requests/packages/charade/eucjpprober.py |
new file mode 100644 |
index 0000000000000000000000000000000000000000..d70cfbbb01761a76903df0c7c3eb1e5f88c82948 |
--- /dev/null |
+++ b/third_party/requests/packages/charade/eucjpprober.py |
@@ -0,0 +1,90 @@ |
+######################## BEGIN LICENSE BLOCK ######################## |
+# The Original Code is mozilla.org code. |
+# |
+# The Initial Developer of the Original Code is |
+# Netscape Communications Corporation. |
+# Portions created by the Initial Developer are Copyright (C) 1998 |
+# the Initial Developer. All Rights Reserved. |
+# |
+# Contributor(s): |
+# Mark Pilgrim - port to Python |
+# |
+# This library is free software; you can redistribute it and/or |
+# modify it under the terms of the GNU Lesser General Public |
+# License as published by the Free Software Foundation; either |
+# version 2.1 of the License, or (at your option) any later version. |
+# |
+# This library is distributed in the hope that it will be useful, |
+# but WITHOUT ANY WARRANTY; without even the implied warranty of |
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
+# Lesser General Public License for more details. |
+# |
+# You should have received a copy of the GNU Lesser General Public |
+# License along with this library; if not, write to the Free Software |
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA |
+# 02110-1301 USA |
+######################### END LICENSE BLOCK ######################### |
+ |
+import sys |
+from . import constants |
+from .mbcharsetprober import MultiByteCharSetProber |
+from .codingstatemachine import CodingStateMachine |
+from .chardistribution import EUCJPDistributionAnalysis |
+from .jpcntx import EUCJPContextAnalysis |
+from .mbcssm import EUCJPSMModel |
+ |
+ |
+class EUCJPProber(MultiByteCharSetProber): |
+ def __init__(self): |
+ MultiByteCharSetProber.__init__(self) |
+ self._mCodingSM = CodingStateMachine(EUCJPSMModel) |
+ self._mDistributionAnalyzer = EUCJPDistributionAnalysis() |
+ self._mContextAnalyzer = EUCJPContextAnalysis() |
+ self.reset() |
+ |
+ def reset(self): |
+ MultiByteCharSetProber.reset(self) |
+ self._mContextAnalyzer.reset() |
+ |
+ def get_charset_name(self): |
+ return "EUC-JP" |
+ |
+ def feed(self, aBuf): |
+ aLen = len(aBuf) |
+ for i in range(0, aLen): |
+ # PY3K: aBuf is a byte array, so aBuf[i] is an int, not a byte |
+ codingState = self._mCodingSM.next_state(aBuf[i]) |
+ if codingState == constants.eError: |
+ if constants._debug: |
+ sys.stderr.write(self.get_charset_name() |
+ + ' prober hit error at byte ' + str(i) |
+ + '\n') |
+ self._mState = constants.eNotMe |
+ break |
+ elif codingState == constants.eItsMe: |
+ self._mState = constants.eFoundIt |
+ break |
+ elif codingState == constants.eStart: |
+ charLen = self._mCodingSM.get_current_charlen() |
+ if i == 0: |
+ self._mLastChar[1] = aBuf[0] |
+ self._mContextAnalyzer.feed(self._mLastChar, charLen) |
+ self._mDistributionAnalyzer.feed(self._mLastChar, charLen) |
+ else: |
+ self._mContextAnalyzer.feed(aBuf[i - 1:i + 1], charLen) |
+ self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], |
+ charLen) |
+ |
+ self._mLastChar[0] = aBuf[aLen - 1] |
+ |
+ if self.get_state() == constants.eDetecting: |
+ if (self._mContextAnalyzer.got_enough_data() and |
+ (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): |
+ self._mState = constants.eFoundIt |
+ |
+ return self.get_state() |
+ |
+ def get_confidence(self): |
+ contxtCf = self._mContextAnalyzer.get_confidence() |
+ distribCf = self._mDistributionAnalyzer.get_confidence() |
+ return max(contxtCf, distribCf) |