golden hour

/lib/python2.7/site-packages/chardet

File/Folder	Size	Actions
__init__.py	1.26 KB
__init__.pyc	797 B
__init__.pyo	797 B
big5freq.py	80.66 KB
big5freq.pyc	111.11 KB
big5freq.pyo	111.11 KB
big5prober.py	1.64 KB
big5prober.pyc	1.07 KB
big5prober.pyo	1.07 KB
chardetect.py	1.09 KB
chardetect.pyc	1.46 KB
chardetect.pyo	1.46 KB
chardistribution.py	9.01 KB
chardistribution.pyc	7.35 KB
chardistribution.pyo	7.35 KB
charsetgroupprober.py	3.7 KB
charsetgroupprober.pyc	2.48 KB
charsetgroupprober.pyo	2.48 KB
charsetprober.py	1.86 KB
charsetprober.pyc	2.08 KB
charsetprober.pyo	2.08 KB
codingstatemachine.py	2.26 KB
codingstatemachine.pyc	1.74 KB
codingstatemachine.pyo	1.74 KB
compat.py	1.13 KB
compat.pyc	559 B
compat.pyo	559 B
constants.py	1.3 KB
constants.pyc	332 B
constants.pyo	332 B
cp949prober.py	1.74 KB
cp949prober.pyc	1.07 KB
cp949prober.pyo	1.07 KB
escprober.py	3.11 KB
escprober.pyc	2.31 KB
escprober.pyo	2.31 KB
escsm.py	7.66 KB
escsm.pyc	7.13 KB
escsm.pyo	7.13 KB
eucjpprober.py	3.59 KB
eucjpprober.pyc	2.71 KB
eucjpprober.pyo	2.71 KB
euckrfreq.py	44.9 KB
euckrfreq.pyc	86.69 KB
euckrfreq.pyo	86.69 KB
euckrprober.py	1.64 KB
euckrprober.pyc	1.08 KB
euckrprober.pyo	1.08 KB
euctwfreq.py	34.05 KB
euctwfreq.pyc	59.74 KB
euctwfreq.pyo	59.74 KB
euctwprober.py	1.64 KB
euctwprober.pyc	1.08 KB
euctwprober.pyo	1.08 KB
gb2312freq.py	35.17 KB
gb2312freq.pyc	67.17 KB
gb2312freq.pyo	67.17 KB
gb2312prober.py	1.64 KB
gb2312prober.pyc	1.08 KB
gb2312prober.pyo	1.08 KB
hebrewprober.py	13.05 KB
hebrewprober.pyc	3.32 KB
hebrewprober.pyo	3.32 KB
jisfreq.py	46.21 KB
jisfreq.pyc	82.03 KB
jisfreq.pyo	82.03 KB
jpcntx.py	18.66 KB
jpcntx.pyc	24.04 KB
jpcntx.pyo	24.04 KB
langbulgarianmodel.py	12.48 KB
langbulgarianmodel.pyc	24.3 KB
langbulgarianmodel.pyo	24.3 KB
langcyrillicmodel.py	17.31 KB
langcyrillicmodel.pyc	29.85 KB
langcyrillicmodel.pyo	29.85 KB
langgreekmodel.py	12.33 KB
langgreekmodel.pyc	24 KB
langgreekmodel.pyo	24 KB
langhebrewmodel.py	11.05 KB
langhebrewmodel.pyc	22.84 KB
langhebrewmodel.pyo	22.84 KB
langhungarianmodel.py	12.24 KB
langhungarianmodel.pyc	24.29 KB
langhungarianmodel.pyo	24.29 KB
langthaimodel.py	11.01 KB
langthaimodel.pyc	22.83 KB
langthaimodel.pyo	22.83 KB
latin1prober.py	5.12 KB
latin1prober.pyc	3.37 KB
latin1prober.pyo	3.37 KB
mbcharsetprober.py	3.19 KB
mbcharsetprober.pyc	2.36 KB
mbcharsetprober.pyo	2.36 KB
mbcsgroupprober.py	1.92 KB
mbcsgroupprober.pyc	1.22 KB
mbcsgroupprober.pyo	1.22 KB
mbcssm.py	19.15 KB
mbcssm.pyc	17.25 KB
mbcssm.pyo	17.25 KB
sbcharsetprober.py	4.68 KB
sbcharsetprober.pyc	3.22 KB
sbcharsetprober.pyo	3.22 KB
sbcsgroupprober.py	3.21 KB
sbcsgroupprober.pyc	1.89 KB
sbcsgroupprober.pyo	1.89 KB
sjisprober.py	3.65 KB
sjisprober.pyc	2.73 KB
sjisprober.pyo	2.73 KB
universaldetector.py	6.67 KB
universaldetector.pyc	4.02 KB
universaldetector.pyo	4.02 KB
utf8prober.py	2.59 KB
utf8prober.pyc	2.14 KB
utf8prober.pyo	2.14 KB

Edit: sbcharsetprober.py

######################## BEGIN LICENSE BLOCK ########################
# The Original Code is Mozilla Universal charset detector code.
#
# The Initial Developer of the Original Code is
# Netscape Communications Corporation.
# Portions created by the Initial Developer are Copyright (C) 2001
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Pilgrim - port to Python
#   Shy Shalom - original C code
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301  USA
######################### END LICENSE BLOCK #########################

import sys
from . import constants
from .charsetprober import CharSetProber
from .compat import wrap_ord

SAMPLE_SIZE = 64
SB_ENOUGH_REL_THRESHOLD = 1024
POSITIVE_SHORTCUT_THRESHOLD = 0.95
NEGATIVE_SHORTCUT_THRESHOLD = 0.05
SYMBOL_CAT_ORDER = 250
NUMBER_OF_SEQ_CAT = 4
POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1
#NEGATIVE_CAT = 0

class SingleByteCharSetProber(CharSetProber):
    def __init__(self, model, reversed=False, nameProber=None):
        CharSetProber.__init__(self)
        self._mModel = model
        # TRUE if we need to reverse every pair in the model lookup
        self._mReversed = reversed
        # Optional auxiliary prober for name decision
        self._mNameProber = nameProber
        self.reset()

def reset(self):
        CharSetProber.reset(self)
        # char order of last character
        self._mLastOrder = 255
        self._mSeqCounters = [0] * NUMBER_OF_SEQ_CAT
        self._mTotalSeqs = 0
        self._mTotalChar = 0
        # characters that fall in our sampling range
        self._mFreqChar = 0

def get_charset_name(self):
        if self._mNameProber:
            return self._mNameProber.get_charset_name()
        else:
            return self._mModel['charsetName']

def feed(self, aBuf):
        if not self._mModel['keepEnglishLetter']:
            aBuf = self.filter_without_english_letters(aBuf)
        aLen = len(aBuf)
        if not aLen:
            return self.get_state()
        for c in aBuf:
            order = self._mModel['charToOrderMap'][wrap_ord(c)]
            if order < SYMBOL_CAT_ORDER:
                self._mTotalChar += 1
            if order < SAMPLE_SIZE:
                self._mFreqChar += 1
                if self._mLastOrder < SAMPLE_SIZE:
                    self._mTotalSeqs += 1
                    if not self._mReversed:
                        i = (self._mLastOrder * SAMPLE_SIZE) + order
                        model = self._mModel['precedenceMatrix'][i]
                    else:  # reverse the order of the letters in the lookup
                        i = (order * SAMPLE_SIZE) + self._mLastOrder
                        model = self._mModel['precedenceMatrix'][i]
                    self._mSeqCounters[model] += 1
            self._mLastOrder = order

if self.get_state() == constants.eDetecting:
            if self._mTotalSeqs > SB_ENOUGH_REL_THRESHOLD:
                cf = self.get_confidence()
                if cf > POSITIVE_SHORTCUT_THRESHOLD:
                    if constants._debug:
                        sys.stderr.write('%s confidence = %s, we have a'
                                         'winner\n' %
                                         (self._mModel['charsetName'], cf))
                    self._mState = constants.eFoundIt
                elif cf < NEGATIVE_SHORTCUT_THRESHOLD:
                    if constants._debug:
                        sys.stderr.write('%s confidence = %s, below negative'
                                         'shortcut threshhold %s\n' %
                                         (self._mModel['charsetName'], cf,
                                          NEGATIVE_SHORTCUT_THRESHOLD))
                    self._mState = constants.eNotMe

return self.get_state()

def get_confidence(self):
        r = 0.01
        if self._mTotalSeqs > 0:
            r = ((1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs
                 / self._mModel['mTypicalPositiveRatio'])
            r = r * self._mFreqChar / self._mTotalChar
            if r >= 1.0:
                r = 0.99
        return r