golden hour
/opt/cloudlinux/venv/lib/python3.11/site-packages/chardet
⬆️ Go Up
Upload
File/Folder
Size
Actions
__init__.py
4.68 KB
Del
OK
__main__.py
123 B
Del
OK
__pycache__
-
Del
OK
big5freq.py
30.54 KB
Del
OK
big5prober.py
1.72 KB
Del
OK
chardistribution.py
9.8 KB
Del
OK
charsetgroupprober.py
3.82 KB
Del
OK
charsetprober.py
5.29 KB
Del
OK
cli
-
Del
OK
codingstatemachine.py
3.64 KB
Del
OK
codingstatemachinedict.py
542 B
Del
OK
cp949prober.py
1.82 KB
Del
OK
enums.py
1.64 KB
Del
OK
escprober.py
3.91 KB
Del
OK
escsm.py
11.89 KB
Del
OK
eucjpprober.py
3.84 KB
Del
OK
euckrfreq.py
13.25 KB
Del
OK
euckrprober.py
1.71 KB
Del
OK
euctwfreq.py
36.05 KB
Del
OK
euctwprober.py
1.71 KB
Del
OK
gb2312freq.py
20.25 KB
Del
OK
gb2312prober.py
1.72 KB
Del
OK
hebrewprober.py
14.2 KB
Del
OK
jisfreq.py
25.19 KB
Del
OK
johabfreq.py
41.5 KB
Del
OK
johabprober.py
1.71 KB
Del
OK
jpcntx.py
26.42 KB
Del
OK
langbulgarianmodel.py
102.1 KB
Del
OK
langgreekmodel.py
96.16 KB
Del
OK
langhebrewmodel.py
95.88 KB
Del
OK
langhungarianmodel.py
98.98 KB
Del
OK
langrussianmodel.py
125.02 KB
Del
OK
langthaimodel.py
100.35 KB
Del
OK
langturkishmodel.py
93.13 KB
Del
OK
latin1prober.py
5.25 KB
Del
OK
macromanprober.py
5.93 KB
Del
OK
mbcharsetprober.py
3.63 KB
Del
OK
mbcsgroupprober.py
2.08 KB
Del
OK
mbcssm.py
29.68 KB
Del
OK
metadata
-
Del
OK
py.typed
0 B
Del
OK
resultdict.py
402 B
Del
OK
sbcharsetprober.py
6.25 KB
Del
OK
sbcsgroupprober.py
4.04 KB
Del
OK
sjisprober.py
3.91 KB
Del
OK
universaldetector.py
14.5 KB
Del
OK
utf1632prober.py
8.31 KB
Del
OK
utf8prober.py
2.75 KB
Del
OK
version.py
244 B
Del
OK
Edit: __init__.py
######################## BEGIN LICENSE BLOCK ######################## # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301 USA ######################### END LICENSE BLOCK ######################### from typing import List, Union from .charsetgroupprober import CharSetGroupProber from .charsetprober import CharSetProber from .enums import InputState from .resultdict import ResultDict from .universaldetector import UniversalDetector from .version import VERSION, __version__ __all__ = ["UniversalDetector", "detect", "detect_all", "__version__", "VERSION"] def detect( byte_str: Union[bytes, bytearray], should_rename_legacy: bool = False ) -> ResultDict: """ Detect the encoding of the given byte string. :param byte_str: The byte sequence to examine. :type byte_str: ``bytes`` or ``bytearray`` :param should_rename_legacy: Should we rename legacy encodings to their more modern equivalents? :type should_rename_legacy: ``bool`` """ if not isinstance(byte_str, bytearray): if not isinstance(byte_str, bytes): raise TypeError( f"Expected object of type bytes or bytearray, got: {type(byte_str)}" ) byte_str = bytearray(byte_str) detector = UniversalDetector(should_rename_legacy=should_rename_legacy) detector.feed(byte_str) return detector.close() def detect_all( byte_str: Union[bytes, bytearray], ignore_threshold: bool = False, should_rename_legacy: bool = False, ) -> List[ResultDict]: """ Detect all the possible encodings of the given byte string. :param byte_str: The byte sequence to examine. :type byte_str: ``bytes`` or ``bytearray`` :param ignore_threshold: Include encodings that are below ``UniversalDetector.MINIMUM_THRESHOLD`` in results. :type ignore_threshold: ``bool`` :param should_rename_legacy: Should we rename legacy encodings to their more modern equivalents? :type should_rename_legacy: ``bool`` """ if not isinstance(byte_str, bytearray): if not isinstance(byte_str, bytes): raise TypeError( f"Expected object of type bytes or bytearray, got: {type(byte_str)}" ) byte_str = bytearray(byte_str) detector = UniversalDetector(should_rename_legacy=should_rename_legacy) detector.feed(byte_str) detector.close() if detector.input_state == InputState.HIGH_BYTE: results: List[ResultDict] = [] probers: List[CharSetProber] = [] for prober in detector.charset_probers: if isinstance(prober, CharSetGroupProber): probers.extend(p for p in prober.probers) else: probers.append(prober) for prober in probers: if ignore_threshold or prober.get_confidence() > detector.MINIMUM_THRESHOLD: charset_name = prober.charset_name or "" lower_charset_name = charset_name.lower() # Use Windows encoding name instead of ISO-8859 if we saw any # extra Windows-specific bytes if lower_charset_name.startswith("iso-8859") and detector.has_win_bytes: charset_name = detector.ISO_WIN_MAP.get( lower_charset_name, charset_name ) # Rename legacy encodings with superset encodings if asked if should_rename_legacy: charset_name = detector.LEGACY_MAP.get( charset_name.lower(), charset_name ) results.append( { "encoding": charset_name, "confidence": prober.get_confidence(), "language": prober.language, } ) if len(results) > 0: return sorted(results, key=lambda result: -result["confidence"]) return [detector.result]
Save