From 4eec9365bd50990d5fb61b8b379c24e055799784 Mon Sep 17 00:00:00 2001 From: Carton He Date: Tue, 29 Dec 2020 00:24:46 +0800 Subject: [PATCH] Support encoding detection Some go servers mistakenly generate sgf files without correct CA properties. Use chardet to detect first 300 bytes. Signed-off-by: Carton He --- katrain/core/sgf_parser.py | 8 ++++++-- setup.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/katrain/core/sgf_parser.py b/katrain/core/sgf_parser.py index 4683689..0e8783e 100644 --- a/katrain/core/sgf_parser.py +++ b/katrain/core/sgf_parser.py @@ -1,4 +1,5 @@ import copy +import chardet import math import re from collections import defaultdict @@ -345,7 +346,7 @@ class SGFNode: class SGF: - DEFAULT_ENCODING = "ISO-8859-1" # as specified by the standard + DEFAULT_ENCODING = "UTF-8" _NODE_CLASS = SGFNode # Class used for SGF Nodes, can change this to something that inherits from SGFNode # https://xkcd.com/1171/ @@ -385,7 +386,10 @@ class SGF: if match: encoding = match[1].decode("ascii", errors="ignore") else: - encoding = cls.DEFAULT_ENCODING + encoding = chardet.detect(bin_contents[:300])['encoding'] + # workaround for some compatibility issues for Windows-1252 and GB2312 encodings + if encoding == "Windows-1252" or encoding == "GB2312": + encoding = "GBK" try: decoded = bin_contents.decode(encoding=encoding, errors="ignore") except LookupError: diff --git a/setup.py b/setup.py index a0bc74a..bcdd9ef 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ setup( "urllib3", "pygame;platform_system=='Darwin'", # some mac versions need this for kivy "screeninfo;platform_system!='Darwin'", # for screen resolution, has problems on macos + "chardet", # for automatic encoding detection ], dependency_links=["https://kivy.org/downloads/simple/"], python_requires=">=3.6, <4",