Support encoding detection
Some go servers mistakenly generate sgf files without correct CA properties. Use chardet to detect first 300 bytes. Signed-off-by: Carton He <cartonhe@gmail.com>
This commit is contained in:
1 parent
bb76630ddc
commit
4eec9365bd
2 files changed
+7
-2
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import copy
|
||||
import chardet
|
||||
import math
|
||||
import re
|
||||
from collections import defaultdict
|
||||
@@ -345,7 +346,7 @@ class SGFNode:
|
||||
|
||||
|
||||
class SGF:
|
||||
DEFAULT_ENCODING = "ISO-8859-1" # as specified by the standard
|
||||
DEFAULT_ENCODING = "UTF-8"
|
||||
|
||||
_NODE_CLASS = SGFNode # Class used for SGF Nodes, can change this to something that inherits from SGFNode
|
||||
# https://xkcd.com/1171/
|
||||
@@ -385,7 +386,10 @@ class SGF:
|
||||
if match:
|
||||
encoding = match[1].decode("ascii", errors="ignore")
|
||||
else:
|
||||
encoding = cls.DEFAULT_ENCODING
|
||||
encoding = chardet.detect(bin_contents[:300])['encoding']
|
||||
# workaround for some compatibility issues for Windows-1252 and GB2312 encodings
|
||||
if encoding == "Windows-1252" or encoding == "GB2312":
|
||||
encoding = "GBK"
|
||||
try:
|
||||
decoded = bin_contents.decode(encoding=encoding, errors="ignore")
|
||||
except LookupError:
|
||||
|
||||
Reference in new issue
Block a user