Merge pull request #299 from Carton/encoding

Add support for "fallback" or "default" encoding for SGFs with no "CA"?

katrain/core/sgf_parser.py

@@ -1,4 +1,5 @@
 import copy
+import chardet
 import math
 import re
 from collections import defaultdict
@@ -368,7 +369,7 @@ class SGFNode:
 
 
 class SGF:
-    DEFAULT_ENCODING = "ISO-8859-1"  # as specified by the standard
+    DEFAULT_ENCODING = "UTF-8"
 
     _NODE_CLASS = SGFNode  # Class used for SGF Nodes, can change this to something that inherits from SGFNode
     # https://xkcd.com/1171/
@@ -408,7 +409,10 @@ class SGF:
                     if match:
                         encoding = match[1].decode("ascii", errors="ignore")
                     else:
-                        encoding = cls.DEFAULT_ENCODING
+                        encoding = chardet.detect(bin_contents[:300])['encoding']
+                        # workaround for some compatibility issues for Windows-1252 and GB2312 encodings
+                        if encoding == "Windows-1252" or encoding == "GB2312":
+                            encoding = "GBK"
             try:
                 decoded = bin_contents.decode(encoding=encoding, errors="ignore")
             except LookupError:

setup.py

@@ -47,6 +47,7 @@ setup(
         "urllib3",
         "pygame;platform_system=='Darwin'",  # some mac versions need this for kivy
         "screeninfo;platform_system!='Darwin'",  # for screen resolution, has problems on macos
+        "chardet",  # for automatic encoding detection
     ],
     dependency_links=["https://kivy.org/downloads/simple/"],
     python_requires=">=3.6, <4",