zero width space markers and base64 encoded data in sgf files are probably terrible ideas

INSTALL.md

@@ -97,8 +97,9 @@ pip3 install screeninfo # Skip on MacOS, not working
 In case the sound is not working, or there is no available wheel for your OS or Python version, try:
 ```bash
 pip3 uninstall kivy
-pip3 install --no-binary kivy kivy==2.0.0rc2
+pip3 install kivy --extra-index-url https://kivy.org/downloads/simple/
 ```
+
 You can now start KaTrain by running `python3 -m katrain`
 
 In case KataGo does not start, an alternative is to go [here](https://github.com/lightvector/KataGo) and compile KataGo yourself.

katrain/__main__.py

@@ -253,14 +253,14 @@ class KaTrainGui(Screen, KaTrainBase):
             else:  # game related actions
                 self.message_queue.put([self.game.game_id, message, args, kwargs])
 
-    def _do_new_game(self, move_tree=None, analyze_fast=False):
+    def _do_new_game(self, move_tree=None, analyze_fast=False, loaded_from_file=None):
         self.idle_analysis = False
         mode = self.play_analyze_mode
         if (move_tree is not None and mode == MODE_PLAY) or (move_tree is None and mode == MODE_ANALYZE):
             self.play_mode.switch_ui_mode()  # for new game, go to play, for loaded, analyze
         self.board_gui.animating_pv = None
         self.engine.on_new_game()  # clear queries
-        self.game = Game(self, self.engine, move_tree=move_tree, analyze_fast=analyze_fast)
+        self.game = Game(self, self.engine, move_tree=move_tree, analyze_fast=analyze_fast,loaded_from_file=loaded_from_file)
         if move_tree:
             for bw, player_info in self.players_info.items():
                 player_info.player_type = PLAYER_HUMAN
@@ -340,7 +340,7 @@ class KaTrainGui(Screen, KaTrainBase):
         self.controls.timer.paused = True
         if not self.teacher_settings_popup:
             self.teacher_settings_popup = I18NPopup(
-                title_key="teacher settings", size=[dp(800), dp(750)], content=ConfigTeacherPopup(self)
+                title_key="teacher settings", size=[dp(800), dp(800)], content=ConfigTeacherPopup(self)
             ).__self__
             self.teacher_settings_popup.content.popup = self.teacher_settings_popup
         self.teacher_settings_popup.open()
@@ -369,7 +369,7 @@ class KaTrainGui(Screen, KaTrainBase):
         except ParseError as e:
             self.log(i18n._("Failed to load SGF").format(error=e), OUTPUT_ERROR)
             return
-        self._do_new_game(move_tree=move_tree, analyze_fast=fast)
+        self._do_new_game(move_tree=move_tree, analyze_fast=fast,loaded_from_file=file)
         if not rewind:
             self.game.redo(999)
 

katrain/config.json

@@ -67,6 +67,7 @@
             true,
             true
         ],
+        "save_analysis": true,
         "low_visits": 25,
         "eval_off_show_last": 3,
         "top_moves_show": "top_move_delta_score",

katrain/core/constants.py

@@ -1,3 +1,4 @@
+PROGRAM_NAME = "KaTrain"
 VERSION = "1.7.0"
 HOMEPAGE = "https://github.com/sanderland/katrain"
 CONFIG_MIN_VERSION = "1.7.0"  # keep config files from this version
@@ -237,3 +238,7 @@ TOP_MOVE_OPTIONS = [
     TOP_MOVE_VISITS,
     TOP_MOVE_NOTHING,
 ]
+
+SGF_COMMENTS_MARKER = "\u200b\u3164"
+SGF_NOTE_MARKER = "\u200b\u200b"
+SGF_SEPARATOR_MARKER = "\u3164\u3164"

katrain/core/game.py

@@ -16,6 +16,7 @@ from katrain.core.constants import (
     STATUS_INFO,
     STATUS_TEACHING,
     PLAYER_HUMAN,
+    VERSION, PROGRAM_NAME,
 )
 from katrain.core.engine import KataGoEngine
 from katrain.core.game_node import GameNode
@@ -35,7 +36,7 @@ class KaTrainSGF(SGF):
 class Game:
     """Represents a game of go, including an implementation of capture rules."""
 
-    DEFAULT_PROPERTIES = {"GM": 1, "FF": 4, "AP": f"KaTrain:{HOMEPAGE}", "CA": "UTF-8"}
+    DEFAULT_PROPERTIES = {"GM": 1, "FF": 4, "AP": f"{PROGRAM_NAME}:{VERSION}", "CA": "UTF-8"}
 
     def __init__(
         self,
@@ -44,6 +45,7 @@ class Game:
         move_tree: GameNode = None,
         analyze_fast=False,
         game_properties: Optional[Dict] = None,
+        loaded_from_file = None
     ):
         self.katrain = katrain
         self._lock = threading.Lock()
@@ -51,6 +53,8 @@ class Game:
             engine = {"B": engine, "W": engine}
         self.engines = engine
         self.game_id = datetime.strftime(datetime.now(), "%Y-%m-%d %H %M %S")
+        self.loaded_from_file = loaded_from_file
+
 
         if move_tree:
             self.root = move_tree
@@ -82,13 +86,16 @@ class Game:
         self.set_current_node(self.root)
         self.main_time_used = 0
         threading.Thread(
-            target=lambda: self.analyze_all_nodes(-1_000_000, analyze_fast=analyze_fast), daemon=True
+            target=lambda: self.analyze_all_nodes(-1_000_000, analyze_fast=analyze_fast, even_if_present=False),
+            daemon=True,
         ).start()  # return faster, but bypass Kivy Clock
 
-    def analyze_all_nodes(self, priority=0, analyze_fast=False):
+    def analyze_all_nodes(self, priority=0, analyze_fast=False, even_if_present=True):
         for node in self.root.nodes_in_tree:
-            node.clear_analysis()
-            node.analyze(self.engines[node.next_player], priority=priority, analyze_fast=analyze_fast)
+            if even_if_present or not node.analysis_loaded:
+                print(even_if_present, node.analysis_loaded, node.move)
+                node.clear_analysis()
+                node.analyze(self.engines[node.next_player], priority=priority, analyze_fast=analyze_fast)
 
     # -- move tree functions --
     def _calculate_groups(self):
@@ -292,9 +299,11 @@ class Game:
         self, path: str, trainer_config: Optional[Dict] = None,
     ):
         if trainer_config is None:
-            trainer_config = self.katrain.config("trainer")
-        save_feedback = trainer_config["save_feedback"]
+            trainer_config = self.katrain.config("trainer", {})
+        save_feedback = trainer_config.get("save_feedback", False)
         eval_thresholds = trainer_config["eval_thresholds"]
+        print(trainer_config)
+        save_analysis = trainer_config.get("save_analysis", False)
 
         def player_name(player_info):
             if player_info.name and player_info.player_type == PLAYER_HUMAN:
@@ -314,15 +323,23 @@ class Game:
             x_properties["RE"] = self.end_result
         self.root.properties = {**root_properties, **{k: [v] for k, v in x_properties.items()}}
         player_names = {bw: re.sub(r"['<>:\"/\\|?*]", "", self.root.get_property("P" + bw, bw)) for bw in "BW"}
-        game_name = f"katrain_{player_names['B']} vs {player_names['W']} {self.game_id}"
-        file_name = os.path.abspath(os.path.join(path, f"{game_name}.sgf"))
+        base_game_name = f"katrain_{player_names['B']} vs {player_names['W']}"
+        game_name = f"{base_game_name} {self.game_id}"
+
+        if self.loaded_from_file and base_game_name in self.loaded_from_file and PROGRAM_NAME in self.root.get_property("AP","") and self.loaded_from_file.endswith('sgf'):
+            file_name = self.loaded_from_file
+        else:
+            file_name = os.path.abspath(os.path.join(path, f"{game_name}.sgf"))
         os.makedirs(os.path.dirname(file_name), exist_ok=True)
 
         show_dots_for = {
             bw: trainer_config.get("eval_show_ai", True) or self.katrain.players_info[bw].human for bw in "BW"
         }
         sgf = self.root.sgf(
-            save_comments_player=show_dots_for, save_comments_class=save_feedback, eval_thresholds=eval_thresholds
+            save_comments_player=show_dots_for,
+            save_comments_class=save_feedback,
+            eval_thresholds=eval_thresholds,
+            save_analysis=save_analysis,
         )
         with open(file_name, "w", encoding="utf-8") as f:
             f.write(sgf)

katrain/core/game_node.py

@@ -2,10 +2,13 @@ import copy
 import random
 from typing import Dict, List, Optional, Tuple
 
+from katrain.core.constants import HOMEPAGE, VERSION, SGF_COMMENTS_MARKER, SGF_SEPARATOR_MARKER, SGF_NOTE_MARKER, PROGRAM_NAME
 from katrain.core.lang import i18n
 from katrain.core.sgf_parser import Move, SGFNode
 from katrain.core.utils import evaluation_class, var_to_grid
 from katrain.gui.style import INFO_PV_COLOR
+import base64
+import gzip
 
 
 class GameNode(SGFNode):
@@ -22,19 +25,56 @@ class GameNode(SGFNode):
         self.time_used = 0
         self.undo_threshold = random.random()  # for fractional undos
         self.end_state = None
+        self.analysis_loaded = False
         self.clear_analysis()
 
+    def add_list_property(self, property: str, values: List):
+        if property == "KT":
+            try:
+                import bson
+
+                self.analysis = bson.loads(gzip.decompress(base64.standard_b64decode(values[0])))
+                self.analysis_loaded = True
+            except Exception as e:
+                print(e)
+        elif property == "C":
+            comments = [  # strip out all previously auto generated comments
+                c for v in values for c in v.split(SGF_SEPARATOR_MARKER)
+            ]
+            self.note = "".join(c for c in comments if SGF_NOTE_MARKER in c)
+            super().add_list_property(
+                property,
+                [c for c in comments if c.strip() and SGF_COMMENTS_MARKER not in c and SGF_NOTE_MARKER not in c],
+            )
+        else:
+            return super().add_list_property(property, values)
+
     def clear_analysis(self):
         self.analysis_visits_requested = 0
         self.analysis = {"moves": {}, "root": None, "completed": False}
 
-    def sgf_properties(self, save_comments_player=None, save_comments_class=None, eval_thresholds=None):
+    def sgf_properties(
+        self, save_comments_player=None, save_comments_class=None, eval_thresholds=None, save_analysis=False
+    ):
         properties = copy.copy(super().sgf_properties())
         note = self.note.strip()
+        print(save_analysis, self.analysis_complete)
+        if save_analysis and self.analysis_complete:
+            try:
+                import bson
+
+                analysis = copy.deepcopy(self.analysis)
+                for movedict in analysis["moves"].values():
+                    if "ownership" in movedict:
+                        del movedict["ownership"]
+                properties["KT"] = [base64.standard_b64encode(gzip.compress(bson.dumps(analysis))).decode("utf-8")]
+            except Exception as e:
+                print(e)
         if self.points_lost and save_comments_class is not None and eval_thresholds is not None:
             show_class = save_comments_class[evaluation_class(self.points_lost, eval_thresholds)]
         else:
             show_class = False
+        comments = properties.get("C", [])
         if (
             self.parent
             and self.parent.analysis_exists
@@ -50,18 +90,19 @@ class GameNode(SGFNode):
                 properties["SQ"] = best_sq
             if top_x and "MA" not in properties:
                 properties["MA"] = [top_x]
-            comment = self.comment(sgf=True, interactive=False)
-            if comment:
-                properties["C"] = ["\n".join(properties.get("C", "")) + comment]
+            comments.append(self.comment(sgf=True, interactive=False))
         if self.is_root:
-            properties["C"] = [
-                i18n._("SGF start message")
-                + "\n"
-                + "\n".join(properties.get("C", ""))
-                + "\nSGF with review generated by KaTrain."
+            comments = [
+                i18n._("SGF start message") + SGF_COMMENTS_MARKER,
+                *comments,
+                f"SGF with generated by {PROGRAM_NAME} {VERSION}{SGF_COMMENTS_MARKER}.",
             ]
         if note:
-            properties["C"] = ["\n".join(properties.get("C", "")) + f"\nNote: {self.note}"]
+            comments.append(f"\nNote: {self.note}{SGF_NOTE_MARKER}")
+        if comments:
+            properties["C"] = [SGF_SEPARATOR_MARKER.join(comments)]
+        elif "C" in properties:
+            del properties["C"]
         return properties
 
     @staticmethod

katrain/gui/popups.py

@@ -274,6 +274,9 @@ class NewGamePopup(QuickConfigGui):
             current = root.get_property(k)
             if current != v:
                 changed = True
+                self.katrain.log(
+                    f"Property {k} changed from {current} to {v}, triggering re-analysis of entire game.", OUTPUT_INFO
+                )
                 self.katrain.game.root.set_property(k, v)
         self.update_playerinfo()
         if changed:

katrain/i18n/locales/cn/LC_MESSAGES/katrain.po

@@ -719,3 +719,7 @@ msgstr "Change in score"
 #. TODO - top move stats show threshold description
 msgid "show stats if"
 msgstr "Show stats if more than ... visits"
+
+#. TODO - analysis option
+msgid "cache analysis to sgf"
+msgstr "Cache analysis in SGF"

katrain/i18n/locales/de/LC_MESSAGES/katrain.po

@@ -764,3 +764,9 @@ msgstr "Change in Score"
 #. TODO - top move stats show threshold description
 msgid "show stats if"
 msgstr "Show stats if more than ... visits"
+
+#. TODO - analysis option
+msgid "cache analysis to sgf"
+msgstr ""
+"cache analysis to sgf\n"
+"(larger files, only compatible with KaTrain)"

katrain/i18n/locales/en/LC_MESSAGES/katrain.po

@@ -378,7 +378,7 @@ msgstr ""
 
 # teacher settings from here
 msgid "teacher settings"
-msgstr "Edit Teacher Settings"
+msgstr "Edit Teaching/Analysis Settings"
 
 msgid "point loss threshold"
 msgstr "Point loss greater than"
@@ -405,6 +405,10 @@ msgstr ""
 "Show dots/SGF comments\n"
 " for AI players"
 
+#. analysis option
+msgid "cache analysis to sgf"
+msgstr "Cache analysis in SGF"
+
 #. removed as an option / can ignore
 msgid "lock ai when playing"
 msgstr ""

katrain/i18n/locales/es/LC_MESSAGES/katrain.po

@@ -779,3 +779,7 @@ msgstr "Change in score"
 #. TODO - top move stats show threshold description
 msgid "show stats if"
 msgstr "Show stats if more than ... visits"
+
+#. TODO - analysis option
+msgid "cache analysis to sgf"
+msgstr "Cache analysis in SGF"

katrain/i18n/locales/fr/LC_MESSAGES/katrain.po

@@ -805,3 +805,7 @@ msgstr "Change in score"
 #. TODO - top move stats show threshold description
 msgid "show stats if"
 msgstr "Show stats if more than ... visits"
+
+#. TODO - analysis option
+msgid "cache analysis to sgf"
+msgstr "Cache analysis in SGF"

katrain/i18n/locales/jp/LC_MESSAGES/katrain.po

@@ -758,3 +758,7 @@ msgstr "Change in score"
 #. TODO - top move stats show threshold description
 msgid "show stats if"
 msgstr "Show stats if more than ... visits"
+
+#. TODO - analysis option
+msgid "cache analysis to sgf"
+msgstr "Cache analysis in SGF"

katrain/i18n/locales/ko/LC_MESSAGES/katrain.po

@@ -733,3 +733,7 @@ msgstr "Change in score"
 #. TODO - top move stats show threshold description
 msgid "show stats if"
 msgstr "Show stats if more than ... visits"
+
+#. TODO - analysis option
+msgid "cache analysis to sgf"
+msgstr "ache analysis in SGF"

katrain/i18n/locales/ru/LC_MESSAGES/katrain.po

@@ -767,3 +767,7 @@ msgstr "Change in score"
 #. TODO - top move stats show threshold description
 msgid "show stats if"
 msgstr "Show stats if more than ... visits"
+
+#. TODO - analysis option
+msgid "cache analysis to sgf"
+msgstr "Cache analysis in SGF"

katrain/popups.kv

@@ -280,10 +280,10 @@
         DescriptionLabel:
             text: i18n._("save dots")
     AnchorLayout:
-        size_hint: 1, 3
+        size_hint: 1, 4
         GridLayout:
             cols: 2
-            rows: 4
+            rows: 5
             padding: 4
             size_hint: 0.95, 1
             DescriptionLabel:
@@ -293,25 +293,25 @@
                 LabelledSpinner:
                     size_hint: 0.95,0.8
                     input_property: 'trainer/theme'
-                    font_size: DESC_FONT_SIZE * 0.8
                     id: themes_spinner
             DescriptionLabel:
                 font_size: DESC_FONT_SIZE * 0.8
                 text: i18n._("stats on top move")
-            BoxLayout:
-                orientation: 'horizontal'
-                size_hint: 1.5,1
-                LabelledSpinner:
-                    id: top_moves_show
-                    value_refs: TOP_MOVE_OPTIONS
-                    input_property: "trainer/top_moves_show"
-                Label:
-                    size_hint: 0.1, 1
-                    text: '&'
-                LabelledSpinner:
-                    id: top_moves_show_secondary
-                    value_refs: TOP_MOVE_OPTIONS
-                    input_property: "trainer/top_moves_show_secondary"
+            AnchorLayout:
+                BoxLayout:
+                    size_hint: 1, 0.8
+                    orientation: 'horizontal'
+                    LabelledSpinner:
+                        id: top_moves_show
+                        value_refs: TOP_MOVE_OPTIONS
+                        input_property: "trainer/top_moves_show"
+                    Label:
+                        size_hint: 0.1, 1
+                        text: '&'
+                    LabelledSpinner:
+                        id: top_moves_show_secondary
+                        value_refs: TOP_MOVE_OPTIONS
+                        input_property: "trainer/top_moves_show_secondary"
             DescriptionLabel:
                 font_size: DESC_FONT_SIZE * 0.8
                 text: i18n._("show stats if")
@@ -332,6 +332,12 @@
             AnchorLayout:
                 LabelledCheckBox:
                     input_property: "trainer/eval_show_ai"
+            DescriptionLabel:
+                font_size: DESC_FONT_SIZE * 0.8
+                text: i18n._("cache analysis to sgf")
+            AnchorLayout:
+                LabelledCheckBox:
+                    input_property: "trainer/save_analysis"
     AnchorLayout:
         size_hint: 1,1.5
         AutoSizedRoundedRectangleButton:

requirements.txt

@@ -1,3 +1,4 @@
+bson==0.5.10
 certifi==2020.6.20
 chardet==3.0.4
 Cython==0.29.14

setup.py

@@ -42,6 +42,7 @@ setup(
         "wheel",
         "setuptools",
         "urllib3",
+        "bson",
         "importlib_resources ;python_version<'3.7'",
         "cython>=0.24,<=0.29.14,!=0.27,!=0.27.2",  # kivy wants this
         "kivy==2.0.0rc2;platform_system=='Darwin'",  # rc3 failing on mac