model downloads

katrain/core/engine.py

@@ -37,7 +37,7 @@ class KataGoEngine:
         self.query_counter = 0
         self.katago_process = None
         self.base_priority = 0
-        self.override_settings = {}  # mainly for bot scripts to hook into
+        self.override_settings = {'reportAnalysisWinratesAs':'BLACK'}  # force these settings
         self._lock = threading.Lock()
         self.analysis_thread = None
         self.stderr_thread = None

katrain/gui/popups.py

@@ -24,9 +24,10 @@ from katrain.core.constants import (
 )
 from katrain.core.engine import KataGoEngine
 from katrain.core.lang import i18n
-from katrain.core.utils import find_package_resource
+from katrain.core.utils import find_package_resource, PATHS
 from katrain.gui.kivyutils import BackgroundMixin, I18NSpinner
 from katrain.gui.style import DEFAULT_FONT, EVAL_COLORS
+from katrain.gui.widgets.progress_loader import ProgressLoader
 
 
 class I18NPopup(Popup):
@@ -306,40 +307,79 @@ class AIPopup(QuickConfigGui):
         for _ in range((self.max_options - len(mode_settings)) * 2):
             self.options_grid.add_widget(Label())
 
-
 class ConfigPopup(QuickConfigGui):
+    def __init__(self, katrain):
+        super().__init__(katrain)
+        self.paths = [self.katrain.config("engine/model"), "katrain/models", "~/.katrain"]
+
+
     def build_and_set_properties(self, *_args):
         super().build_and_set_properties()
         # self.check_models()
 
     def check_models(self, *args):  # WIP
-        try:
-            model = self.collect_properties(self)["engine/model"]
-        except InputParseError:
-            self.model_files.values = []
-            return
+        done = set()
+        model_files = []
+        for path in self.paths + [self.model_path.text]:
+            path = path.rstrip("/\\")
+            if path.startswith("katrain"):
+                path = path.replace("katrain", PATHS["PACKAGE"].rstrip("/\\"), 1)
+            path = os.path.expanduser(path)
+            if not os.path.isdir(path):
+                path, _file = os.path.split(path)
+            slashpath = path.replace("\\", "/")
+            if slashpath in done or not os.path.isdir(path):
+                continue
+            done.add(slashpath)
+            files = [
+                f.replace("/", os.path.sep).replace(PATHS["PACKAGE"], "katrain")
+                for ftype in ["*.bin.gz", "*.txt.gz"]
+                for f in glob.glob(slashpath + "/" + ftype)
+            ]
+            print(path,files)
+            if files and path not in self.paths:
+                self.paths.append(path)  # persistent on paths with models found
+            model_files += files
+        models_available_msg = i18n._("models available").format(num=len(model_files))
+        self.model_files.values = [models_available_msg] + model_files
+        self.model_files.text = models_available_msg
 
-        if os.path.exists(model):
-            if os.path.isdir(model):
-                path = model.rstrip("\\/")
-                file = None
-            else:
-                if model.startswith("katrain"):
-                    model = find_package_resource(model)
-                path, file = os.path.split(model)
-            files = sorted(
-                [
-                    os.path.split(f)[1]
-                    for ftype in ["*.bin.gz", "*.txt.gz"]
-                    for f in glob.glob(path + os.path.sep + ftype)
-                ]
-            )
-            self.model_files.values = files
-            print(file, files, file in files)
-            if file in files:
-                self.model_files.text = file
-        else:
-            self.model_files.values = []
+    MODELS = {
+#        "pure 20b": "https://github.com/lightvector/KataGo/releases/download/v1.4.0/g170-b20c256x2-s4384473088-d968438914.bin.gz",
+#        "pure 30b": "https://github.com/lightvector/KataGo/releases/download/v1.4.0/g170-b30c320x2-s3530176512-d968463914.bin.gz",
+#        "pure 40b": "https://github.com/lightvector/KataGo/releases/download/v1.4.0/g170-b40c256x2-s3708042240-d967973220.bin.gz",
+        "final 20b":"https://github.com/lightvector/KataGo/releases/download/v1.4.5/g170e-b20c256x2-s5303129600-d1228401921.bin.gz",
+        "final 30b": "https://github.com/lightvector/KataGo/releases/download/v1.4.5/g170-b30c320x2-s4824661760-d1229536699.bin.gz",
+        "final 40b":"https://github.com/lightvector/KataGo/releases/download/v1.4.5/g170-b40c256x2-s5095420928-d1229425124.bin.gz"
+    }
+
+    def download_models(self, *_largs):
+        def download_complete(req, tmp_path, path, model):
+            try:
+                os.rename(tmp_path, path)
+                self.katrain.log(f"Download of {model} model complete -> {path}", OUTPUT_INFO)
+            except Exception as e:
+                self.katrain.log(f"Download of {model} model complete, but could not move file: {e}", OUTPUT_ERROR)
+            self.check_models()
+
+        for name, url in self.MODELS.items():
+            filename = os.path.split(url)[1]
+            if not any(os.path.split(f)[1] == filename for f in self.model_files.values):
+                savepath = os.path.expanduser(os.path.join("~/.katrain", filename))
+                savepath_tmp = savepath + ".part"
+                self.katrain.log(f"Downloading {name} model from {url} to {savepath_tmp}", OUTPUT_INFO)
+                progress = ProgressLoader(
+                    download_url=url,
+                    path_to_file=savepath_tmp,
+                    downloading_text=f"Downloading {name} model: " + "{}%",
+                    download_complete=lambda req, tmp=savepath_tmp, path=savepath, model=name: download_complete(
+                        req, tmp, path, model
+                    ),
+                    download_redirected=lambda req: self.katrain.log(
+                        f"Download {name} redirected {req.resp_headers}", OUTPUT_DEBUG
+                    ),
+                )
+                progress.start(self.download_progress_box)
 
     def update_config(self, save_to_file=True):
         updated = super().update_config(save_to_file=save_to_file)

katrain/gui/widgets/progress_loader.py

@@ -0,0 +1,109 @@
+# From KivyMD which will remove it in their next version, with some fixes
+from kivy.clock import Clock
+from kivy.animation import Animation
+from kivy.network.urlrequest import UrlRequest
+from kivy.lang import Builder
+from kivy.properties import StringProperty, ObjectProperty
+from kivy.uix.boxlayout import BoxLayout
+
+Builder.load_string(
+    """
+#:import Window kivy.core.window.Window
+
+
+<ProgressLoader>
+    opacity: 0
+    spacing: 10
+    MDSpinner
+        id: spinner
+        size_hint: None, 0.8
+        width: dp(32)
+        color: 0.95,0.95,0.95,1
+    MDLabel:
+        id: label_download
+        shorten: True
+        max_lines: 1
+        halign: 'left'
+        valign: 'center'
+        text_size: self.size
+        color: 0.95,0.95,0.95,1
+        text: root.label_downloading_text
+"""
+)
+
+
+class ProgressLoader(BoxLayout):
+    path_to_file = StringProperty()
+    """The path to which the uploaded file will be saved."""
+
+    download_url = StringProperty()
+    """Link to uploaded file."""
+
+    label_downloading_text = StringProperty("Downloading...")
+    """Default text before downloading."""
+
+    downloading_text = StringProperty("Downloading: {}%")
+    """Progress text of the downloaded file."""
+
+    download_complete = ObjectProperty()
+    """Function, called after a successful file upload."""
+    download_redirected = ObjectProperty()
+    """Function, called after a redirect event."""
+
+    request = ObjectProperty()
+    """UrlRequest object."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.root_instance = None
+
+    def start(self, root_instance):
+        self.root_instance = root_instance
+        self.root_instance.add_widget(self)
+        self.request_download_file(self.download_url, self.path_to_file)
+        Clock.schedule_once(self.animation_show, 1)
+
+    def animation_show(self, _dt):
+        animation = Animation(opacity=1, d=0.2, t="out_quad",)
+        animation.start(self)
+
+    def request_download_file(self, url, path):
+        """
+        :type url: str;
+        :param url: link to content;
+
+        :type path: str;
+        :param path: path to save content;
+        """
+
+        self.request = UrlRequest(
+            url,
+            file_path=path,
+            chunk_size=102400,
+            on_progress=self.update_progress,
+            on_success=self.on_success,
+            on_redirect=self.redirected,
+            on_error=self.download_error,
+        )
+
+    def redirected(self, request, *_args):
+        new_url = request.resp_headers.get("location")
+        if new_url:
+            self.download_url = new_url
+            self.request_download_file(self.download_url, self.path_to_file)
+        if self.download_redirected:
+            self.download_redirected(request)
+
+    def download_error(self, request, *_args):
+        pass
+
+    def update_progress(self, request, current_size, total_size):
+        if total_size < 1e4:
+            current_size = 0
+        percent = current_size * 100 // max(total_size, 1)
+        self.label_downloading_text = self.downloading_text.format(percent)
+
+    def on_success(self, request, result):
+        self.root_instance.remove_widget(self)
+        if self.download_complete:
+            self.download_complete(request)

katrain/i18n/locales/cn/LC_MESSAGES/katrain.po

@@ -556,3 +556,11 @@ msgstr ""
 msgid "Starting Kata failed"
 msgstr ""
 "用'{command}'命令启动katago出错 {error}. 请确认引擎设置里'katago'的值指向了 正确的KataGo运行文件."
+
+#. TODO
+msgid "models available"
+msgstr "{num} available model(s) found"
+
+#. TODO
+msgid "download models button"
+msgstr "Download Models"

katrain/i18n/locales/de/LC_MESSAGES/katrain.po

@@ -599,3 +599,11 @@ msgstr "Analyse der Kandidatenzüge wird abgeglichen mit {visits} Visits"
 
 msgid "sweep analysis"
 msgstr "Analysiere gesamtes Brett mit {visits} Visits/Zug"
+
+#. TODO
+msgid "models available"
+msgstr "{num} available model(s) found"
+
+#. TODO
+msgid "download models button"
+msgstr "Download Models"

katrain/i18n/locales/en/LC_MESSAGES/katrain.po

@@ -592,3 +592,9 @@ msgstr ""
 "Picks moves at random from a limited selection of moves and plays the best "
 "one. Stronger settings select the best move from a larger selection. Since "
 "there is no 0 kyu/dan, 3 dan = -2 kyu."
+
+msgid "download models button"
+msgstr "Download Models"
+
+msgid "models available"
+msgstr "{num} available model(s) found"

katrain/i18n/locales/es/LC_MESSAGES/katrain.po

@@ -594,3 +594,11 @@ msgstr ""
 
 msgid "extra analysis"
 msgstr "Realizar análisis adicional a {visits} visitas"
+
+#. TODO
+msgid "models available"
+msgstr "{num} available model(s) found"
+
+#. TODO
+msgid "download models button"
+msgstr "Download Models"

katrain/i18n/locales/fr/LC_MESSAGES/katrain.po

@@ -633,3 +633,11 @@ msgstr ""
 "Plus kyu_rank (-3~18) est bas, plus l'échantillon sera large et le niveau de jeu élevé. \n"
 "Une valeur positive équivaudra à un niveau en kyu, 0 à 1er dan, -1 à 2e dan, etc. \n"
 "(Niveaux estimés sur OGS avec le réseau neuronal utilisé PAR DÉFAUT)"
+
+#. TODO
+msgid "models available"
+msgstr "{num} available model(s) found"
+
+#. TODO
+msgid "download models button"
+msgstr "Download Models"

katrain/i18n/locales/ko/LC_MESSAGES/katrain.po

@@ -571,3 +571,11 @@ msgstr "모든 후보수를 균일하게 {visits} visits으로 분석합니다."
 
 msgid "extra analysis"
 msgstr "{visits} visits까지 추가적으로 분석하고 있습니다."
+
+#. TODO
+msgid "models available"
+msgstr "{num} available model(s) found"
+
+#. TODO
+msgid "download models button"
+msgstr "Download Models"

katrain/i18n/locales/ru/LC_MESSAGES/katrain.po

@@ -596,3 +596,11 @@ msgstr ""
 "Выбирает случайные ходы из ограниченного множества и играет лучший из них. "
 "Более сильные настройки расширяют множество для выбора. Т.к. на шкале "
 "отсутствуют 0 кю/дан, то 3 дан = -2 кю."
+
+#. TODO
+msgid "models available"
+msgstr "{num} available model(s) found"
+
+#. TODO
+msgid "download models button"
+msgstr "Download Models"

katrain/popups.kv

@@ -80,6 +80,9 @@
 
 <ConfigPopup>:
     configfile: configfile
+    model_path: model_path
+    model_files: model_files
+    download_progress_box: download_progress_box
     DescriptionLabel:
         text: i18n._('katago settings')
         font_size: DESC_FONT_SIZE * 1.5
@@ -87,6 +90,8 @@
         cols: 2
         rows: 3
         size_hint: 1,3
+        rows: 4
+        size_hint: 1,4
         spacing: CP_SPACING
         DescriptionLabel:
             text: i18n._("engine:katago")
@@ -97,16 +102,37 @@
                 hint_text: i18n._("engine:katago:hint")
         DescriptionLabel:
             text: i18n._("engine:model")
-            size_hint: 0.33, 1
-        AnchorLayout:
-            LabelledPathInput:
-                input_property: "engine/model"
-        DescriptionLabel:
             text: i18n._("engine:config")
             size_hint: 0.33, 1
         AnchorLayout:
             LabelledPathInput:
+                input_property: "engine/model"
                 input_property: "engine/config"
+        DescriptionLabel:
+            text: i18n._("engine:config")
+            text: i18n._("engine:model")
+            size_hint: 0.33, 1
+        AnchorLayout:
+            LabelledPathInput:
+                input_property: "engine/config"
+                id: model_path
+                input_property: "engine/model"
+                on_text: root.check_models()
+        AnchorLayout:
+            size_hint: 0.33, 1
+            AutoSizedRoundedRectangleButton:
+                text: i18n._("download models button")
+                on_press: root.download_models()
+                size_hint_y: 0.7
+        AnchorLayout:
+            Spinner:
+                id: model_files
+                text: ''
+                on_text: if self.text and self.text != self.values[0]: model_path.text = args[1]
+                size_hint_y: 0.7
+                sync_height_frac: 1.0
+                -font_size: self.height * 0.5
+                -background_color: [*[c*255/88 for c in BOX_BACKGROUND_COLOR[:3]], 1] # compensate for texture
     BoxLayout:
         size_hint: 1,1
         orientation: 'horizontal'
@@ -177,13 +203,18 @@
                 LabelledFloatInput:
                     input_property: "engine/wide_root_noise"
                     hint_text: i18n._("engine:wide_root_noise:hint")
-    AnchorLayout:
+    BoxLayout:
         size_hint: 1,1.5
-        AutoSizedRoundedRectangleButton:
-            padding_x: 15
-            size_hint: None,0.5
-            text: i18n._("update settings")
-            on_press: root.update_config(True)
+        BoxLayout:
+            orientation: 'vertical'
+            id: download_progress_box
+        AnchorLayout:
+            AutoSizedRoundedRectangleButton:
+                padding_x: 15
+                size_hint: None,0.5
+                text: i18n._("update settings")
+                on_press: root.update_config(True)
+        Widget: # for centering
 
 <ConfigTeacherPopup>
     options_grid: options_grid