diff --git a/.gitignore b/.gitignore index f17c9bba..d710546a 100644 --- a/.gitignore +++ b/.gitignore @@ -255,3 +255,7 @@ gha-creds-*.json tmp/ assets/ experiments/ + +# CodeQL build artifacts +cpp/_codeql_build_dir/ +_codeql_detected_source_root diff --git a/cpp/python/CMakeLists.txt b/cpp/python/CMakeLists.txt index 99a1a755..e21b4ccd 100644 --- a/cpp/python/CMakeLists.txt +++ b/cpp/python/CMakeLists.txt @@ -53,7 +53,6 @@ set(VSHOGI_INCLUDE_DIRS nanobind_add_module( _vshogi STABLE_ABI - NB_STATIC ${VSHOGI_SOURCES} vshogi_wrapper.cpp ) @@ -69,7 +68,6 @@ install(TARGETS _vshogi LIBRARY DESTINATION .) nanobind_add_module( _minishogi STABLE_ABI - NB_STATIC ${VSHOGI_SOURCES} minishogi_wrapper.cpp ) @@ -85,7 +83,6 @@ install(TARGETS _minishogi LIBRARY DESTINATION minishogi) nanobind_add_module( _judkins_shogi STABLE_ABI - NB_STATIC ${VSHOGI_SOURCES} judkins_shogi_wrapper.cpp ) @@ -101,7 +98,6 @@ install(TARGETS _judkins_shogi LIBRARY DESTINATION judkins_shogi) nanobind_add_module( _shogi STABLE_ABI - NB_STATIC ${VSHOGI_SOURCES} shogi_wrapper.cpp ) diff --git a/cpp/python/vshogi_wrapper.hpp b/cpp/python/vshogi_wrapper.hpp index 1bc05fc2..7d76b26e 100644 --- a/cpp/python/vshogi_wrapper.hpp +++ b/cpp/python/vshogi_wrapper.hpp @@ -354,8 +354,8 @@ inline void export_game(nanobind::module_& m) return self.apply_discard(m.m_value); }) .def("undo_discard", &Game::undo_discard) - .def_static("ranks", []() { return &Game::num_ranks; }) - .def_static("files", []() { return &Game::num_files; }) + .def_static("ranks", []() { return Game::num_ranks; }) + .def_static("files", []() { return Game::num_files; }) .def_static("feature_channels", &Game::feature_channels) .def_static("num_dlshogi_policy", &Game::num_dlshogi_policy) .def_static( diff --git a/python/tests/test_vshogi/test_dlshogi/test_replay_buffer.py b/python/tests/test_vshogi/test_dlshogi/test_replay_buffer.py index c9919d31..b65c4c96 100644 --- a/python/tests/test_vshogi/test_dlshogi/test_replay_buffer.py +++ b/python/tests/test_vshogi/test_dlshogi/test_replay_buffer.py @@ -7,7 +7,7 @@ def test_deduplicate(): legal_moves = Game("rbsgk/4p/5/P4/KGSBR b -").get_legal_moves() - buffer = ReplayBuffer() + buffer = ReplayBuffer(1, alpha=0.9) buffer.add( Data( sfen="rbsgk/4p/5/P4/KGSBR b -", @@ -24,11 +24,30 @@ def test_deduplicate(): weight=1.0, ) ) + assert len(buffer) == 2 p, v = buffer.__getitem__(0)[1:3] - assert np.isclose(p, 0.5).sum() == 2 + assert np.isclose(p, 0.9).sum() == 1 + assert np.isclose(p, 0.1).sum() == 1 assert np.isclose(p, 0.0).sum() == 12 assert np.isclose(p, -1.0e05).sum() == p.size - (2 + 12) - assert np.allclose(v, 0.25) + assert np.allclose(v, 0.05) + assert np.isclose(buffer.get("rbsgk/4p/5/P4/KGSBR b -").value01, 0.05) + + buffer.add( + Data( + sfen="4k/4p/5/P4/K4 b -", + policy={ + m: float(m == Move("5e4d")) + for m in Game("4k/4p/5/P4/K4 b -").get_legal_moves() + }, + value01=0.5, + weight=1.0, + ) + ) + assert len(buffer) == 2 + assert np.isclose(buffer[0][2], 0.5) + assert buffer.get("rbsgk/4p/5/P4/KGSBR b -") is None + assert np.isclose(buffer.get("4k/4p/5/P4/K4 b -").value01, 0.5) if __name__ == "__main__": diff --git a/python/vshogi/dlshogi/_cli/_cycler.py b/python/vshogi/dlshogi/_cli/_cycler.py index 8d8e3a39..5bcf8056 100644 --- a/python/vshogi/dlshogi/_cli/_cycler.py +++ b/python/vshogi/dlshogi/_cli/_cycler.py @@ -6,6 +6,7 @@ import click as cl +from vshogi.dlshogi import ReplayBuffer from vshogi.dlshogi._cli._nn_trainer import _train_step, _trainer_parameters from vshogi.dlshogi._cli._self_play_worker import ( _compute_random_moves, @@ -15,9 +16,10 @@ ) -def _train(nth_cycle: int, **kwargs): +def _train(buffer: ReplayBuffer, nth_cycle: int, **kwargs): weight_path = os.path.join(kwargs['output'], 'models/model_{:04d}.pth') _train_step( + buffer=buffer, model_path=weight_path.format(nth_cycle), prev_model_path=( None if nth_cycle == 0 else weight_path.format(nth_cycle - 1) @@ -26,7 +28,6 @@ def _train(nth_cycle: int, **kwargs): network_hidden_channels=kwargs['train_hidden_channels'], network_bottleneck_channels=kwargs['train_bottleneck_channels'], network_backbone_blocks=kwargs['train_backbone_blocks'], - max_dataset_size=kwargs['train_dataset_size'], kifu_path_pattern=os.path.join( kwargs['output'], "datasets/dataset_*/kifu_*.tsv", @@ -116,8 +117,12 @@ def _resume_from() -> int: return int(tflite_list[-1].split('_')[-1].split('.')[0]) + 1 start = _resume_from() + buffer = ReplayBuffer( + buffer_size=kwargs["train_buffer_size"], + alpha=kwargs["train_buffer_decay"], + ) if start == 0: - _train(nth_cycle=0, **kwargs) + _train(buffer=buffer, nth_cycle=0, **kwargs) start += 1 else: print(f"Resume cycle from {start}") @@ -145,6 +150,6 @@ def _resume_from() -> int: max_random_moves=max_random_moves, **kwargs, ) - _train(nth_cycle=i, **kwargs) + _train(buffer=buffer, nth_cycle=i, **kwargs) if os.path.exists(tflite_path.format(i)): break diff --git a/python/vshogi/dlshogi/_cli/_nn_trainer.py b/python/vshogi/dlshogi/_cli/_nn_trainer.py index 43e36367..9015ab8b 100644 --- a/python/vshogi/dlshogi/_cli/_nn_trainer.py +++ b/python/vshogi/dlshogi/_cli/_nn_trainer.py @@ -6,6 +6,7 @@ from collections.abc import Callable from datetime import datetime from glob import glob +from time import time import click as cl import numpy as np @@ -72,7 +73,19 @@ def _trainer_parameters(prefix: str = "") -> callable: ), ), cl.option( - f"--{prefix}dataset-size", default=100000, show_default=True + f"--{prefix}buffer-size", + default=100000, + show_default=True, + help=( + "Maximum size of the replay buffer for storing training " + "samples." + ), + ), + cl.option( + f"--{prefix}buffer-decay", + default=0.9, + show_default=True, + help="Decay factor for the replay buffer.", ), cl.option( f"--{prefix}per/--{prefix}no-per", @@ -209,7 +222,7 @@ def _average_kifu_length(kifu_dir: str) -> float | None: def _dataset( - max_dataset_size: int, + buffer: vs.dlshogi.ReplayBuffer, kifu_path_pattern: str, *, kifu_fraction: float = 1.0, @@ -218,32 +231,31 @@ def _dataset( default_result_rate: float = 1.0, value_func: Callable[[str], float] | None = None, ) -> th.utils.data.Dataset: - buffer = vs.dlshogi.ReplayBuffer(buffer_size=max_dataset_size) + buffer._first = None + if not hasattr(buffer, "_last"): + buffer._last = None + count: dict[str, int] = {} kifu_dir_list = sorted( glob('/'.join(kifu_path_pattern.split('/')[:-1])), reverse=True, ) + start = time() for kifu_dir, fr in zip( kifu_dir_list, (0.8**i for i in range(len(kifu_dir_list))), ): - if fr < 0.01: + if fr < 0.01 or (time() - start) > 60: break kifu_list = sorted( glob(kifu_dir + '/' + kifu_path_pattern.split('/')[-1]), reverse=True, ) - kifu_length = _average_kifu_length(kifu_dir=kifu_dir) * fr - sample_frac = max( - min( - kifu_fraction, - (max_dataset_size * 2) / (len(kifu_list) * kifu_length), - ), - 0.1, - ) - if kifu_dir == kifu_dir_list[0]: - print(f"{sample_frac=}") for kifu_path in kifu_list: + if buffer._first is None: + buffer._first = kifu_path + if buffer._last == kifu_path: + buffer._last = buffer._first + break df = vs.dlshogi.read_kifu( kifu_path, discount_factor=discount_factor, @@ -266,35 +278,35 @@ def _dataset( df = df.sample( **( {"n": 1} - if len(df) * sample_frac < 1 - else {"frac": sample_frac} + if len(df) * kifu_fraction < 1 + else {"frac": kifu_fraction} ), weights="priority", ) for _, row in df.iterrows(): - buffer.add( - vs.dlshogi.Data( - sfen=row['sfen'], - policy=row['policy'], - value01=row['value01'], - weight=row['weight'], - ) + data = vs.dlshogi.Data( + sfen=row['sfen'], + policy=row['policy'], + value01=row['value01'], + weight=row['weight'], ) - if buffer.is_full(): - break - if buffer.is_full(): + buffer.add(data) + if data.sfen not in count: + count[data.sfen] = 0 + count[data.sfen] += 1 + if (time() - start) > 60: break - if buffer.is_full(): + if (buffer._last == buffer._first) or (time() - start) > 60: break - summary = buffer.deduplicate() + buffer._last = buffer._first df_summary = pd.DataFrame( [ { - 'sfen': s, - 'count': data['count'], - 'value': 2 * data['value01'] - 1, + 'sfen': sfen, + 'count': v, + 'value': 2 * getattr(buffer.get(sfen), "value01", np.nan) - 1, } - for s, data in summary.items() + for sfen, v in count.items() ], columns=['sfen', 'count', 'value'], ) @@ -392,13 +404,13 @@ def _get_best_player_index( def _train_step( + buffer: vs.dlshogi.ReplayBuffer, model_path: str, prev_model_path: str | None, shogi_variant: tp.Literal['minishogi', 'judkins_shogi', 'shogi'], network_hidden_channels: int, network_bottleneck_channels: int, network_backbone_blocks: int, - max_dataset_size: int, kifu_path_pattern: str, prioritized_experience_replay: bool, kifu_fraction: float, @@ -452,8 +464,8 @@ def value_func(sfen: str) -> float: else: value_func = None - dataset = _dataset( - max_dataset_size=max_dataset_size, + buffer = _dataset( + buffer=buffer, kifu_path_pattern=kifu_path_pattern, kifu_fraction=kifu_fraction, discount_factor=discount_factor, @@ -461,7 +473,7 @@ def value_func(sfen: str) -> float: default_result_rate=default_result_rate, value_func=value_func, ) - if len(dataset) != 0: + if len(buffer) != 0: print(f"Start training: {model_path}") network.to(th.device(device)) for state in optimizer.state.values(): @@ -470,7 +482,7 @@ def value_func(sfen: str) -> float: state[k] = v.to(th.device(device)) _train( network, - dataset, + buffer, optimizer, minibatch_size=minibatch_size, epochs=epochs, @@ -534,16 +546,20 @@ def _resume_from() -> int: return int(tflite_list[-1].split('_')[-1].split('.')[0]) + 1 ii = _resume_from() + buffer = vs.dlshogi.ReplayBuffer( + buffer_size=kwargs["buffer_size"], + alpha=kwargs["buffer_decay"], + ) model_path = 'models/model_{:04d}.pth' while ii < 10000: _train_step( + buffer=buffer, model_path=model_path.format(ii), prev_model_path=None if ii == 0 else model_path.format(ii - 1), shogi_variant=kwargs['shogi'], network_hidden_channels=kwargs['hidden_channels'], network_bottleneck_channels=kwargs['bottleneck_channels'], network_backbone_blocks=kwargs['backbone_blocks'], - max_dataset_size=kwargs['dataset_size'], kifu_path_pattern="datasets/dataset_*/kifu_*.tsv", prioritized_experience_replay=kwargs["per"], kifu_fraction=kwargs['kifu_fraction'], diff --git a/python/vshogi/dlshogi/_data.py b/python/vshogi/dlshogi/_data.py index 950eaeee..68bb6618 100644 --- a/python/vshogi/dlshogi/_data.py +++ b/python/vshogi/dlshogi/_data.py @@ -22,7 +22,6 @@ class Data: policy: dict[Move, float] value01: float weight: float = 1.0 - count: int = 1 def __post_init__(self): if self.sfen.count(' ') == 3: diff --git a/python/vshogi/dlshogi/_replay_buffer.py b/python/vshogi/dlshogi/_replay_buffer.py index e905d1ad..f2c1b5e2 100644 --- a/python/vshogi/dlshogi/_replay_buffer.py +++ b/python/vshogi/dlshogi/_replay_buffer.py @@ -7,19 +7,6 @@ from vshogi.shogi._game import Game as StandardGame # noqa: F401 -def _add_dicts(d1: dict, d2: dict) -> dict: - if (not d1) or (not d2): - return {} - out = { - k: d1.get(k, 0.0) + d2.get(k, 0.0) - for k in (set(d1.keys()) | set(d2.keys())) - } - if np.isclose(sum(out.values()), 0): - msg = f"Adding two invalid dictionaries: {d1} and {d2}" - raise ValueError(msg) - return out - - def _normalize(d: dict) -> dict: if not d: return d @@ -35,28 +22,31 @@ class ReplayBuffer(th.utils.data.Dataset): Examples -------- >>> from vshogi.minishogi import Move; import numpy as np - >>> b = ReplayBuffer(buffer_size=2) + >>> b = ReplayBuffer(buffer_size=2, alpha=0.5) >>> b.add(Data('4k/5/4P/5/5 b G 1', {Move('1c1b'): 1}, 1., 1.)) >>> len(b) # Note that the length is doubled 2 - >>> b.add(Data('4k/5/4P/5/5 b G 3', {}, 0.5, 0.9)) + >>> b.add(Data('4k/5/4P/5/5 b G 3', {}, 0.5, 0.6)) >>> len(b) 2 >>> b[0][2:] # value01, weight - (array([0.75], dtype=float32), array(1.9, dtype=float32)) + (array([0.75], dtype=float32), array(0.8, dtype=float32)) """ - def __init__(self, buffer_size: int = 100000): + def __init__(self, buffer_size: int = 100000, alpha: float = 0.9): """Initialize dataset class. Parameters ---------- buffer_size : int Max size of the buffer + alpha : float + Decay rate to update duplicating data. """ super().__init__() self._buffer: list[Data] = [] self._buffer_size = buffer_size + self._alpha: float = alpha self._game_variant: str | None = None def add(self, data: Data): @@ -73,28 +63,44 @@ def add(self, data: Data): ) if index is not None: old = self._buffer.pop(index) - data = self._merge(old, data) + data = self._update(old, data) self._buffer.append(data) if self._game_variant is None: self._game_variant = self._infer_game_variant(data.sfen) while len(self._buffer) > self._buffer_size: self._buffer.pop(0) # FIFO - @staticmethod - def _merge(a: Data, b: Data) -> Data: + def get(self, sfen: str) -> Data | None: + """Get data of the given SFEN game position. + + Parameters + ---------- + sfen : str + SFEN representation of the game position. + + Returns + ------- + Data | None + Data of the game position if found, otherwise None. + """ + return next( + (b for b in self._buffer if b.sfen == sfen), + None, + ) + + def _update(self, a: Data, b: Data) -> Data: assert a.sfen == b.sfen - s = a.count + b.count + r: float = self._alpha # rate of `a` return Data( sfen=a.sfen, policy={} if (not a.policy) or (not b.policy) else { - m: (a.count * a.policy[m] + b.count * b.policy[m]) / s + m: (r * a.policy[m] + (1 - r) * b.policy[m]) for m in set(a.policy.keys()) | set(b.policy.keys()) }, - value01=(a.value01 * a.count + b.value01 * b.count) / s, - weight=a.weight + b.weight, - count=s, + value01=(r * a.value01 + (1 - r) * b.value01), + weight=(r * a.weight + (1 - r) * b.weight), ) def is_full(self) -> bool: @@ -107,28 +113,10 @@ def is_full(self) -> bool: """ return len(self._buffer) == self._buffer_size - def deduplicate(self) -> dict: - """Update value01 and policy by averaging all the data. - - Returns - ------- - dict - Summary. - """ - summary: dict[str, dict] = { - d.sfen: { - "value01": d.value01, - "policy": d.policy, - "count": d.count, - "weight": d.weight, - } - for d in self._buffer - } - scale = 1 / np.mean([v["weight"] for v in summary.values()]) + def normalize(self) -> None: + """Normalize policy distributions of all data.""" for d in self._buffer: - d.weight /= scale d.policy = _normalize(d.policy) - return summary def __len__(self): """Return the length of the dataset."""