Dataset generator

.gitignore

@@ -49,3 +49,7 @@ config.json
 *.epd
 *.book
 
+# Tuner outputs
+tuned_values*.txt
+tuner*.log
+

Makefile

@@ -88,6 +88,22 @@ tuner: clean ./src/tuner.o $(SRCS)
 	@ echo "================="
 	@ echo "capi_tuner compilado com sucesso"
 
+# Self-play data collector — Phase A. Plays engine-vs-engine games, samples
+# quiet positions during play, writes them out labeled with the game's WDL
+# result. Output feeds capi_resolve (Phase B) and then capi_tuner.
+selfplay: clean ./src/selfplay.o ./src/fen_serializer.o $(SRCS)
+	@ $(COMP) $(CXXFLAGS) -o capi_selfplay ./src/selfplay.o ./src/fen_serializer.o $(SRCS)
+	@ echo "================="
+	@ echo "capi_selfplay compilado com sucesso"
+
+# PV-resolver — Phase B. For each input position, runs a high-depth search,
+# walks the PV through the TT, and emits the leaf position with the original
+# game's WDL. Produces the final dataset for capi_tuner.
+resolve: clean ./src/resolve.o ./src/fen_serializer.o $(SRCS)
+	@ $(COMP) $(CXXFLAGS) -o capi_resolve ./src/resolve.o ./src/fen_serializer.o $(SRCS)
+	@ echo "================="
+	@ echo "capi_resolve compilado com sucesso"
+
 magics: ./src/generate_magics.cpp
 	@ $(COMP) -c $(CXXFLAGS) ./src/generate_magics.cpp -o ./src/generate_magics.o
 	@ $(COMP) -o generate_magics ./src/generate_magics.o 

scripts/run_selfplay.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+#
+# Two-phase self-play data collector orchestrator.
+#
+#   Phase A: launches N parallel `capi_selfplay` workers, each playing
+#            $GAMES games, writing to its own raw file.
+#   Phase B: concatenates Phase A outputs, splits into N chunks, launches
+#            N parallel `capi_resolve` workers, concatenates resolved chunks
+#            into the final dataset.
+#
+# The output dataset is named ${OUT_PREFIX}_D${DEPTH}_Resolved.book —
+# tuner-compatible, ready for `./capi_tuner <file>`.
+#
+# Usage:
+#   ./scripts/run_selfplay.sh [workers] [games_per_worker] [out_prefix] [extra_selfplay_args...]
+#
+# Examples:
+#   ./scripts/run_selfplay.sh 8 1000 selfplay_run1
+#   ./scripts/run_selfplay.sh 8 500 myrun --time-per-move 200 --book openings.epd
+#
+# Tuning notes:
+#   - --time-per-move (default 100ms) controls Phase A search quality vs throughput.
+#   - --depth-fixed N (Phase A alternative) for reproducible runs.
+#   - Resolver depth (Phase B) is set via RESOLVE_DEPTH below, default 12 to
+#     match the E12.33 dataset convention.
+
+set -euo pipefail
+
+WORKERS=${1:-8}
+GAMES=${2:-1000}
+OUT_PREFIX=${3:-selfplay}
+shift 3 2>/dev/null || true
+EXTRA_ARGS=("$@")
+
+RESOLVE_DEPTH=${RESOLVE_DEPTH:-12}
+RESOLVE_TT_MB=${RESOLVE_TT_MB:-16}
+
+if [[ ! -x ./capi_selfplay ]]; then
+    echo "error: ./capi_selfplay not found — run 'make selfplay' first" >&2
+    exit 1
+fi
+if [[ ! -x ./capi_resolve ]]; then
+    echo "error: ./capi_resolve not found — run 'make resolve' first" >&2
+    exit 1
+fi
+
+echo "==== Phase A: ${WORKERS} workers × ${GAMES} games ===="
+phase_a_start=$(date +%s)
+
+pids=()
+for i in $(seq 1 "${WORKERS}"); do
+    seed=$((i * 1000 + RANDOM))
+    out_file="${OUT_PREFIX}_raw_${i}.txt"
+    ./capi_selfplay --games "${GAMES}" --out "${out_file}" --seed "${seed}" "${EXTRA_ARGS[@]}" \
+        > "${OUT_PREFIX}_worker_${i}.log" 2>&1 &
+    pids+=($!)
+done
+for pid in "${pids[@]}"; do
+    wait "${pid}"
+done
+
+phase_a_end=$(date +%s)
+echo "Phase A done in $((phase_a_end - phase_a_start))s"
+
+# Concatenate raw outputs.
+raw_combined="${OUT_PREFIX}_raw.txt"
+cat "${OUT_PREFIX}"_raw_*.txt > "${raw_combined}"
+rm -f "${OUT_PREFIX}"_raw_*.txt
+echo "Phase A produced $(wc -l < "${raw_combined}") raw positions in ${raw_combined}"
+
+echo "==== Phase B: ${WORKERS} resolvers at depth ${RESOLVE_DEPTH} ===="
+phase_b_start=$(date +%s)
+
+# Split raw file into N roughly-equal chunks. macOS `split` lacks `-n l/N`, so
+# we use awk's NR modulo to deal lines round-robin into N output files — same
+# net effect (lines distributed evenly, no input duplication).
+chunk_prefix="${OUT_PREFIX}_chunk_"
+awk -v n="${WORKERS}" -v prefix="${chunk_prefix}" '
+    { print > sprintf("%s%02d", prefix, (NR - 1) % n) }
+' "${raw_combined}"
+
+pids=()
+for i in $(seq 0 $((WORKERS - 1))); do
+    chunk=$(printf "%s%02d" "${chunk_prefix}" "${i}")
+    [[ -f "${chunk}" ]] || continue
+    ./capi_resolve --in "${chunk}" --out "${chunk}.resolved" \
+                   --depth "${RESOLVE_DEPTH}" --tt-mb "${RESOLVE_TT_MB}" \
+        > "${chunk}.log" 2>&1 &
+    pids+=($!)
+done
+for pid in "${pids[@]}"; do
+    wait "${pid}"
+done
+
+phase_b_end=$(date +%s)
+echo "Phase B done in $((phase_b_end - phase_b_start))s"
+
+final_out="${OUT_PREFIX}_D${RESOLVE_DEPTH}_Resolved.book"
+cat "${chunk_prefix}"*.resolved > "${final_out}"
+rm -f "${chunk_prefix}"*
+
+echo "==== Done ===="
+echo "Final dataset: ${final_out} ($(wc -l < "${final_out}") positions)"
+echo "Feed to tuner: ./capi_tuner ${final_out} --max 0 --passes 1"

src/fen_serializer.cpp

@@ -0,0 +1,97 @@
+#include "fen_serializer.h"
+
+#include <cstdio>
+#include <cstdlib>
+
+#include "consts.h"
+#include "bitboard.h"
+#include "game.h"
+
+void Fen::serialize(char* out) {
+    int pos = 0;
+
+    // 1. Piece placement: ranks 7 (rank 8 in algebraic) → 0, files 0 → 7.
+    //    Square index = rank * 8 + file (matches the convention in
+    //    Update::setar_posicao). Empty runs compress to a single digit.
+    for (int rank = 7; rank >= 0; rank--) {
+        int empty = 0;
+        for (int file = 0; file < 8; file++) {
+            const int sq = rank * 8 + file;
+            const int piece = Bitboard::tabuleiro[sq];
+            if (piece == VAZIO) {
+                empty++;
+                continue;
+            }
+            if (empty > 0) {
+                out[pos++] = '0' + empty;
+                empty = 0;
+            }
+            char c = Consts::piece_char[piece];
+            // Consts::piece_char is uppercase by default. Lowercase for black.
+            if (Bitboard::bit_lados[PRETAS] & Bitboard::mask[sq]) {
+                c = c + ('a' - 'A');
+            }
+            out[pos++] = c;
+        }
+        if (empty > 0) {
+            out[pos++] = '0' + empty;
+        }
+        if (rank > 0) {
+            out[pos++] = '/';
+        }
+    }
+
+    out[pos++] = ' ';
+
+    // 2. Side to move.
+    out[pos++] = (Game::lado == BRANCAS) ? 'w' : 'b';
+
+    out[pos++] = ' ';
+
+    // 3. Castling rights, in canonical KQkq order.
+    if (Game::roque == 0) {
+        out[pos++] = '-';
+    } else {
+        if (Game::roque & BRANCAS_ROQUE_MENOR) out[pos++] = 'K';
+        if (Game::roque & BRANCAS_ROQUE_MAIOR) out[pos++] = 'Q';
+        if (Game::roque & PRETAS_ROQUE_MENOR)  out[pos++] = 'k';
+        if (Game::roque & PRETAS_ROQUE_MAIOR)  out[pos++] = 'q';
+    }
+
+    out[pos++] = ' ';
+
+    // 4. En passant target. setar_posicao stashes the (inicio, destino) of a
+    //    synthetic double-pawn-push in lista_do_jogo[hply-1] when parsing an
+    //    EP square, and real double pushes leave the same shape there. So if
+    //    the last record looks like a 2-square pawn advance (piece at destino
+    //    is a pawn, |destino - inicio| == 16), the EP square sits between them.
+    bool ep_emitted = false;
+    if (Game::hply > 1) {
+        const Game::jogo& last = Game::lista_do_jogo[Game::hply - 1];
+        const int piece_at_dest = Bitboard::tabuleiro[last.destino];
+        if (piece_at_dest == P && std::abs(last.destino - last.inicio) == 16) {
+            const int ep_sq = (last.inicio + last.destino) / 2;
+            const int ep_file = ep_sq & 7;
+            const int ep_rank = ep_sq >> 3;
+            out[pos++] = 'a' + ep_file;
+            out[pos++] = '1' + ep_rank;
+            ep_emitted = true;
+        }
+    }
+    if (!ep_emitted) {
+        out[pos++] = '-';
+    }
+
+    out[pos++] = ' ';
+
+    // 5. Halfmove clock.
+    pos += std::sprintf(out + pos, "%d", Game::cinquenta);
+
+    out[pos++] = ' ';
+
+    // 6. Fullmove number. White-to-move at hply=1 → fullmove=1; the formula
+    //    (hply + 1) / 2 advances on each black→white transition.
+    pos += std::sprintf(out + pos, "%d", (Game::hply + 1) / 2);
+
+    out[pos] = '\0';
+}

src/fen_serializer.h

@@ -0,0 +1,15 @@
+#ifndef FEN_SERIALIZER
+#define FEN_SERIALIZER
+
+namespace Fen {
+    // Writes a FEN string for the current global engine state into `out`.
+    // Caller owns the buffer; ≥90 bytes is safe for any legal position.
+    // Reads: Bitboard::tabuleiro, Bitboard::bit_lados, Game::lado,
+    // Game::roque, Game::cinquenta, Game::hply, Game::lista_do_jogo[hply-1].
+    // Inverse of Update::setar_posicao for positions reachable via real play
+    // (note: setar_posicao discards the FEN's fullmove counter, so a round-trip
+    // through that function may shift the fullmove field).
+    void serialize(char* out);
+}
+
+#endif