From 312318ad6548f65cc6af118c24811dfd40280b27 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 8 Nov 2026 17:32:09 +1300 Subject: feat: Resume training from any weights; balance training by outcome - train --from starts from any saved network; without it, training still carries on from the out dir's newest generation - Each epoch takes as many positions from sheep-win games as from tiger-win games (Config::balance_outcomes, on by default) - Training output shows the error on tiger-win and sheep-win positions separately, before and after training - Fix a broken sentence in CLAUDE.md's cache description Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 8 ++--- plans.org | 11 +++++++ src/main.rs | 13 ++++++-- src/train.rs | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++--------- 4 files changed, 109 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 42cf0cc..bae782b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ The toolchain comes from a Nix flake (`flake.nix`, loaded via direnv `.envrc` ### Benchmarking and profiling the search -- Train: `cargo run --release -- train [generations] [games] [out-dir]` (defaults 50, 500, `weights/`, which is gitignored), then play it with `cargo run --release -- neural weights/latest.bin [sheep|tiger]`. +- Train: `cargo run --release -- train [generations] [games] [out-dir] [--from ]` (defaults 50, 500, `weights/`, which is gitignored). It starts from `--from` if given, otherwise from the newest `gen-NNN.bin` in the out dir, otherwise from random weights. Then play it with `cargo run --release -- neural weights/latest.bin [sheep|tiger]`. - Bench: `cargo run --profile profiling -- bench [depth]` (default depth 6; 9 takes about 0.6 s) searches six fixed positions to a fixed depth without the TUI and prints the move, score and time for each. The positions come from `ClaudePlayer` playing itself to depth 4, so they are deterministic. An optimisation that only makes the search faster must leave every move and score unchanged. - Counters: add `--features search-stats` to print nodes, nodes/s, cache hit/overwrite rates and the share of cutoffs on the first move. They are thread-local, bumped with the `count!` macro in `player.rs` (also used in `board.rs`), and compile to nothing without the feature. Run `cargo test` with and without the feature. - The `profiling` Cargo profile is release plus debug info. The dev shell has `perf` and `samply`. As a non-root user, `perf record` needs `-m 16` (a small ring buffer) with `--call-graph dwarf`, otherwise mmap fails on the `perf_event_mlock_kb` limit: `perf record -e cycles:u -m 16 --call-graph dwarf target/profiling/tigers-and-sheep-rs bench 9`. @@ -30,7 +30,7 @@ The toolchain comes from a Nix flake (`flake.nix`, loaded via direnv `.envrc` - `src/board.rs` — all game logic and its unit tests (`#[cfg(test)] mod tests` at the bottom). - `src/player.rs` — the `Player` trait. Players are polled, not blocking: the coordinator calls `update(board, turn)` before each draw, `pick_move(board, turn) -> Option` once per loop (computer players search and return `Some`), and passes clicks/keys to `handle_input(Input, board, turn) -> Option`. `highlights()` gives points for the board to highlight, `reset()` is called on both players after a revert (drop any selection and last-move comments; it is *not* called after a normal move, so players clear their own selection when they return a move), `render(area, buf)` draws the current player's panel (default: its name), and `render_last_turn(area, buf)` draws end-of-turn comments on the move it just made (default: nothing). `impl Widget for &dyn Player` makes every player a ratatui widget. Also the private-ish `MinimaxPlayer` trait: implementors supply `evaluate(board, turn)` (positive favours tigers; `turn` is the side to move) and optionally `depth()` (at most `MAX_DEPTH` = 64)/`time_limit()`/`name()`, plus `set_last_search`/`last_search` to keep a `SearchReport` (move, depth reached, score) that the blanket impl's `render_last_turn` shows; a blanket impl makes every `MinimaxPlayer` a `Player`. - Search: alpha-beta (`minimax` checks the cache, `minimax_uncached` searches) with iterative deepening in `pick_move` until half the time limit has passed. Moves are tried in this order: the cache's best move (the `hint`), then tiger jumps (`get_moves` lists them first), then killer moves, then the rest. This is done by rotating moves to the front, because a `sort_by_key` here cost more than it saved. - - `cache()` returns an optional `ScoreCache`, keyed by board, side to move and remaining depth. `minimax` passes it `board.canonical()`, so the 8 symmetric boards share one entry, and turns the entry's best move between orientations with `transform_move`/`inverse_symmetry`. This assumes `evaluate` gives symmetric boards the same score. `ClaudePlayer`'s does; a `Network` only does so approximately, since it learns symmetry from augmented data. storing `CacheEntry` bounds. `TranspositionTable` is the fixed-size one (`new(bits)`: `1 << bits` slots of ~24 bytes, in two-slot buckets where one slot keeps the deepest entry; `clear()` bumps a generation number instead of wiping). The `HashMap` impl is kept as a simple reference for tests. + - `cache()` returns an optional `ScoreCache`, keyed by board, side to move and remaining depth. `minimax` passes it `board.canonical()`, so the 8 symmetric boards share one entry, and turns the entry's best move between orientations with `transform_move`/`inverse_symmetry`. This assumes `evaluate` gives symmetric boards the same score. `ClaudePlayer`'s does; a `Network` only does so approximately, since it learns symmetry from augmented data. Entries are `CacheEntry` bounds. `TranspositionTable` is the fixed-size one (`new(bits)`: `1 << bits` slots of ~24 bytes, in two-slot buckets where one slot keeps the deepest entry; `clear()` bumps a generation number instead of wiping). The `HashMap` impl is kept as a simple reference for tests. - `killers()` returns optional `Killers`: two quiet moves per remaining depth that last caused a cutoff. `pick_move` clears both the cache and the killers. - Tests compare the search with plain minimax (`reference`) for every cache kind (`CacheKind::None`/`Map`/`TinyTable`, a table small enough to collide constantly) with and without killers. - `src/game.rs` — `GameHandler::play_game(sheep_player, tiger_player)` runs a ratatui TUI (mouse capture on) until the user quits, returning the winner if there was one. Layout: board | move history on top; below, the current player's panel (70%) | the previous player's last-turn panel (30%). Panes overlap by one cell (`Layout::spacing(-1)`) and are built with `panel()` (`MergeStrategy::Exact`) so shared borders join up. The bottom row is drawn first and keeps its titles on its bottom border, because its top border is the same line as the board's bottom border, which lists the keybinds (keep them in step with `handle_event`). History is `Vec<(Move, GameBoard)>` (board *after* each move); the side to move comes from its length's parity (sheep first). Clicking a history row calls `revert(i)`, which goes back to the board *before* move `i`. The loop only waits for input (`event::read`) when `pick_move` returns `None`. After a computer's search, `pending_action` handles any input that came in meanwhile, so quit and revert still work in computer-vs-computer games. They take effect once the current search finishes. @@ -40,8 +40,8 @@ The toolchain comes from a Nix flake (`flake.nix`, loaded via direnv `.envrc` - `src/bench.rs` — the `bench` mode (see above). - `src/network.rs` — `Network`, a small MLP (52 inputs → 64 ReLU → 32 ReLU → tanh, positive favours tigers) with hand-written backprop and an `Adam` optimiser, all in one flat `params` Vec at the `W1`/`B1`/… offsets. Inputs are only the raw position: a bit per point for tigers and for sheep, sheep in hand / 20, and whether tigers are to move. `forward` adds up the layer-1 weight rows of the set bits rather than multiplying through the zeros. Also `Rng` (xorshift64*) and `Sample` (board, turn, target). File format (`save`/`load`): `TSNN`, then version, inputs, hidden1, hidden2 as little-endian u32s, then every param as a little-endian f32; changing the layer sizes makes old files fail to load. - `src/neural-player.rs` — `NeuralPlayer`, a `MinimaxPlayer` like `ClaudePlayer` whose `evaluate` is a `Network`. -- `src/train.rs` — self-play training. Each generation plays `games_per_generation` games of the net against itself at a fixed depth (default 3) across all cores. Moves are random for the first `random_plies`, and with chance `epsilon` after that. A game reaching `ply_cap` is a draw, since the rules have no draw. Each position is labelled with a TD(λ) target worked backwards from the result (±1, or 0) through the searched scores. A position where a random move was played is labelled with its own search score instead, which cuts the chain there. Without that cut, the sheep's random moves (which usually hang a sheep) taught the net that tigers nearly always win. Self-play still ends mostly in tiger wins, because of those random moves, so judge progress by the match against `ClaudePlayer` (`match_games` = 100). The net then trains on the last `buffer_generations` generations, each position shown in a random one of the board's 8 symmetries (`GameBoard::transformed`). After that it plays matches against the previous generation and against `ClaudePlayer`, which is only a yardstick and never a training signal. Each generation is saved as `out_dir/gen-NNN.bin` and `latest.bin`, and a rerun carries on from the newest `gen-NNN.bin`. -- `src/main.rs` — `bench [depth]` runs the benchmark; `train [generations] [games per generation] [out dir]` trains a network (use `--release`); `neural [sheep|tiger]` plays you, on that side (default sheep), against a `NeuralPlayer`; otherwise it plays a game in the TUI (the sheep player first, then the tigers) and prints the winner. Which players it uses changes: `LocalPlayer` or `ClaudePlayer` on either side. +- `src/train.rs` — self-play training. Each generation plays `games_per_generation` games of the net against itself at a fixed depth (default 3) across all cores. Moves are random for the first `random_plies`, and with chance `epsilon` after that. A game reaching `ply_cap` is a draw, since the rules have no draw. Each position is labelled with a TD(λ) target worked backwards from the result (±1, or 0) through the searched scores. A position where a random move was played is labelled with its own search score instead, which cuts the chain there. Without that cut, the sheep's random moves (which usually hang a sheep) taught the net that tigers nearly always win. Self-play still ends mostly in tiger wins, because of those random moves, so judge progress by the match against `ClaudePlayer` (`match_games` = 100). The net then trains on the last `buffer_generations` generations, each position shown in a random one of the board's 8 symmetries (`GameBoard::transformed`). With `balance_outcomes` (the default), `balanced` builds each epoch from as many positions of sheep-win games as of tiger-win games: their mean, so the epoch stays the same size. Tiger-win positions are subsampled and sheep-win ones repeated; draws are kept as they are. The output shows the mean error on tiger-win (T) and sheep-win (S) positions separately, before and after training. After that it plays matches against the previous generation and against `ClaudePlayer`, which is only a yardstick and never a training signal. Each generation is saved as `out_dir/gen-NNN.bin` and `latest.bin`, and a rerun carries on from the newest `gen-NNN.bin`. `Config::from` (`--from`) starts from any weights file instead. Generation numbers still follow on from the out dir. The Adam state and the sample buffer are not saved, so they start empty either way. +- `src/main.rs` — `bench [depth]` runs the benchmark; `train [generations] [games per generation] [out dir] [--from ]` trains a network (use `--release`; `--from` may go anywhere); `neural [sheep|tiger]` plays you, on that side (default sheep), against a `NeuralPlayer`; otherwise it plays a game in the TUI (the sheep player first, then the tigers) and prints the winner. Which players it uses changes: `LocalPlayer` or `ClaudePlayer` on either side. Notation (`pos_name`, `impl Display for Move` in board.rs): files `a`–`e` are x 0–4, ranks are `5 - y` (y = 0 is drawn at the top as rank 5). Moves are `sc3` (place), `sc3c2` (sheep move), `ta5b4` (tiger step), `ta5xc3` (tiger jump). diff --git a/plans.org b/plans.org index 5a06798..8f53255 100644 --- a/plans.org +++ b/plans.org @@ -67,3 +67,14 @@ Before the fix, tigers won ~97% of self-play games and the net only reached ~15- *** Next candidates - Pick exploratory moves among near-best ones (softmax over root scores) rather than uniformly - Sheep are still the weaker side: report the match score for each side separately +** DONE Balance tiger-win and sheep-win positions +The unbalanced net (gen 100) had 7x the error on positions from sheep-win games (0.045) as on +tiger-win ones (0.0065), so it leaned towards tiger wins, though not a constant prediction (that +would give errors near 2). =balance_outcomes= trains on equal numbers of each. +| vs =ClaudePlayer= (depth 3, 100 games) | gen 25 | gen 50 | gen 75 | gen 100 | +|----------------------------------------+--------+--------+--------+---------| +| Unbalanced | 25% | 50% | 54% | 62% | +| Balanced | 22% | 19% | 51% | 62% | +- The T and S errors are now similar (both ~0.02), but the strength reached is the same; balanced was slower + in the middle. Self-play is still ~95% tiger wins, from the random sheep moves. +- "vs prev" stays at ~50% because after 4 random plies both sides play deterministically; it says little. diff --git a/src/main.rs b/src/main.rs index 8f6968d..1404a9f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,14 @@ mod neural_player; mod train; fn main() -> std::io::Result<()> { - let args: Vec = std::env::args().collect(); + let mut args: Vec = std::env::args().collect(); + // `--from ` can go anywhere after `train` + let from = args.iter().position(|a| a == "--from").map(|i| + { + let path = args.get(i + 1).cloned().expect("usage: --from "); + args.drain(i..=i + 1); + path + }); let arg = |i: usize| args.get(i).map(String::as_str); let number = |i: usize, default: usize| arg(i).map_or(default, |n| n.parse().expect("expected a number")); match arg(1) @@ -29,7 +36,8 @@ fn main() -> std::io::Result<()> { bench::run(number(2, 6)); return Ok(()); } - // `train [generations] [games per generation] [out dir]` trains a network by self-play + // `train [generations] [games per generation] [out dir] [--from ]` trains a network + // by self-play, starting from `--from`, or the newest generation in the out dir, or random weights Some("train") => { let defaults = train::Config::default(); @@ -38,6 +46,7 @@ fn main() -> std::io::Result<()> { generations: number(2, defaults.generations), games_per_generation: number(3, defaults.games_per_generation), out_dir: arg(4).map_or(defaults.out_dir.clone(), Into::into), + from: from.map(Into::into), ..defaults }; return train::run(&cfg); diff --git a/src/train.rs b/src/train.rs index 1ee40c0..e6ed822 100644 --- a/src/train.rs +++ b/src/train.rs @@ -9,6 +9,8 @@ pub struct Config pub generations: usize, pub games_per_generation: usize, pub out_dir: PathBuf, + /// Start from this network instead of the newest one in `out_dir` (or a random one). + pub from: Option, /// Search depth for every move in self-play and matches. The depth cap, not a time limit, /// ends each search, so runs are repeatable. pub depth: usize, @@ -22,6 +24,8 @@ pub struct Config pub lambda: f32, /// Train on the samples of this many of the latest generations. pub buffer_generations: usize, + /// Train on as many positions from sheep-win games as from tiger-win games (see `balanced`). + pub balance_outcomes: bool, pub epochs: usize, pub batch_size: usize, pub learning_rate: f32, @@ -38,12 +42,14 @@ impl Default for Config generations: 50, games_per_generation: 500, out_dir: PathBuf::from("weights"), + from: None, depth: 3, ply_cap: 200, random_plies: 10, epsilon: 0.05, lambda: 0.7, buffer_generations: 5, + balance_outcomes: true, epochs: 2, batch_size: 256, learning_rate: 1e-3, @@ -62,6 +68,8 @@ pub enum Outcome impl Outcome { + const ALL: [Outcome; 3] = [Outcome::Tiger, Outcome::Sheep, Outcome::Draw]; + /// +1 for a tiger win, -1 for a sheep win, matching the sign of the scores. fn value(self) -> f32 { @@ -212,6 +220,39 @@ fn match_score(net: &Network, opponent: impl Fn() -> points / cfg.match_games.max(1) as f32 } +/// One epoch's training positions, taken from the positions of tiger-win, sheep-win and drawn +/// games. Self-play ends mostly in tiger wins, so training on everything would teach the net +/// mostly what tiger wins look like. Instead this takes the same number from each kind of win: +/// their mean, so the epoch is the same size. The bigger pool gives a random subset, and the +/// smaller one gives every position plus random repeats (shown in different orientations by +/// the caller). Draws are kept as they are. If either kind of win has no positions, everything is used. +fn balanced(pools: &[Vec; 3], rng: &mut Rng) -> Vec +{ + let [tigers, sheep, draws] = pools; + if tigers.is_empty() || sheep.is_empty() + { + return pools.concat(); + } + let n = (tigers.len() + sheep.len()) / 2; + let mut out = draws.clone(); + for pool in [tigers, sheep] + { + // Every position once while there are enough, picked by a partial shuffle of the indices + let mut idx: Vec = (0..pool.len()).collect(); + for i in 0..n.min(pool.len()) + { + let j = i + rng.below(pool.len() - i); + idx.swap(i, j); + out.push(pool[idx[i]]); + } + for _ in pool.len()..n + { + out.push(pool[rng.below(pool.len())]); + } + } + out +} + /// The newest `gen-NNN.bin` in `dir`, as (generation, path). fn latest_generation(dir: &Path) -> Option<(usize, PathBuf)> { @@ -226,24 +267,29 @@ fn latest_generation(dir: &Path) -> Option<(usize, PathBuf)> } /// Trains a network by self-play, saving each generation to `out_dir/gen-NNN.bin` and `out_dir/latest.bin`. -/// Carries on from the newest generation already in `out_dir`, if there is one. +/// Starts from `cfg.from` if given, otherwise carries on from the newest generation already in +/// `out_dir`, if there is one. Generations are numbered on from the newest in `out_dir` either way. pub fn run(cfg: &Config) -> std::io::Result<()> { fs::create_dir_all(&cfg.out_dir)?; - let (first, mut net) = match latest_generation(&cfg.out_dir) + let latest = latest_generation(&cfg.out_dir); + let first = latest.as_ref().map_or(1, |(n, _)| n + 1); + let mut net = match cfg.from.as_ref().or(latest.as_ref().map(|(_, path)| path)) { - Some((n, path)) => + Some(path) => { println!("Carrying on from {}", path.display()); - (n + 1, Network::load(&path)?) + Network::load(path)? } - None => (1, Network::random(0x5EED)), + None => Network::random(0x5EED), }; let mut adam = Adam::new(cfg.learning_rate); - let mut buffer: VecDeque> = VecDeque::new(); + // For each generation, its positions from tiger-win, sheep-win and drawn games + let mut buffer: VecDeque<[Vec; 3]> = VecDeque::new(); let mut rng = Rng::new(first as u64); - println!("{:>4} {:>7} {:>6} {:>6} {:>6} {:>8} {:>8} {:>7} {:>7} {:>7}", - "gen", "samples", "tigers", "sheep", "draws", "loss", "after", "vs prev", "vs Claude", "time s"); + println!("Self-play results, then the mean error on positions from tiger-win (T) and sheep-win (S) games, before and after training"); + println!("{:>4} {:>7} {:>6} {:>6} {:>6} {:>8} {:>8} {:>8} {:>8} {:>7} {:>9} {:>7}", + "gen", "trained", "tigers", "sheep", "draws", "T before", "S before", "T after", "S after", "vs prev", "vs Claude", "time s"); for generation in first..first + cfg.generations { let start = Instant::now(); @@ -251,17 +297,21 @@ pub fn run(cfg: &Config) -> std::io::Result<()> let games = parallel(cfg.games_per_generation, |i| self_play(&net, cfg, seed_base | i as u64)); let count = |o: Outcome| games.iter().filter(|(g, _)| *g == o).count(); let (tigers, sheep, draws) = (count(Outcome::Tiger), count(Outcome::Sheep), count(Outcome::Draw)); - buffer.push_back(games.into_iter().flat_map(|(_, s)| s).collect()); + buffer.push_back(Outcome::ALL.map(|o| games.iter().filter(|(g, _)| *g == o).flat_map(|(_, s)| s).copied().collect())); while buffer.len() > cfg.buffer_generations { buffer.pop_front(); } - let mut samples: Vec = buffer.iter().flatten().copied().collect(); - let loss_before = net.loss(&samples); + let pools: [Vec; 3] = [0, 1, 2].map(|k| buffer.iter().flat_map(|g| &g[k]).copied().collect()); + let losses = |net: &Network| (net.loss(&pools[0]), net.loss(&pools[1])); + let before = losses(&net); let previous = net.clone(); + let mut trained = 0; for _ in 0..cfg.epochs { + let mut samples = if cfg.balance_outcomes { balanced(&pools, &mut rng) } else { pools.concat() }; + trained = samples.len(); // Shuffle, and show each position in a random one of its 8 orientations for i in (1..samples.len()).rev() { @@ -276,14 +326,14 @@ pub fn run(cfg: &Config) -> std::io::Result<()> net.train_batch(batch, &mut adam); } } - let loss_after = net.loss(&samples); + let after = losses(&net); let vs_prev = match_score(&net, || neural(&previous, cfg.depth), cfg, seed_base ^ 0xA); let vs_claude = match_score(&net, || claude(cfg.depth), cfg, seed_base ^ 0xB); net.save(&cfg.out_dir.join(format!("gen-{generation:03}.bin")))?; net.save(&cfg.out_dir.join("latest.bin"))?; - println!("{generation:>4} {:>7} {tigers:>6} {sheep:>6} {draws:>6} {loss_before:>8.4} {loss_after:>8.4} {:>6.0}% {:>8.0}% {:>7.1}", - samples.len(), vs_prev * 100.0, vs_claude * 100.0, start.elapsed().as_secs_f64()); + println!("{generation:>4} {trained:>7} {tigers:>6} {sheep:>6} {draws:>6} {:>8.4} {:>8.4} {:>8.4} {:>8.4} {:>6.0}% {:>8.0}% {:>7.1}", + before.0, before.1, after.0, after.1, vs_prev * 100.0, vs_claude * 100.0, start.elapsed().as_secs_f64()); } Ok(()) } @@ -345,6 +395,25 @@ mod tests } } + #[test] + fn balanced_takes_as_many_of_each_win() + { + let sample = |target: f32| Sample { board: GameBoard::default(), turn: Turn::Sheep, target }; + let pools = [vec![sample(1.0); 90], (0..10).map(|i| sample(-(i as f32) / 10.0)).collect(), vec![sample(0.5); 4]]; + let mut rng = Rng::new(1); + let out = balanced(&pools, &mut rng); + assert_eq!(out.len(), 50 + 50 + 4); + assert_eq!(out.iter().filter(|s| s.target == 1.0).count(), 50); + assert_eq!(out.iter().filter(|s| s.target == 0.5).count(), 4); + // The smaller pool's positions are all there, before any repeats + let sheep: Vec = out.iter().map(|s| s.target).filter(|&t| t <= 0.0).collect(); + assert_eq!(sheep.len(), 50); + assert!((0..10).all(|i| sheep[..10].contains(&(-(i as f32) / 10.0)))); + // Without sheep wins there is nothing to balance + let lopsided = [vec![sample(1.0); 5], vec![], vec![sample(0.0); 2]]; + assert_eq!(balanced(&lopsided, &mut rng).len(), 7); + } + #[test] fn parallel_keeps_the_order() { -- cgit v1.3.1