diff --git a/.circleci/config.yml b/.circleci/config.yml index 18233fd872..6404b6bb28 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,39 +2,52 @@ version: 2 jobs: build: docker: - - image: danieluranga/leela_chess_zero-lc0_ubuntu_builder:0.0.8 + - image: nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04 steps: - checkout - run: - name: "Pull Submodules" - command: git submodule update --init + name: Install build tools + command: | + apt-get update + apt-get -y install git python3-pip gcc-10 g++-10 clang-12 zlib1g zlib1g-dev wget + pip3 install meson==0.63 + pip3 install ninja - run: - name: Update Meson - command: pip3 install --upgrade meson==0.58.1 + name: Install onnxruntime + command: | + wget https://github.com/microsoft/onnxruntime/releases/download/v1.22.0/onnxruntime-linux-x64-1.22.0.tgz -P /tmp + tar xzf /tmp/onnxruntime-linux-x64-1.22.0.tgz -C /tmp - run: name: Meson GCC environment: - CC: gcc-8 - CXX: g++-8 - command: meson build-gcc -Dgtest=false + CC: gcc-10 + CXX: g++-10 + command: meson build-gcc -Dgtest=false -Donnx_include=/tmp/onnxruntime-linux-x64-1.22.0/include -Donnx_libdir=/tmp/onnxruntime-linux-x64-1.22.0/lib + - run: + name: Meson Clang + environment: + CC: clang-12 + CXX: clang++-12 + command: meson build-clang -Dgtest=false -Db_lto=false -Donnx_include=/tmp/onnxruntime-linux-x64-1.22.0/include -Donnx_libdir=/tmp/onnxruntime-linux-x64-1.22.0/lib - run: name: Build GCC command: | cd build-gcc ninja -j 4 + - run: + name: Build Clang + command: | + cd build-clang + ninja -j 4 "mac": macos: - xcode: 14.1.0 - resource_class: macos.m1.medium.gen1 + xcode: 14.3.1 steps: - checkout - - run: - name: "Pull Submodules" - command: git submodule update --init - run: name: Install build tools command: | - pip3 install meson==0.63 + pip3 install meson==1.3.0 pip3 install ninja curl -LJOs https://github.com/ispc/ispc/releases/download/v1.21.0/ispc-v1.21.0-macOS.universal.tar.gz tar xf ispc-v1.21.0-macOS.universal.tar.gz @@ -56,10 +69,68 @@ jobs: command: lipo -create -o /tmp/lc0 build/lc0 build-arm/lc0 - store_artifacts: path: /tmp/lc0 - destination: lc0-macos_12.6.1 + destination: lc0-macos_13.2.1 + - run: + name: Prepare Workspace + command: | + mkdir -p workspace + mv /tmp/lc0 workspace + - persist_to_workspace: + root: workspace + paths: + - lc0 + "mac latest": + macos: + xcode: 26.1.0 + steps: + - checkout + - run: + name: Install build tools + command: | + pip3 install meson + pip3 install ninja + - run: + name: Build lc0 arm + command: | + meson build-arm --buildtype=release -Dgtest=false -Dopencl=false + cd build-arm + ninja + "upload-github-release": + macos: + xcode: 14.3.1 + steps: + - attach_workspace: + at: /tmp/workspace + - run: + name: Install GitHub CLI + command: brew install gh + - run: + name: Verify Workspace + command: | + ls -lah /tmp/workspace + - run: + name: Upload to GitHub Release + command: | + mv /tmp/workspace/lc0 /tmp/lc0-$CIRCLE_TAG-macos_13.2.1 + gh release upload \ + "$CIRCLE_TAG" \ + /tmp/lc0-$CIRCLE_TAG-macos_13.2.1 \ + --clobber --repo LeelaChessZero/lc0 workflows: version: 2 builds: jobs: - build - - "mac" + - "mac": + filters: + tags: + only: /v[0-9]+(\.[0-9]+)*(\-.+)?/ + - "mac latest" + - "upload-github-release": + requires: + - "mac" + filters: + tags: + only: /v[0-9]+(\.[0-9]+)*(\-.+)?/ + branches: + ignore: /.*/ diff --git a/.gitignore b/.gitignore index ea18c9ee58..ae0570869a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .cache/ .clangd/ build/ +builddir/ __pycache__/ compile_commands.json CUDA_NN/ diff --git a/.gitmodules b/.gitmodules index 6575e63266..e69de29bb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "libs/lczero-common"] - path = libs/lczero-common - url = https://github.com/LeelaChessZero/lczero-common.git diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000000..b7d6010ae3 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,129 @@ +A-childs-encyclopedia +Adam Treat +Alex Greason +Alexander Lyashuk +Alexis Olson +alice +almaudoh +Aloril +Andrew Grant +Andy Olsen +Aniebiet Udoh +Ankan +Ankan Banerjee +Anson Hu +Asger Alstrup Palm +baum +benini +borg323 +Boštjan Mejak +Brandon Lin +Brett Holman +Carlo Wood +Chin-Chang Yang +cn4750 +Cong +Contrad Namiseb (Bonan) +Copilot (bot) +cwbriscoe +danegraphics +Daniel Monroe +Daniel Uranga +Dieter Dobbelaere +dje-dev +Dominik Schlösser +DU-jdto +dubslow +Dubslow +Ed Lee +Epanek +Error323 +evalon32 +exa +exoticorn +F. Huizinga +fischerandom +fohristiwhirl +Francis +Francis Li +Francois +Francois Pays +François Pays +Gabe +Ganesh Krishnan +GBeauregard +Gergely Fülöp +Gian-Carlo Pascutto +gmorenz +Google LLC +gsobala +Hace +Hans Ekbrand +Henrik Forstén +Ikko Eltociear Ashimine +Jack L +Jack Thomson +James Horsfall Thomas +jamie +jjoshua2 +John Newlin +john-sp +Julian-Dominik Helmsen +Karl Kfoury +Kathleen Mcgrievy +kiilas +Kip Hamiltons +Kovax +Leandro Álvarez González +Linmiao Xu +Lisa Butterfly +Luka Rahne +Marcin Stefaniuk +Martin +Martin Senft +masterkni6 +masterkni666 +Menkib +Mike Roberts +Naphthalin +nathan-lc0 +Neelesh Jain +nguyenpham +noobpwnftw +Ofek Shochat +oscardssmith +Pan +patrik-ha +PaulJeFi +Pratik Dixit +psykose +QxC4eva +Rafal Bielski +Raj +Reece H. Dunn +Ron Wolf +Sami Kiminki +Sherman Siu +Shreyas Kapur +shtayerc +Shukant Pal +Simon +slash +students +SunnyWar +TesseractA +Tilps +Timofey Kondrashov +Ting-Hsuan Huang +Tony Su +trre123 +Usman Haider +uyhcire +Valentin +Valeriy Huz +Victor Popovici +Videodr0me +Viet-Anh Tran +Viren6 +Yan Zhang +zz4032 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 90c7d4aa63..0249700c0a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,81 +1,135 @@ -# Contributing to lc0 - -These are the guidelines and standards followed by this codebase. - -The language is C++, specifically C++17. As such, manual `new` and `delete` memory mangement is strongly discouraged; use the standard library tools for managing memory (such as `unique_ptr`, `shared_ptr` etc.). - -This codebase uses semantic versioning. A release is the final commit for that version number, and all subsequent commits are development for the next version. `master` is the default branch, and the active development branch (as such, all Pull Requests go here); it always targets a minor (or major) version which succeeds the current relase. `release` is always equivalent to the latest tag. - - -### Style - -Style is of course the first guideline on every new contributor's mind :) - -This codebase largely complies with the [Google C++ style guide](https://google.github.io/styleguide/cppguide.html). The maintainers recommend the use of [Clang's auto formatter](https://clang.llvm.org/docs/ClangFormatStyleOptions.html). - -Notable exceptions: - 1. C++ exceptions are allowed (in fact, only `lczero::Exception`, defined in `utils/exception.h`, is allowed) - 2. We use `#pragma once` instead of header guards. - 3. Default function parameters are sometimes allowed. - 4. Rvalue reference function params are sometimes allowed, not only for constructors and assignment operators. - -For items (3) and (4), usage of those are discouraged, only use them if they benefit readability or have significant performance gain. It's possible that those exceptions (3) and (4) will be disallowed in future. - -The most important rule to follow is consistency: look at the surrounding code when doing changes and follow similar style. - -These are the most important parts of the codebase style (as a sort of tl;dr): - - * Comments must be full sentences, i.e. capitalized and ending in a period. (Sentences with elided subjects are fine.) Only `//` style comments are allowed, `/* */` style comments aren't. - - * Braces are a variant of K&R style, as can be gleaned from existing code. All `if` statements must use braces, with the possible exception of single statement `if`s, which *may* omit if the braces *if* the conditional and following statement are on the same line. Again, see surrounding code for examples. - - * Indentation is two spaces; \t characters are disallowed. - - * Code line length is strictly capped at 80 characters. - - * Using non-`const` references as function parameters is disallowed; use pointers instead. (Using `const` references as parameters is fine.) - - * Identifier style: - - `kLikeThis` for constants and enum values - - `like_this` for variables - - `like_this_` for member variables - - `LikeThis` for function and class names - - * All code should be inside `namespace lczero` - -The internal code dependency structure looks like this: - - * Code in `src/utils` is not allowed to depend on any other code. - - * Code in `src/chess` only depends on `src/utils` - - * Code in `src/neural` only depends on `src/utils` and `src/chess` - - * Code in `src/mcts` only depends on `src/utils`, `src/chess` and `src/neural` - - -### Git history - -Pull Requests are squashed when merged. This means all commits in the branch will be squashed into one commit applied onto master, so branches and their PRs should stick to *one* topic only. If you think changes deserve separate commits, make separate PRs for each commit. - -This also means it's not possible to reuse one branch for multiple PRs; new PRs must either use entirely new branches, or else you could use `git reset --hard` on the current branch. - - -### Allowed features - -Lc0 is still in early stages of development, and has not yet reached the point where we are ready to add small tweaks to add few points of a rating. Large code changes still happen, and having lots of small optimizations adds overhead to larger changes, slowing development. - -Therefore, as a rule, search algorithm tweaks that give a gain of less than ~20 Elo points are discouraged at this point. (This limit will gradually be lowered as Lc0 code matures, eventually to 0.0 Elo). - - -#### Adding new command line flags/UCI parameters - -Only add new parameters if users can significantly (>20 Elo) benefit by tweaking it. We don't want to make every single constant configurable (or rather, users don't want to see hundreds of parameters which don't do anything). - -Try to minimize number of parameters that your feature introduces. If your feature introduces several parameters, every individual parameter should be significant (i.e. tweaking it with other fixes will give >20 Elo). - - -#### Adding features for testing - -It is fine to temporarily commit a feature of unknown Elo gain so that people may test it. It's also fine to expose many parameters for the feature initially so that people can tune them. However, if the tweak doesn't prove to be significant, it should be removed after a few weeks. - +# Contributing to Leela Chess Zero (Lc0) + +Last updated: June 2025 + +Thank you for your interest in contributing to LCZero! This document provides +guidelines for contributing to the codebase. + +## Before you start + +* All contributors are encouraged to join our Discord server at + . +* Refer to [README.md](README.md) for building and running instructions. + * The protobufs that are shared with the training code are located in a + separate repository. Don't forget to run `git submodule update --init` to + fetch them. +* Familiarize yourself with the developer documentation at + . + +We use the [Meson](https://mesonbuild.com/) build system. + +* In Linux, using `builddir/` is recommended for development + (`meson setup builddir/`), as VSCode recognizes it and all the development and + debugging tools work there (ask in Discord if you have issues). + * In the `builddir/`, run `ninja lc0` (which is faster than just `ninja`). Run + `ninja test` to run the tests. +* In Windows, `meson setup build/debug` generates a Visual Studio solution that + can be used for development. +* Check `meson_options.txt` for the available build options (to use them, pass + `-D=` to `meson setup`). + +## Sending Pull Requests + +* We use GitHub for managing contributions. Please fork the repository and + create a new branch for your changes. +* It's encouraged to discuss your changes in the Discord server before + starting work. + * Small bug fixes are fine to submit without prior discussion. + * Large changes that add code rather than modifying existing code (e.g. new + backends, new search algorithms) are generally fine as well. Use your best + judgement on whether your change may be controversial. + * Changes that modify existing code (e.g. search algorithm tweaks, API + changes) should be discussed first. + +Changes that may affect playing strength **must** be tested. + +* Unfortunately, we don't have a robust strength testing framework yet (working + on it), so ask in the #testing-discuss channel on Discord for help with + testing. +* Even for Elo-positive changes, we need to balance the strength and + maintainability of the code. If your change is Elo-positive but makes the code + more complex, please discuss it first. Recently, we added an option to + clone the search algorithm in extreme cases. +* Elo-neutral simplifications are always welcome. + +Pull Requests are squashed when merged. This means all commits in the branch +will be squashed into one commit applied onto master. This makes it tricky to +reuse the branch and continue to work on it after the PR is merged. + +**Note:** This section only applies if you have dependent branches that were +built on top of your merged PR branch. If you only had one branch, you can +simply delete it after merging. + +**Example scenario:** You had a branch `add-feature` that got merged, and you +have another branch `extend-feature` that was based on `add-feature`. After +`add-feature` is merged, you need to rebase `extend-feature` onto the new +master. Here's what to do after your PR is merged: + +```shell +git fetch upstream # Update your local master +git checkout extend-feature # Switch to your dependent branch +git rebase --update-refs --onto upstream/master add-feature # Rebase onto the updated master +``` + +The `--update-refs` flag will also update any branches between your leaf branch +and the merged branch if you have any. + +## C++ Standard and Libraries + +* We use most C++20 features. However, supported compilers are GCC 10 and clang + 10, so some features may not be available. +* We use protocol buffers. However, we don't use any external library for it, + but rather generate the code from `.proto` files using the script in + `scripts/`. +* Since v0.32, we use Abseil (`absl::`). +* Use `CERR` for logging (goes to stderr and log), or `LOGFILE` (goes to log + file only). +* Writing tests is encouraged. We use `gtest`/`gmock` for unit tests. Tests are + located in the same directory as the code they test, in a file with the same + name but ending with `_test.cc`. + +## Style Guidelines + +We follow the +[Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html) with +these modifications: + +* **Header guards**: Use `#pragma once` instead of traditional header guards. +* **Exceptions**: are allowed, but only one: `lczero::Exception`. +* **References vs Pointers**: Non-const reference parameters are neither + encouraged nor discouraged over pointers (in the Google style guide, they used + to be discouraged, and now they are encouraged). +* **Formatting**: Run `clang-format` on all code before committing. +* **RValue references**: Rvalue reference function parameters are allowed, not + only for constructors and assignment operators. However, use them only if they + benefit readability or have significant performance gain. +* Every new file should contain a GPLv3 banner with an additional exception + under GNU GPL version 3 section 7 regarding NVIDIA libraries (see examples in + existing files). + * It's allowed to write new backends without the NVIDIA exception, but that + means that we won't be able to include NVIDIA libraries if we link with that + code. + +## AI-Assisted Development + +AI tools and coding assistants are allowed for contributing to Leela Chess Zero, +provided you follow these guidelines: + +* **Disclose AI usage**: Clearly mention in your PR description if you used AI + tools, LLMs, or agentic coding approaches (beyond simple code completion). +* **Maintain code ownership**: You must thoroughly read, understand, and review + all AI-generated code in detail before submitting. +* **Ensure quality**: Take all appropriate steps to verify the code quality, + correctness, and adherence to project standards. +* **Submit only what you could write yourself**: Use AI as a productivity + booster, not as a replacement for proper programming skills and domain + knowledge. +* **Core vs. auxiliary code**: While AI assistance works well for auxiliary code + (website, documentation, simple tools), it has failed so far on core lc0 + engine code. We discourage using agentic coding for core engine components but + welcome it for simpler projects within the Leela Chess Zero organization. + +Remember: You are responsible for the quality and correctness of all code you +submit, regardless of how it was generated. + +Thank you for helping make Leela Chess Zero better! diff --git a/OpenBench/Makefile b/OpenBench/Makefile new file mode 100644 index 0000000000..6147fd9125 --- /dev/null +++ b/OpenBench/Makefile @@ -0,0 +1,24 @@ +ifndef EXE + EXE = $(CURDIR)/lc0 +endif + +BUILD_FLAGS = +ifdef EVALFILE + BUILD_FLAGS += -Dembed=true +endif +ifdef SEARCH + BUILD_FLAGS += -Ddefault_search=$(SEARCH) +endif + +POST_BUILD_COMMANDS = +ifdef EVALFILE + POST_BUILD_COMMANDS = \ + cat $(EVALFILE) >> $(EXE); \ + perl -e "printf '%sLc0!', pack('V', -s '$(EVALFILE)')" >> $(EXE) +endif + +.PHONY: all +all: + chmod +x ../build.sh + ../build.sh $(BUILD_FLAGS) && mv ../build/release/lc0 $(EXE) + $(POST_BUILD_COMMANDS) \ No newline at end of file diff --git a/README.md b/README.md index a56da72740..0ce7a2a125 100644 --- a/README.md +++ b/README.md @@ -7,33 +7,28 @@ Lc0 is a UCI-compliant chess engine designed to play chess via neural network, s ## Downloading source -Lc0 can be acquired either via a git clone or an archive download from GitHub. Be aware that there is a required submodule which isn't included in source archives. +Lc0 can be acquired either via a git clone or an archive download from GitHub. -For essentially all purposes, including selfplay game generation and match play, we highly recommend using the latest `release/version` branch (for example `release/0.31`), which is equivalent to using the latest version tag. +For essentially all purposes, including selfplay game generation and match play, we highly recommend using the latest `release/version` branch (for example `release/0.32`), which is equivalent to using the latest version tag. Versioning follows the Semantic Versioning guidelines, with major, minor and patch sections. The training server enforces game quality using the versions output by the client and engine. - Download using git: ```shell -git clone -b release/0.31 --recurse-submodules https://github.com/LeelaChessZero/lc0.git +git clone -b release/0.32 https://github.com/LeelaChessZero/lc0.git ``` If you have cloned already an old version, fetch, view and checkout a new branch: ```shell git fetch --all git branch --all -git checkout -t remotes/origin/release/0.31 +git checkout -t remotes/origin/release/0.32 ``` - -If you prefer to download an archive, you need to also download and place the submodule: - * Download the [.zip](https://api.github.com/repos/LeelaChessZero/lc0/zipball/release/0.31) file ([.tar.gz](https://api.github.com/repos/LeelaChessZero/lc0/tarball/release/0.31) archive is also available) +If you prefer to download an archive: + * Download the [.zip](https://api.github.com/repos/LeelaChessZero/lc0/zipball/release/0.32) file ([.tar.gz](https://api.github.com/repos/LeelaChessZero/lc0/tarball/release/0.32) archive is also available) * Extract - * Download https://github.com/LeelaChessZero/lczero-common/archive/master.zip (also available as [.tar.gz](https://github.com/LeelaChessZero/lczero-common/archive/master.tar.gz)) - * Move the second archive into the first archive's `libs/lczero-common/` folder and extract - * The final form should look like `/libs/lczero-common/proto/` Having successfully acquired Lc0 via either of these methods, proceed to the build section below and follow the instructions for your OS. @@ -42,13 +37,11 @@ Having successfully acquired Lc0 via either of these methods, proceed to the bui Building should be easier now than it was in the past. Please report any problems you have. -Aside from the git submodule, lc0 requires the Meson build system and at least one backend library for evaluating the neural network, as well as the required `zlib`. (`gtest` is optionally used for the test suite.) If your system already has this library installed, they will be used; otherwise Meson will generate its own copy of the two (a "subproject"), which in turn requires that git is installed (yes, separately from cloning the actual lc0 repository). Meson also requires python and Ninja. +Building lc0 requires the Meson build system and at least one backend library for evaluating the neural network, as well as a few libraries. If your system already has these libraries installed, they will be used; otherwise Meson will generate its own copy (a "subproject"), which in turn requires that git is installed (yes, separately from cloning the actual lc0 repository). Meson also requires python and Ninja. -Backend support includes (in theory) any CBLAS-compatible library for CPU usage, such as OpenBLAS or Intel's DNNL or MKL. For GPUs, OpenCL and CUDA+cudnn are supported, while DX-12 can be used in Windows 10 with latest drivers. +Backend support includes (in theory) any CBLAS-compatible library for CPU usage, but OpenBLAS or Intel's DNNL are the main ones. For GPUs, the following are supported: CUDA (with optional cuDNN), various flavors of onnxruntime, and Apple's Metal Performance Shaders. There is also experimental SYCL support for AMD and Intel GPUs. -Finally, lc0 requires a compiler supporting C++17. Minimal versions seem to be g++ v8.0, clang v5.0 (with C++17 stdlib) or Visual Studio 2017. - -*Note* that cuda checks the compiler version and stops even with newer compilers, and to work around this we have added the `nvcc_ccbin` build option. This is more of an issue with new Linux versions, but you can get around it by using an earlier version of gcc just for cuda. As an example, adding `-Dnvcc_ccbin=g++-9` to the `build.sh` command line will use g++-9 with cuda instead of the system compiler. +Finally, lc0 requires a compiler supporting C++20. Minimal versions tested are g++ v10.0, clang v12.0 and Visual Studio 2019 version 16.11. Given those basics, the OS and backend specific instructions are below. @@ -56,160 +49,125 @@ Given those basics, the OS and backend specific instructions are below. #### Generic -1. Install backend: - - If you want to use NVidia graphics cards Install [CUDA](https://developer.nvidia.com/cuda-zone) and [cuDNN](https://developer.nvidia.com/cudnn). - - If you want to use AMD graphics cards install OpenCL. - - if you want OpenBLAS version Install OpenBLAS (`libopenblas-dev`). +1. Install backend (also read the detailed instructions in later sections): + - If you want to use NVidia graphics cards Install [CUDA](https://developer.nvidia.com/cuda-zone) (and optionally [cuDNN](https://developer.nvidia.com/cudnn)). + - If you want to use AMD or Intel graphics cards you can try SYCL. + - if you want BLAS install either OpenBLAS or DNNL. 2. Install ninja build (`ninja-build`), meson, and (optionally) gtest (`libgtest-dev`). 3. Go to `lc0/` 4. Run `./build.sh` 5. `lc0` will be in `lc0/build/release/` directory -6. Unzip a [neural network](https://lczero.org/play/networks/bestnets/) in the same directory as the binary. +6. Download a [neural network](https://lczero.org/play/networks/bestnets/) in the same directory as the binary (no need to unpack it). If you want to build with a different compiler, pass the `CC` and `CXX` environment variables: +```shell +CC=clang CXX=clang++ ./build.sh +``` - CC=clang-6.0 CXX=clang++-6.0 ./build.sh - -#### Note on installing CUDA on Ubuntu - -Nvidia provides .deb packages. CUDA will be installed in `/usr/local/cuda-10.0` and requires 3GB of diskspace. -If your `/usr/local` partition doesn't have that much space left you can create a symbolic link before -doing the install; for example: `sudo ln -s /opt/cuda-10.0 /usr/local/cuda-10.0` - -The instructions given on the nvidia website tell you to finish with `apt install cuda`. However, this -might not work (missing dependencies). In that case use `apt install cuda-10-0`. Afterwards you can -install the meta package `cuda` which will cause an automatic upgrade to a newer version when that -comes available (assuming you use `Installer Type deb (network)`, if you'd want that (just cuda-10-0 will -stay at version 10). If you don't know what to do, only install cuda-10-0. - -cuDNN exists of two packages, the Runtime Library and the Developer Library (both a .deb package). +#### Ubuntu 20.04 -Before you can download the latter you need to create a (free) "developer" account with nvidia for -which at least a legit email address is required (their website says: The e-mail address is not made public -and will only be used if you wish to receive a new password or wish to receive certain news or notifications -by e-mail.). Further they ask for a name, date of birth (not visible later on), country, organisation ("LeelaZero" -if you have none), primary industry segment ("Other"/none) and which development areas you are interested -in ("Deep Learning"). +For Ubuntu 20.04 you need meson, ninja and gcc-10 before performing the steps above. The following should work: +```shell +apt-get update +apt-get -y install git python3-pip gcc-10 g++-10 zlib1g zlib1g-dev +pip3 install meson +pip3 install ninja +CC=gcc-10 CXX=g++-10 INSTALL_PREFIX=~/.local ./build.sh +``` -#### Ubuntu 18.04 +Make sure that `~/.local/bin` is in your `PATH` environment variable. You can now type `lc0 --help` and start. -For Ubuntu 18.04 you need the latest version of meson, libstdc++-8-dev, and clang-6.0 before performing the steps above: +### Windows - sudo apt-get install libstdc++-8-dev clang-6.0 ninja-build pkg-config - pip3 install meson --user - CC=clang-6.0 CXX=clang++-6.0 INSTALL_PREFIX=~/.local ./build.sh +Here are the brief instructions for CUDA/cuDNN, for details and other options see `windows-build.md` and the instructions in the following sections. -Make sure that `~/.local/bin` is in your `PATH` environment variable. You can now type `lc0 --help` and start. +1. Install Microsoft Visual Studio (2019 version 16.11 or later) +2. Install [CUDA](https://developer.nvidia.com/cuda-zone) +3. (Optionally install [cuDNN](https://developer.nvidia.com/cudnn)). +4. Install Python3 if you didn't install it with Visual Studio. +5. Install Meson: `pip3 install --upgrade meson` +6. If `CUDA_PATH` is not set (run the `set` command to see the full list of variables), edit `build.cmd` and set the `CUDA_PATH` with your CUDA directory +* If you also want cuDNN, set `CUDNN_PATH` with your cuDNN directory (not needed if it is the same with `CUDA_PATH`). -#### Ubuntu 16.04 +7. Run `build.cmd`. It will ask permission to delete the build directory, then generate MSVS project and pause. -For Ubuntu 16.04 you need the latest version of meson, ninja, clang-6.0, and libstdc++-8: +Then either: - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - - sudo apt-add-repository 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main' - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt-get update - sudo apt-get install clang-6.0 libstdc++-8-dev - pip3 install meson ninja --user - CC=clang-6.0 CXX=clang++-6.0 INSTALL_PREFIX=~/.local ./build.sh +8. Hit `Enter` to build it. +9. Resulting binary will be `build/lc0.exe` -Make sure that `~/.local/bin` is in your `PATH` environment variable. You can now type `lc0 --help` and start. +Or. -#### openSUSE (all versions) +8. Open generated solution `build/lc0.sln` in Visual Studio and build it yourself. -Instructions, packages and tools for building on openSUSE are at [openSUSE_install.md](openSUSE_install.md) +### Mac -#### Docker +You will need xcode and python3 installed. Then you need to install some required packages through Terminal: -Use https://github.com/vochicong/lc0-docker -to run latest releases of lc0 and the client inside a Docker container. +1. Install meson: `pip3 install meson` +2. Install ninja: `pip3 install ninja` +Now download the lc0 source, if you haven't already done so, following the instructions earlier in the page. -### Windows +3. Go to the lc0 directory. +4. Run `./build.sh -Dgtest=false` -Here are the brief instructions for CUDA/CuDNN, for details and other options see `windows-build.md`. +The compiled Lc0 will be in `build/release` -0. Install Microsoft Visual Studio (2017 or later) -1. Install [CUDA](https://developer.nvidia.com/cuda-zone) -2. Install [cuDNN](https://developer.nvidia.com/cudnn). -3. Install Python3 -4. Install Meson: `pip3 install --upgrade meson` -5. Edit `build.cmd`: +Starting with v0.32.0, we are also offering a pre-compiled version that can be downloaded from the [release page](https://github.com/LeelaChessZero/lc0/releases). -* Set `CUDA_PATH` with your CUDA directory -* Set `CUDNN_PATH` with your cuDNN directory (may be the same with CUDA_PATH) +### CUDA -6. Run `build.cmd`. It will ask permission to delete the build directory, then generate MSVS project and pause. +CUDA can be downloaded and installed following the instructions in from . The build in most cases will pick it up with no further action. However if the cuda compiler (`nvcc`) is not found you can call the build like this: `PATH=/usr/local/cuda/bin:$PATH ./build.sh`, replacing the path with the correct one for `nvcc`. -Then either: +*Note* that CUDA uses the system compiler and stops if it doesn't recognize the version, even if newer. This is more of an issue with new Linux versions, but you can get around with the `nvcc_ccbin` build option to specify a different compiler just for cuda. As an example, adding `-Dnvcc_ccbin=g++-11` to the build command line will use g++-11 with cuda instead of the system compiler. -7. Hit `Enter` to build it. -8. Resulting binary will be `build/lc0.exe` +### ONNX -Or. +Lc0 offers several ONNX based backends, namely onnx-cpu, onnx-cuda, onnx-trt, onnx-rocm and on Windows onnx-dml, utilizing the execution providers offered by onnxruntime. -7. Open generated solution `build/lc0.sln` in Visual Studio and build yourself. +Some Linux systems are starting to offer onnxruntime packages, so after installing this there is a good chance the Lc0 build will pick it up with no further action required. Otherwise you can set the `onnx_libdir` and `onnx_include` build options to point to the onnxruntime libraries and include directories respectively. The same options are used if you unpack a package downloaded from . -### Mac +For Windows, we offer pre-compiled packages for onnx-dml and onnx-trt, see the included README for installation instructions. -First you need to install some required packages through Terminal: -1. Install brew as per the instructions at https://brew.sh/ -2. Install python3: `brew install python3` -3. Install meson: `brew install meson` -4. Install ninja: `brew install ninja` -5. (For Mac OS 10.14 Mojave, or if the other step 5 fails): - * Install developer tools: ``xcode-select --install`` - * When using Mojave install SDK headers: `installer -pkg /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg -target /` (if this doesn't work, use `sudo installer` instead of just `installer`.) +### SYCL -Or. +*Note* that SYCL support is new in v0.32.0 and as such is still considered experimental. -5. (For MacOS 10.15 Catalina, or if the other step 5 fails): - * Install Xcode command-line tools: ``xcode-select --install`` - * Install "XCode Developer Tools" through the app store. (First one on the list of Apps if searched.) - * Associate the SDK headers in XCode with a command: export CPATH=\`xcrun --show-sdk-path\`/usr/include - -Now download the lc0 source, if you haven't already done so, following the instructions earlier in the page. +You will need the Intel "oneAPI DPC++/C++ Compiler", "DPC++ Compatibility Tool" and (for an Intel GPU) "oneAPI Math Kernel Library (oneMKL)" or (for an AMD GPU) hipBLAS. -6. Go to the lc0 directory. -7. Run `./build.sh -Dgtest=false` (needs step 5) +The Intel tools can be found in either the "oneAPI Base Toolkit" or "C++ Essentials" packages that can be downloaded from +, while hipBLAS can be downloaded from + -### Raspberry Pi +The compiler for C code is icx and for C++ code is icx on Windows but icpx on Linux. -You'll need to be running the latest Raspberry Pi OS "buster". +To build Lc0 with SYCL you need to set the `sycl` build option using `-Dsycl=l0` (that is el zero) for an Intel GPU or `-Dsycl=amd` for (you guessed it) an AMD GPU. -1. Install OpenBLAS +You may also have to set the `dpct_include` option to point to the DPC++ Compatibility Tool includes, the `onemkl_include` similarly for the oneMKL includes, or `hip_libdirs` and `hip_include` to the AMD HIP libraries and includes respectively. +On Linux, a typical session would go like this: ```shell -git clone https://github.com/xianyi/OpenBLAS.git -cd OpenBLAS/ -make -sudo make PREFIX=/usr install -cd .. +. /opt/intel/oneapi/setvars.sh --include-intel-llvm +CC=icx CXX=icpx AR=llvm-ar ./build.sh release -Dgtest=false -Dsycl=l0 ``` +The first line is to initialize the build environment and is only needed once per session, while the build line may need modification as described above. -2. Install Meson +On windows you will have to build using `ninja`, this is provided by Visual Studio if you install the CMake component. We provide a `build-sycl.cmd` script that should build just fine for an Intel GPU. This script has not yet been tested with and AMD GPU, some editing will be required. -```shell -pip install meson -pip install ninja -``` +You can also install the [oneAPI DPC++/C++ Compiler Runtime](https://www.intel.com/content/www/us/en/developer/articles/tool/compilers-redistributable-libraries-by-version.html) so you can run Lc0 without needing to initialize the build environment every time. -3. Install compiler and standard libraries +### BLAS -```shell -sudo apt install clang-6.0 libstdc++-8-dev -``` +Lc0 can also run (a bit slow) on CPU, using matrix multiplication functions from a BLAS library. By default OpenBLAS is used if available as it seems to offer good performance on a wide range of processors. If your system doesn't offer an OpenBLAS package (e.g. `libopenblas-dev`), or you have a recent processor you can get DNNL from [here](). To use DNNL you have to pass `-Ddnnl=true` to the build and specify the directory where it was installed using the `-Ddnnl_dir=` option. For macs, the Accelerate library will be used. -4. Clone lc0 and compile +If the "Intel Implicit SPMD Program Compiler" (`ispc`) is [installed](), some performance critical functions will use vectorized code for faster execution. -```shell -git clone https://github.com/LeelaChessZero/lc0.git -cd lc0 -git submodule update --init --recursive -CC=clang-6.0 CXX=clang++-6.0 ./build.sh -Ddefault_library=static -``` +*Note* that Lc0 is not able to control the number of threads with all BLAS libraries. Some libraries try to exploit cores aggressively, in which case it may be best to leave the threads set to the default (i.e. automatic) setting. + +## Getting help -5. The resulting binary will be in build/release +If there is an issue or the above instructions were not clear, you can always ask for help. The fastest way is to ask in the help channel of our [discord chat](http://lc0.org/chat), but you can also open a [github issue](https://github.com/LeelaChessZero/lc0/issues) (after checking the issue hasn't already been reported). ## Python bindings @@ -240,8 +198,8 @@ along with Leela Chess. If not, see . ### Additional permission under GNU GPL version 3 section 7 -_The source files of Lc0 with the exception of the BLAS and OpenCL -backends (all files in the `blas` and `opencl` sub-directories) have +_The source files of Lc0 with the exception of the BLAS, OpenCL and SYCL +backends (all files in the `blas`, `opencl` and `sycl` sub-directories) have the following additional permission, as allowed under GNU GPL version 3 section 7:_ diff --git a/appveyor.yml b/appveyor.yml index 543b60f4bf..e68f9f136e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,24 +2,28 @@ version: '{build}' configuration: Release platform: x64 image: -- Visual Studio 2017 +- Visual Studio 2019 environment: matrix: - NAME: gpu-nvidia-cudnn - - NAME: gpu-nvidia-cuda + - NAME: gpu-nvidia-cuda12 # - NAME: gpu-dx12 # - NAME: gpu-opencl - NAME: cpu-dnnl - NAME: cpu-openblas # - NAME: onednn - - NAME: onnx-dml + - NAME: onnx - NAME: android + - NAME: gpu-nvidia-cuda11 for: - matrix: only: + - NAME: gpu-nvidia-cudnn + - NAME: gpu-nvidia-cuda11 # - NAME: gpu-opencl - NAME: cpu-dnnl + - NAME: cpu-openblas skip_non_tags: true clone_folder: c:\projects\lc0 install: @@ -29,34 +33,38 @@ install: - cmd: set OPENCL=false - cmd: set BLAS=false - cmd: set ONEDNN=false -- cmd: set ONNX_DML=false +- cmd: set ONNX=false - cmd: set GTEST=false - cmd: set ANDROID=false - cmd: IF %NAME%==android set ANDROID=true - cmd: IF %NAME%==gpu-nvidia-cudnn set CUDNN=true - cmd: IF %NAME%==gpu-nvidia-cudnn set CUDA=true -- cmd: IF %NAME%==gpu-nvidia-cuda set CUDA=true +- cmd: IF %NAME%==gpu-nvidia-cuda11 set CUDA=true +- cmd: IF %NAME%==gpu-nvidia-cuda12 set CUDA=true - cmd: IF %NAME%==gpu-dx12 set DX=true - cmd: IF %NAME%==gpu-opencl set OPENCL=true - cmd: IF %NAME%==cpu-dnnl set BLAS=true - cmd: IF %NAME%==cpu-openblas set BLAS=true -- cmd: IF %NAME%==cpu-openblas set GTEST=true - cmd: IF %NAME%==onednn set ONEDNN=true -- cmd: IF %NAME%==onnx-dml set ONNX_DML=true +- cmd: IF %NAME%==onnx set ONNX=true +- cmd: IF %NAME%==onnx set GTEST=true - cmd: set NET=753723 - cmd: set NET_HASH=3e3444370b9fe413244fdc79671a490e19b93d3cca1669710ffeac890493d198 - cmd: IF NOT %OPENCL%==true IF NOT %DX%==true set NET=791556 - cmd: IF NOT %OPENCL%==true IF NOT %DX%==true set NET_HASH=f404e156ceb2882470fd8c032b8754af0fa0b71168328912eaef14671a256e34 -- cmd: call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64 +#- cmd: call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64 - cmd: set DNNL_NAME=dnnl_win_1.5.0_cpu_vcomp - cmd: IF %NAME%==cpu-dnnl IF NOT EXIST C:\cache\%DNNL_NAME% appveyor DownloadFile https://github.com/oneapi-src/oneDNN/releases/download/v1.5/dnnl_win_1.5.0_cpu_vcomp.zip - cmd: IF %NAME%==cpu-dnnl IF NOT EXIST C:\cache\%DNNL_NAME% 7z x dnnl_win_1.5.0_cpu_vcomp.zip -oC:\cache - cmd: IF %NAME%==onednn set DNNL_NAME=dnnl_win_2.7.2_cpu_vcomp_gpu_vcomp - cmd: IF %NAME%==onednn IF NOT EXIST C:\cache\%DNNL_NAME% appveyor DownloadFile https://github.com/borg323/oneDNN/releases/download/v2.7.2/dnnl_win_2.7.2_cpu_vcomp_gpu_vcomp.zip - cmd: IF %NAME%==onednn IF NOT EXIST C:\cache\%DNNL_NAME% 7z x dnnl_win_2.7.2_cpu_vcomp_gpu_vcomp.zip -oC:\cache -- cmd: IF %NAME%==onnx-dml IF NOT EXIST C:\cache\onnxruntime-win-x64-dml-1.13.1 appveyor DownloadFile https://github.com/borg323/onnxruntime/releases/download/v1.13.1/onnxruntime-win-x64-dml-1.13.1.zip -- cmd: IF %NAME%==onnx-dml IF NOT EXIST C:\cache\onnxruntime-win-x64-dml-1.13.1 7z x onnxruntime-win-x64-dml-1.13.1.zip -oC:\cache -- cmd: IF %NAME%==onnx-dml set ONNX_NAME=onnxruntime-win-x64-dml-1.13.1 +- cmd: IF %NAME%==onnx set ONNX_NAME=onnxruntime-win-x64-dml-1.22.1 +- cmd: IF %NAME%==onnx set ONNX_NAME_TWO=onnxruntime-win-x64-gpu-1.22.1 +- cmd: IF %NAME%==onnx IF NOT EXIST C:\cache\%ONNX_NAME% appveyor DownloadFile https://github.com/microsoft/onnxruntime/releases/download/v1.22.1/Microsoft.ML.OnnxRuntime.DirectML.1.22.1.nupkg +- cmd: IF %NAME%==onnx IF NOT EXIST C:\cache\%ONNX_NAME% 7z x Microsoft.ML.OnnxRuntime.DirectML.1.22.1.nupkg -oC:\cache\%ONNX_NAME% +- cmd: IF %NAME%==onnx IF NOT EXIST C:\cache\%ONNX_NAME_TWO% appveyor DownloadFile https://github.com/microsoft/onnxruntime/releases/download/v1.22.1/onnxruntime-win-x64-gpu-1.22.1.zip +- cmd: IF %NAME%==onnx IF NOT EXIST C:\cache\%ONNX_NAME_TWO% 7z x onnxruntime-win-x64-gpu-1.22.1.zip -oC:\cache - cmd: IF %NAME%==cpu-openblas IF NOT EXIST C:\cache\OpenBLAS appveyor DownloadFile https://sjeng.org/ftp/OpenBLAS-0.3.3-win-oldthread.zip - cmd: IF %NAME%==cpu-openblas IF NOT EXIST C:\cache\OpenBLAS 7z x OpenBLAS-0.3.3-win-oldthread.zip -oC:\cache\OpenBLAS - cmd: IF %OPENCL%==true nuget install opencl-nug -Version 0.777.77 -OutputDirectory C:\cache @@ -65,26 +73,36 @@ install: - cmd: IF %ISPC%==true IF NOT EXIST C:\cache\ispc-v1.13.0-windows appveyor DownloadFile https://github.com/ispc/ispc/releases/download/v1.13.0/ispc-v1.13.0-windows.zip - cmd: IF %ISPC%==true IF NOT EXIST C:\cache\ispc-v1.13.0-windows 7z x ispc-v1.13.0-windows.zip -oC:\cache\ispc-v1.13.0-windows - cmd: IF %ISPC%==true set PATH=C:\cache\ispc-v1.13.0-windows\bin;%PATH% -- cmd: set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0" +- cmd: set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1" - cmd: IF %CUDNN%==true IF NOT EXIST "%CUDA_PATH%\cuda" set CUDNN_INSTALL=1 -- cmd: IF DEFINED CUDNN_INSTALL appveyor DownloadFile https://developer.nvidia.com/compute/cuda/10.0/Prod/network_installers/cuda_10.0.130_win10_network -- cmd: IF DEFINED CUDNN_INSTALL cuda_10.0.130_win10_network -s nvcc_10.0 cublas_dev_10.0 cublas_10.0 cudart_10.0 -- cmd: IF DEFINED CUDNN_INSTALL appveyor DownloadFile http://developer.download.nvidia.com/compute/redist/cudnn/v7.4.2/cudnn-10.0-windows10-x64-v7.4.2.24.zip -- cmd: IF DEFINED CUDNN_INSTALL 7z x cudnn-10.0-windows10-x64-v7.4.2.24.zip -o"%CUDA_PATH%" -- cmd: IF %CUDNN%==false set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1" +- cmd: IF DEFINED CUDNN_INSTALL appveyor DownloadFile https://developer.download.nvidia.com/compute/cuda/10.1/Prod/network_installers/cuda_10.1.243_win10_network.exe +- cmd: IF DEFINED CUDNN_INSTALL cuda_10.1.243_win10_network -s nvcc_10.1 cublas_dev_10.1 cublas_10.1 cudart_10.1 +- cmd: IF DEFINED CUDNN_INSTALL appveyor DownloadFile https://developer.download.nvidia.com/compute/redist/cudnn/v7.5.1/cudnn-10.1-windows10-x64-v7.5.1.10.zip +- cmd: IF DEFINED CUDNN_INSTALL 7z x cudnn-10.1-windows10-x64-v7.5.1.10.zip -o"%CUDA_PATH%" +- cmd: IF %NAME%==gpu-nvidia-cuda11 set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1" +- cmd: IF %NAME%==gpu-nvidia-cuda12 set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.9" +- cmd: IF %NAME%==onnx set "CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.9" - cmd: IF %CUDA%==true IF NOT EXIST "%CUDA_PATH%" set CUDA_INSTALL=1 -- cmd: IF DEFINED CUDA_INSTALL appveyor DownloadFile https://developer.download.nvidia.com/compute/cuda/11.1.0/network_installers/cuda_11.1.0_win10_network.exe -- cmd: IF DEFINED CUDA_INSTALL cuda_11.1.0_win10_network.exe -s nvcc_11.1 cublas_dev_11.1 cublas_11.1 cudart_11.1 documentation_11.1 +- cmd: IF %ONNX%==true IF NOT EXIST "%CUDA_PATH%" set CUDA_INSTALL=1 +- cmd: IF DEFINED CUDA_INSTALL IF %NAME%==gpu-nvidia-cuda11 appveyor DownloadFile https://developer.download.nvidia.com/compute/cuda/11.1.0/network_installers/cuda_11.1.0_win10_network.exe +- cmd: IF DEFINED CUDA_INSTALL IF %NAME%==gpu-nvidia-cuda11 cuda_11.1.0_win10_network.exe -s nvcc_11.1 cublas_dev_11.1 cublas_11.1 cudart_11.1 documentation_11.1 +- cmd: IF DEFINED CUDA_INSTALL IF %NAME%==gpu-nvidia-cuda12 appveyor DownloadFile https://developer.download.nvidia.com/compute/cuda/12.9.0/network_installers/cuda_12.9.0_windows_network.exe +- cmd: IF DEFINED CUDA_INSTALL IF %NAME%==gpu-nvidia-cuda12 cuda_12.9.0_windows_network.exe -s nvcc_12.9 cublas_dev_12.9 cublas_12.9 curand_dev_12.9 cudart_12.9 documentation_12.9 +- cmd: IF %NAME%==gpu-nvidia-cuda12 IF NOT EXIST C:\cache\cutlass-2.11.0 appveyor DownloadFile https://github.com/NVIDIA/cutlass/archive/refs/tags/v2.11.0.zip +- cmd: IF %NAME%==gpu-nvidia-cuda12 IF NOT EXIST C:\cache\cutlass-2.11.0 7z x v2.11.0.zip -oC:\cache\ +- cmd: IF DEFINED CUDA_INSTALL IF %NAME%==onnx appveyor DownloadFile https://developer.download.nvidia.com/compute/cuda/12.9.0/network_installers/cuda_12.9.0_windows_network.exe +- cmd: IF DEFINED CUDA_INSTALL IF %NAME%==onnx cuda_12.9.0_windows_network.exe -s nvcc_12.9 cudart_12.9 - cmd: IF %CUDA%==true set PATH=%CUDA_PATH%\bin;%PATH% -- cmd: set PATH=C:\Python36;C:\Python36\scripts;%PATH% -- cmd: pip3 install --upgrade meson==0.55.3 -- cmd: set MIMALLOC_PATH=C:\cache\mimalloc-1.7.1 -- cmd: IF %ANDROID%==false IF NOT EXIST "%MIMALLOC_PATH%" appveyor DownloadFile https://github.com/microsoft/mimalloc/archive/refs/tags/v1.7.1.zip -- cmd: IF %ANDROID%==false IF NOT EXIST "%MIMALLOC_PATH%" 7z x v1.7.1.zip -oC:\cache\ -- cmd: IF %ANDROID%==false IF NOT EXIST "%MIMALLOC_PATH%"\out msbuild "%MIMALLOC_PATH%"\ide\vs2017\mimalloc-override.vcxproj /p:Configuration=Release /m -- cmd: IF %NAME%==android IF NOT EXIST C:\ndk\android-ndk-r19c\toolchains\llvm\prebuilt\windows-x86_64 appveyor DownloadFile https://dl.google.com/android/repository/android-ndk-r19c-windows-x86_64.zip -- cmd: IF %NAME%==android IF NOT EXIST C:\ndk\android-ndk-r19c\toolchains\llvm\prebuilt\windows-x86_64 7z x android-ndk-r19c-windows-x86_64.zip -oC:\ndk -- cmd: IF %NAME%==android set PATH=C:\ndk\android-ndk-r19c\toolchains\llvm\prebuilt\windows-x86_64\bin;%PATH% +- cmd: IF %ONNX%==true set PATH=%CUDA_PATH%\bin;%PATH% +- cmd: set PATH=C:\Python310;C:\Python310\scripts;%PATH% +#- cmd: pip3 install --upgrade meson==0.55.3 +- cmd: set MIMALLOC_PATH=C:\cache\mimalloc-1.8.7 +- cmd: IF %ANDROID%==false IF NOT EXIST "%MIMALLOC_PATH%" appveyor DownloadFile https://github.com/microsoft/mimalloc/archive/refs/tags/v1.8.7.zip +- cmd: IF %ANDROID%==false IF NOT EXIST "%MIMALLOC_PATH%" 7z x v1.8.7.zip -oC:\cache\ +- cmd: IF %ANDROID%==false IF NOT EXIST "%MIMALLOC_PATH%"\out msbuild "%MIMALLOC_PATH%"\ide\vs2019\mimalloc-override.vcxproj /p:Configuration=Release /m +- cmd: IF %NAME%==android IF NOT EXIST C:\ndk\android-ndk-r27c\toolchains\llvm\prebuilt\windows-x86_64 appveyor DownloadFile https://dl.google.com/android/repository/android-ndk-r27c-windows.zip +- cmd: IF %NAME%==android IF NOT EXIST C:\ndk\android-ndk-r27c\toolchains\llvm\prebuilt\windows-x86_64 7z x android-ndk-r27c-windows.zip -oC:\ndk +- cmd: IF %NAME%==android set PATH=C:\ndk\android-ndk-r27c\toolchains\llvm\prebuilt\windows-x86_64\bin;%PATH% - cmd: IF %NAME%==android sed "s/clang+*/&.cmd/" cross-files/aarch64-linux-android >crossfile-aarch64 - cmd: IF %NAME%==android IF NOT EXIST C:\cache\OpenBLAS\android-aarch64 appveyor DownloadFile https://github.com/borg323/OpenBLAS/releases/download/android-0.3.27/openblas-android-aarch64.zip - cmd: IF %NAME%==android IF NOT EXIST C:\cache\OpenBLAS\android-aarch64 7z x openblas-android-aarch64.zip -oC:\cache\OpenBLAS @@ -97,18 +115,21 @@ install: - cmd: touch -t 201801010000.00 c:\cache\%NET%.pb.gz - cmd: IF %GTEST%==true IF NOT EXIST C:\cache\syzygy mkdir C:\cache\syzygy - cmd: IF %GTEST%==true cd C:\cache\syzygy -- cmd: IF %GTEST%==true IF NOT EXIST KQvK.rtbz curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5/K{P,N,R,B,Q}vK.rtb{w,z} -- cmd: IF %GTEST%==true IF NOT EXIST KQQvK.rtbz curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5/K{P,N,R,B,Q}{P,N,R,B,Q}vK.rtb{w,z} -- cmd: IF %GTEST%==true IF NOT EXIST KQvKQ.rtbz curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5/K{P,N,R,B,Q}vK{P,N,R,B,Q}.rtb{w,z} +- cmd: IF %GTEST%==true IF NOT EXIST KQvK.rtbz curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5-dtz/K{P,N,R,B,Q}vK.rtbz +- cmd: IF %GTEST%==true IF NOT EXIST KQQvK.rtbz curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5-dtz/K{P,N,R,B,Q}{P,N,R,B,Q}vK.rtbz +- cmd: IF %GTEST%==true IF NOT EXIST KQvKQ.rtbz curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5-dtz/K{P,N,R,B,Q}vK{P,N,R,B,Q}.rtbz +- cmd: IF %GTEST%==true IF NOT EXIST KQvK.rtbw curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5-wdl/K{P,N,R,B,Q}vK.rtbw +- cmd: IF %GTEST%==true IF NOT EXIST KQQvK.rtbw curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5-wdl/K{P,N,R,B,Q}{P,N,R,B,Q}vK.rtbw +- cmd: IF %GTEST%==true IF NOT EXIST KQvKQ.rtbw curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5-wdl/K{P,N,R,B,Q}vK{P,N,R,B,Q}.rtbw - cmd: cd C:\projects\lc0 cache: - C:\cache - - 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0' + - 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1' - 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1' + - 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.9 -> appveyor.yml' - C:\projects\lc0\subprojects\packagecache - - C:\ndk\android-ndk-r19c\toolchains\llvm\prebuilt\windows-x86_64 + - C:\ndk\android-ndk-r27c\toolchains\llvm\prebuilt\windows-x86_64 before_build: -- cmd: git submodule update --init --recursive - cmd: IF %BLAS%==true (echo.#define DEFAULT_MAX_PREFETCH 0 & echo.#define DEFAULT_TASK_WORKERS 0) > params_override.h - cmd: IF %ANDROID%==true (echo.#define DEFAULT_MAX_PREFETCH 0 & echo.#define DEFAULT_TASK_WORKERS 0) > params_override.h - cmd: SET BUILD_BLAS=%BLAS% @@ -123,8 +144,9 @@ before_build: - cmd: IF %CUDA%==true SET F16C=false - cmd: SET EXTRA= - cmd: IF %ANDROID%==false SET EXTRA=-Db_vscrt=md -- cmd: IF %ONNX_DML%==true SET EXTRA=-Db_vscrt=md -Donnx_libdir=C:\cache\%ONNX_NAME%\lib -Donnx_include=C:\cache\%ONNX_NAME%\include -- cmd: IF %ANDROID%==false meson build --backend vs2017 --buildtype release -Dgtest=%GTEST% -Dopencl=%OPENCL% -Dblas=%BUILD_BLAS% -Ddnnl=true -Ddx=%DX% -Dcudnn=%CUDNN% -Donednn=%ONEDNN% -Dispc_native_only=false -Dnative_cuda=false -Dpopcnt=%POPCNT% -Df16c=%F16C% -Dcudnn_include="%CUDA_PATH%\include","%CUDA_PATH%\cuda\include" -Dcudnn_libdirs="%CUDA_PATH%\lib\x64","%CUDA_PATH%\cuda\lib\x64" -Dopenblas_include="%PKG_FOLDER%\OpenBLAS\dist64\include" -Dopenblas_libdirs="%PKG_FOLDER%\OpenBLAS\dist64\lib" -Ddnnl_dir="%PKG_FOLDER%\%DNNL_NAME%" -Dopencl_include="%PKG_FOLDER%\opencl-nug.0.777.77\build\native\include" -Dopencl_libdirs="%PKG_FOLDER%\opencl-nug.0.777.77\build\native\lib\x64" -Ddefault_library=static -Dmalloc=mimalloc -Dmimalloc_libdir="%MIMALLOC_PATH%"\out\msvc-x64\Release %EXTRA% +- cmd: IF %ONNX%==true SET EXTRA=-Db_vscrt=md -Donnx_libdir=C:\cache\%ONNX_NAME%\runtimes\win-x64\native\ -Donnx_include=C:\cache\%ONNX_NAME%\build\native\include -Ddefault_backend=onnx-trt -Dplain_cuda=false +- cmd: IF %NAME%==gpu-nvidia-cuda12 SET EXTRA=-Db_vscrt=md -Dcutlass=true -Dcutlass_include=C:\cache\cutlass-2.11.0\include +- cmd: IF %ANDROID%==false meson build --backend vs2019 --buildtype release -Dgtest=false -Dopencl=%OPENCL% -Dblas=%BUILD_BLAS% -Ddnnl=true -Ddx=%DX% -Dcudnn=%CUDNN% -Donednn=%ONEDNN% -Dispc_native_only=false -Dnative_cuda=false -Dpopcnt=%POPCNT% -Df16c=%F16C% -Dcudnn_include="%CUDA_PATH%\include","%CUDA_PATH%\cuda\include" -Dcudnn_libdirs="%CUDA_PATH%\lib\x64","%CUDA_PATH%\cuda\lib\x64" -Dopenblas_include="%PKG_FOLDER%\OpenBLAS\dist64\include" -Dopenblas_libdirs="%PKG_FOLDER%\OpenBLAS\dist64\lib" -Ddnnl_dir="%PKG_FOLDER%\%DNNL_NAME%" -Dopencl_include="%PKG_FOLDER%\opencl-nug.0.777.77\build\native\include" -Dopencl_libdirs="%PKG_FOLDER%\opencl-nug.0.777.77\build\native\lib\x64" -Ddefault_library=static -Dmalloc=mimalloc -Dmimalloc_libdir="%MIMALLOC_PATH%"\out\msvc-x64\Release %EXTRA% - cmd: IF %ANDROID%==true meson arm64-v8a --buildtype release -Dgtest=false -Dopenblas_include="%PKG_FOLDER%\OpenBLAS\android-aarch64\include" -Dopenblas_libdirs="%PKG_FOLDER%\OpenBLAS\android-aarch64\lib" -Dembed=%EMBED% -Ddefault_library=static --cross-file crossfile-aarch64 - cmd: IF %ANDROID%==true meson armeabi-v7a --buildtype release -Dgtest=false -Dopenblas_include="%PKG_FOLDER%\OpenBLAS\android-armv7a\include" -Dopenblas_libdirs="%PKG_FOLDER%\OpenBLAS\android-armv7a\lib" -Dembed=%EMBED% -Ddefault_library=static --cross-file crossfile-armv7a -Dispc=false -Dneon=false build_script: @@ -136,7 +158,7 @@ after_build: - cmd: IF %APPVEYOR_REPO_TAG%==true IF %ANDROID%==true call scripts\appveyor_android_package.cmd - cmd: cd C:\projects\lc0 artifacts: - - path: build/lc0.exe + - path: /build/lc0*.exe/ name: lc0-$(NAME) - path: arm64-v8a/lc0 name: lc0-android-arm64-v8a @@ -166,6 +188,7 @@ deploy: test_script: - cmd: IF %GTEST%==true cd build - cmd: IF %GTEST%==true xcopy /s /i C:\cache\syzygy syzygy +- cmd: IF %GTEST%==true IF %ONNX%==true copy %PKG_FOLDER%\%ONNX_NAME%\runtimes\win-x64\native\onnxruntime.dll - cmd: IF %GTEST%==true meson test --print-errorlogs - cmd: cd C:\projects\lc0 on_finish: diff --git a/build-sycl.cmd b/build-sycl.cmd new file mode 100644 index 0000000000..7f4d626d77 --- /dev/null +++ b/build-sycl.cmd @@ -0,0 +1,61 @@ +@echo off +setlocal + +rem 1. Set the following for the options you want to build. +rem SYCL can be off, l0, amd or nvidia. +set SYCL=l0 +set CUDNN=true +set CUDA=true +set DX12=false +set OPENCL=false +set MKL=false +set DNNL=true +set OPENBLAS=false +set EIGEN=false +set TEST=false + +rem 2. Edit the paths for the build dependencies. +set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0 +set CUDNN_PATH=%CUDA_PATH% +set OPENBLAS_PATH=C:\OpenBLAS +set MKL_PATH=C:\Program Files (x86)\Intel\oneAPI\mkl\latest\ +set DNNL_PATH=C:\Program Files (x86)\Intel\oneAPI\dnnl\latest\cpu_iomp +set OPENCL_LIB_PATH=%CUDA_PATH%\lib\x64 +set OPENCL_INCLUDE_PATH=%CUDA_PATH%\include + +rem 3. In most cases you won't need to change anything further down. +echo Deleting build directory: +rd /s build + +rem Use cl for C files to get a resource compiler as needed for zlib. +set CC=cl +set CXX=icx + +set BLAS=true +if %MKL%==false if %DNNL%==false if %OPENBLAS%==false if %EIGEN%==false set BLAS=false + +if "%CUDA_PATH%"=="%CUDNN_PATH%" ( + set CUDNN_LIB_PATH=%CUDNN_PATH%\lib\x64 + set CUDNN_INCLUDE_PATH=%CUDNN_PATH%\include +) else ( + set CUDNN_LIB_PATH=%CUDA_PATH%\lib\x64,%CUDNN_PATH%\lib\x64 + set CUDNN_INCLUDE_PATH=%CUDA_PATH%\include,%CUDNN_PATH%\include +) + +if %CUDNN%==true set PATH=%CUDA_PATH%\bin;%PATH% + +meson setup build --buildtype release -Ddx=%DX12% -Dcudnn=%CUDNN% -Dplain_cuda=%CUDA% ^ +-Dopencl=%OPENCL% -Dblas=%BLAS% -Dmkl=%MKL% -Dopenblas=%OPENBLAS% -Ddnnl=%DNNL% -Dgtest=%TEST% ^ +-Dcudnn_include="%CUDNN_INCLUDE_PATH%" -Dcudnn_libdirs="%CUDNN_LIB_PATH%" ^ +-Dmkl_include="%MKL_PATH%\include" -Dmkl_libdirs="%MKL_PATH%\lib\intel64" -Ddnnl_dir="%DNNL_PATH%" ^ +-Dopencl_libdirs="%OPENCL_LIB_PATH%" -Dopencl_include="%OPENCL_INCLUDE_PATH%" ^ +-Dopenblas_include="%OPENBLAS_PATH%\include" -Dopenblas_libdirs="%OPENBLAS_PATH%\lib" ^ +-Ddefault_library=static -Dsycl=%SYCL% -Db_vscrt=md + +if errorlevel 1 exit /b + +pause + +cd build + +ninja \ No newline at end of file diff --git a/build.cmd b/build.cmd index 9646029711..262b5ee00e 100644 --- a/build.cmd +++ b/build.cmd @@ -2,7 +2,7 @@ setlocal rem 1. Set the following for the options you want to build. -set CUDNN=true +set CUDNN=false set CUDA=true set DX12=false set OPENCL=false @@ -11,15 +11,24 @@ set DNNL=false set OPENBLAS=false set EIGEN=false set TEST=false +set CUTLASS=true + +if "%CUDA%"=="true" ( + if not defined CUDA_PATH ( + echo WARNING: CUDA_PATH environment variable not found. Using default value. + ) else ( + echo CUDA_PATH found in system environment: "%CUDA_PATH%" + ) +) rem 2. Edit the paths for the build dependencies. -set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0 +if not defined CUDA_PATH set CUDA_PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.9 set CUDNN_PATH=%CUDA_PATH% +set OPENCL_LIB_PATH=%CUDA_PATH%\lib\x64 +set OPENCL_INCLUDE_PATH=%CUDA_PATH%\include set OPENBLAS_PATH=C:\OpenBLAS set MKL_PATH=C:\Program Files (x86)\IntelSWTools\compilers_and_libraries\windows\mkl set DNNL_PATH=C:\dnnl_win_1.1.1_cpu_vcomp -set OPENCL_LIB_PATH=%CUDA_PATH%\lib\x64 -set OPENCL_INCLUDE_PATH=%CUDA_PATH%\include rem 3. In most cases you won't need to change anything further down. echo Deleting build directory: @@ -63,6 +72,7 @@ meson setup build --backend %backend% --buildtype release -Ddx=%DX12% -Dcudnn=%C -Dmkl_include="%MKL_PATH%\include" -Dmkl_libdirs="%MKL_PATH%\lib\intel64" -Ddnnl_dir="%DNNL_PATH%" ^ -Dopencl_libdirs="%OPENCL_LIB_PATH%" -Dopencl_include="%OPENCL_INCLUDE_PATH%" ^ -Dopenblas_include="%OPENBLAS_PATH%\include" -Dopenblas_libdirs="%OPENBLAS_PATH%\lib" ^ +-Dcutlass="%CUTLASS%" ^ -Ddefault_library=static if errorlevel 1 exit /b diff --git a/build.sh b/build.sh index fa30e5c3df..8eb935c926 100755 --- a/build.sh +++ b/build.sh @@ -24,7 +24,7 @@ if [ -f "${BUILDDIR}/build.ninja" ] then "${MESON}" configure "${BUILDDIR}" -Dbuildtype="${BUILDTYPE}" -Dprefix="${INSTALL_PREFIX:-/usr/local}" "$@" else - "${MESON}" "${BUILDDIR}" --buildtype "${BUILDTYPE}" --prefix "${INSTALL_PREFIX:-/usr/local}" "$@" + "${MESON}" setup "${BUILDDIR}" --buildtype "${BUILDTYPE}" --prefix "${INSTALL_PREFIX:-/usr/local}" "$@" fi "${MESON}" compile -C "${BUILDDIR}" diff --git a/build_rescorer.cmd b/build_rescorer.cmd index ee20db0d58..9d3897c543 100644 --- a/build_rescorer.cmd +++ b/build_rescorer.cmd @@ -23,7 +23,7 @@ if exist "C:\Program Files\Microsoft Visual Studio\2022" ( set backend=vs2017 ) -meson build --backend %backend% --buildtype release -Drescorer=true -Dlc0=false -Dgtest=false -Ddefault_library=static +meson setup build --backend %backend% --buildtype release -Drescorer=true -Dlc0=false -Dgtest=false -Ddefault_library=static if errorlevel 1 exit /b @@ -32,4 +32,4 @@ pause cd build msbuild /m /p:Configuration=Release /p:Platform=x64 /p:WholeProgramOptimization=true ^ -/p:PreferredToolArchitecture=x64 rescorer.sln /filelogger +/p:PreferredToolArchitecture=x64 lc0.sln /filelogger diff --git a/changelog.txt b/changelog.txt index 5d208674ac..cdfec68116 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,101 @@ -v0.31.0-rc1 (2024-03-25) +v0.32.0 (2025-08-21) +~~~~~~~ +* Support for building with cuda 13. +* README update. +* Build system improvements. + +v0.32.0-rc2 (2025-08-12) +~~~~~~~ +* Fix for onnx-trt bug, where the wrong network could be used from the cache. +* Added code to detect RPE nets and give an error instead of bad results. +* Better instructions in the readme and install script for onnx-trt. +* Made `UCI_ShowWDL` again off by default again as some GUIs have issues. +* Fixed a long standing issue when compiled with `-ffast-math` (or `icx -O3`). +* Several improvements to the sycl backend. +* Several improvements to the metal backend. +* Refactored the rescorer code and training data header to make them usable by + external tools. +* Relaxed cuda/cudnn version checks so that no warnings are shown for mismatched + versions that are supported. +* Several build system updates. +* Assorted small fixes and improvements. + +v0.32.0-rc1 (2025-07-18) +~~~~~~~ +The code has been reorganized and undergone major changes. Therefore this +changelog will be less detailed and describe the changes in major groups. +* We have a new search API that allows search algorithms to co-exist. Currently + available are `classic` (the default), `dag-preview` (more later), + `valuehead` and `policyhead`. The default algorithm can be changed either at + build time by the `default_search` option or by renaming the executable to + include the algorithm name (e.g. lc0-valuehead). +* We also have a new backend interface that is chess oriented and not tied to + the network architecture. The existing backends still use the old interface + through a wrapper. +* The source code is reorganized, with a more logical directory structure. +* The original search was ported to the new search and backend interfaces and + is renamed to `classic`. This has allowed some streamlining and + simplifications. +* The `dag-preview` search is the DAG algorithm that lived in a separate branch + up to now. It hasn't been so well tested, that's why it has "preview" in its + name for now, but lives in the `src/search/dag-classic` directory. +* The `valuehead` search replaces `ValueOnly` mode and selects the move with the + best value head evaluation. +* The `policyhead` search is equivalent to a single node search, selecting the + best move using just the policy head. +* The new `default_backend` build option allows to override the fixed priority + for the backend used by default. +* The new `native_arch` build option to override the `-march=native` compiler + default for linux release builds, to help with distribution package creation. +* We have a new `sycl` backend that will work with amd, intel and nvidia gpus. +* There is also a new `onnx-trt` backend, using tensorrt on nvidia gpus. +* Support simple/normal/pro mode in options was cleaned up, using a common + mechanism. +* Added the `wait` uci extension command to allow running simple tests from the + command line. +* Removed the `fen` uci extension command as it was unnecessarily complicating + things. +* Some preliminary fp8 support was added for onnx and xla. This is not + functional, just there to make experimentation easier. +* Several build system changes and improvements. +* We now generate binaries for cuda 12, onnx-trt and macos. +* Support for using lc0 with openbench. +* New `bench` mode for a quicker benchmark. +* Assorted small fixes and improvements. + +v0.31.2 (2024-10-20) +~~~~~~~ +* Updated the WDL_mu centipawn fallback. +* Fix for build issues with newer Linux c++ libraries. +* Fix for an XLA Mish bug. +* Minor README.md update. + +v0.31.1 (2024-08-11) +~~~~~~~ +* Make WDL_mu score type work as intended. +* Fix macos CI builds. + +v0.31.0 (2024-06-16) +~~~~~~~ +* No changes from rc3. + +v0.31.0-rc3 (2024-05-29) +~~~~~~~ +* The `WDLDrawRateTarget` option now accepts the value 0 (new default) to retain + raw WDL values if `WDLCalibrationElo` is set to 0 (default). +* Improvements to the verbose move stats if `WDLEvalObjectivity` is used. +* The centipawn score is displayed by default for old nets without WDL output. +* Some build system improvements. + +v0.31.0-rc2 (2024-04-16) +~~~~~~~ +* Changed cuda compilation options to use `-arch=native` or `-arch=all-major` + if no specific version is requested, with fallback for older cuda that don't + support those options. +* Updated android builds to use openblas 0.3.27. +* A few small fixes. + +v0.31.0-rc1 (2024-03-25) ~~~~~~~ * The blas, cuda, eigen, metal and onnx backends now have support for multihead network architecture and can run BT3/BT4 nets. @@ -39,6 +136,9 @@ natively higher draw rates. * Made the WDL Rescale sharpness limit configurable via the `--wdl-max-s` hidden option. +* The search task workers can be set automatically, to either 0 for cpu backends + or up to 4 depending on the number of cpu cores. This is enabled by + `--task-workers=-1` (the new default). * Several assorted fixes and code cleanups. v0.30.0 (2023-07-21) diff --git a/cross-files/aarch64-linux-android b/cross-files/aarch64-linux-android index 4a55d838de..75e9e63de9 100644 --- a/cross-files/aarch64-linux-android +++ b/cross-files/aarch64-linux-android @@ -1,5 +1,5 @@ -# Tested with Android NDK r19c, default toolchain +# Tested with Android NDK r27c, default toolchain # Targeting API level 21 # Set the toolchain path on your environment @@ -17,8 +17,8 @@ cpp_link_args = ['-llog', '-static-libstdc++'] [binaries] c = 'aarch64-linux-android21-clang' cpp = 'aarch64-linux-android21-clang++' -ar = 'aarch64-linux-android-ar' -strip = 'aarch64-linux-android-strip' -ld = 'aarch64-linux-android-ld' -ranlib = 'aarch64-linux-android-ranlib' -as = 'aarch64-linux-android-as' +ar = 'llvm-ar' +strip = 'llvm-strip' +ld = 'ld' +ranlib = 'llvm-ranlib' +as = 'aarch64-linux-android21-clang' diff --git a/cross-files/armv7a-linux-android b/cross-files/armv7a-linux-android index 16b3e93f90..3fed7aee8b 100644 --- a/cross-files/armv7a-linux-android +++ b/cross-files/armv7a-linux-android @@ -1,5 +1,5 @@ -# Tested with Android NDK r19c, default toolchain +# Tested with Android NDK r27c, default toolchain # Targeting API level 21 # When targeting API levels < 24 the build fails unless _FILE_OFFSET_BITS is unset. @@ -24,8 +24,8 @@ cpp_link_args = ['-llog', '-static-libstdc++'] [binaries] c = 'armv7a-linux-androideabi21-clang' cpp = 'armv7a-linux-androideabi21-clang++' -ar = 'arm-linux-androideabi-ar' -strip = 'arm-linux-androideabi-strip' -ld = 'arm-linux-androideabi-ld' -ranlib = 'arm-linux-androideabi-ranlib' -as = 'arm-linux-androideabi-as' +ar = 'llvm-ar' +strip = 'llvm-strip' +ld = 'ld' +ranlib = 'llvm-ranlib' +as = 'armv7a-linux-androideabi21-clang' diff --git a/dist/README-onnx-dml.txt b/dist/README-onnx-dml.txt index 5e34b3eb52..c86029e5cb 100644 --- a/dist/README-onnx-dml.txt +++ b/dist/README-onnx-dml.txt @@ -7,7 +7,7 @@ neural network, specifically those of the LeelaChessZero project To run this version you will most likely need a very recent DirectML dll, which you can get by running the included `install.cmd` script. Alternatively, you can download the currently latest nuget installer package from -. +. If you don't know how to use nuget installer packages, you can change the extension to .zip and open it as a normal zip file, the dll you need is `/bin/x64-win/DirectML.dll`. diff --git a/dist/README-onnx-trt.txt b/dist/README-onnx-trt.txt new file mode 100644 index 0000000000..8a50b2689e --- /dev/null +++ b/dist/README-onnx-trt.txt @@ -0,0 +1,88 @@ +# Lc0 + +Lc0 is a UCI-compliant chess engine designed to play chess via +neural network, specifically those of the LeelaChessZero project +(https://lczero.org). + +# Installation + +Summary: run `instrall.cmd` and follow the instructions. + +To run this version you will also need several dll files from NVIDA's +CUDA, cuDNN and TensorRT. Those dlls can either be on the system path +from a separate installation of these libraries, or can be placed +directly in the Lc0 folder. Either way, you will get an error message +for any that isn't found. + +The dlls needed are the following: + +1. CUDA +* cublas64_12.dll +* cublasLt64_12.dll +* cudart64_12.dll +* cufft64_11.dll + +2. cuDNN +* cudnn64_9.dll +* cudnn_graph64_9.dll + +3. TensorRT: +* nvinfer_10.dll +* nvinfer_builder_resource_10.dll +* nvinfer_plugin_10.dll +* nvonnxparser_10.dll + +The install.cmd script included in this package will download the +CUDA and cuDNN files needed and will open the TensorRT download page +using your browser. If it fails, you can download the files manually +using the following addresses, the dlls are in the `bin` directory +in the CUDA/cuDNN zips and the `lib` directory in the TensorRT zip. + +* https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-12.9.79-archive.zip +* https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-12.9.1.4-archive.zip +* https://developer.download.nvidia.com/compute/cuda/redist/libcufft/windows-x86_64/libcufft-windows-x86_64-11.4.1.4-archive.zip +* https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/windows-x86_64/cudnn-windows-x86_64-9.11.0.98_cuda12-archive.zip +* https://developer.nvidia.com/tensorrt/download/10x#trt1012 + +The TensorRT link will take you to the download page, after +registering go to the "TensorRT 10.12 GA for x86_64 Architecture" +section and get the "TensorRT 10.12 GA for Windows 10, 11, +Server 2022 and CUDA 12.0 to 12.9 ZIP Package". + +Finally, if Lc0 still won't run, get the latest Visual C++ +redistributable from: https://aka.ms/vs/17/release/vc_redist.x64.exe + +# Running + +When running Lc0 with a new network file, it will take some time to +create the optimized model to use. This is normal. The model will be +cached for future runs in the `trt_cache` folder, so next time it will +be faster. If you want to experiment you can rename the `trt_cache` +folder and rerun, sometimes TensorRT will generate a different model +that may be faster. Moreover, if you are having issues, you can +delete/rename the cache and rerun. + +# License + +Leela Chess is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +Leela Chess is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Leela Chess. If not, see . + +Additional permission under GNU GPL version 3 section 7 + +If you modify this Program, or any covered work, by linking or +combining it with NVIDIA Corporation's libraries from the NVIDIA CUDA +Toolkit and the NVIDIA CUDA Deep Neural Network library (or a +modified version of those libraries), containing parts covered by the +terms of the respective license agreement, the licensors of this +Program grant you additional permission to convey the resulting work. + diff --git a/dist/install-cuda_12_9.cmd b/dist/install-cuda_12_9.cmd new file mode 100644 index 0000000000..c5a253093b --- /dev/null +++ b/dist/install-cuda_12_9.cmd @@ -0,0 +1,43 @@ +@echo off +where /q tar +if errorlevel 1 goto error + +cd /d %~dp0 + +cls +echo Installing the CUDA dlls required by the Lc0 cuda backend. + +echo 1/4. Downloading cudart. +curl -# --ssl-no-revoke -o tmp_cudart.zip https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-12.9.37-archive.zip" +if errorlevel 1 goto error + +echo 2/4. Extracting files. +tar -xzOf tmp_cudart.zip cuda_cudart-windows-x86_64-12.9.37-archive/bin/cudart64_12.dll >cudart64_12.dll +if errorlevel 1 goto error + +tar -xzOf tmp_cudart.zip cuda_cudart-windows-x86_64-12.9.37-archive/LICENSE >CUDA.txt + +del /q tmp_cudart.zip + +echo 3/4. Downloading cublas. +curl -# --ssl-no-revoke -o tmp_cublas.zip https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-12.9.0.13-archive.zip" +if errorlevel 1 goto error + +echo 4/4. Extracting files. +tar -xzOf tmp_cublas.zip libcublas-windows-x86_64-12.9.0.13-archive/bin/cublas64_12.dll >cublas64_12.dll +if errorlevel 1 goto error + +tar -xzOf tmp_cublas.zip libcublas-windows-x86_64-12.9.0.13-archive/bin/cublasLt64_12.dll >cublasLt64_12.dll +if errorlevel 1 goto error + +del /q tmp_cublas.zip + +echo Installation successful. +pause +exit /b + +:error +cls +echo Installation failed - you will have to download cuda 12.9 yourself. +pause + diff --git a/dist/install-dml.cmd b/dist/install-dml.cmd index ca93411a55..d223925866 100644 --- a/dist/install-dml.cmd +++ b/dist/install-dml.cmd @@ -6,7 +6,7 @@ cd /d %~dp0 cls echo Installing the DirectML.dll version required by the Lc0 onnx-dml backend. -curl -# --ssl-no-revoke -o tmp_directml.zip https://globalcdn.nuget.org/packages/microsoft.ai.directml.1.10.0.nupkg" +curl -# --ssl-no-revoke -o tmp_directml.zip https://globalcdn.nuget.org/packages/microsoft.ai.directml.1.15.4.nupkg" if errorlevel 1 goto error tar -xzOf tmp_directml.zip bin/x64-win/DirectML.dll >DirectML.dll diff --git a/dist/install-trt.cmd b/dist/install-trt.cmd new file mode 100644 index 0000000000..3538c30b66 --- /dev/null +++ b/dist/install-trt.cmd @@ -0,0 +1,99 @@ +@echo off +where /q tar +if errorlevel 1 goto error + +cd /d %~dp0 + +cls + +echo This script will download and install the CUDA/cuDNN/tensorRT dlls required by the Lc0 onnx-trt backend. +echo( +echo If you are using a metered internet connection, be aware the download will be arounbd 3 Gb. +echo( +pause + +echo Installing the CUDA dlls required by the Lc0 onnx-trt backend. + +echo 1/6. Downloading cudart. +curl -# --ssl-no-revoke -o tmp_cudart.zip https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-12.9.79-archive.zip" +if errorlevel 1 goto error + +echo 2/6. Extracting files. +tar -xzOf tmp_cudart.zip cuda_cudart-windows-x86_64-12.9.79-archive/bin/cudart64_12.dll >cudart64_12.dll +if errorlevel 1 goto error + +tar -xzOf tmp_cudart.zip cuda_cudart-windows-x86_64-12.9.79-archive/LICENSE >CUDA.txt + +del /q tmp_cudart.zip + +echo 3/6. Downloading cublas. +curl -# --ssl-no-revoke -o tmp_cublas.zip https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-x86_64/libcublas-windows-x86_64-12.9.1.4-archive.zip" +if errorlevel 1 goto error + +echo 4/6. Extracting files. +tar -xzOf tmp_cublas.zip libcublas-windows-x86_64-12.9.1.4-archive/bin/cublas64_12.dll >cublas64_12.dll +if errorlevel 1 goto error + +tar -xzOf tmp_cublas.zip libcublas-windows-x86_64-12.9.1.4-archive/bin/cublasLt64_12.dll >cublasLt64_12.dll +if errorlevel 1 goto error + +del /q tmp_cublas.zip + +echo 5/6. Downloading cufft. +curl -# --ssl-no-revoke -o tmp_cufft.zip https://developer.download.nvidia.com/compute/cuda/redist/libcufft/windows-x86_64/libcufft-windows-x86_64-11.4.1.4-archive.zip" +if errorlevel 1 goto error + +echo 6/6. Extracting files. +tar -xzOf tmp_cufft.zip libcufft-windows-x86_64-11.4.1.4-archive/bin/cufft64_11.dll >cufft64_11.dll +if errorlevel 1 goto error + +del /q tmp_cufft.zip + +echo Installing the cuDNN dlls required by the Lc0 onnx-trt backend. + +echo 1/2. Downloading cudnn. +curl -# --ssl-no-revoke -o tmp_cudnn.zip https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/windows-x86_64/cudnn-windows-x86_64-9.11.0.98_cuda12-archive.zip" +if errorlevel 1 goto error + +echo 2/2. Extracting files. +tar -xzOf tmp_cudnn.zip cudnn-windows-x86_64-9.11.0.98_cuda12-archive/bin/cudnn64_9.dll >cudnn64_9.dll +if errorlevel 1 goto error + +tar -xzOf tmp_cudnn.zip cudnn-windows-x86_64-9.11.0.98_cuda12-archive/bin/cudnn_graph64_9.dll >cudnn_graph64_9.dll +if errorlevel 1 goto error + +tar -xzOf tmp_cudnn.zip cudnn-windows-x86_64-9.11.0.98_cuda12-archive/LICENSE >CUDNN.txt + +del /q tmp_cudnn.zip + +echo Installing the tensorRT dlls required by the Lc0 onnx-trt backend. + +echo 1/2. Downloading tensorRT. +curl -# --ssl-no-revoke -o tmp_tensorrt.zip https://developer.download.nvidia.com/compute/machine-learning/tensorrt/10.12.0/zip/TensorRT-10.12.0.36.Windows.win10.cuda-12.9.zip" +if errorlevel 1 goto error + +echo 2/2. Extracting files. +tar -xzOf tmp_tensorrt.zip TensorRT-10.12.0.36/lib/nvinfer_10.dll >nvinfer_10.dll +if errorlevel 1 goto error + +tar -xzOf tmp_tensorrt.zip TensorRT-10.12.0.36/lib/nvinfer_builder_resource_10.dll >nvinfer_builder_resource_10.dll +if errorlevel 1 goto error + +tar -xzOf tmp_tensorrt.zip TensorRT-10.12.0.36/lib/nvinfer_plugin_10.dll >nvinfer_plugin_10.dll +if errorlevel 1 goto error + +tar -xzOf tmp_tensorrt.zip TensorRT-10.12.0.36/lib/nvonnxparser_10.dll >nvonnxparser_10.dll +if errorlevel 1 goto error + +tar -xzOf tmp_tensorrt.zip TensorRT-10.12.0.36/doc/Readme.txt >TENSORRT.txt + +del /q tmp_tensorrt.zip + +pause +exit /b + +:error +cls +echo Installation failed - see the README for alternative download instructions. +pause + diff --git a/libs/lczero-common b/libs/lczero-common deleted file mode 160000 index 55e1b382ef..0000000000 --- a/libs/lczero-common +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 55e1b382efadd57903e37f2a2e29caef3ea85799 diff --git a/meson.build b/meson.build index 27d6c6cb63..63613fb618 100644 --- a/meson.build +++ b/meson.build @@ -15,30 +15,22 @@ # along with Leela Chess. If not, see . project('lc0', 'cpp', - default_options : ['cpp_std=c++17', 'b_ndebug=if-release', 'warning_level=3', 'b_lto=true', 'b_vscrt=mt'], - meson_version: '>=0.55') + default_options : ['cpp_std=c++20', 'b_ndebug=if-release', 'warning_level=3', 'b_lto=true', 'b_vscrt=mt'], + meson_version: '>=0.57') -cc = meson.get_compiler('cpp') - -if not cc.has_header('optional') or not cc.has_header('string_view') - error('Lc0 requires a compiler supporting C++17, for example g++ v8.0, ' + - 'clang v5.0 or later (with C++17 stdlib) and Visual Studio 2017 or ' + - 'later.') -endif +fs = import('fs') -if not cc.has_header('charconv') - warning('Your compiler or library does not have full C++17 support. ' + - 'See the README for compilers that are known to be working. ' + - 'This will become an error in the future.') -endif +cc = meson.get_compiler('cpp') if cc.get_id() == 'clang' # Thread safety annotation add_project_arguments('-Wthread-safety', language : 'cpp') endif -if cc.get_id() == 'clang' or cc.get_id() == 'gcc' +if cc.get_id() != 'msvc' if get_option('buildtype') == 'release' - add_project_arguments(cc.get_supported_arguments(['-march=native']), language : 'cpp') + if get_option('native_arch') + add_project_arguments(cc.get_supported_arguments(['-march=native']), language : 'cpp') + endif endif endif if cc.get_id() == 'msvc' @@ -47,6 +39,7 @@ if cc.get_id() == 'msvc' endif if host_machine.system() == 'windows' add_project_arguments('-DNOMINMAX', language : 'cpp') + add_project_arguments(cc.get_supported_arguments(['/source-charset:utf-8']), language : 'cpp') endif if ['arm', 'aarch64'].contains(host_machine.cpu_family()) if get_option('neon') @@ -69,37 +62,19 @@ includes += include_directories('third_party', is_system: true) compile_proto = find_program('scripts/compile_proto.py') gen = generator(compile_proto, output: ['@BASENAME@.pb.h'], arguments : [ - '--proto_path=@CURRENT_SOURCE_DIR@/libs/lczero-common', + '--proto_path=@CURRENT_SOURCE_DIR@', '--cpp_out=@BUILD_DIR@', '@INPUT@']) -# Handle submodules. -git = find_program('git', required: false) -if run_command('scripts/checkdir.py', 'libs/lczero-common/proto', check : false).returncode() != 0 - if git.found() - if run_command(git, 'status', check : false).returncode() == 0 - message('updating git submodule libs/lczero-common') - run_command(git, 'submodule', 'update', '--init', '--recursive', check : false) - else - message('cloning lczero-common.git into libs/lczero-common') - run_command(git, 'clone', '--depth=1', - 'https://github.com/LeelaChessZero/lczero-common.git', - 'libs/lczero-common/', check : false) - endif - else - error('Please install git to automatically fetch submodules or download the archives manually from GitHub.') - endif -endif - pb_files = [ 'src/utils/protomessage.cc', - gen.process('libs/lczero-common/proto/net.proto', - preserve_path_from : meson.current_source_dir() + '/libs/lczero-common/') + gen.process('proto/net.proto', preserve_path_from : meson.current_source_dir()) ] common_files += pb_files # Extract git short revision. short_rev = 'unknown' +git = find_program('git', required: false) if git.found() r = run_command(git, 'rev-parse', '--short', 'HEAD', check : false) if r.returncode() == 0 @@ -141,29 +116,30 @@ elif get_option('malloc') != '' endif # ONNX and HLO protobufs. -gen_proto_src = generator(compile_proto, output: ['@BASENAME@.pb.h'], - arguments : [ - '--proto_path=@CURRENT_SOURCE_DIR@/src', - '--cpp_out=@BUILD_DIR@', - '@INPUT@']) +files += gen.process('proto/onnx.proto', + preserve_path_from : meson.current_source_dir()) -files += gen_proto_src.process('src/neural/onnx/onnx.proto', - preserve_path_from : meson.current_source_dir() + '/src/') - -files += gen_proto_src.process('src/neural/xla/hlo.proto', - preserve_path_from : meson.current_source_dir() + '/src/') +files += gen.process('proto/hlo.proto', + preserve_path_from : meson.current_source_dir()) ############################################################################# ## Main files ############################################################################# common_files += [ - 'src/chess/bitboard.cc', 'src/chess/board.cc', + 'src/chess/gamestate.cc', 'src/chess/position.cc', 'src/chess/uciloop.cc', - 'src/mcts/node.cc', + 'src/neural/backend.cc', + 'src/neural/batchsplit.cc', 'src/neural/decoder.cc', 'src/neural/encoder.cc', + 'src/neural/factory.cc', + 'src/neural/loader.cc', + 'src/neural/register.cc', + 'src/neural/shared_params.cc', + 'src/neural/wrapper.cc', + 'src/search/classic/node.cc', 'src/syzygy/syzygy.cc', 'src/trainingdata/reader.cc', 'src/trainingdata/trainingdata.cc', @@ -181,49 +157,61 @@ common_files += [ ] files += [ - 'src/benchmark/backendbench.cc', - 'src/benchmark/benchmark.cc', + 'src/engine_loop.cc', 'src/engine.cc', - 'src/lc0ctl/describenet.cc', - 'src/lc0ctl/leela2onnx.cc', - 'src/lc0ctl/onnx2leela.cc', - 'src/mcts/params.cc', - 'src/mcts/search.cc', - 'src/mcts/stoppers/alphazero.cc', - 'src/mcts/stoppers/common.cc', - 'src/mcts/stoppers/factory.cc', - 'src/mcts/stoppers/legacy.cc', - 'src/mcts/stoppers/simple.cc', - 'src/mcts/stoppers/smooth.cc', - 'src/mcts/stoppers/stoppers.cc', - 'src/mcts/stoppers/timemgr.cc', - 'src/neural/cache.cc', - 'src/neural/factory.cc', - 'src/neural/loader.cc', - 'src/neural/network_check.cc', - 'src/neural/network_demux.cc', + 'src/neural/backends/network_check.cc', + 'src/neural/backends/network_demux.cc', + 'src/neural/backends/network_mux.cc', + 'src/neural/backends/network_random.cc', + 'src/neural/backends/network_record.cc', + 'src/neural/backends/network_rr.cc', + 'src/neural/backends/network_trivial.cc', + 'src/neural/memcache.cc', 'src/neural/network_legacy.cc', - 'src/neural/network_mux.cc', - 'src/neural/network_random.cc', - 'src/neural/network_record.cc', - 'src/neural/network_rr.cc', - 'src/neural/network_trivial.cc', 'src/neural/onnx/adapters.cc', 'src/neural/onnx/builder.cc', 'src/neural/onnx/converter.cc', 'src/neural/xla/hlo_builder.cc', 'src/neural/xla/onnx2hlo.cc', 'src/neural/xla/print_hlo.cc', + 'src/neural/xla/xla_tensor.cc', + 'src/search/classic/params.cc', + 'src/search/classic/search.cc', + 'src/search/classic/stoppers/alphazero.cc', + 'src/search/classic/stoppers/common.cc', + 'src/search/classic/stoppers/factory.cc', + 'src/search/classic/stoppers/legacy.cc', + 'src/search/classic/stoppers/simple.cc', + 'src/search/classic/stoppers/smooth.cc', + 'src/search/classic/stoppers/stoppers.cc', + 'src/search/classic/stoppers/timemgr.cc', + 'src/search/classic/wrapper.cc', + 'src/search/register.cc', 'src/selfplay/game.cc', 'src/selfplay/loop.cc', 'src/selfplay/multigame.cc', 'src/selfplay/tournament.cc', + 'src/tools/backendbench.cc', + 'src/tools/benchmark.cc', + 'src/tools/describenet.cc', + 'src/tools/leela2onnx.cc', + 'src/tools/onnx2leela.cc', 'src/utils/histogram.cc', 'src/utils/numa.cc', 'src/utils/weights_adapter.cc', ] + +files += [ + 'src/search/instamove/instamove.cc', +] + includes += include_directories('src') +deps += dependency('absl_flat_hash_map', + include_type: 'system', + fallback: ['abseil-cpp', 'absl_container_dep'], + default_options : ['warning_level=0', 'cpp_std=c++20']) + deps += dependency('threads') ############################################################################# @@ -235,6 +223,17 @@ else common_files += 'src/utils/filesystem.posix.cc' endif +############################################################################# +## DAG CLASSIC SEARCH +############################################################################# +if get_option('dag_classic') + files += [ + 'src/search/dag_classic/node.cc', + 'src/search/dag_classic/search.cc', + 'src/search/dag_classic/wrapper.cc', + ] +endif + ############################################################################# ## BACKENDS ############################################################################# @@ -248,7 +247,7 @@ if get_option('build_backends') tf_tensorflow_cc_lib = dependency('tensorflow_cc', required: false) if get_option('tensorflow') and tf_dl_lib.found() and tf_tensorflow_cc_lib.found() deps += [tf_dl_lib, tf_tensorflow_cc_lib] - files += 'src/neural/network_tf_cc.cc' + files += 'src/neural/backends/network_tf_cc.cc' has_backends = true endif @@ -319,7 +318,13 @@ if get_option('build_backends') endif - deps += dependency('eigen3', fallback: ['eigen', 'eigen_dep']).as_system() + eigen_dep = dependency('eigen3') + # Check for needed header, bad dependency seen in the widl. + if eigen_dep.found() and cc.has_header('Eigen/Core', dependencies: eigen_dep) + deps += eigen_dep.as_system() + else + deps += subproject('eigen').get_variable('eigen_dep').as_system() + endif ispc = find_program('ispc', required: false) ispc_arch = 'x86-64' @@ -373,25 +378,25 @@ if get_option('build_backends') endif blas_files = [ - 'src/neural/blas/convolution1.cc', - 'src/neural/blas/fully_connected_layer.cc', - 'src/neural/blas/se_unit.cc', - 'src/neural/blas/network_blas.cc', - 'src/neural/blas/winograd_convolution3.cc' + 'src/neural/backends/blas/convolution1.cc', + 'src/neural/backends/blas/fully_connected_layer.cc', + 'src/neural/backends/blas/se_unit.cc', + 'src/neural/backends/blas/network_blas.cc', + 'src/neural/backends/blas/winograd_convolution3.cc' ] shared_files = [ - 'src/neural/shared/activation.cc', - 'src/neural/shared/winograd_filter.cc', + 'src/neural/backends/shared/activation.cc', + 'src/neural/backends/shared/winograd_filter.cc', ] files += blas_files has_backends = true if get_option('ispc') and ispc.found() - files += iscp_gen.process('src/neural/blas/winograd_transform.ispc') - files += iscp_gen.process('src/neural/blas/layer_norm.ispc') - files += iscp_gen.process('src/neural/shared/activation.ispc') + files += iscp_gen.process('src/neural/backends/blas/winograd_transform.ispc') + files += iscp_gen.process('src/neural/backends/blas/layer_norm.ispc') + files += iscp_gen.process('src/neural/backends/shared/activation.ispc') add_project_arguments('-DUSE_ISPC', language : 'cpp') endif @@ -421,15 +426,15 @@ if get_option('build_backends') if get_option('opencl') and has_opencl opencl_files = [ - 'src/neural/opencl/network_opencl.cc', - 'src/neural/opencl/OpenCL.cc', - 'src/neural/opencl/OpenCLTuner.cc', - 'src/neural/opencl/OpenCLBuffers.cc', + 'src/neural/backends/opencl/network_opencl.cc', + 'src/neural/backends/opencl/OpenCL.cc', + 'src/neural/backends/opencl/OpenCLTuner.cc', + 'src/neural/backends/opencl/OpenCLBuffers.cc', ] shared_files = [ - 'src/neural/shared/activation.cc', - 'src/neural/shared/winograd_filter.cc', + 'src/neural/backends/shared/activation.cc', + 'src/neural/backends/shared/winograd_filter.cc', ] if not opencl_framework.found() @@ -447,48 +452,45 @@ if get_option('build_backends') ## cuDNN ## ~~~~~ cudnn_libdirs = get_option('cudnn_libdirs') + nvcc_paths = [] + foreach p : cudnn_libdirs + nvcc_paths += fs.parent(p) + '/bin/nvcc' + endforeach + nvcc_paths += ['nvcc', '/usr/local/cuda/bin/nvcc', '/opt/cuda/bin/nvcc'] + message('Looking for nvcc in: ' + ', '.join(nvcc_paths)) cu_blas = cc.find_library('cublas', dirs: cudnn_libdirs, required: false) cu_dnn = cc.find_library('cudnn', dirs: cudnn_libdirs, required: false) cu_dart = cc.find_library('cudart', dirs: cudnn_libdirs, required: false) - nvcc = find_program('nvcc', '/usr/local/cuda/bin/nvcc', '/opt/cuda/bin/nvcc', + nvcc = find_program(nvcc_paths, required: false) - - if (get_option('cudnn') or get_option('plain_cuda')) and cu_blas.found() and cu_dart.found() and nvcc.found() - deps += [cu_blas, cu_dart] - cuda_files = ['src/neural/cuda/layers.cc'] - if get_option('cudnn') and cu_dnn.found() - deps += cu_dnn - cuda_files += 'src/neural/cuda/network_cudnn.cc' - cuda_files += 'src/neural/cuda/network_cuda.cc' # To support newer nets. - add_project_arguments('-DUSE_CUDNN', language : 'cpp') - elif get_option('plain_cuda') - cuda_files += 'src/neural/cuda/network_cuda.cc' - endif + nvcc_ok = false + if get_option('nvcc') and nvcc.found() foreach d : get_option('cudnn_include') if run_command('scripts/checkdir.py', d, check : false).returncode() == 0 includes += include_directories(d, is_system: true) endif endforeach - includes += include_directories('src/neural/cuda/') - - cuda_arguments = ['-c', '@INPUT@', '-o', '@OUTPUT@', + nvcc_arguments = ['-c', '@INPUT@', '-o', '@OUTPUT@', '-I', meson.current_source_dir() + '/src'] nvcc_help = run_command(nvcc, '-h', check : false).stdout() if host_machine.system() == 'windows' if get_option('b_vscrt') == 'mt' - cuda_arguments += ['-Xcompiler', '-MT'] + nvcc_arguments += ['-Xcompiler', '-MT'] elif get_option('b_vscrt') == 'mtd' - cuda_arguments += ['-Xcompiler', '-MTd'] + nvcc_arguments += ['-Xcompiler', '-MTd'] elif get_option('b_vscrt') == 'mdd' or (get_option('b_vscrt') == 'from_buildtype' and get_option('buildtype') == 'debug') - cuda_arguments += ['-Xcompiler', '-MDd'] + nvcc_arguments += ['-Xcompiler', '-MDd'] elif get_option('b_vscrt') != 'none' - cuda_arguments += ['-Xcompiler', '-MD'] + nvcc_arguments += ['-Xcompiler', '-MD'] endif else - cuda_arguments += ['--std=c++14', '-Xcompiler', '-fPIC'] + nvcc_arguments += ['--std=c++17', '-Xcompiler', '-fPIC'] + if get_option('debug') + nvcc_arguments += ['-g'] + endif endif if get_option('nvcc_ccbin') != '' - cuda_arguments += ['-ccbin=' + get_option('nvcc_ccbin')] + nvcc_arguments += ['-ccbin=' + get_option('nvcc_ccbin')] endif cuda_cc = get_option('cc_cuda') # Unfortunately option cuda_cc is reserved. nvcc_extra_args = [] @@ -514,26 +516,68 @@ if get_option('build_backends') endif endif foreach x : get_option('cudnn_include') - cuda_arguments += ['-I', x] + nvcc_arguments += ['-I', x] endforeach if host_machine.system() == 'windows' outputname = '@BASENAME@.obj' else outputname = '@BASENAME@.o' endif + nvcc_ok = true + + max_cuda = 0 + nvcc_dryrun = run_command(nvcc, '--dryrun', nvcc_extra_args, 'foo.cu', check : false).stderr() + foreach x : nvcc_dryrun.split() + if x.contains('-D__CUDA_ARCH__=') + arch = x.substring(16).to_int() + if arch > max_cuda + max_cuda = arch + endif + endif + endforeach + endif + if (get_option('cudnn') or get_option('plain_cuda')) and cu_dart.found() and cu_blas.found() and nvcc_ok + deps += [cu_blas, cu_dart] + cuda_files = ['src/neural/backends/cuda/layers.cc'] + if get_option('cudnn') and cu_dnn.found() + deps += cu_dnn + cuda_files += 'src/neural/backends/cuda/network_cudnn.cc' + cuda_files += 'src/neural/backends/cuda/network_cuda.cc' # To support newer nets. + add_project_arguments('-DUSE_CUDNN', language : 'cpp') + elif get_option('plain_cuda') + cuda_files += 'src/neural/backends/cuda/network_cuda.cc' + endif + includes += include_directories('src/neural/backends/cuda/') files += cuda_files + + if get_option('cutlass') and max_cuda >= 800 + add_project_arguments('-DUSE_CUTLASS', language : 'cpp') + nvcc_arguments += ['-DUSE_CUTLASS'] + if get_option('cutlass_include') != '' + nvcc_arguments += ['-I', get_option('cutlass_include')] + else + nvcc_arguments += ['-I', subproject('cutlass').get_variable('include_directory')] + endif + nvcc_arguments += ['-isystem=@CURRENT_SOURCE_DIR@/third_party'] + files += custom_target('cuda cutlass code', + input : 'src/neural/backends/cuda/cutlass_kernels.cu', + output : outputname, + command : [nvcc, nvcc_extra_args, nvcc_arguments] + ) + endif + files += custom_target('cuda fp32 code', - input : 'src/neural/cuda/common_kernels.cu', + input : 'src/neural/backends/cuda/common_kernels.cu', output : outputname, - depend_files: 'src/neural/cuda/winograd_helper.inc', - command : [nvcc, nvcc_extra_args, cuda_arguments] + depend_files: 'src/neural/backends/cuda/winograd_helper.inc', + command : [nvcc, nvcc_extra_args, nvcc_arguments] ) files += custom_target('cuda fp16 code', - input : 'src/neural/cuda/fp16_kernels.cu', + input : 'src/neural/backends/cuda/fp16_kernels.cu', output : outputname, - depend_files: 'src/neural/cuda/winograd_helper.inc', - command : [nvcc, nvcc_extra_args, cuda_arguments] + depend_files: 'src/neural/backends/cuda/winograd_helper.inc', + command : [nvcc, nvcc_extra_args, nvcc_arguments] ) has_backends = true endif @@ -548,14 +592,14 @@ if get_option('build_backends') dx_dxgi = cc.find_library('dxgi') dx_files = [ - 'src/neural/dx/network_dx.cc', - 'src/neural/dx/shader_wrapper.cc', - 'src/neural/dx/layers_dx.cc', + 'src/neural/backends/dx/network_dx.cc', + 'src/neural/backends/dx/shader_wrapper.cc', + 'src/neural/backends/dx/layers_dx.cc', ] files += dx_files deps += [dx_d3d12, dx_dxgi] - subdir('src/neural/dx/shaders') + subdir('src/neural/backends/dx/shaders') has_backends = true endif @@ -564,8 +608,8 @@ if get_option('build_backends') includes += include_directories(get_option('dnnl_dir') + '/include') deps += [ dnnl_lib, dependency('openmp', required:true) ] files += [ - 'src/neural/onednn/network_onednn.cc', - 'src/neural/onednn/layers.cc', + 'src/neural/backends/onednn/network_onednn.cc', + 'src/neural/backends/onednn/layers.cc', ] has_backends = true endif @@ -573,24 +617,47 @@ if get_option('build_backends') ## ~~~~~~~~~~ ## ONNX ## ~~~~~~~~~~ - if get_option('onnx_libdir') != '' and get_option('onnx_include') != '' - deps += cc.find_library('onnxruntime', dirs: get_option('onnx_libdir'), - required: true) - includes += include_directories(get_option('onnx_include'), is_system: true) + onnxruntime = cc.find_library('onnxruntime', dirs: get_option('onnx_libdir'), + required: false) + if get_option('onnx') and onnxruntime.found() + deps += onnxruntime + onnx_inc_dir = get_option('onnx_include') + if fs.is_dir(onnx_inc_dir + '/onnxruntime/core/session') + # Top level of source dir. + onnx_inc_dir += '/onnxruntime/core/session' + elif fs.is_dir(onnx_inc_dir + '/onnxruntime') + onnx_inc_dir += '/onnxruntime' + endif + includes += include_directories(onnx_inc_dir, is_system: true) cc.has_header('onnxruntime_cxx_api.h', required: true, - args: '-I' + get_option('onnx_include')) - if not cc.has_header('cpu_provider_factory.h', - args: '-I' + get_option('onnx_include')) - cc.has_header('../providers/cpu/cpu_provider_factory.h', required: true, - args: '-I' + get_option('onnx_include')) - includes += include_directories(get_option('onnx_include') + '/../providers/cpu', - is_system: true) + include_directories: includes) + files += 'src/neural/backends/onnx/network_onnx.cc' + onnx_conf = configuration_data() + if cc.has_header('dml_provider_factory.h', required: false, + include_directories: includes) + # The header is not actually needed, used here to detect DML onnxruntime. + onnx_conf.set('USE_DML', true) endif - files += 'src/neural/onnx/network_onnx.cc' if cc.find_library('onnxruntime_providers_rocm', dirs: get_option('onnx_libdir'), required: false).found() - add_project_arguments('-DUSE_ROCM', language : 'cpp') + onnx_conf.set('USE_ROCM', true) + endif + if cc.find_library('onnxruntime_providers_migraphx', + dirs: get_option('onnx_libdir'), required: false).found() + onnx_conf.set('USE_MIGRAPHX', true) endif + if cu_dart.found() and nvcc_ok + onnx_conf.set('USE_ONNX_CUDART', true) + deps += cu_dart + files += custom_target('cuda onnx code', + input : 'src/neural/backends/onnx/onnx_kernels.cu', + output : outputname, + command : [nvcc, nvcc_extra_args, nvcc_arguments] + ) + else + warning('No CUDA support available. Using compatibility implementation for onnx-trt and onnx-cuda.') + endif + configure_file(output : 'onnx_conf.h', configuration : onnx_conf) has_backends = true endif @@ -603,35 +670,130 @@ if get_option('build_backends') modules : ['Foundation', 'Metal', 'MetalPerformanceShaders', 'MetalPerformanceShadersGraph'], required: get_option('metal')) - if (metal_frameworks.found() and add_languages('objc', 'objcpp')) + if metal_frameworks.found() and add_languages('objc', 'objcpp', native: false) deps += metal_frameworks files += [ - 'src/neural/metal/network_metal.cc', - 'src/neural/metal/mps/NetworkGraph.mm', - 'src/neural/metal/mps/MetalNetworkBuilder.mm', + 'src/neural/backends/metal/network_metal.cc', + 'src/neural/backends/metal/mps/NetworkGraph.mm', + 'src/neural/backends/metal/mps/MetalNetworkBuilder.mm', ] has_backends = true add_project_arguments('-fobjc-arc', language : 'objc') add_project_arguments('-fobjc-arc', language : 'objcpp') - endif + # Minimum MacOS version = 12.6.1 + macos_min_version = '12.6' + add_project_arguments( + '-mmacosx-version-min=' + macos_min_version, + language: ['c', 'cpp', 'objc', 'objcpp'] + ) + endif ## ~~~~~~~~ ## XLA ## ~~~~~~~~ if get_option('xla') files += [ - 'src/neural/xla/network_xla.cc', - 'src/neural/xla/pjrt.cc', - 'src/neural/xla/xla_runner.cc', - 'src/neural/xla/xla_tensor.cc', + 'src/neural/backends/xla/network_xla.cc', + 'src/neural/backends/xla/pjrt.cc', + 'src/neural/backends/xla/xla_runner.cc', ] deps += cc.find_library('dl', required: false) has_backends = true endif + ## ~~~~ + ## Sycl + ## ~~~~ + if get_option('sycl') != 'off' + has_backends = true + message('Building SYCL') + add_project_arguments('-fsycl', language : 'cpp') + add_project_link_arguments('-fsycl', language : 'cpp') + + files += 'src/neural/backends/sycl/layers.cc.dp.cpp' + files += 'src/neural/backends/sycl/network_sycl.cc.dp.cpp' + files += 'src/neural/backends/sycl/common_kernels.dp.cpp' + files += 'src/neural/backends/sycl/fp16_kernels.dp.cpp' + + if get_option('sycl') == 'l0' + message('Building SYCL for the L0 backend') + add_project_arguments('-DMKL_ILP64', language : 'cpp') + deps += cc.find_library('mkl_sycl', required: true) + deps += cc.find_library('mkl_intel_ilp64', required: true) + deps += cc.find_library('mkl_sequential', required: true) + deps += cc.find_library('mkl_core', required: true) + deps += cc.find_library('OpenCL', required: true) + elif get_option('sycl') == 'amd' + hip_libdirs = get_option('hip_libdirs') + hip_args = [] + foreach hip_include : get_option('hip_include') + if run_command('scripts/checkdir.py', hip_include, check : false).returncode() == 0 + includes += include_directories(hip_include, is_system: true) + hip_args += '-I' + hip_include + endif + endforeach + deps += cc.find_library('hipblas', dirs: hip_libdirs, required: true) + cc.has_header('hipblas/hipblas.h', required: true, args: hip_args) + deps += cc.find_library('amdhip64', dirs: hip_libdirs, required: true) + cc.has_header('hip/hip_runtime.h', required: true, args: hip_args) + add_project_arguments('-DUSE_HIPBLAS=ON', language : 'cpp') + add_project_arguments('-D__HIP_PLATFORM_AMD__', language : 'cpp') + amd_gfx = get_option('amd_gfx') + if amd_gfx == '' + amd_gfx = [] + agent_enum = find_program('rocm_agent_enumerator', '/opt/rocm/bin/rocm_agent_enumerator', + required: false) + if not agent_enum.found() + warning( '\'rocm_agent_enumerator\' not found. AMD GPU detection doesn\'t work. You can install rocminfo or set -Damd_gfx.') + elif meson.version().version_compare('<1.2.0') + warning( 'Automatic AMD GPU detection requires Meson 1.2.0') + else + agents = run_command(agent_enum, check : false).stdout() + agent_list = agents.splitlines() + foreach agent : agent_list + if agent.startswith('gfx') + amd_gfx += 'amd_gpu_' + agent + else + error( '\'' + agent_enum.full_path() + '\' unexpected output: ' + agent) + endif + endforeach + if amd_gfx.length() == 0 + warning( '\'' + agent_enum.full_path() + '\' failed to detect any AMD GPUs in the system.') + else + message( 'Detected AMD GPU cores: ' + ','.join(amd_gfx)) + endif + endif + else + amd_gfx = ['amd_gpu_' + amd_gfx] + endif + if amd_gfx.length() == 0 + error('-Dsycl=amd requires specifying -Damd_gfx architecture identifier (e.g. gfx90a, gfx1100 or similar)') + endif + add_project_arguments('-fsycl-targets=' + ','.join(amd_gfx), language : 'cpp') + add_project_link_arguments('-fsycl-targets=' + ','.join(amd_gfx), language : 'cpp') + else + deps += cc.find_library('cublas', required: true) + deps += cc.find_library('cudart', required: true) + add_project_arguments('-DUSE_CUBLAS=ON', language : 'cpp') + if get_option('cc_cuda') != '' + sycl_nvidia_target = 'nvidia_gpu_sm_' + get_option('cc_cuda') + else + sycl_nvidia_target = 'nvptx64-nvidia-cuda' + endif + add_project_arguments('-fsycl-targets='+sycl_nvidia_target, language : 'cpp') + add_project_link_arguments('-fsycl-targets='+sycl_nvidia_target, language : 'cpp') + endif + if host_machine.system() == 'windows' + # For sycl under windows we need to link using icx to generate the device code. + # This script edits build.ninja for this and for an icx dependency issue. + meson.add_postconf_script('scripts/sycl_build_hack.py') + add_project_link_arguments('-rtlib=compiler-rt', language : 'cpp') + endif + endif + endif # if get_option('build_backends') if not has_backends and get_option('lc0') and get_option('build_backends') @@ -659,15 +821,53 @@ endif deps += dependency('zlib', fallback: ['zlib', 'zlib_dep']) endif + trace_lib = get_option('trace_library') + trace_config = configuration_data() + + common_files += 'src/utils/trace.cc' + ## ~~~~~~~~ + ## perfetto + ## ~~~~~~~~ + if trace_lib == 'perfetto' + perfetto_dep = dependency('perfetto', required: true, + fallback: ['perfetto', 'dep_perfetto']) + deps += perfetto_dep + trace_config.set('USE_PERFETTO_TRACE', 1) + endif + + ## ~~~~ + ## nvtx + ## ~~~~ + if trace_lib == 'nvtx' + nvtx_includes = get_option('cudnn_include') + nvtx_header_found = false + foreach d : nvtx_includes + if run_command('scripts/checkdir.py', d, check : false).returncode() == 0 + if cc.has_header('nvtx3/nvtx3.hpp', args: '-I' + d) + includes += include_directories(d) + nvtx_header_found = true + break + endif + endif + endforeach + if not nvtx_header_found + error('nvtx3/nvtx3.hpp header not found in cudnn_include paths') + endif + # This could support other tracing apis like systemtap. + trace_config.set('USE_NVTX_TRACE', 1) + endif + configure_file(output : 'trace_config.h', + configuration : trace_config) + ## ~~~~~~~~ ## Profiler ## ~~~~~~~~ if get_option('buildtype') != 'release' - deps += cc.find_library('libprofiler', + deps += cc.find_library('profiler', dirs: ['/usr/local/lib'], required: false) endif - deps += cc.find_library('libatomic', required: false) + deps += cc.find_library('atomic', required: false) ############################################################################# ## Main Executable @@ -681,6 +881,10 @@ if not get_option('f16c') add_project_arguments('-DNO_F16C', language : 'cpp') endif +if cc.has_type('_Float16') + add_project_arguments('-DHAS_FLOAT16', language : 'cpp') +endif + if not get_option('pext') add_project_arguments('-DNO_PEXT', language : 'cpp') endif @@ -689,6 +893,20 @@ if get_option('embed') add_project_arguments('-DEMBED', language : 'cpp') endif +default_search_h = configuration_data() +if get_option('default_search') != '' + default_search_h.set_quoted('DEFAULT_SEARCH', get_option('default_search')) +endif +configure_file(output : 'default_search.h', + configuration : default_search_h) + +default_backend_h = configuration_data() +if get_option('default_backend') != '' + default_backend_h.set_quoted('DEFAULT_BACKEND', get_option('default_backend')) +endif +configure_file(output : 'default_backend.h', + configuration : default_backend_h) + if get_option('lc0') files += common_files executable('lc0', 'src/main.cc', @@ -700,10 +918,10 @@ endif ############################################################################# if get_option('rescorer') - deps += subproject('gaviotatb').get_variable('gaviotatb_dep') + gaviota_dep = subproject('gaviotatb').get_variable('gaviotatb_dep') executable('rescorer', 'src/rescorer_main.cc', - [common_files, 'src/rescorer/rescoreloop.cc'], - include_directories: includes, dependencies: deps, install: true) + [common_files, 'src/trainingdata/rescorer.cc'], + include_directories: includes, dependencies: [deps, gaviota_dep], install: true) endif ############################################################################# @@ -712,13 +930,19 @@ endif if get_option('gtest') gtest = dependency('gtest', fallback: ['gtest', 'gtest_dep']) - lc0_lib = library('lc0_lib', files, include_directories: includes, dependencies: deps) + gmock = dependency('gmock', fallback: ['gtest', 'gmock_dep']) + lc0_lib = library('lc0_lib', common_files, include_directories: includes, dependencies: deps) test('ChessBoard', executable('chessboard_test', 'src/chess/board_test.cc', include_directories: includes, link_with: lc0_lib, dependencies: gtest ), args: '--gtest_output=xml:chessboard.xml', timeout: 90) + test('FP16', + executable('fp16_test', 'src/utils/fp16_utils_test.cc', + include_directories: includes, link_with: lc0_lib, dependencies: gtest + ), args: '--gtest_output=xml:fp16.xml', timeout: 90) + test('HashCat', executable('hashcat_test', 'src/utils/hashcat_test.cc', include_directories: includes, link_with: lc0_lib, dependencies: gtest @@ -744,6 +968,12 @@ if get_option('gtest') include_directories: includes, link_with: lc0_lib, dependencies: [gtest] ), args: '--gtest_output=xml:encoder.xml', timeout: 90) + + test('EngineTest', + executable('engine_test', 'src/engine_test.cc', 'src/engine.cc', + 'src/neural/memcache.cc', pb_files, + include_directories: includes, link_with: lc0_lib, dependencies: [gtest, gmock]), + args: '--gtest_output=xml:engine_test.xml', timeout: 90) endif diff --git a/meson_options.txt b/meson_options.txt index a9f820947b..ec5c53917a 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -43,6 +43,11 @@ option('cudnn_include', value: ['/opt/cuda/include/', '/usr/local/cuda/include/', '/usr/lib/cuda/include/'], description: 'Paths to cudnn include directory') +option('cutlass_include', + type: 'string', + value: '', + description: 'Paths to cutlass include directory') + option('build_backends', type: 'boolean', value: true, @@ -68,9 +73,14 @@ option('native_cuda', value: true, description: 'build cuda code for native arch only (if supported)') +option('native_arch', + type: 'boolean', + value: true, + description: 'build code for native arch only') + option('cudnn', type: 'boolean', - value: true, + value: false, description: 'Enable cuDNN backend') option('plain_cuda', @@ -78,14 +88,19 @@ option('plain_cuda', value: true, description: 'Enable CUDA backend') -option('opencl', +option('cutlass', type: 'boolean', value: true, + description: 'Enable cutlass lib for cuda backend. Only supports Ampere+ right now') + +option('opencl', + type: 'boolean', + value: false, description: 'Enable OpenCL backend') option('dx', type: 'boolean', - value: true, + value: false, description: 'Enable DirectX12 backend') option('tensorflow', @@ -105,7 +120,7 @@ option('openblas', option('mkl', type: 'boolean', - value: true, + value: false, description: 'Enable MKL BLAS support') option('dnnl', @@ -178,14 +193,24 @@ option('cc_cuda', value: '', description: 'Build for a specific cuda CC, e.g. -Dcc_cuda=35 for CC 3.5') -option('onnx_libdir', +option('amd_gfx', type: 'string', value: '', + description: 'Build for a specific AMD GPU architecture, e.g. -Damd_gfx=gfx90a for gfx90a') + +option('onnx', + type: 'boolean', + value: true, + description: 'Enable ONNX backends') + +option('onnx_libdir', + type: 'string', + value: '/usr/lib/', description: 'Paths to ONNX runtime libraries') option('onnx_include', type: 'string', - value: '', + value: '/usr/include/onnxruntime/', description: 'Paths to ONNX runtime includes') option('xla', @@ -193,6 +218,28 @@ option('xla', value: false, description: 'Enable XLA backend') +option('sycl', + type: 'combo', + choices : ['off', 'l0', 'amd', 'nvidia'], + value: 'off', + description: 'Enable SYCL backend') + +option('hip_libdirs', + type: 'array', + value: ['/opt/rocm/lib'], + description: 'Paths to AMD HIP libraries') + +option('hip_include', + type: 'array', + value: ['/opt/rocm/include'], + description: 'Path to AMD HIP includes') + +option('trace_library', + type: 'combo', + choices: ['off', 'perfetto', 'nvtx'], + value: 'off', + description: 'Enable trace library support') + option('lc0', type: 'boolean', value: true, @@ -202,3 +249,23 @@ option('rescorer', type: 'boolean', value: false, description: 'Build rescorer') + +option('default_search', + type: 'string', + value: '', + description: 'Default search algorithm to use, e.g. -Ddefault_search=classic') + +option('default_backend', + type: 'string', + value: '', + description: 'Default backend to use, e.g. -Ddefault_backend=onnx-trt') + +option('dag_classic', + type: 'boolean', + value: true, + description: 'Enable dag-classic search algorithm') + +option('nvcc', + type: 'boolean', + value: true, + description: 'Use nvcc: required for cuda, optional for onnx') diff --git a/src/neural/xla/hlo.proto b/proto/hlo.proto similarity index 96% rename from src/neural/xla/hlo.proto rename to proto/hlo.proto index 6ced6f938d..ba1fe21653 100644 --- a/src/neural/xla/hlo.proto +++ b/proto/hlo.proto @@ -354,6 +354,15 @@ message CompileEnvOptionProto { required OptionOverrideProto value = 2; } +message XlaDeviceAssignmentProto { + optional int32 replica_count = 1; + optional int32 computation_count = 2; + message ComputationDevice { + repeated int64 replica_device_ids = 1; + } + repeated ComputationDevice computation_devices = 3; +} + message ExecutableBuildOptionsProto { // If set, this is the device to build the computation for. Valid // device_ordinal values are: 0 to # of devices - 1. These values are @@ -386,6 +395,12 @@ message ExecutableBuildOptionsProto { // Whether HLOs should be deduplicated. optional bool deduplicate_hlo = 8; + // If set, this specifies a static device assignment for the computation. + // Otherwise, the computation will be compiled generically and can be run with + // any device assignment compatible with the computation's replica and + // partition counts. + optional XlaDeviceAssignmentProto device_assignment = 9; + // Whether input and output buffers are aliased if the associated parameter is // passed-through XLA modules without being changed. optional bool alias_passthrough_params = 10; diff --git a/proto/net.proto b/proto/net.proto new file mode 100644 index 0000000000..961a73992a --- /dev/null +++ b/proto/net.proto @@ -0,0 +1,411 @@ +/* + This file is part of Leela Chess Zero. + Copyright (C) 2018 The LCZero Authors + + Leela Chess is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Leela Chess is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Leela Chess. If not, see . + + Additional permission under GNU GPL version 3 section 7 + + If you modify this Program, or any covered work, by linking or + combining it with NVIDIA Corporation's libraries from the NVIDIA CUDA + Toolkit and the NVIDIA CUDA Deep Neural Network library (or a + modified version of those libraries), containing parts covered by the + terms of the respective license agreement, the licensors of this + Program grant you additional permission to convey the resulting work. +*/ +syntax = "proto2"; + +package pblczero; + +message EngineVersion { + optional uint32 major = 1; + optional uint32 minor = 2; + optional uint32 patch = 3; +} + +message Weights { + message Layer { + optional float min_val = 1; + optional float max_val = 2; + optional bytes params = 3; + enum Encoding { + UNKNOWN_ENCODING = 0; + LINEAR16 = 1; + FLOAT16 = 2; + BFLOAT16 = 3; + FLOAT32 = 4; + } + optional Encoding encoding = 4; + repeated uint32 dims = 5; + } + + message ConvBlock { + optional Layer weights = 1; + optional Layer biases = 2; + optional Layer bn_means = 3; + optional Layer bn_stddivs = 4; + optional Layer bn_gammas = 5; + optional Layer bn_betas = 6; + } + + message SEunit { + // Squeeze-excitation unit (https://arxiv.org/abs/1709.01507) + // weights and biases of the two fully connected layers. + optional Layer w1 = 1; + optional Layer b1 = 2; + optional Layer w2 = 3; + optional Layer b2 = 4; + } + + message Residual { + optional ConvBlock conv1 = 1; + optional ConvBlock conv2 = 2; + optional SEunit se = 3; + } + + message Smolgen { + // For NETWORK_ATTENTIONBODY_WITH_HEADFORMAT. + optional Layer compress = 1; + optional Layer dense1_w = 2; + optional Layer dense1_b = 3; + optional Layer ln1_gammas = 4; + optional Layer ln1_betas = 5; + optional Layer dense2_w = 6; + optional Layer dense2_b = 7; + optional Layer ln2_gammas = 8; + optional Layer ln2_betas = 9; + } + + message MHA { + optional Layer q_w = 1; + optional Layer q_b = 2; + optional Layer k_w = 3; + optional Layer k_b = 4; + optional Layer v_w = 5; + optional Layer v_b = 6; + optional Layer dense_w = 7; + optional Layer dense_b = 8; + optional Smolgen smolgen = 9; + + optional Layer rpe_q = 10; + optional Layer rpe_k = 11; + optional Layer rpe_v = 12; + + // reserved 13 - 22 for int8 quantization + } + + message FFN { + optional Layer dense1_w = 1; + optional Layer dense1_b = 2; + optional Layer dense2_w = 3; + optional Layer dense2_b = 4; + // reserved 5 - 10 for int8 quantization + } + + message EncoderLayer { + optional MHA mha = 1; + optional Layer ln1_gammas = 2; + optional Layer ln1_betas = 3; + optional FFN ffn = 4; + optional Layer ln2_gammas = 5; + optional Layer ln2_betas = 6; + } + + message PolicyHead { + optional Layer ip_pol_w = 1; + optional Layer ip_pol_b = 2; + optional Layer ip2_pol_w = 3; // "wq" in policy attention + optional Layer ip2_pol_b = 4; + optional Layer ip3_pol_w = 5; // "wk" in policy attention + optional Layer ip3_pol_b = 6; + optional Layer ip4_pol_w = 7; // "ppo" in policy attention + + // Optional policy encoders for policy head. + repeated EncoderLayer pol_encoder = 8; + optional uint32 pol_headcount = 9; + + // Convolutions for legacy policy head. + optional ConvBlock policy1 = 10; + optional ConvBlock policy = 11; + } + + message ValueHead { + optional Layer ip_val_w = 1; // "embedding" for attention body value + optional Layer ip_val_b = 2; + optional Layer ip1_val_w = 3; + optional Layer ip1_val_b = 4; + optional Layer ip2_val_w = 5; + optional Layer ip2_val_b = 6; + optional Layer ip_val_err_w = 7; + optional Layer ip_val_err_b = 8; + optional Layer ip_val_cat_w = 9; + optional Layer ip_val_cat_b = 10; + + // Legacy value head support. + optional ConvBlock value = 11; + } + + message PolicyHeadMap { + required string key = 1; // name of the policy head + required PolicyHead value = 2; + } + + message PolicyHeads { + optional Layer ip_pol_w = 1; // "embedding" in policy attention + optional Layer ip_pol_b = 2; + optional PolicyHead vanilla = 3; + optional PolicyHead optimistic_st = 4; + optional PolicyHead soft = 5; + optional PolicyHead opponent = 6; + // map policy_head_map = 7; + repeated PolicyHeadMap policy_head_map = 7; + } + + message ValueHeadMap { + required string key = 1; // name of the value head + required ValueHead value = 2; + } + + message ValueHeads { + optional ValueHead winner = 1; + optional ValueHead q = 2; + optional ValueHead st = 3; + // map value_head_map = 4; + repeated ValueHeadMap value_head_map = 4; + } + + // Input convnet. + optional ConvBlock input = 1; + + // Residual tower. + repeated Residual residual = 2; + + // Embedding layer for attention body encoders + // (NETWORK_ATTENTIONBODY_WITH_HEADFORMAT). + + optional Layer ip_emb_preproc_w = 37; + optional Layer ip_emb_preproc_b = 38; + + optional Layer ip_emb_w = 25; + optional Layer ip_emb_b = 26; + + optional Layer ip_emb_ln_gammas = 39; + optional Layer ip_emb_ln_betas = 40; + + + + // Input gating (NETWORK_ATTENTIONBODY_WITH_HEADFORMAT). + optional Layer ip_mult_gate = 33; + optional Layer ip_add_gate = 34; + + optional FFN ip_emb_ffn = 41; + optional Layer ip_emb_ffn_ln_gammas = 42; + optional Layer ip_emb_ffn_ln_betas = 43; + + // Encoder stack (NETWORK_ATTENTIONBODY_WITH_HEADFORMAT). + repeated EncoderLayer encoder = 27; + optional uint32 headcount = 28; + + // Policy encoder stack + // The ffn activation up to and including NETWORK_SE_WITH_HEADFORMAT is SELU, + // otherwise it follows the ffn activation setting. + repeated EncoderLayer pol_encoder = 21; + optional uint32 pol_headcount = 24; + + // Policy head + // Extra convolution for AZ-style policy head + optional ConvBlock policy1 = 11; + optional ConvBlock policy = 3; + optional Layer ip_pol_w = 4; // "embedding" in policy attention + optional Layer ip_pol_b = 5; + // For policy attention, up to and including NETWORK_SE_WITH_HEADFORMAT the + // "embedding" activation is SELU, otherwise it is the default activation. + optional Layer ip2_pol_w = 17; // "wq" in policy attention + optional Layer ip2_pol_b = 18; + optional Layer ip3_pol_w = 19; // "wk" in policy attention + optional Layer ip3_pol_b = 20; + optional Layer ip4_pol_w = 22; // "ppo" in policy attention + + // Value head + optional ConvBlock value = 6; + optional Layer ip_val_w = 29; // "embedding" for attention body value + optional Layer ip_val_b = 30; + optional Layer ip1_val_w = 7; + optional Layer ip1_val_b = 8; + optional Layer ip2_val_w = 9; + optional Layer ip2_val_b = 10; + + optional ValueHeads value_heads = 44; + optional PolicyHeads policy_heads = 45; + + // Moves left head + optional ConvBlock moves_left = 12; + optional Layer ip_mov_w = 31; // "embedding" for attention body moves left + optional Layer ip_mov_b = 32; + optional Layer ip1_mov_w = 13; + optional Layer ip1_mov_b = 14; + optional Layer ip2_mov_w = 15; + optional Layer ip2_mov_b = 16; + + // Global smolgen weights (NETWORK_ATTENTIONBODY_WITH_HEADFORMAT). + optional Layer smolgen_w = 35; + optional Layer smolgen_b = 36; +} + +message TrainingParams { + optional uint32 training_steps = 1; + optional float learning_rate = 2; + optional float mse_loss = 3; + optional float policy_loss = 4; + optional float accuracy = 5; + optional string lc0_params = 6; +} + +message NetworkFormat { + // Format to encode the input planes with. Used by position encoder. + enum InputFormat { + INPUT_UNKNOWN = 0; + INPUT_CLASSICAL_112_PLANE = 1; + INPUT_112_WITH_CASTLING_PLANE = 2; + INPUT_112_WITH_CANONICALIZATION = 3; + INPUT_112_WITH_CANONICALIZATION_HECTOPLIES = 4; + INPUT_112_WITH_CANONICALIZATION_HECTOPLIES_ARMAGEDDON = 132; + INPUT_112_WITH_CANONICALIZATION_V2 = 5; + INPUT_112_WITH_CANONICALIZATION_V2_ARMAGEDDON = 133; + } + optional InputFormat input = 1; + + // Output format of the NN. Used by search code to interpret results. + enum OutputFormat { + OUTPUT_UNKNOWN = 0; + OUTPUT_CLASSICAL = 1; + OUTPUT_WDL = 2; + } + optional OutputFormat output = 2; + + // Network architecture. Used by backends to build the network. + enum NetworkStructure { + // Networks without PolicyFormat or ValueFormat specified + NETWORK_UNKNOWN = 0; + NETWORK_CLASSICAL = 1; + NETWORK_SE = 2; + // Networks with PolicyFormat and ValueFormat specified + NETWORK_CLASSICAL_WITH_HEADFORMAT = 3; + NETWORK_SE_WITH_HEADFORMAT = 4; + NETWORK_ONNX = 5; + NETWORK_ATTENTIONBODY_WITH_HEADFORMAT = 6; + NETWORK_ATTENTIONBODY_WITH_MULTIHEADFORMAT = 7; + NETWORK_AB_LEGACY_WITH_MULTIHEADFORMAT = 134; + } + optional NetworkStructure network = 3; + + // Policy head architecture + enum PolicyFormat { + POLICY_UNKNOWN = 0; + POLICY_CLASSICAL = 1; + POLICY_CONVOLUTION = 2; + POLICY_ATTENTION = 3; + } + optional PolicyFormat policy = 4; + + // Value head architecture + enum ValueFormat { + VALUE_UNKNOWN = 0; + VALUE_CLASSICAL = 1; + VALUE_WDL = 2; + VALUE_PARAM = 3; + } + optional ValueFormat value = 5; + + // Moves left head architecture + enum MovesLeftFormat { + MOVES_LEFT_NONE = 0; + MOVES_LEFT_V1 = 1; + } + optional MovesLeftFormat moves_left = 6; + + enum ActivationFunction { + ACTIVATION_DEFAULT = 0; + ACTIVATION_MISH = 1; + ACTIVATION_RELU = 2; + ACTIVATION_NONE = 3; + ACTIVATION_TANH = 4; + ACTIVATION_SIGMOID = 5; + ACTIVATION_SELU = 6; + ACTIVATION_SWISH = 7; + ACTIVATION_RELU_2 = 8; + ACTIVATION_SOFTMAX = 9; + } + + // Activation used everywhere except head outputs or otherwise specified. + enum DefaultActivation { + DEFAULT_ACTIVATION_RELU = 0; + DEFAULT_ACTIVATION_MISH = 1; + } + optional DefaultActivation default_activation = 7; + + optional ActivationFunction smolgen_activation = 8; + optional ActivationFunction ffn_activation = 9; + + enum InputEmbeddingFormat { + INPUT_EMBEDDING_NONE = 0; + INPUT_EMBEDDING_PE_MAP = 1; + INPUT_EMBEDDING_PE_DENSE = 2; + } + optional InputEmbeddingFormat input_embedding = 10; +} + +message Format { + enum Encoding { + UNKNOWN = 0; + LINEAR16 = 1; + } + // Any encoding specified in a Layer overides this. + optional Encoding weights_encoding = 1; + // If network_format is missing, it's assumed to have + // INPUT_CLASSICAL_112_PLANE / OUTPUT_CLASSICAL / NETWORK_CLASSICAL format. + optional NetworkFormat network_format = 2; +} + +message OnnxModel { + enum DataType { + UNKNOWN_DATATYPE = 0; + FLOAT = 1; + FLOAT16 = 10; + BFLOAT16 = 16; + } + + // Serialized OnnxProto model. + optional bytes model = 1; + optional DataType data_type = 2; + // Name of the input tensor to populate. + optional string input_planes = 3; + // Names of the output tensors to get results from. + // If some feature is not present, corresponding values are not set. + optional string output_value = 4; + optional string output_wdl = 5; + optional string output_policy = 6; + optional string output_mlh = 7; +} + +message Net { + optional fixed32 magic = 1; + optional string license = 2; + optional EngineVersion min_version = 3; + optional Format format = 4; + optional TrainingParams training_params = 5; + // Either weights or onnx_model is set, but not both. + optional Weights weights = 10; + optional OnnxModel onnx_model = 11; +} diff --git a/src/neural/onnx/onnx.proto b/proto/onnx.proto similarity index 100% rename from src/neural/onnx/onnx.proto rename to proto/onnx.proto diff --git a/scripts/appveyor_android_build.cmd b/scripts/appveyor_android_build.cmd index 9f2f79665a..a9f3f01860 100644 --- a/scripts/appveyor_android_build.cmd +++ b/scripts/appveyor_android_build.cmd @@ -1,7 +1,7 @@ cd arm64-v8a ninja -aarch64-linux-android-strip lc0 +llvm-strip lc0 cd C:\projects\lc0 cd armeabi-v7a ninja -arm-linux-androideabi-strip lc0 +llvm-strip lc0 diff --git a/scripts/appveyor_win_build.cmd b/scripts/appveyor_win_build.cmd index 43ab5f211a..00e739d567 100644 --- a/scripts/appveyor_win_build.cmd +++ b/scripts/appveyor_win_build.cmd @@ -1,5 +1,5 @@ SET PGO=false -IF %APPVEYOR_REPO_TAG%==true IF %DX%==false IF %ONNX_DML%==false SET PGO=true +IF %APPVEYOR_REPO_TAG%==true IF %DX%==false IF %ONNX%==false SET PGO=true IF %PGO%==false msbuild "C:\projects\lc0\build\lc0.sln" /m /p:WholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" IF EXIST build\lc0.pdb del build\lc0.pdb IF %PGO%==true msbuild "C:\projects\lc0\build\lc0.sln" /m /p:WholeProgramOptimization=PGInstrument /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" @@ -19,3 +19,12 @@ IF %PGO%==true ( ) cd .. IF %PGO%==true msbuild "C:\projects\lc0\build\lc0.sln" /m /p:WholeProgramOptimization=PGOptimize /p:DebugInformationFormat=ProgramDatabase /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" +IF %NAME%==onnx ( + ren build\lc0.exe lc0-trt.exe + meson configure build -Ddefault_backend= -Dcudnn_libdirs= -Dgtest=%GTEST% + # This is needed as a separate step. + msbuild "C:\projects\lc0\build\lc0.sln" /target:REGEN + IF %PGO%==true msbuild "C:\projects\lc0\build\lc0.sln" /m /p:WholeProgramOptimization=PGOptimize /p:DebugInformationFormat=ProgramDatabase /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + IF %PGO%==false msbuild "C:\projects\lc0\build\lc0.sln" /m /p:WholeProgramOptimization=true /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + ren build\lc0.exe lc0-dml.exe +) diff --git a/scripts/appveyor_win_package.cmd b/scripts/appveyor_win_package.cmd index 36f98d8eef..eaf1ba73b7 100644 --- a/scripts/appveyor_win_package.cmd +++ b/scripts/appveyor_win_package.cmd @@ -1,6 +1,6 @@ 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip %APPVEYOR_BUILD_FOLDER%\build\lc0.exe -IF %NAME%==gpu-nvidia-cuda appveyor DownloadFile "https://github.com/LeelaChessZero/lczero-client/releases/latest/download/lc0-training-client.exe" -IF %NAME%==gpu-nvidia-cuda 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip lc0-training-client.exe +IF %NAME%==gpu-nvidia-cuda12 appveyor DownloadFile "https://github.com/LeelaChessZero/lczero-client/releases/latest/download/lc0-training-client.exe" +IF %NAME%==gpu-nvidia-cuda12 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip lc0-training-client.exe type COPYING |more /P > dist\COPYING 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\COPYING 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip c:\cache\%NET%.pb.gz @@ -15,26 +15,50 @@ IF %NAME%==cpu-openblas 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip C:\ IF %NAME%==cpu-dnnl 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip C:\cache\%DNNL_NAME%\bin\dnnl.dll IF %NAME%==onednn 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip C:\cache\%DNNL_NAME%\bin\dnnl.dll IF %OPENCL%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip C:\cache\opencl-nug.0.777.77\build\native\bin\OpenCL.dll -IF %CUDNN%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip "%CUDA_PATH%\bin\cudart64_100.dll" "%CUDA_PATH%\bin\cublas64_100.dll" +IF %CUDNN%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip "%CUDA_PATH%\bin\cudart64_101.dll" "%CUDA_PATH%\bin\cublas64_10.dll" "%CUDA_PATH%\bin\cublasLt64_10.dll" IF %CUDNN%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip "%CUDA_PATH%\cuda\bin\cudnn64_7.dll" -IF %CUDA%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip "%CUDA_PATH%\bin\cudart64_110.dll" "%CUDA_PATH%\bin\cublas64_11.dll" "%CUDA_PATH%\bin\cublasLt64_11.dll" -IF %NAME%==cpu-dnnl copy "%PKG_FOLDER%\%DNNL_NAME%\LICENSE" dist\DNNL-LICENSE -IF %NAME%==cpu-dnnl copy "%PKG_FOLDER%\%DNNL_NAME%\THIRD-PARTY-PROGRAMS" dist\DNNL-THIRD-PARTY-PROGRAMS -IF %NAME%==cpu-dnnl 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\DNNL-LICENSE -IF %NAME%==cpu-dnnl 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\DNNL-THIRD-PARTY-PROGRAMS -IF %NAME%==onednn copy "%PKG_FOLDER%\%DNNL_NAME%\LICENSE" dist\DNNL-LICENSE -IF %NAME%==onednn copy "%PKG_FOLDER%\%DNNL_NAME%\THIRD-PARTY-PROGRAMS" dist\DNNL-THIRD-PARTY-PROGRAMS -IF %NAME%==onednn 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\DNNL-LICENSE -IF %NAME%==onednn 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\DNNL-THIRD-PARTY-PROGRAMS -IF %ONNX_DML%==true type dist\README-onnx-dml.txt |more /P > dist\README.txt -IF %ONNX_DML%==true type dist\install-dml.cmd |more /P > dist\install.cmd -IF %ONNX_DML%==true copy "%PKG_FOLDER%\%ONNX_NAME%\LICENSE" dist\ONNX-DML-LICENSE -IF %ONNX_DML%==true copy "%PKG_FOLDER%\%ONNX_NAME%\ThirdPartyNotices.txt" dist\ONNX-DML-ThirdPartyNotices.txt -IF %ONNX_DML%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip "%PKG_FOLDER%\%ONNX_NAME%\lib\onnxruntime.dll" -IF %ONNX_DML%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\README.txt -IF %ONNX_DML%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\install.cmd -IF %ONNX_DML%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\ONNX-DML-LICENSE -IF %ONNX_DML%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\ONNX-DML-ThirdPartyNotices.txt +IF %NAME%==gpu-nvidia-cuda11 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip "%CUDA_PATH%\bin\cudart64_110.dll" "%CUDA_PATH%\bin\cublas64_11.dll" "%CUDA_PATH%\bin\cublasLt64_11.dll" +IF %NAME%==gpu-nvidia-cuda12 ( + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip "%CUDA_PATH%\bin\cudart64_12.dll" "%CUDA_PATH%\bin\cublas64_12.dll" "%CUDA_PATH%\bin\cublasLt64_12.dll" + type dist\install-cuda_12_9.cmd |more /P > dist\install.cmd + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-nodll.zip .\dist\install.cmd +) +IF %NAME%==cpu-dnnl ( + copy "%PKG_FOLDER%\%DNNL_NAME%\LICENSE" dist\DNNL-LICENSE + copy "%PKG_FOLDER%\%DNNL_NAME%\THIRD-PARTY-PROGRAMS" dist\DNNL-THIRD-PARTY-PROGRAMS + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\DNNL-LICENSE + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\DNNL-THIRD-PARTY-PROGRAMS +) +IF %NAME%==onednn ( + copy "%PKG_FOLDER%\%DNNL_NAME%\LICENSE" dist\DNNL-LICENSE + copy "%PKG_FOLDER%\%DNNL_NAME%\THIRD-PARTY-PROGRAMS" dist\DNNL-THIRD-PARTY-PROGRAMS + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\DNNL-LICENSE + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\DNNL-THIRD-PARTY-PROGRAMS +) +IF %ONNX%==true ( + copy "%PKG_FOLDER%\%ONNX_NAME%\LICENSE" dist\ONNX-LICENSE + copy "%PKG_FOLDER%\%ONNX_NAME%\ThirdPartyNotices.txt" dist\ONNX-ThirdPartyNotices.txt + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip "%PKG_FOLDER%\%ONNX_NAME%\runtimes\win-x64\native\onnxruntime.dll" + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\ONNX-LICENSE + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\ONNX-ThirdPartyNotices.txt + copy lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-trt.zip + ren lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-dml.zip + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-dml.zip %APPVEYOR_BUILD_FOLDER%\build\lc0-dml.exe + 7z rn lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-dml.zip lc0-dml.exe lc0.exe + type dist\README-onnx-dml.txt |more /P > dist\README.txt + type dist\install-dml.cmd |more /P > dist\install.cmd + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-dml.zip .\dist\README.txt + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-dml.zip .\dist\install.cmd + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-trt.zip %APPVEYOR_BUILD_FOLDER%\build\lc0-trt.exe + 7z rn lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-trt.zip lc0-trt.exe lc0.exe + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-trt.zip "%PKG_FOLDER%\%ONNX_NAME%\runtimes\win-x64\native\onnxruntime_providers_shared.dll" + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-trt.zip "%PKG_FOLDER%\%ONNX_NAME_TWO%\lib\onnxruntime_providers_cuda.dll" + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-trt.zip "%PKG_FOLDER%\%ONNX_NAME_TWO%\lib\onnxruntime_providers_tensorrt.dll" + type dist\README-onnx-trt.txt |more /P > dist\README.txt + type dist\install-trt.cmd |more /P > dist\install.cmd + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-trt.zip .\dist\README.txt + 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%-trt.zip .\dist\install.cmd +) IF %OPENCL%==true type scripts\check_opencl.bat |more /P > dist\check_opencl.bat IF %OPENCL%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\check_opencl.bat IF %DX%==true type scripts\check_dx.bat |more /P > dist\check_dx.bat diff --git a/scripts/compile_proto.py b/scripts/compile_proto.py index cb7d0450b2..c6a81996d9 100755 --- a/scripts/compile_proto.py +++ b/scripts/compile_proto.py @@ -29,60 +29,73 @@ import os import re import sys +from typing import Any VARINT_TYPES = { - 'int32': 'std::int32_t', - 'int64': 'std::int64_t', - 'uint32': 'std::uint32_t', - 'uint64': 'std::uint64_t', - 'sint32': 'std::uint32_t', - 'sint64': 'std::uint64_t', - 'bool': 'bool', + "int32": "std::int32_t", + "int64": "std::int64_t", + "uint32": "std::uint32_t", + "uint64": "std::uint64_t", + "sint32": "std::uint32_t", + "sint64": "std::uint64_t", + "bool": "bool", } FIXED64_TYPES = { - 'fixed64': 'std::uint64_t', - 'sfixed64': 'std::int64_t', - 'double': 'double', + "fixed64": "std::uint64_t", + "sfixed64": "std::int64_t", + "double": "double", } FIXED32_TYPES = { - 'fixed32': 'std::uint32_t', - 'sfixed32': 'std::int32_t', - 'float': 'float', + "fixed32": "std::uint32_t", + "sfixed32": "std::int32_t", + "float": "float", } BYTES_TYPES = { - 'string': 'std::string_view', - 'bytes': 'std::string_view', + "string": "std::string_view", + "bytes": "std::string_view", } -ZIGZAG_TYPES = set(['sint32', 'sint64']) -FLOAT_TYPES = set(['float', 'double']) +ZIGZAG_TYPES = set(["sint32", "sint64"]) +FLOAT_TYPES = set(["float", "double"]) TYPES = {**VARINT_TYPES, **FIXED32_TYPES, **FIXED64_TYPES, **BYTES_TYPES} RESERVED_WORDS = [ - 'syntax', - 'package', - 'message', - 'optional', - 'required', - 'repeated', - 'enum', + "enum", + "message", + "optional", + "package", + "repeated", + "required", + "reserved", + "syntax", + "to", ] + list(TYPES.keys()) -GRAMMAR = ([(r'%s\b' % x, x) - for x in RESERVED_WORDS] + [('\\' + x, x) for x in '=;{}.'] + [ - (r'/\*.*?\*/', None), # /* Comment */ - (r'//.*?$', None), # // Comment - (r'\s+', None), # Whitespace - (r'$', 'EOF'), - (r'"((?:[^"\\]|\\.)*)"', 'string'), - (r'\d+', 'number'), - (r'\w+', 'identifier'), - ]) +GRAMMAR = ( + [(r"%s\b" % x, x) for x in RESERVED_WORDS] + + [("\\" + x, x) for x in "=;{}.,[]"] + + [ + (r"/\*.*?\*/", None), # /* Comment */ + (r"//.*?$", None), # // Comment + (r"\s+", None), # Whitespace + (r"$", "EOF"), + (r'"((?:[^"\\]|\\.)*)"', "string"), + ( + r"[-+]?(?:[0-9]*\.[0-9]+(?:[eE][-+]?[0-9]+)?|[0-9]+[eE][-+]?[0-9]+)", + "fnumber", + ), + (r"[-+]?\d+", "number"), + (r"(\w+)", "identifier"), + ] +) + +ALLOWED_ATTRIBUTES = { + "default", +} class Lexer: - def __init__(self, text): self.text = text self.grammar = [(re.compile(x, re.S + re.M), y) for x, y in GRAMMAR] @@ -90,31 +103,31 @@ def __init__(self, text): self.cur_offset = 0 def Pick(self): - '''Picks the last token in queue. Doesn't advance the queue.''' + """Picks the last token in queue. Doesn't advance the queue.""" if self.cur_token is None: self.cur_token = self.NextToken() return self.cur_token def Consume(self, expected_token, value=None, group=0): - '''Gets the token from the queue and advances the queue. + """Gets the token from the queue and advances the queue. If @expected_token if of wrong type, or @value is not equal to regexes @group, throws an error. - ''' + """ token, match = self.Pick() if expected_token != token: - self.Error('Expected token type [%s]' % expected_token) + self.Error(f"Expected token type [{expected_token}], got [{token}]") if value is not None and value != match.group(group): - self.Error('Expected value [%s]' % value) + self.Error("Expected value [%s]" % value) self.cur_offset = match.span()[1] self.cur_token = None return match def NextToken(self): - '''Reads the stream and returns the next token. + """Reads the stream and returns the next token. (which is not whitespace or comment) - ''' + """ while True: token, match = self.NextTokenOrWhitespace() if token is None: @@ -123,39 +136,42 @@ def NextToken(self): return token, match def NextTokenOrWhitespace(self): - '''Reads the stream and returns the next token (possibly whitespace).''' + """Reads the stream and returns the next token (possibly whitespace).""" for r, token in self.grammar: m = r.match(self.text, self.cur_offset) if m: return (token, m) - self.Error('Unexpected token') + token_snippet = self.text[self.cur_offset : self.cur_offset + 10] + self.Error(f"Unparseable token [{token_snippet}...]") def Error(self, text): - '''Throws an error with context in the file read.''' - line = self.text[:self.cur_offset].count('\n') + 1 - line_start = self.text.rfind('\n', 0, self.cur_offset) + 1 - line_end = self.text.find('\n', line_start) + """Throws an error with context in the file read.""" + line = self.text[: self.cur_offset].count("\n") + 1 + line_start = self.text.rfind("\n", 0, self.cur_offset) + 1 + line_end = self.text.find("\n", line_start) if line_end == -1: line_end = len(self.text) - sys.stderr.write('%s:\n' % text) - sys.stderr.write(self.text[line_start:line_end] + '\n') - sys.stderr.write(' ' * (self.cur_offset - line_start) + '^^^\n') - raise ValueError("Parse error: %s at line %d column %d." % - (text, line, (self.cur_offset - line_start))) + sys.stderr.write("%s:\n" % text) + sys.stderr.write(self.text[line_start:line_end] + "\n") + sys.stderr.write(" " * (self.cur_offset - line_start) + "^^^\n") + raise ValueError( + "Parse error: %s at line %d column %d." + % (text, line, (self.cur_offset - line_start)) + ) def ReadIdentifierPath(lexer): - '''Reads qualified identifier a.b.d into ['a', 'b', 'd'] list''' + """Reads qualified identifier a.b.d into ['a', 'b', 'd'] list""" path = [] while True: - path.append(lexer.Consume('identifier').group(0)) - if lexer.Pick()[0] != '.': + path.append(lexer.Consume("identifier").group(0)) + if lexer.Pick()[0] != ".": return path - lexer.Consume('.') + lexer.Consume(".") def LookupType(name, stack): - '''Looks up the (possibly qualified) from the innermost scope first.''' + """Looks up the (possibly qualified) from the innermost scope first.""" for y in stack: for x in y: if x.GetName() == name[0]: @@ -163,7 +179,7 @@ def LookupType(name, stack): return x else: return LookupType(name[1:], [x.GetTypes()]) - raise ValueError("Cannot find type: %s." % '.'.join(name)) + raise ValueError("Cannot find type: %s." % ".".join(name)) # All *Parser classes have the following semantics: @@ -172,18 +188,17 @@ def LookupType(name, stack): class ProtoTypeParser: - def __init__(self, lexer, object_stack): token, match = lexer.Pick() if token in TYPES: - self.typetype = 'basic' + self.typetype = "basic" self.name = token lexer.Consume(token) - elif token == 'identifier': + elif token == "identifier": self.name = ReadIdentifierPath(lexer) - self.typetype = 'forward' + self.typetype = "forward" else: - lexer.Error('Type expected') + lexer.Error("Type expected") def LookupForwardFieldType(self, object_stack): if self.IsForward(): @@ -192,41 +207,43 @@ def LookupForwardFieldType(self, object_stack): self.name = [typ.GetFullName()] def IsZigzag(self): - if self.typetype == 'basic': + if self.typetype == "basic": return self.name in ZIGZAG_TYPES return False def GetCppType(self): - if self.typetype == 'basic': + if self.typetype == "basic": return TYPES[self.name] else: - return '_'.join(self.name) + return "_".join(self.name) def GetVariableCppType(self): if self.IsBytesType(): - return 'std::string' + return "std::string" else: return self.GetCppType() def IsEnumType(self): - return self.typetype == 'enum' + return self.typetype == "enum" def IsVarintType(self): - return self.typetype == 'enum' or (self.typetype == 'basic' - and self.name in VARINT_TYPES) + return self.typetype == "enum" or ( + self.typetype == "basic" and self.name in VARINT_TYPES + ) def IsFixedType(self): - return self.typetype == 'basic' and (self.name in FIXED64_TYPES - or self.name in FIXED32_TYPES) + return self.typetype == "basic" and ( + self.name in FIXED64_TYPES or self.name in FIXED32_TYPES + ) def IsBytesType(self): - return self.typetype == 'basic' and self.name in BYTES_TYPES + return self.typetype == "basic" and self.name in BYTES_TYPES def IsFloatType(self): - return self.typetype == 'basic' and self.name in FLOAT_TYPES + return self.typetype == "basic" and self.name in FLOAT_TYPES def GetWireType(self): - if self.typetype == 'basic': + if self.typetype == "basic": if self.name in VARINT_TYPES: return 0 if self.name in FIXED64_TYPES: @@ -235,52 +252,84 @@ def GetWireType(self): return 2 if self.name in FIXED32_TYPES: return 5 - raise ValueError('Unknown type %s' % self.name) - elif self.typetype == 'enum': + raise ValueError("Unknown type %s" % self.name) + elif self.typetype == "enum": return 0 - elif self.typetype == 'message': + elif self.typetype == "message": return 2 else: - raise ValueError('Unknown typetype %s' % self.typetype) + raise ValueError("Unknown typetype %s" % self.typetype) def IsMessage(self): - return self.typetype == 'message' + return self.typetype == "message" def IsForward(self): - return self.typetype == 'forward' + return self.typetype == "forward" def IsIntegralType(self): - if self.typetype == 'basic': - if self.name == 'double': + if self.typetype == "basic": + if self.name == "double": return False - if self.name == 'float': + if self.name == "float": return False if self.name in BYTES_TYPES: return False if self.name in TYPES: return True - raise ValueError('Unknown type %s' % self.name) - elif self.typetype == 'enum': + raise ValueError("Unknown type %s" % self.name) + elif self.typetype == "enum": return True - elif self.typetype == 'message': + elif self.typetype == "message": return False else: - raise ValueError('Unknown typetype %s' % self.typetype) + raise ValueError("Unknown typetype %s" % self.typetype) class ProtoFieldParser: - def __init__(self, lexer, object_stack): token, match = lexer.Pick() - if token not in ['repeated', 'optional', 'required']: - lexer.Error('repeated, optional or required expected') + if token not in ["repeated", "optional", "required"]: + lexer.Error("repeated, optional or required expected") self.category = token lexer.Consume(token) self.type = ProtoTypeParser(lexer, object_stack) - self.name = lexer.Consume('identifier') - lexer.Consume('=') - self.number = int(lexer.Consume('number').group(0)) - lexer.Consume(';') + self.name = lexer.Consume("identifier") + lexer.Consume("=") + self.number = int(lexer.Consume("number").group(0)) + self.attributes = ProtoFieldParser.ParseAttributes(lexer) + lexer.Consume(";") + + @staticmethod + def ParseAttributes(lexer): + attributes = {} + token, match = lexer.Pick() + if token != "[": + return attributes + lexer.Consume("[") + while True: + name = lexer.Consume("identifier").group(0) + if name not in ALLOWED_ATTRIBUTES: + lexer.Error("Unknown attribute %s" % name) + lexer.Consume("=") + token, match = lexer.Pick() + value = None + if token == "string": + value = lexer.Consume("string").group(0) + elif token == "fnumber": + value = float(lexer.Consume("fnumber").group(0)) + elif token == "number": + value = int(lexer.Consume("number").group(0)) + else: + lexer.Error("Expected string or number as default value") + attributes[name] = value + token, _ = lexer.Pick() + if token == "]": + lexer.Consume("]") + return attributes + elif token == ",": + lexer.Consume(",") + else: + lexer.Error("Expected ']' or ','") def IsType(self): return False @@ -291,96 +340,96 @@ def LookupForwardFieldType(self, object_stack): def GetParser(self): name = self.name.group(0) if self.type.IsMessage(): - if self.category == 'repeated': - return 'add_%s()->MergeFromString(val)' % name + if self.category == "repeated": + return "add_%s()->MergeFromString(val)" % name else: - return 'mutable_%s()->MergeFromString(val)' % name + return "mutable_%s()->MergeFromString(val)" % name cpp_type = self.type.GetCppType() - val = 'NOT IMPLEMENTED!' + val = "NOT IMPLEMENTED!" if self.type.IsVarintType(): - val_val = 'UnZigZag(val)' if self.type.IsZigzag() else 'val' - val = 'static_cast<%s>(%s)' % (cpp_type, val_val) + val_val = "UnZigZag(val)" if self.type.IsZigzag() else "val" + val = "static_cast<%s>(%s)" % (cpp_type, val_val) elif self.type.IsFixedType(): if self.type.IsFloatType(): - val = 'bit_cast<%s>(val)' % cpp_type + val = "bit_cast<%s>(val)" % cpp_type else: - val = 'static_cast<%s>(val)' % cpp_type + val = "static_cast<%s>(val)" % cpp_type elif self.type.IsBytesType(): - val = 'val' + val = "val" - if self.category == 'repeated': - return '%s_.emplace_back(%s)' % (name, val) + if self.category == "repeated": + return "%s_.emplace_back(%s)" % (name, val) else: - return 'set_%s(%s)' % (name, val) + return "set_%s(%s)" % (name, val) def GenerateCaseClause(self, w): - w.Write('case %d: %s; break;' % (self.number, self.GetParser())) + w.Write("case %d: %s; break;" % (self.number, self.GetParser())) def GenerateClear(self, w): name = self.name.group(0) - if self.category == 'repeated': - w.Write('%s_.clear();' % name) + if self.category == "repeated": + w.Write("%s_.clear();" % name) else: - w.Write('has_%s_ = false;' % name) - w.Write('%s_ = {};' % name) + w.Write("has_%s_ = false;" % name) + if "default" in self.attributes: + w.Write("%s_ = %s;" % (name, self.attributes["default"])) + else: + w.Write("%s_ = {};" % name) def GenerateOutput(self, w): fname = { - 0: 'AppendVarInt', - 1: 'AppendInt64', - 2: 'AppendString', - 5: 'AppendInt32' + 0: "AppendVarInt", + 1: "AppendInt64", + 2: "AppendString", + 5: "AppendInt32", } tname = { - 0: 'std::uint64_t', - 1: 'std::uint64_t', - 2: 'std::string_view', - 5: 'std::uint32_t' + 0: "std::uint64_t", + 1: "std::uint64_t", + 2: "std::string_view", + 5: "std::uint32_t", } wire_id = self.type.GetWireType() - if self.category == 'repeated': - prefix = 'for (const auto& x : %s)' % (self.name.group(0) + '_') - name = 'x' + if self.category == "repeated": + prefix = "for (const auto& x : %s)" % (self.name.group(0) + "_") + name = "x" else: - name = self.name.group(0) + '_' - prefix = 'if (has_%s)' % (name) + name = self.name.group(0) + "_" + prefix = "if (has_%s)" % (name) if self.type.IsMessage(): - name += '.OutputAsString()' + name += ".OutputAsString()" elif self.type.IsFloatType(): - name = 'bit_cast<%s>(%s)' % (tname[wire_id], name) + name = "bit_cast<%s>(%s)" % (tname[wire_id], name) - w.Write('%s %s(%d, %s, &out);' % - (prefix, fname[wire_id], self.number, name)) + w.Write("%s %s(%d, %s, &out);" % (prefix, fname[wire_id], self.number, name)) def GenerateJsonOutput(self, w): name = self.name.group(0) - if self.category == 'repeated': - prefix = 'if (!%s_.empty())' % name - funcname = 'AppendJsonRepeatedField' + if self.category == "repeated": + prefix = "if (!%s_.empty())" % name + funcname = "AppendJsonRepeatedField" else: - prefix = 'if (has_%s_)' % name - funcname = 'AppendJsonField' + prefix = "if (has_%s_)" % name + funcname = "AppendJsonField" if self.type.IsEnumType(): - value = '%s_Name(%s_)' % (self.type.GetCppType(), name) + value = "%s_Name(%s_)" % (self.type.GetCppType(), name) else: value = name + "_" - w.Write('%s %s("%s", %s, &first, &out);' % - (prefix, funcname, name, value)) + w.Write('%s %s("%s", %s, &first, &out);' % (prefix, funcname, name, value)) def GenerateFunctionDeclarations(self, w): name = self.name.group(0) cpp_type = self.type.GetCppType() var_cpp_type = self.type.GetVariableCppType() - if self.category == 'repeated': + if self.category == "repeated": if self.type.IsMessage(): w.Write("%s* add_%s();" % (cpp_type, name)) else: w.Write("void add_%s(%s val);" % (name, cpp_type)) # Using a vector here breaks API compatibility with the standard # protobuf library, but it is more convenient. - w.Write("const std::vector<%s>& %s() const;" % - (var_cpp_type, name)) + w.Write("const std::vector<%s>& %s() const;" % (var_cpp_type, name)) w.Write("std::vector<%s>* mutable_%s();" % (var_cpp_type, name)) if self.type.IsMessage(): w.Write("const %s& %s(size_t idx) const;" % (cpp_type, name)) @@ -392,8 +441,9 @@ def GenerateFunctionDeclarations(self, w): w.Write("bool has_%s() const;" % (name)) if self.type.IsMessage(): w.Write("const %s& %s() const;" % (cpp_type, name)) - w.Write("%s* mutable_%s();" % (cpp_type, name)) - else: + if self.type.IsMessage() or self.type.IsBytesType(): + w.Write("%s* mutable_%s();" % (var_cpp_type, name)) + if not self.type.IsMessage(): w.Write("%s %s() const;" % (cpp_type, name)) w.Write("void set_%s(%s val);" % (name, cpp_type)) @@ -401,53 +451,70 @@ def GenerateFunctionDefinitions(self, w, class_name): name = self.name.group(0) cpp_type = self.type.GetCppType() var_cpp_type = self.type.GetVariableCppType() - if self.category == 'repeated': + if self.category == "repeated": if self.type.IsMessage(): w.Write( - "inline %s* %s::add_%s() { return &%s_.emplace_back(); }" % - (cpp_type, class_name, name, name)) + "inline %s* %s::add_%s() { return &%s_.emplace_back(); }" + % (cpp_type, class_name, name, name) + ) else: w.Write( "inline void %s::add_%s(%s val) { %s_.emplace_back(val); }" - % (class_name, name, cpp_type, name)) + % (class_name, name, cpp_type, name) + ) w.Write( "inline const std::vector<%s>& %s::%s() const { return %s_; }" - % (var_cpp_type, class_name, name, name)) + % (var_cpp_type, class_name, name, name) + ) w.Write( "inline std::vector<%s>* %s::mutable_%s() { return &%s_; }" - % (var_cpp_type, class_name, name, name)) + % (var_cpp_type, class_name, name, name) + ) if self.type.IsMessage(): w.Write( "inline const %s& %s::%s(size_t idx) const { return %s_[idx]; }" - % (cpp_type, class_name, name, name)) + % (cpp_type, class_name, name, name) + ) w.Write( "inline %s* %s::mutable_%s(size_t idx) { return &%s_[idx]; }" - % (cpp_type, class_name, name, name)) + % (cpp_type, class_name, name, name) + ) else: w.Write( - "inline %s %s::%s(size_t idx) const { return %s_[idx]; }" % - (cpp_type, class_name, name, name)) + "inline %s %s::%s(size_t idx) const { return %s_[idx]; }" + % (cpp_type, class_name, name, name) + ) w.Write( - "inline size_t %s::%s_size() const { return %s_.size(); }" % - (class_name, name, name)) + "inline size_t %s::%s_size() const { return %s_.size(); }" + % (class_name, name, name) + ) else: - w.Write("inline bool %s::has_%s() const { return has_%s_; }" % - (class_name, name, name)) + w.Write( + "inline bool %s::has_%s() const { return has_%s_; }" + % (class_name, name, name) + ) if self.type.IsMessage(): - w.Write("inline const %s& %s::%s() const { return %s_; }" % - (cpp_type, class_name, name, name)) - w.Write("inline %s* %s::mutable_%s() {" % - (cpp_type, class_name, name)) + w.Write( + "inline const %s& %s::%s() const { return %s_; }" + % (cpp_type, class_name, name, name) + ) + if self.type.IsMessage() or self.type.IsBytesType(): + w.Write( + "inline %s* %s::mutable_%s() {" % (var_cpp_type, class_name, name) + ) w.Indent() - w.Write('has_%s_ = true;' % (name)) - w.Write('return &%s_;' % name) + w.Write("has_%s_ = true;" % (name)) + w.Write("return &%s_;" % name) w.Unindent() w.Write("}") - else: - w.Write("inline %s %s::%s() const { return %s_; }" % - (cpp_type, class_name, name, name)) - w.Write("inline void %s::set_%s(%s val) {" % - (class_name, name, cpp_type)) + if not self.type.IsMessage(): + w.Write( + "inline %s %s::%s() const { return %s_; }" + % (cpp_type, class_name, name, name) + ) + w.Write( + "inline void %s::set_%s(%s val) {" % (class_name, name, cpp_type) + ) w.Indent() w.Write("has_%s_ = true;" % name) w.Write("%s_ = val;" % name) @@ -457,41 +524,43 @@ def GenerateFunctionDefinitions(self, w, class_name): def GenerateVariable(self, w): name = self.name.group(0) cpp_type = self.type.GetVariableCppType() - if self.category == 'repeated': + if self.category == "repeated": w.Write("std::vector<%s> %s_;" % (cpp_type, name)) else: w.Write("bool has_%s_{};" % (name)) - w.Write("%s %s_{};" % (cpp_type, name)) + if "default" in self.attributes: + w.Write("%s %s_{%s};" % (cpp_type, name, self.attributes["default"])) + else: + w.Write("%s %s_{};" % (cpp_type, name)) return class ProtoEnumParser: - def __init__(self, lexer, scope): - lexer.Consume('enum') - self.name = lexer.Consume('identifier').group(0) + lexer.Consume("enum") + self.name = lexer.Consume("identifier").group(0) self.values = [] self.scope = scope[:] - lexer.Consume('{') + lexer.Consume("{") while True: token, match = lexer.Pick() - if token == '}': + if token == "}": break - key = lexer.Consume('identifier').group(0) - lexer.Consume('=') - value = int(lexer.Consume('number').group(0)) - lexer.Consume(';') + key = lexer.Consume("identifier").group(0) + lexer.Consume("=") + value = int(lexer.Consume("number").group(0)) + lexer.Consume(";") self.values.append((key, value)) - lexer.Consume('}') + lexer.Consume("}") def GetName(self): return self.name def GetFullName(self): - return '_'.join([x.GetName() for x in self.scope] + [self.name]) + return "_".join([x.GetName() for x in self.scope] + [self.name]) def GetType(self): - return 'enum' + return "enum" def IsType(self): return True @@ -510,81 +579,112 @@ def GenerateFunctionDefinitions(self, w): def GenerateEnumDefinitions(self, w): # Protobuf enum is mapped directly to C++ enum. - w.Write('enum %s : int {' % self.GetFullName()) + w.Write("enum %s : int {" % self.GetFullName()) w.Indent() for key, value in self.values: - w.Write('%s_%s = %d,' % (self.GetFullName(), key, value)) + w.Write("%s_%s = %d," % (self.GetFullName(), key, value)) w.Unindent() - w.Write('};') - w.Write('inline std::string %s_Name(%s val) {' % - (self.GetFullName(), self.GetFullName())) + w.Write("};") + w.Write( + "inline std::string %s_Name(%s val) {" + % (self.GetFullName(), self.GetFullName()) + ) w.Indent() - w.Write('switch (val) {') + w.Write("switch (val) {") w.Indent() for key, _ in self.values: - w.Write('case %s_%s:' % (self.GetFullName(), key)) + w.Write("case %s_%s:" % (self.GetFullName(), key)) w.Write(' return "%s";' % key) w.Unindent() - w.Write('};') + w.Write("};") w.Write('return "%s(" + std::to_string(val) + ")";' % self.name) w.Unindent() - w.Write('}') + w.Write("}") def GenerateUsingDirectives(self, w): - w.Write('using %s = %s;' % (self.name, self.GetFullName())) + w.Write("using %s = %s;" % (self.name, self.GetFullName())) for key, _ in self.values: - w.Write('static constexpr %s %s =' % (self.name, key)) - w.Write(' %s_%s;' % (self.GetFullName(), key)) - w.Write('static constexpr std::array<%s,%d> %s_AllValues = {' % - (self.name, len(self.values), self.name)) + w.Write("static constexpr %s %s =" % (self.name, key)) + w.Write(" %s_%s;" % (self.GetFullName(), key)) + w.Write( + "static constexpr std::array<%s,%d> %s_AllValues = {" + % (self.name, len(self.values), self.name) + ) w.Indent() for key, _ in self.values: - w.Write('%s,' % key) + w.Write("%s," % key) w.Unindent() - w.Write('};') + w.Write("};") # Static function to convert an enum value to its name. - w.Write('static std::string %s_Name(%s val) {' % - (self.name, self.name)) + w.Write("static std::string %s_Name(%s val) {" % (self.name, self.name)) w.Indent() - w.Write('return %s_Name(val);' % (self.GetFullName())) + w.Write("return %s_Name(val);" % (self.GetFullName())) w.Unindent() - w.Write('}') + w.Write("}") -class ProtoMessageParser: +def ParseReservedFields(lexer): + res = set() + lexer.Consume("reserved") + while True: + token, match = lexer.Pick() + if token == "number": + num = int(lexer.Consume("number").group(0)) + if lexer.Pick()[0] == "to": + lexer.Consume("to") + end = int(lexer.Consume("number").group(0)) + res.add(range(num, end + 1)) + else: + res.add(num) + elif token in ["identifier", "string"]: + res.add(lexer.Consume(token).group(1)) + else: + lexer.Error("Expected number or identifier") + token, _ = lexer.Pick() + if token == ";": + lexer.Consume(";") + break + lexer.Consume(",") + return res + +class ProtoMessageParser: def __init__(self, lexer, type_stack, scope): type_stack[0].append(self) + self.reserved = set() self.types = [] self.fields = [] self.scope = scope[:] - lexer.Consume('message') - self.name = lexer.Consume('identifier').group(0) - lexer.Consume('{') + lexer.Consume("message") + self.name = lexer.Consume("identifier").group(0) + lexer.Consume("{") while True: token, match = lexer.Pick() - if token == '}': + if token == "}": break - elif token == 'message': - ProtoMessageParser(lexer, [self.types, *type_stack], - self.scope + [self]) - elif token == 'enum': + elif token == "message": + ProtoMessageParser( + lexer, [self.types, *type_stack], self.scope + [self] + ) + elif token == "enum": self.types.append(ProtoEnumParser(lexer, self.scope + [self])) - elif token in ['repeated', 'optional', 'required']: - self.fields.append( - ProtoFieldParser(lexer, [self.types, *type_stack])) + elif token in ["repeated", "optional", "required"]: + self.fields.append(ProtoFieldParser(lexer, [self.types, *type_stack])) + elif token == "reserved": + self.reserved.update(ParseReservedFields(lexer)) else: - lexer.Error('Expected field or type') - lexer.Consume('}') + lexer.Error("Expected field or type") + lexer.Consume("}") + self.CheckReserved() def GetName(self): return self.name def GetFullName(self): - return '_'.join([x.GetName() for x in self.scope] + [self.name]) + return "_".join([x.GetName() for x in self.scope] + [self.name]) def GetType(self): - return 'message' + return "message" def IsType(self): return True @@ -598,6 +698,20 @@ def GetFieldsGruppedByWireType(self): type_to_fields.setdefault(x.type.GetWireType(), []).append(x) return type_to_fields + def CheckReserved(self): + for r in self.reserved: + if isinstance(r, int): + if any(x.number == r for x in self.fields): + raise ValueError(f"Field number [{r}] is reserved.") + elif isinstance(r, range): + if any(x.number in r for x in self.fields): + raise ValueError( + f"Field range [{r.start} to {r.stop - 1}] is reserved." + ) + else: + if any(x.name.group(0) == r for x in self.fields): + raise ValueError(f"Field name [{r}] is reserved.") + def ResolveForwardDeclarations(self, type_stack): type_stack.append(self.types) for x in self.types: @@ -607,41 +721,44 @@ def ResolveForwardDeclarations(self, type_stack): type_stack.pop() def WriteFieldParserDeclaration(self, w, wire_id, fields): - fname = {0: 'SetVarInt', 1: 'SetInt64', 2: 'SetString', 5: 'SetInt32'} + fname = {0: "SetVarInt", 1: "SetInt64", 2: "SetString", 5: "SetInt32"} tname = { - 0: 'std::uint64_t', - 1: 'std::uint64_t', - 2: 'std::string_view', - 5: 'std::uint32_t' + 0: "std::uint64_t", + 1: "std::uint64_t", + 2: "std::string_view", + 5: "std::uint32_t", } - w.Write('void %s(int field_id, %s val) final;' % - (fname[wire_id], tname[wire_id])) + w.Write( + "void %s(int field_id, %s val) final;" % (fname[wire_id], tname[wire_id]) + ) def WriteFieldParserDefinition(self, w, wire_id, fields): - fname = {0: 'SetVarInt', 1: 'SetInt64', 2: 'SetString', 5: 'SetInt32'} + fname = {0: "SetVarInt", 1: "SetInt64", 2: "SetString", 5: "SetInt32"} tname = { - 0: 'std::uint64_t', - 1: 'std::uint64_t', - 2: 'std::string_view', - 5: 'std::uint32_t' + 0: "std::uint64_t", + 1: "std::uint64_t", + 2: "std::string_view", + 5: "std::uint32_t", } - w.Write('inline void %s::%s(int field_id, %s val) {' % - (self.GetFullName(), fname[wire_id], tname[wire_id])) + w.Write( + "inline void %s::%s(int field_id, %s val) {" + % (self.GetFullName(), fname[wire_id], tname[wire_id]) + ) w.Indent() - w.Write('switch (field_id) {') + w.Write("switch (field_id) {") w.Indent() for field in fields: field.GenerateCaseClause(w) w.Unindent() - w.Write('}') + w.Write("}") w.Unindent() - w.Write('}') + w.Write("}") def GenerateUsingDirectives(self, w): - w.Write('using %s = %s;' % (self.name, self.GetFullName())) + w.Write("using %s = %s;" % (self.name, self.GetFullName())) def GenerateMessageDeclarations(self, w): - w.Write(f'class %s;' % self.GetFullName()) + w.Write(f"class %s;" % self.GetFullName()) for x in self.types: x.GenerateMessageDeclarations(w) @@ -652,42 +769,41 @@ def GenerateEnumDefinitions(self, w): def GenerateMessageDefinitions(self, w): # Writing nested messages. for x in self.types: - if x.GetType() == 'message': + if x.GetType() == "message": x.GenerateMessageDefinitions(w) # Protobuf message is a C++ class. - w.Write('class %s final : public lczero::ProtoMessage {' % - self.GetFullName()) - w.Write(' public:') + w.Write("class %s final : public lczero::ProtoMessage {" % self.GetFullName()) + w.Write(" public:") w.Indent() # Writing using directives. for x in self.types: x.GenerateUsingDirectives(w) # Writing function declarations. for x in self.fields: - w.Write('') + w.Write("") x.GenerateFunctionDeclarations(w) - w.Write('') - w.Write('std::string OutputAsString() const final;') - w.Write('std::string OutputAsJson() const final;') - w.Write('void Clear() final;') + w.Write("") + w.Write("std::string OutputAsString() const final;") + w.Write("std::string OutputAsJson() const final;") + w.Write("void Clear() final;") w.Unindent() - w.Write('') - w.Write(' private:') + w.Write("") + w.Write(" private:") w.Indent() for k, v in self.GetFieldsGruppedByWireType().items(): self.WriteFieldParserDeclaration(w, k, v) - w.Write('') + w.Write("") for x in self.fields: x.GenerateVariable(w) w.Unindent() - w.Write('};') - w.Write('') + w.Write("};") + w.Write("") def GenerateFunctionDefinitions(self, w): # Writing nested messages. for x in self.types: - if x.GetType() == 'message': + if x.GetType() == "message": x.GenerateFunctionDefinitions(w) self.GenerateOutputAsStringFunc(w) self.GenerateOutputAsJsonFunc(w) @@ -696,37 +812,35 @@ def GenerateFunctionDefinitions(self, w): self.GenerateFieldAccessorFuncs(w) def GenerateOutputAsStringFunc(self, w): - w.Write('inline std::string %s::OutputAsString() const {' % - self.GetFullName()) + w.Write("inline std::string %s::OutputAsString() const {" % self.GetFullName()) w.Indent() - w.Write('std::string out;') + w.Write("std::string out;") for x in sorted(self.fields, key=lambda x: x.number): x.GenerateOutput(w) - w.Write('return out;') + w.Write("return out;") w.Unindent() - w.Write('}') + w.Write("}") def GenerateOutputAsJsonFunc(self, w): - w.Write('inline std::string %s::OutputAsJson() const {' % - self.GetFullName()) + w.Write("inline std::string %s::OutputAsJson() const {" % self.GetFullName()) w.Indent() if self.fields: - w.Write('bool first = true;') + w.Write("bool first = true;") w.Write('std::string out = "{";') for x in self.fields: x.GenerateJsonOutput(w) w.Write('out += "}";') - w.Write('return out;') + w.Write("return out;") w.Unindent() - w.Write('}') + w.Write("}") def GenerateClearFunc(self, w): - w.Write('inline void %s::Clear() {' % self.GetFullName()) + w.Write("inline void %s::Clear() {" % self.GetFullName()) w.Indent() for x in self.fields: x.GenerateClear(w) w.Unindent() - w.Write('}') + w.Write("}") def GenerateParserFuncs(self, w): for k, v in self.GetFieldsGruppedByWireType().items(): @@ -738,38 +852,38 @@ def GenerateFieldAccessorFuncs(self, w): class ProtoFileParser: - '''Root grammar of .proto file''' + """Root grammar of .proto file""" def __init__(self, lexer): self.package = None self.types = [] while True: token, match = lexer.Pick() - if token == 'EOF': + if token == "EOF": return - elif token == 'syntax': + elif token == "syntax": self.ParseSyntax(lexer) - elif token == 'package': + elif token == "package": self.ParsePackage(lexer) - elif token == 'message': + elif token == "message": self.ParseMessage(lexer) - elif token == 'enum': + elif token == "enum": self.ParseEnum(lexer) else: - lexer.Error('Expected message or something similar') + lexer.Error("Expected message or something similar") def ParseSyntax(self, lexer): - lexer.Consume('syntax') - lexer.Consume('=') - lexer.Consume('string', 'proto2', 1) - lexer.Consume(';') + lexer.Consume("syntax") + lexer.Consume("=") + lexer.Consume("string", "proto2", 1) + lexer.Consume(";") def ParsePackage(self, lexer): - lexer.Consume('package') + lexer.Consume("package") if self.package is not None: - lexer.Error('Package was already defined') + lexer.Error("Package was already defined") self.package = ReadIdentifierPath(lexer) - lexer.Consume(';') + lexer.Consume(";") def ParseMessage(self, lexer): ProtoMessageParser(lexer, [self.types], []) @@ -778,27 +892,27 @@ def ParseEnum(self, lexer): self.types.append(ProtoEnumParser(lexer, [])) def Generate(self, w): - w.Write('// This file is AUTOGENERATED, do not edit.') - w.Write('#pragma once') + w.Write("// This file is AUTOGENERATED, do not edit.") + w.Write("#pragma once") w.Write('#include "utils/protomessage.h"') for x in self.package: - w.Write('namespace %s {' % x) - w.Write('') - w.Write('// Forward declarations.') + w.Write("namespace %s {" % x) + w.Write("") + w.Write("// Forward declarations.") for object in self.types: object.GenerateMessageDeclarations(w) for object in self.types: object.GenerateEnumDefinitions(w) - w.Write('') - w.Write('// Class declarations.') + w.Write("") + w.Write("// Class declarations.") for object in self.types: object.GenerateMessageDefinitions(w) - w.Write('') - w.Write('// Function definitions.') + w.Write("") + w.Write("// Function definitions.") for object in self.types: object.GenerateFunctionDefinitions(w) for x in reversed(self.package): - w.Write('} // namespace %s' % x) + w.Write("} // namespace %s" % x) def ResolveForwardDeclarations(self): type_stack = [self.types] @@ -807,7 +921,7 @@ def ResolveForwardDeclarations(self): class Writer: - '''A helper class for writing file line by line with indent.''' + """A helper class for writing file line by line with indent.""" def __init__(self, fo): self.fo = fo @@ -821,26 +935,26 @@ def Unindent(self): def Write(self, text): if text: - self.fo.write(' ' * self.indent + text + '\n') + self.fo.write(" " * self.indent + text + "\n") else: - self.fo.write('\n') + self.fo.write("\n") if __name__ == "__main__": # Have the same flags as protoc has. parser = argparse.ArgumentParser(description="Compile protobuf files.") - parser.add_argument('input', type=str) - parser.add_argument('--proto_path', type=str) - parser.add_argument('--cpp_out', type=str) + parser.add_argument("input", type=str) + parser.add_argument("--proto_path", type=str) + parser.add_argument("--cpp_out", type=str) args = parser.parse_args() rel_path = os.path.relpath(args.input, args.proto_path) - dest_name = os.path.splitext(rel_path)[0] + '.pb.h' + dest_name = os.path.splitext(rel_path)[0] + ".pb.h" dest_path = os.path.join(args.cpp_out, dest_name) dest_dir = os.path.dirname(dest_path) os.makedirs(dest_dir, exist_ok=True) - with open(args.input, 'r') as input, open(dest_path, 'w') as output: + with open(args.input, "r") as input, open(dest_path, "w") as output: proto_file = ProtoFileParser(Lexer(input.read())) proto_file.ResolveForwardDeclarations() writer = Writer(output) diff --git a/scripts/sycl_build_hack.py b/scripts/sycl_build_hack.py new file mode 100644 index 0000000000..e7e3478875 --- /dev/null +++ b/scripts/sycl_build_hack.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import os + +dir = os.getenv('MESON_BUILD_ROOT') + +with open(dir + '/build.ninja', 'r') as file: + lines = file.readlines() + +updated = [] +dep_flag = False +link_flag = False + +for line in lines: + # Replace xilink with icx as the linker. + if not link_flag: + link_flag = 'xilink.exe' in line + if link_flag: + line = line.replace('xilink.exe', 'icx') + line = line.replace('/MACHINE:x64', '') + line = line.replace('/OUT:', '-o ') + line = line.replace('/SUBSYSTEM:CONSOLE', '') + line = line.replace('/OPT:REF', '') + line = line.replace('/PDB:', '/Fd') + # Replace msvc compatible dependencies with gcc ones as icx output with /showincludes includes + # temporary header files causing full project rebuilds. + if line.startswith('rule') or line.startswith('build'): + dep_flag = 'cpp_COMPILER' in line + if dep_flag: + line = line.replace('deps = msvc', 'deps = gcc\n depfile = $out.d') + line = line.replace('/showIncludes', '/QMD') + if 'icx' in line: + line = line.replace('/Fo$out', '/Fo$out /QMF$out.d') + updated.append(line) + +with open(dir + '/build.ninja', 'w') as file: + file.writelines(updated) diff --git a/src/benchmark/backendbench.cc b/src/benchmark/backendbench.cc deleted file mode 100644 index 6792f9b778..0000000000 --- a/src/benchmark/backendbench.cc +++ /dev/null @@ -1,189 +0,0 @@ -/* - This file is part of Leela Chess Zero. - Copyright (C) 2020-2021 The LCZero Authors - - Leela Chess is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Leela Chess is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Leela Chess. If not, see . - - Additional permission under GNU GPL version 3 section 7 - - If you modify this Program, or any covered work, by linking or - combining it with NVIDIA Corporation's libraries from the NVIDIA CUDA - Toolkit and the NVIDIA CUDA Deep Neural Network library (or a - modified version of those libraries), containing parts covered by the - terms of the respective license agreement, the licensors of this - Program grant you additional permission to convey the resulting work. -*/ - -#include "benchmark/backendbench.h" - -#include "chess/board.h" -#include "mcts/node.h" -#include "neural/factory.h" -#include "utils/optionsparser.h" - -namespace lczero { -namespace { -const int kDefaultThreads = 1; - -const OptionId kThreadsOptionId{"threads", "Threads", - "Number of (CPU) worker threads to use.", 't'}; -const OptionId kBatchesId{"batches", "", - "Number of batches to run as a benchmark."}; -const OptionId kStartBatchSizeId{"start-batch-size", "", - "Start benchmark from this batch size."}; -const OptionId kMaxBatchSizeId{"max-batch-size", "", - "Maximum batch size to benchmark."}; -const OptionId kBatchStepId{"batch-step", "", - "Step of batch size in benchmark."}; -const OptionId kFenId{"fen", "", "Benchmark initial position FEN."}; - -const OptionId kClippyId{"clippy", "", "Enable helpful assistant."}; - -void Clippy(std::string title, - std::string msg3, std::string best3, std::string msg2, - std::string best2, std::string msg, std::string best) { - std::cout << " __" << std::endl; - std::cout << " / \\" << std::endl; - std::cout << " | | " << std::string(title.length()+2, '_') << std::endl; - std::cout << " + + | " << std::string(title.length()+1, ' ') - << "|" << std::endl; - std::cout << "(@)(@) _| " - << title << " |" - << std::endl; - std::cout << " | | \\ " << std::string(6, ' ') << msg3 - << std::string(4 - best3.length(), ' ') << best3 - << std::string(title.length()-33, ' ') << "|" << std::endl; - std::cout << " || |/ | " << std::string(6, ' ') << msg2 - << std::string(4 - best2.length(), ' ') << best2 - << std::string(title.length()-33, ' ') << "|" << std::endl; - std::cout << " || || | " << std::string(6, ' ') << msg - << std::string(4 - best.length(), ' ') << best - << std::string(title.length()-33, ' ') << "|" << std::endl; - std::cout << " |\\_/| |" << std::string(title.length()+2, '_') << "|" - << std::endl; - std::cout << " \\___/" << std::endl; -} -} // namespace - -void BackendBenchmark::Run() { - OptionsParser options; - NetworkFactory::PopulateOptions(&options); - options.Add(kThreadsOptionId, 1, 128) = kDefaultThreads; - - options.Add(kBatchesId, 1, 999999999) = 100; - options.Add(kStartBatchSizeId, 1, 1024) = 1; - options.Add(kMaxBatchSizeId, 1, 1024) = 256; - options.Add(kBatchStepId, 1, 256) = 1; - options.Add(kFenId) = ChessBoard::kStartposFen; - options.Add(kClippyId) = false; - - if (!options.ProcessAllFlags()) return; - - try { - auto option_dict = options.GetOptionsDict(); - - auto network = NetworkFactory::LoadNetwork(option_dict); - - NodeTree tree; - tree.ResetToPosition(option_dict.Get(kFenId), {}); - - // Do any backend initialization outside the loop. - auto warmup = network->NewComputation(); - warmup->AddInput(EncodePositionForNN( - network->GetCapabilities().input_format, tree.GetPositionHistory(), 8, - FillEmptyHistory::ALWAYS, nullptr)); - warmup->ComputeBlocking(); - - const int batches = option_dict.Get(kBatchesId); - - int best = 1; int best2 = 1; int best3 = 1; - float best_nps = 0.0f; float best_nps2 = 0.0f; float best_nps3 = 0.0f; - std::optional> pending; - - for (int i = option_dict.Get(kStartBatchSizeId); - i <= option_dict.Get(kMaxBatchSizeId); - i += option_dict.Get(kBatchStepId)) { - const auto start = std::chrono::steady_clock::now(); - // TODO: support threads not equal to 1 to be able to more sensibly test - // multiplexing backend. - for (int j = 0; j < batches; j++) { - // Put i copies of tree root node into computation and compute. - auto computation = network->NewComputation(); - for (int k = 0; k < i; k++) { - computation->AddInput(EncodePositionForNN( - network->GetCapabilities().input_format, - tree.GetPositionHistory(), 8, FillEmptyHistory::ALWAYS, nullptr)); - } - computation->ComputeBlocking(); - } - - const auto end = std::chrono::steady_clock::now(); - std::chrono::duration time = end - start; - const auto nps = i * batches / time.count(); - std::cout << "Benchmark batch size " << i - << " with inference average time " - << time.count() / batches * 1000 << "ms - throughput " << nps - << " nps." << std::endl; - - if (option_dict.Get(kClippyId)) { - float nps_ingame = std::pow((nps + best_nps) / 2, 1.085); - float nps_ingame2 = std::pow((nps + best_nps2) / 2, 1.085); - float nps_ingame3 = std::pow((nps + best_nps3) / 2, 1.085); - float threshold = 0.16947 * exp(-4.1695e-6 * nps_ingame * 180) + 0.02; - float threshold2 = 0.16947 * exp(-4.1695e-6 * nps_ingame2 * 15) + 0.02; - float threshold3 = 0.16947 * exp(-4.1695e-6 * nps_ingame3 * 1) + 0.02; - - if (nps > best_nps && - threshold * (i - best) * best_nps < (nps - best_nps) * best) { - best_nps = nps; - best = i; - if (threshold2 * (i - best2) * best_nps2 < - (nps - best_nps2) * best2) { - best_nps2 = nps; - best2 = i; - if (threshold3 * (i - best3) * best_nps3 < - (nps - best_nps3) * best3) { - best_nps3 = nps; - best3 = i; - } - } - if (!pending) { - pending = std::chrono::steady_clock::now(); - } - } - if (pending) { - time = std::chrono::steady_clock::now() - *pending; - if (time.count() > 10) { - Clippy( - "Recommended minibatch-size for this net (so far):", - "1s/move (Bullet): ", std::to_string(best3), - "15s/move (Rapid): ", std::to_string(best2), - "3min/move (Tournament): ", std::to_string(best)); - pending.reset(); - } - } - } - } - if (option_dict.Get(kClippyId)) { - Clippy( - "Recommended minibatch-size for this net:", - "1s/move (Bullet): ", std::to_string(best3), - "15s/move (Rapid): ", std::to_string(best2), - "3min/move (Tournament): ", std::to_string(best)); - } - } catch (Exception& ex) { - std::cerr << ex.what() << std::endl; - } -} -} // namespace lczero diff --git a/src/chess/bitboard.cc b/src/chess/bitboard.cc deleted file mode 100644 index 3402775eec..0000000000 --- a/src/chess/bitboard.cc +++ /dev/null @@ -1,365 +0,0 @@ -/* - This file is part of Leela Chess Zero. - Copyright (C) 2018 The LCZero Authors - - Leela Chess is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Leela Chess is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Leela Chess. If not, see . - - Additional permission under GNU GPL version 3 section 7 - - If you modify this Program, or any covered work, by linking or - combining it with NVIDIA Corporation's libraries from the NVIDIA CUDA - Toolkit and the NVIDIA CUDA Deep Neural Network library (or a - modified version of those libraries), containing parts covered by the - terms of the respective license agreement, the licensors of this - Program grant you additional permission to convey the resulting work. -*/ - -#include "chess/bitboard.h" - -#include "utils/exception.h" - -namespace lczero { - -namespace { - -const Move kIdxToMove[] = { - "a1b1", "a1c1", "a1d1", "a1e1", "a1f1", "a1g1", "a1h1", "a1a2", - "a1b2", "a1c2", "a1a3", "a1b3", "a1c3", "a1a4", "a1d4", "a1a5", - "a1e5", "a1a6", "a1f6", "a1a7", "a1g7", "a1a8", "a1h8", "b1a1", - "b1c1", "b1d1", "b1e1", "b1f1", "b1g1", "b1h1", "b1a2", "b1b2", - "b1c2", "b1d2", "b1a3", "b1b3", "b1c3", "b1d3", "b1b4", "b1e4", - "b1b5", "b1f5", "b1b6", "b1g6", "b1b7", "b1h7", "b1b8", "c1a1", - "c1b1", "c1d1", "c1e1", "c1f1", "c1g1", "c1h1", "c1a2", "c1b2", - "c1c2", "c1d2", "c1e2", "c1a3", "c1b3", "c1c3", "c1d3", "c1e3", - "c1c4", "c1f4", "c1c5", "c1g5", "c1c6", "c1h6", "c1c7", "c1c8", - "d1a1", "d1b1", "d1c1", "d1e1", "d1f1", "d1g1", "d1h1", "d1b2", - "d1c2", "d1d2", "d1e2", "d1f2", "d1b3", "d1c3", "d1d3", "d1e3", - "d1f3", "d1a4", "d1d4", "d1g4", "d1d5", "d1h5", "d1d6", "d1d7", - "d1d8", "e1a1", "e1b1", "e1c1", "e1d1", "e1f1", "e1g1", "e1h1", - "e1c2", "e1d2", "e1e2", "e1f2", "e1g2", "e1c3", "e1d3", "e1e3", - "e1f3", "e1g3", "e1b4", "e1e4", "e1h4", "e1a5", "e1e5", "e1e6", - "e1e7", "e1e8", "f1a1", "f1b1", "f1c1", "f1d1", "f1e1", "f1g1", - "f1h1", "f1d2", "f1e2", "f1f2", "f1g2", "f1h2", "f1d3", "f1e3", - "f1f3", "f1g3", "f1h3", "f1c4", "f1f4", "f1b5", "f1f5", "f1a6", - "f1f6", "f1f7", "f1f8", "g1a1", "g1b1", "g1c1", "g1d1", "g1e1", - "g1f1", "g1h1", "g1e2", "g1f2", "g1g2", "g1h2", "g1e3", "g1f3", - "g1g3", "g1h3", "g1d4", "g1g4", "g1c5", "g1g5", "g1b6", "g1g6", - "g1a7", "g1g7", "g1g8", "h1a1", "h1b1", "h1c1", "h1d1", "h1e1", - "h1f1", "h1g1", "h1f2", "h1g2", "h1h2", "h1f3", "h1g3", "h1h3", - "h1e4", "h1h4", "h1d5", "h1h5", "h1c6", "h1h6", "h1b7", "h1h7", - "h1a8", "h1h8", "a2a1", "a2b1", "a2c1", "a2b2", "a2c2", "a2d2", - "a2e2", "a2f2", "a2g2", "a2h2", "a2a3", "a2b3", "a2c3", "a2a4", - "a2b4", "a2c4", "a2a5", "a2d5", "a2a6", "a2e6", "a2a7", "a2f7", - "a2a8", "a2g8", "b2a1", "b2b1", "b2c1", "b2d1", "b2a2", "b2c2", - "b2d2", "b2e2", "b2f2", "b2g2", "b2h2", "b2a3", "b2b3", "b2c3", - "b2d3", "b2a4", "b2b4", "b2c4", "b2d4", "b2b5", "b2e5", "b2b6", - "b2f6", "b2b7", "b2g7", "b2b8", "b2h8", "c2a1", "c2b1", "c2c1", - "c2d1", "c2e1", "c2a2", "c2b2", "c2d2", "c2e2", "c2f2", "c2g2", - "c2h2", "c2a3", "c2b3", "c2c3", "c2d3", "c2e3", "c2a4", "c2b4", - "c2c4", "c2d4", "c2e4", "c2c5", "c2f5", "c2c6", "c2g6", "c2c7", - "c2h7", "c2c8", "d2b1", "d2c1", "d2d1", "d2e1", "d2f1", "d2a2", - "d2b2", "d2c2", "d2e2", "d2f2", "d2g2", "d2h2", "d2b3", "d2c3", - "d2d3", "d2e3", "d2f3", "d2b4", "d2c4", "d2d4", "d2e4", "d2f4", - "d2a5", "d2d5", "d2g5", "d2d6", "d2h6", "d2d7", "d2d8", "e2c1", - "e2d1", "e2e1", "e2f1", "e2g1", "e2a2", "e2b2", "e2c2", "e2d2", - "e2f2", "e2g2", "e2h2", "e2c3", "e2d3", "e2e3", "e2f3", "e2g3", - "e2c4", "e2d4", "e2e4", "e2f4", "e2g4", "e2b5", "e2e5", "e2h5", - "e2a6", "e2e6", "e2e7", "e2e8", "f2d1", "f2e1", "f2f1", "f2g1", - "f2h1", "f2a2", "f2b2", "f2c2", "f2d2", "f2e2", "f2g2", "f2h2", - "f2d3", "f2e3", "f2f3", "f2g3", "f2h3", "f2d4", "f2e4", "f2f4", - "f2g4", "f2h4", "f2c5", "f2f5", "f2b6", "f2f6", "f2a7", "f2f7", - "f2f8", "g2e1", "g2f1", "g2g1", "g2h1", "g2a2", "g2b2", "g2c2", - "g2d2", "g2e2", "g2f2", "g2h2", "g2e3", "g2f3", "g2g3", "g2h3", - "g2e4", "g2f4", "g2g4", "g2h4", "g2d5", "g2g5", "g2c6", "g2g6", - "g2b7", "g2g7", "g2a8", "g2g8", "h2f1", "h2g1", "h2h1", "h2a2", - "h2b2", "h2c2", "h2d2", "h2e2", "h2f2", "h2g2", "h2f3", "h2g3", - "h2h3", "h2f4", "h2g4", "h2h4", "h2e5", "h2h5", "h2d6", "h2h6", - "h2c7", "h2h7", "h2b8", "h2h8", "a3a1", "a3b1", "a3c1", "a3a2", - "a3b2", "a3c2", "a3b3", "a3c3", "a3d3", "a3e3", "a3f3", "a3g3", - "a3h3", "a3a4", "a3b4", "a3c4", "a3a5", "a3b5", "a3c5", "a3a6", - "a3d6", "a3a7", "a3e7", "a3a8", "a3f8", "b3a1", "b3b1", "b3c1", - "b3d1", "b3a2", "b3b2", "b3c2", "b3d2", "b3a3", "b3c3", "b3d3", - "b3e3", "b3f3", "b3g3", "b3h3", "b3a4", "b3b4", "b3c4", "b3d4", - "b3a5", "b3b5", "b3c5", "b3d5", "b3b6", "b3e6", "b3b7", "b3f7", - "b3b8", "b3g8", "c3a1", "c3b1", "c3c1", "c3d1", "c3e1", "c3a2", - "c3b2", "c3c2", "c3d2", "c3e2", "c3a3", "c3b3", "c3d3", "c3e3", - "c3f3", "c3g3", "c3h3", "c3a4", "c3b4", "c3c4", "c3d4", "c3e4", - "c3a5", "c3b5", "c3c5", "c3d5", "c3e5", "c3c6", "c3f6", "c3c7", - "c3g7", "c3c8", "c3h8", "d3b1", "d3c1", "d3d1", "d3e1", "d3f1", - "d3b2", "d3c2", "d3d2", "d3e2", "d3f2", "d3a3", "d3b3", "d3c3", - "d3e3", "d3f3", "d3g3", "d3h3", "d3b4", "d3c4", "d3d4", "d3e4", - "d3f4", "d3b5", "d3c5", "d3d5", "d3e5", "d3f5", "d3a6", "d3d6", - "d3g6", "d3d7", "d3h7", "d3d8", "e3c1", "e3d1", "e3e1", "e3f1", - "e3g1", "e3c2", "e3d2", "e3e2", "e3f2", "e3g2", "e3a3", "e3b3", - "e3c3", "e3d3", "e3f3", "e3g3", "e3h3", "e3c4", "e3d4", "e3e4", - "e3f4", "e3g4", "e3c5", "e3d5", "e3e5", "e3f5", "e3g5", "e3b6", - "e3e6", "e3h6", "e3a7", "e3e7", "e3e8", "f3d1", "f3e1", "f3f1", - "f3g1", "f3h1", "f3d2", "f3e2", "f3f2", "f3g2", "f3h2", "f3a3", - "f3b3", "f3c3", "f3d3", "f3e3", "f3g3", "f3h3", "f3d4", "f3e4", - "f3f4", "f3g4", "f3h4", "f3d5", "f3e5", "f3f5", "f3g5", "f3h5", - "f3c6", "f3f6", "f3b7", "f3f7", "f3a8", "f3f8", "g3e1", "g3f1", - "g3g1", "g3h1", "g3e2", "g3f2", "g3g2", "g3h2", "g3a3", "g3b3", - "g3c3", "g3d3", "g3e3", "g3f3", "g3h3", "g3e4", "g3f4", "g3g4", - "g3h4", "g3e5", "g3f5", "g3g5", "g3h5", "g3d6", "g3g6", "g3c7", - "g3g7", "g3b8", "g3g8", "h3f1", "h3g1", "h3h1", "h3f2", "h3g2", - "h3h2", "h3a3", "h3b3", "h3c3", "h3d3", "h3e3", "h3f3", "h3g3", - "h3f4", "h3g4", "h3h4", "h3f5", "h3g5", "h3h5", "h3e6", "h3h6", - "h3d7", "h3h7", "h3c8", "h3h8", "a4a1", "a4d1", "a4a2", "a4b2", - "a4c2", "a4a3", "a4b3", "a4c3", "a4b4", "a4c4", "a4d4", "a4e4", - "a4f4", "a4g4", "a4h4", "a4a5", "a4b5", "a4c5", "a4a6", "a4b6", - "a4c6", "a4a7", "a4d7", "a4a8", "a4e8", "b4b1", "b4e1", "b4a2", - "b4b2", "b4c2", "b4d2", "b4a3", "b4b3", "b4c3", "b4d3", "b4a4", - "b4c4", "b4d4", "b4e4", "b4f4", "b4g4", "b4h4", "b4a5", "b4b5", - "b4c5", "b4d5", "b4a6", "b4b6", "b4c6", "b4d6", "b4b7", "b4e7", - "b4b8", "b4f8", "c4c1", "c4f1", "c4a2", "c4b2", "c4c2", "c4d2", - "c4e2", "c4a3", "c4b3", "c4c3", "c4d3", "c4e3", "c4a4", "c4b4", - "c4d4", "c4e4", "c4f4", "c4g4", "c4h4", "c4a5", "c4b5", "c4c5", - "c4d5", "c4e5", "c4a6", "c4b6", "c4c6", "c4d6", "c4e6", "c4c7", - "c4f7", "c4c8", "c4g8", "d4a1", "d4d1", "d4g1", "d4b2", "d4c2", - "d4d2", "d4e2", "d4f2", "d4b3", "d4c3", "d4d3", "d4e3", "d4f3", - "d4a4", "d4b4", "d4c4", "d4e4", "d4f4", "d4g4", "d4h4", "d4b5", - "d4c5", "d4d5", "d4e5", "d4f5", "d4b6", "d4c6", "d4d6", "d4e6", - "d4f6", "d4a7", "d4d7", "d4g7", "d4d8", "d4h8", "e4b1", "e4e1", - "e4h1", "e4c2", "e4d2", "e4e2", "e4f2", "e4g2", "e4c3", "e4d3", - "e4e3", "e4f3", "e4g3", "e4a4", "e4b4", "e4c4", "e4d4", "e4f4", - "e4g4", "e4h4", "e4c5", "e4d5", "e4e5", "e4f5", "e4g5", "e4c6", - "e4d6", "e4e6", "e4f6", "e4g6", "e4b7", "e4e7", "e4h7", "e4a8", - "e4e8", "f4c1", "f4f1", "f4d2", "f4e2", "f4f2", "f4g2", "f4h2", - "f4d3", "f4e3", "f4f3", "f4g3", "f4h3", "f4a4", "f4b4", "f4c4", - "f4d4", "f4e4", "f4g4", "f4h4", "f4d5", "f4e5", "f4f5", "f4g5", - "f4h5", "f4d6", "f4e6", "f4f6", "f4g6", "f4h6", "f4c7", "f4f7", - "f4b8", "f4f8", "g4d1", "g4g1", "g4e2", "g4f2", "g4g2", "g4h2", - "g4e3", "g4f3", "g4g3", "g4h3", "g4a4", "g4b4", "g4c4", "g4d4", - "g4e4", "g4f4", "g4h4", "g4e5", "g4f5", "g4g5", "g4h5", "g4e6", - "g4f6", "g4g6", "g4h6", "g4d7", "g4g7", "g4c8", "g4g8", "h4e1", - "h4h1", "h4f2", "h4g2", "h4h2", "h4f3", "h4g3", "h4h3", "h4a4", - "h4b4", "h4c4", "h4d4", "h4e4", "h4f4", "h4g4", "h4f5", "h4g5", - "h4h5", "h4f6", "h4g6", "h4h6", "h4e7", "h4h7", "h4d8", "h4h8", - "a5a1", "a5e1", "a5a2", "a5d2", "a5a3", "a5b3", "a5c3", "a5a4", - "a5b4", "a5c4", "a5b5", "a5c5", "a5d5", "a5e5", "a5f5", "a5g5", - "a5h5", "a5a6", "a5b6", "a5c6", "a5a7", "a5b7", "a5c7", "a5a8", - "a5d8", "b5b1", "b5f1", "b5b2", "b5e2", "b5a3", "b5b3", "b5c3", - "b5d3", "b5a4", "b5b4", "b5c4", "b5d4", "b5a5", "b5c5", "b5d5", - "b5e5", "b5f5", "b5g5", "b5h5", "b5a6", "b5b6", "b5c6", "b5d6", - "b5a7", "b5b7", "b5c7", "b5d7", "b5b8", "b5e8", "c5c1", "c5g1", - "c5c2", "c5f2", "c5a3", "c5b3", "c5c3", "c5d3", "c5e3", "c5a4", - "c5b4", "c5c4", "c5d4", "c5e4", "c5a5", "c5b5", "c5d5", "c5e5", - "c5f5", "c5g5", "c5h5", "c5a6", "c5b6", "c5c6", "c5d6", "c5e6", - "c5a7", "c5b7", "c5c7", "c5d7", "c5e7", "c5c8", "c5f8", "d5d1", - "d5h1", "d5a2", "d5d2", "d5g2", "d5b3", "d5c3", "d5d3", "d5e3", - "d5f3", "d5b4", "d5c4", "d5d4", "d5e4", "d5f4", "d5a5", "d5b5", - "d5c5", "d5e5", "d5f5", "d5g5", "d5h5", "d5b6", "d5c6", "d5d6", - "d5e6", "d5f6", "d5b7", "d5c7", "d5d7", "d5e7", "d5f7", "d5a8", - "d5d8", "d5g8", "e5a1", "e5e1", "e5b2", "e5e2", "e5h2", "e5c3", - "e5d3", "e5e3", "e5f3", "e5g3", "e5c4", "e5d4", "e5e4", "e5f4", - "e5g4", "e5a5", "e5b5", "e5c5", "e5d5", "e5f5", "e5g5", "e5h5", - "e5c6", "e5d6", "e5e6", "e5f6", "e5g6", "e5c7", "e5d7", "e5e7", - "e5f7", "e5g7", "e5b8", "e5e8", "e5h8", "f5b1", "f5f1", "f5c2", - "f5f2", "f5d3", "f5e3", "f5f3", "f5g3", "f5h3", "f5d4", "f5e4", - "f5f4", "f5g4", "f5h4", "f5a5", "f5b5", "f5c5", "f5d5", "f5e5", - "f5g5", "f5h5", "f5d6", "f5e6", "f5f6", "f5g6", "f5h6", "f5d7", - "f5e7", "f5f7", "f5g7", "f5h7", "f5c8", "f5f8", "g5c1", "g5g1", - "g5d2", "g5g2", "g5e3", "g5f3", "g5g3", "g5h3", "g5e4", "g5f4", - "g5g4", "g5h4", "g5a5", "g5b5", "g5c5", "g5d5", "g5e5", "g5f5", - "g5h5", "g5e6", "g5f6", "g5g6", "g5h6", "g5e7", "g5f7", "g5g7", - "g5h7", "g5d8", "g5g8", "h5d1", "h5h1", "h5e2", "h5h2", "h5f3", - "h5g3", "h5h3", "h5f4", "h5g4", "h5h4", "h5a5", "h5b5", "h5c5", - "h5d5", "h5e5", "h5f5", "h5g5", "h5f6", "h5g6", "h5h6", "h5f7", - "h5g7", "h5h7", "h5e8", "h5h8", "a6a1", "a6f1", "a6a2", "a6e2", - "a6a3", "a6d3", "a6a4", "a6b4", "a6c4", "a6a5", "a6b5", "a6c5", - "a6b6", "a6c6", "a6d6", "a6e6", "a6f6", "a6g6", "a6h6", "a6a7", - "a6b7", "a6c7", "a6a8", "a6b8", "a6c8", "b6b1", "b6g1", "b6b2", - "b6f2", "b6b3", "b6e3", "b6a4", "b6b4", "b6c4", "b6d4", "b6a5", - "b6b5", "b6c5", "b6d5", "b6a6", "b6c6", "b6d6", "b6e6", "b6f6", - "b6g6", "b6h6", "b6a7", "b6b7", "b6c7", "b6d7", "b6a8", "b6b8", - "b6c8", "b6d8", "c6c1", "c6h1", "c6c2", "c6g2", "c6c3", "c6f3", - "c6a4", "c6b4", "c6c4", "c6d4", "c6e4", "c6a5", "c6b5", "c6c5", - "c6d5", "c6e5", "c6a6", "c6b6", "c6d6", "c6e6", "c6f6", "c6g6", - "c6h6", "c6a7", "c6b7", "c6c7", "c6d7", "c6e7", "c6a8", "c6b8", - "c6c8", "c6d8", "c6e8", "d6d1", "d6d2", "d6h2", "d6a3", "d6d3", - "d6g3", "d6b4", "d6c4", "d6d4", "d6e4", "d6f4", "d6b5", "d6c5", - "d6d5", "d6e5", "d6f5", "d6a6", "d6b6", "d6c6", "d6e6", "d6f6", - "d6g6", "d6h6", "d6b7", "d6c7", "d6d7", "d6e7", "d6f7", "d6b8", - "d6c8", "d6d8", "d6e8", "d6f8", "e6e1", "e6a2", "e6e2", "e6b3", - "e6e3", "e6h3", "e6c4", "e6d4", "e6e4", "e6f4", "e6g4", "e6c5", - "e6d5", "e6e5", "e6f5", "e6g5", "e6a6", "e6b6", "e6c6", "e6d6", - "e6f6", "e6g6", "e6h6", "e6c7", "e6d7", "e6e7", "e6f7", "e6g7", - "e6c8", "e6d8", "e6e8", "e6f8", "e6g8", "f6a1", "f6f1", "f6b2", - "f6f2", "f6c3", "f6f3", "f6d4", "f6e4", "f6f4", "f6g4", "f6h4", - "f6d5", "f6e5", "f6f5", "f6g5", "f6h5", "f6a6", "f6b6", "f6c6", - "f6d6", "f6e6", "f6g6", "f6h6", "f6d7", "f6e7", "f6f7", "f6g7", - "f6h7", "f6d8", "f6e8", "f6f8", "f6g8", "f6h8", "g6b1", "g6g1", - "g6c2", "g6g2", "g6d3", "g6g3", "g6e4", "g6f4", "g6g4", "g6h4", - "g6e5", "g6f5", "g6g5", "g6h5", "g6a6", "g6b6", "g6c6", "g6d6", - "g6e6", "g6f6", "g6h6", "g6e7", "g6f7", "g6g7", "g6h7", "g6e8", - "g6f8", "g6g8", "g6h8", "h6c1", "h6h1", "h6d2", "h6h2", "h6e3", - "h6h3", "h6f4", "h6g4", "h6h4", "h6f5", "h6g5", "h6h5", "h6a6", - "h6b6", "h6c6", "h6d6", "h6e6", "h6f6", "h6g6", "h6f7", "h6g7", - "h6h7", "h6f8", "h6g8", "h6h8", "a7a1", "a7g1", "a7a2", "a7f2", - "a7a3", "a7e3", "a7a4", "a7d4", "a7a5", "a7b5", "a7c5", "a7a6", - "a7b6", "a7c6", "a7b7", "a7c7", "a7d7", "a7e7", "a7f7", "a7g7", - "a7h7", "a7a8", "a7b8", "a7c8", "b7b1", "b7h1", "b7b2", "b7g2", - "b7b3", "b7f3", "b7b4", "b7e4", "b7a5", "b7b5", "b7c5", "b7d5", - "b7a6", "b7b6", "b7c6", "b7d6", "b7a7", "b7c7", "b7d7", "b7e7", - "b7f7", "b7g7", "b7h7", "b7a8", "b7b8", "b7c8", "b7d8", "c7c1", - "c7c2", "c7h2", "c7c3", "c7g3", "c7c4", "c7f4", "c7a5", "c7b5", - "c7c5", "c7d5", "c7e5", "c7a6", "c7b6", "c7c6", "c7d6", "c7e6", - "c7a7", "c7b7", "c7d7", "c7e7", "c7f7", "c7g7", "c7h7", "c7a8", - "c7b8", "c7c8", "c7d8", "c7e8", "d7d1", "d7d2", "d7d3", "d7h3", - "d7a4", "d7d4", "d7g4", "d7b5", "d7c5", "d7d5", "d7e5", "d7f5", - "d7b6", "d7c6", "d7d6", "d7e6", "d7f6", "d7a7", "d7b7", "d7c7", - "d7e7", "d7f7", "d7g7", "d7h7", "d7b8", "d7c8", "d7d8", "d7e8", - "d7f8", "e7e1", "e7e2", "e7a3", "e7e3", "e7b4", "e7e4", "e7h4", - "e7c5", "e7d5", "e7e5", "e7f5", "e7g5", "e7c6", "e7d6", "e7e6", - "e7f6", "e7g6", "e7a7", "e7b7", "e7c7", "e7d7", "e7f7", "e7g7", - "e7h7", "e7c8", "e7d8", "e7e8", "e7f8", "e7g8", "f7f1", "f7a2", - "f7f2", "f7b3", "f7f3", "f7c4", "f7f4", "f7d5", "f7e5", "f7f5", - "f7g5", "f7h5", "f7d6", "f7e6", "f7f6", "f7g6", "f7h6", "f7a7", - "f7b7", "f7c7", "f7d7", "f7e7", "f7g7", "f7h7", "f7d8", "f7e8", - "f7f8", "f7g8", "f7h8", "g7a1", "g7g1", "g7b2", "g7g2", "g7c3", - "g7g3", "g7d4", "g7g4", "g7e5", "g7f5", "g7g5", "g7h5", "g7e6", - "g7f6", "g7g6", "g7h6", "g7a7", "g7b7", "g7c7", "g7d7", "g7e7", - "g7f7", "g7h7", "g7e8", "g7f8", "g7g8", "g7h8", "h7b1", "h7h1", - "h7c2", "h7h2", "h7d3", "h7h3", "h7e4", "h7h4", "h7f5", "h7g5", - "h7h5", "h7f6", "h7g6", "h7h6", "h7a7", "h7b7", "h7c7", "h7d7", - "h7e7", "h7f7", "h7g7", "h7f8", "h7g8", "h7h8", "a8a1", "a8h1", - "a8a2", "a8g2", "a8a3", "a8f3", "a8a4", "a8e4", "a8a5", "a8d5", - "a8a6", "a8b6", "a8c6", "a8a7", "a8b7", "a8c7", "a8b8", "a8c8", - "a8d8", "a8e8", "a8f8", "a8g8", "a8h8", "b8b1", "b8b2", "b8h2", - "b8b3", "b8g3", "b8b4", "b8f4", "b8b5", "b8e5", "b8a6", "b8b6", - "b8c6", "b8d6", "b8a7", "b8b7", "b8c7", "b8d7", "b8a8", "b8c8", - "b8d8", "b8e8", "b8f8", "b8g8", "b8h8", "c8c1", "c8c2", "c8c3", - "c8h3", "c8c4", "c8g4", "c8c5", "c8f5", "c8a6", "c8b6", "c8c6", - "c8d6", "c8e6", "c8a7", "c8b7", "c8c7", "c8d7", "c8e7", "c8a8", - "c8b8", "c8d8", "c8e8", "c8f8", "c8g8", "c8h8", "d8d1", "d8d2", - "d8d3", "d8d4", "d8h4", "d8a5", "d8d5", "d8g5", "d8b6", "d8c6", - "d8d6", "d8e6", "d8f6", "d8b7", "d8c7", "d8d7", "d8e7", "d8f7", - "d8a8", "d8b8", "d8c8", "d8e8", "d8f8", "d8g8", "d8h8", "e8e1", - "e8e2", "e8e3", "e8a4", "e8e4", "e8b5", "e8e5", "e8h5", "e8c6", - "e8d6", "e8e6", "e8f6", "e8g6", "e8c7", "e8d7", "e8e7", "e8f7", - "e8g7", "e8a8", "e8b8", "e8c8", "e8d8", "e8f8", "e8g8", "e8h8", - "f8f1", "f8f2", "f8a3", "f8f3", "f8b4", "f8f4", "f8c5", "f8f5", - "f8d6", "f8e6", "f8f6", "f8g6", "f8h6", "f8d7", "f8e7", "f8f7", - "f8g7", "f8h7", "f8a8", "f8b8", "f8c8", "f8d8", "f8e8", "f8g8", - "f8h8", "g8g1", "g8a2", "g8g2", "g8b3", "g8g3", "g8c4", "g8g4", - "g8d5", "g8g5", "g8e6", "g8f6", "g8g6", "g8h6", "g8e7", "g8f7", - "g8g7", "g8h7", "g8a8", "g8b8", "g8c8", "g8d8", "g8e8", "g8f8", - "g8h8", "h8a1", "h8h1", "h8b2", "h8h2", "h8c3", "h8h3", "h8d4", - "h8h4", "h8e5", "h8h5", "h8f6", "h8g6", "h8h6", "h8f7", "h8g7", - "h8h7", "h8a8", "h8b8", "h8c8", "h8d8", "h8e8", "h8f8", "h8g8", - "a7a8q", "a7a8r", "a7a8b", "a7b8q", "a7b8r", "a7b8b", "b7a8q", "b7a8r", - "b7a8b", "b7b8q", "b7b8r", "b7b8b", "b7c8q", "b7c8r", "b7c8b", "c7b8q", - "c7b8r", "c7b8b", "c7c8q", "c7c8r", "c7c8b", "c7d8q", "c7d8r", "c7d8b", - "d7c8q", "d7c8r", "d7c8b", "d7d8q", "d7d8r", "d7d8b", "d7e8q", "d7e8r", - "d7e8b", "e7d8q", "e7d8r", "e7d8b", "e7e8q", "e7e8r", "e7e8b", "e7f8q", - "e7f8r", "e7f8b", "f7e8q", "f7e8r", "f7e8b", "f7f8q", "f7f8r", "f7f8b", - "f7g8q", "f7g8r", "f7g8b", "g7f8q", "g7f8r", "g7f8b", "g7g8q", "g7g8r", - "g7g8b", "g7h8q", "g7h8r", "g7h8b", "h7g8q", "h7g8r", "h7g8b", "h7h8q", - "h7h8r", "h7h8b"}; - -std::vector BuildMoveIndices() { - std::vector res(4 * 64 * 64); - for (size_t i = 0; i < sizeof(kIdxToMove) / sizeof(kIdxToMove[0]); ++i) { - res[kIdxToMove[i].as_packed_int()] = i; - } - return res; -} - -const std::vector kMoveToIdx = BuildMoveIndices(); -const int kKingCastleIndex = - kMoveToIdx[BoardSquare("e1").as_int() * 64 + BoardSquare("h1").as_int()]; -const int kQueenCastleIndex = - kMoveToIdx[BoardSquare("e1").as_int() * 64 + BoardSquare("a1").as_int()]; - -BoardSquare Transform(BoardSquare sq, int transform) { - if ((transform & FlipTransform) != 0) { - sq.set(sq.row(), 7 - sq.col()); - } - if ((transform & MirrorTransform) != 0) { - sq.set(7 - sq.row(), sq.col()); - } - if ((transform & TransposeTransform) != 0) { - sq.set(7 - sq.col(), 7 - sq.row()); - } - return sq; -} -} // namespace - -Move::Move(const std::string& str, bool black) { - if (str.size() < 4) throw Exception("Bad move: " + str); - SetFrom(BoardSquare(str.substr(0, 2), black)); - SetTo(BoardSquare(str.substr(2, 2), black)); - if (str.size() != 4) { - if (str.size() != 5) throw Exception("Bad move: " + str); - switch (str[4]) { - case 'q': - case 'Q': - SetPromotion(Promotion::Queen); - break; - case 'r': - case 'R': - SetPromotion(Promotion::Rook); - break; - case 'b': - case 'B': - SetPromotion(Promotion::Bishop); - break; - case 'n': - case 'N': - SetPromotion(Promotion::Knight); - break; - default: - throw Exception("Bad move: " + str); - } - } -} - -uint16_t Move::as_packed_int() const { - if (promotion() == Promotion::Knight) { - return from().as_int() * 64 + to().as_int(); - } else { - return static_cast(promotion()) * 64 * 64 + from().as_int() * 64 + - to().as_int(); - } -} - -uint16_t Move::as_nn_index(int transform) const { - if (transform == 0) { - return kMoveToIdx[as_packed_int()]; - } - Move transformed = *this; - transformed.SetTo(Transform(to(), transform)); - transformed.SetFrom(Transform(from(), transform)); - return transformed.as_nn_index(0); -} - -Move MoveFromNNIndex(int idx, int transform) { - Move m = kIdxToMove[idx]; - if (transform == 0) { - return m; - } - int inv_transform; - if (transform & TransposeTransform) { - inv_transform = TransposeTransform; - if (transform & FlipTransform) inv_transform |= MirrorTransform; - if (transform & MirrorTransform) inv_transform |= FlipTransform; - } else { - inv_transform = transform; - } - m.SetTo(Transform(m.to(), inv_transform)); - m.SetFrom(Transform(m.from(), inv_transform)); - return m; -} - -} // namespace lczero diff --git a/src/chess/bitboard.h b/src/chess/bitboard.h index b6e6394d0f..e01b87dd93 100644 --- a/src/chess/bitboard.h +++ b/src/chess/bitboard.h @@ -32,59 +32,11 @@ #include #include +#include "chess/types.h" #include "utils/bititer.h" namespace lczero { -// Stores a coordinates of a single square. -class BoardSquare { - public: - constexpr BoardSquare() {} - // As a single number, 0 to 63, bottom to top, left to right. - // 0 is a1, 8 is a2, 63 is h8. - constexpr BoardSquare(std::uint8_t num) : square_(num) {} - // From row(bottom to top), and col(left to right), 0-based. - constexpr BoardSquare(int row, int col) : BoardSquare(row * 8 + col) {} - // From Square name, e.g e4. Only lowercase. - BoardSquare(const std::string& str, bool black = false) - : BoardSquare(black ? '8' - str[1] : str[1] - '1', str[0] - 'a') {} - constexpr std::uint8_t as_int() const { return square_; } - constexpr std::uint64_t as_board() const { return 1ULL << square_; } - void set(int row, int col) { square_ = row * 8 + col; } - - // 0-based, bottom to top. - int row() const { return square_ / 8; } - // 0-based, left to right. - int col() const { return square_ % 8; } - - // Row := 7 - row. Col remains the same. - void Mirror() { square_ = square_ ^ 0b111000; } - - // Checks whether coordinate is within 0..7. - static bool IsValidCoord(int x) { return x >= 0 && x < 8; } - - // Checks whether coordinates are within 0..7. - static bool IsValid(int row, int col) { - return IsValidCoord(row) && IsValidCoord(col); - } - - constexpr bool operator==(const BoardSquare& other) const { - return square_ == other.square_; - } - - constexpr bool operator!=(const BoardSquare& other) const { - return square_ != other.square_; - } - - // Returns the square in algebraic notation (e.g. "e4"). - std::string as_string() const { - return std::string(1, 'a' + col()) + std::string(1, '1' + row()); - } - - private: - std::uint8_t square_ = 0; // Only lower six bits should be set. -}; - // Represents a board as an array of 64 bits. // Bit enumeration goes from bottom to top, from left to right: // Square a1 is bit 0, square h1 is bit 7, square a2 is bit 8. @@ -92,8 +44,9 @@ class BitBoard { public: constexpr BitBoard(std::uint64_t board) : board_(board) {} BitBoard() = default; - BitBoard(const BitBoard&) = default; - BitBoard& operator=(const BitBoard&) = default; + constexpr static BitBoard FromSquare(Square square) { + return BitBoard(1ULL << square.as_idx()); + } std::uint64_t as_int() const { return board_; } void clear() { board_ = 0; } @@ -134,30 +87,15 @@ class BitBoard { // Sets the value for given square to 1 if cond is true. // Otherwise does nothing (doesn't reset!). - void set_if(BoardSquare square, bool cond) { set_if(square.as_int(), cond); } - void set_if(std::uint8_t pos, bool cond) { - board_ |= (std::uint64_t(cond) << pos); - } - void set_if(int row, int col, bool cond) { - set_if(BoardSquare(row, col), cond); + void set_if(Square square, bool cond) { + board_ |= (static_cast(cond) << square.as_idx()); } - // Sets value of given square to 1. - void set(BoardSquare square) { set(square.as_int()); } - void set(std::uint8_t pos) { board_ |= (std::uint64_t(1) << pos); } - void set(int row, int col) { set(BoardSquare(row, col)); } - + void set(Square square) { board_ |= (1ULL << square.as_idx()); } // Sets value of given square to 0. - void reset(BoardSquare square) { reset(square.as_int()); } - void reset(std::uint8_t pos) { board_ &= ~(std::uint64_t(1) << pos); } - void reset(int row, int col) { reset(BoardSquare(row, col)); } - + void reset(Square square) { board_ &= ~(1ULL << square.as_idx()); } // Gets value of a square. - bool get(BoardSquare square) const { return get(square.as_int()); } - bool get(std::uint8_t pos) const { - return board_ & (std::uint64_t(1) << pos); - } - bool get(int row, int col) const { return get(BoardSquare(row, col)); } + bool get(Square square) const { return board_ & (1ULL << square.as_idx()); } // Returns whether all bits of a board are set to 0. bool empty() const { return board_ == 0; } @@ -168,26 +106,21 @@ class BitBoard { // Flips black and white side of a board. void Mirror() { board_ = ReverseBytesInBytes(board_); } - bool operator==(const BitBoard& other) const { - return board_ == other.board_; - } + bool operator==(const BitBoard& other) const = default; + bool operator!=(const BitBoard& other) const = default; - bool operator!=(const BitBoard& other) const { - return board_ != other.board_; - } - - BitIterator begin() const { return board_; } - BitIterator end() const { return 0; } + struct Uin64ToSquare { + constexpr Square operator()(uint64_t x) { return Square::FromIdx(x); } + }; + using Iterator = BitIterator; + Iterator begin() const { return board_; } + Iterator end() const { return 0; } std::string DebugString() const { std::string res; for (int i = 7; i >= 0; --i) { - for (int j = 0; j < 8; ++j) { - if (get(i, j)) - res += '#'; - else - res += '.'; - } + for (int j = 0; j < 8; ++j) + res += get({File::FromIdx(i), Rank::FromIdx(j)}) ? '#' : '.'; res += '\n'; } return res; @@ -215,8 +148,8 @@ class BitBoard { } // Returns bitboard with one bit reset. - friend BitBoard operator-(const BitBoard& a, const BoardSquare& b) { - return {a.board_ & ~b.as_board()}; + friend BitBoard operator-(const BitBoard& a, const Square& b) { + return {a.board_ & ~(1ULL << b.as_idx())}; } // Returns difference (bitwise AND-NOT) of two boards. @@ -228,77 +161,4 @@ class BitBoard { std::uint64_t board_ = 0; }; -class Move { - public: - enum class Promotion : std::uint8_t { None, Queen, Rook, Bishop, Knight }; - Move() = default; - constexpr Move(BoardSquare from, BoardSquare to) - : data_(to.as_int() + (from.as_int() << 6)) {} - constexpr Move(BoardSquare from, BoardSquare to, Promotion promotion) - : data_(to.as_int() + (from.as_int() << 6) + - (static_cast(promotion) << 12)) {} - Move(const std::string& str, bool black = false); - Move(const char* str, bool black = false) : Move(std::string(str), black) {} - - BoardSquare to() const { return BoardSquare(data_ & kToMask); } - BoardSquare from() const { return BoardSquare((data_ & kFromMask) >> 6); } - Promotion promotion() const { return Promotion((data_ & kPromoMask) >> 12); } - - void SetTo(BoardSquare to) { data_ = (data_ & ~kToMask) | to.as_int(); } - void SetFrom(BoardSquare from) { - data_ = (data_ & ~kFromMask) | (from.as_int() << 6); - } - void SetPromotion(Promotion promotion) { - data_ = (data_ & ~kPromoMask) | (static_cast(promotion) << 12); - } - // 0 .. 16384, knight promotion and no promotion is the same. - uint16_t as_packed_int() const; - - // 0 .. 1857, to use in neural networks. - // Transform is a bit field which describes a transform to be applied to the - // the move before converting it to an index. - uint16_t as_nn_index(int transform) const; - - explicit operator bool() const { return data_ != 0; } - bool operator==(const Move& other) const { return data_ == other.data_; } - - void Mirror() { data_ ^= 0b111000111000; } - - std::string as_string() const { - std::string res = from().as_string() + to().as_string(); - switch (promotion()) { - case Promotion::None: - return res; - case Promotion::Queen: - return res + 'q'; - case Promotion::Rook: - return res + 'r'; - case Promotion::Bishop: - return res + 'b'; - case Promotion::Knight: - return res + 'n'; - } - assert(false); - return "Error!"; - } - - private: - uint16_t data_ = 0; - // Move, using the following encoding: - // bits 0..5 "to"-square - // bits 6..11 "from"-square - // bits 12..14 promotion value - - enum Masks : uint16_t { - kToMask = 0b0000000000111111, - kFromMask = 0b0000111111000000, - kPromoMask = 0b0111000000000000, - }; -}; - -using MoveList = std::vector; - -// Gets the move from the NN move index, undoing the given transform. -Move MoveFromNNIndex(int idx, int transform); - } // namespace lczero diff --git a/src/chess/board.cc b/src/chess/board.cc index 083b6b13ad..59bc0c39cd 100644 --- a/src/chess/board.cc +++ b/src/chess/board.cc @@ -29,9 +29,12 @@ #include #include +#include #include #include #include +#include +#include #include "utils/exception.h" @@ -49,9 +52,7 @@ const ChessBoard ChessBoard::kStartposBoard(ChessBoard::kStartposFen); const BitBoard ChessBoard::kPawnMask = 0x00FFFFFFFFFFFF00ULL; -void ChessBoard::Clear() { - *this = ChessBoard(); -} +void ChessBoard::Clear() { *this = ChessBoard(); } void ChessBoard::Mirror() { our_pieces_.Mirror(); @@ -60,8 +61,8 @@ void ChessBoard::Mirror() { rooks_.Mirror(); bishops_.Mirror(); pawns_.Mirror(); - our_king_.Mirror(); - their_king_.Mirror(); + our_king_.Flip(); + their_king_.Flip(); std::swap(our_king_, their_king_); castlings_.Mirror(); flipped_ = !flipped_; @@ -174,12 +175,7 @@ static const BitBoard kPawnAttacks[] = { 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL, 0x0000000000000000ULL}; -static const Move::Promotion kPromotions[] = { - Move::Promotion::Queen, - Move::Promotion::Rook, - Move::Promotion::Bishop, - Move::Promotion::Knight, -}; +static constexpr PieceType kPromotions[] = {kQueen, kRook, kBishop, kKnight}; // Magic bitboard routines and structures. // We use so-called "fancy" magic bitboards. @@ -258,6 +254,11 @@ static MagicParams bishop_magic_params[64]; static BitBoard rook_attacks_table[102400]; static BitBoard bishop_attacks_table[5248]; +namespace { +constexpr bool IsOnBoard(int x) { return x >= 0 && x < 8; } +constexpr bool IsOnBoard(int x, int y) { return IsOnBoard(x) && IsOnBoard(y); } +} // namespace + // Builds rook or bishop attacks table. static void BuildAttacksTable(MagicParams* magic_params, BitBoard* attacks_table, @@ -267,24 +268,24 @@ static void BuildAttacksTable(MagicParams* magic_params, // Initialize for all board squares. for (unsigned square = 0; square < 64; square++) { - const BoardSquare b_sq(square); + const Square b_sq = Square::FromIdx(square); // Calculate relevant occupancy masks. BitBoard mask = {0}; for (int j = 0; j < 4; j++) { auto direction = directions[j]; - auto dst_row = b_sq.row(); - auto dst_col = b_sq.col(); + auto dst_row = b_sq.rank().idx; + auto dst_col = b_sq.file().idx; while (true) { dst_row += direction.first; dst_col += direction.second; // If the next square in this direction is invalid, the current square // is at the board's edge and should not be added. - if (!BoardSquare::IsValid(dst_row + direction.first, - dst_col + direction.second)) + if (!IsOnBoard(dst_row + direction.first, dst_col + direction.second)) break; - const BoardSquare destination(dst_row, dst_col); + const Square destination(File::FromIdx(dst_col), + Rank::FromIdx(dst_row)); mask.set(destination); } } @@ -293,7 +294,7 @@ static void BuildAttacksTable(MagicParams* magic_params, magic_params[square].mask_ = mask.as_int(); // Cache relevant occupancy board squares. - std::vector occupancy_squares; + std::vector occupancy_squares; for (auto occ_sq : BitBoard(magic_params[square].mask_)) { occupancy_squares.emplace_back(occ_sq); @@ -327,13 +328,14 @@ static void BuildAttacksTable(MagicParams* magic_params, for (int j = 0; j < 4; j++) { auto direction = directions[j]; - auto dst_row = b_sq.row(); - auto dst_col = b_sq.col(); + auto dst_row = b_sq.rank().idx; + auto dst_col = b_sq.file().idx; while (true) { dst_row += direction.first; dst_col += direction.second; - if (!BoardSquare::IsValid(dst_row, dst_col)) break; - const BoardSquare destination(dst_row, dst_col); + if (!IsOnBoard(dst_row, dst_col)) break; + const Square destination(File::FromIdx(dst_col), + Rank::FromIdx(dst_row)); attacks.set(destination); if (occupancy.get(destination)) break; } @@ -369,10 +371,10 @@ static void BuildAttacksTable(MagicParams* magic_params, // Returns the rook attacks bitboard for the given rook board square and the // given occupied piece bitboard. -static inline BitBoard GetRookAttacks(const BoardSquare rook_square, +static inline BitBoard GetRookAttacks(const Square rook_square, const BitBoard pieces) { // Calculate magic index. - const uint8_t square = rook_square.as_int(); + const uint8_t square = rook_square.as_idx(); #if defined(NO_PEXT) uint64_t index = pieces.as_int() & rook_magic_params[square].mask_; @@ -388,10 +390,10 @@ static inline BitBoard GetRookAttacks(const BoardSquare rook_square, // Returns the bishop attacks bitboard for the given bishop board square and // the given occupied piece bitboard. -static inline BitBoard GetBishopAttacks(const BoardSquare bishop_square, +static inline BitBoard GetBishopAttacks(const Square bishop_square, const BitBoard pieces) { // Calculate magic index. - const uint8_t square = bishop_square.as_int(); + const uint8_t square = bishop_square.as_idx(); #if defined(NO_PEXT) uint64_t index = pieces.as_int() & bishop_magic_params[square].mask_; @@ -432,50 +434,53 @@ MoveList ChessBoard::GeneratePseudolegalMoves() const { // King if (source == our_king_) { for (const auto& delta : kKingMoves) { - const auto dst_row = source.row() + delta.first; - const auto dst_col = source.col() + delta.second; - if (!BoardSquare::IsValid(dst_row, dst_col)) continue; - const BoardSquare destination(dst_row, dst_col); + const Rank dst_rank = source.rank() + delta.first; + if (!dst_rank.IsValid()) continue; + const File dst_file = source.file() + delta.second; + if (!dst_file.IsValid()) continue; + const Square destination(dst_file, dst_rank); if (our_pieces_.get(destination)) continue; if (IsUnderAttack(destination)) continue; - result.emplace_back(source, destination); + result.emplace_back(Move::White(source, destination)); } // Castlings. - auto walk_free = [this](int from, int to, int rook, int king) { - for (int i = from; i <= to; ++i) { + auto walk_free = [this](File from, File to, File rook, File king) { + for (File i = from; i <= to; ++i) { if (i == rook || i == king) continue; - if (our_pieces_.get(i) || their_pieces_.get(i)) return false; + if (our_pieces_.get({i, kRank1}) || their_pieces_.get({i, kRank1})) { + return false; + } } return true; }; // @From may be less or greater than @to. @To is not included in check // unless it is the same with @from. - auto range_attacked = [this](int from, int to) { - if (from == to) return IsUnderAttack(from); + auto range_attacked = [this](File from, File to) { + if (from == to) return IsUnderAttack(Square(from, kRank1)); const int increment = from < to ? 1 : -1; while (from != to) { - if (IsUnderAttack(from)) return true; + if (IsUnderAttack(Square(from, kRank1))) return true; from += increment; } return false; }; - const uint8_t king = source.col(); + const File king = source.file(); // For castlings we don't check destination king square for checks, it // will be done in legal move check phase. if (castlings_.we_can_000()) { - const uint8_t qrook = castlings_.our_queenside_rook(); - if (walk_free(std::min(static_cast(C1), qrook), - std::max(static_cast(D1), king), qrook, king) && - !range_attacked(king, C1)) { - result.emplace_back(source, BoardSquare(RANK_1, qrook)); + const File qrook = castlings_.our_queenside_rook; + if (walk_free(std::min(kFileC, qrook), std::max(kFileD, king), qrook, + king) && + !range_attacked(king, kFileC)) { + result.emplace_back(Move::WhiteCastling(king, qrook)); } } if (castlings_.we_can_00()) { - const uint8_t krook = castlings_.our_kingside_rook(); - if (walk_free(std::min(static_cast(F1), king), - std::max(static_cast(G1), krook), krook, king) && - !range_attacked(king, G1)) { - result.emplace_back(source, BoardSquare(RANK_1, krook)); + const File krook = castlings_.our_kingside_rook; + if (walk_free(std::min(kFileF, king), std::max(kFileG, krook), krook, + king) && + !range_attacked(king, kFileG)) { + result.emplace_back(Move::WhiteCastling(king, krook)); } } continue; @@ -488,7 +493,7 @@ MoveList ChessBoard::GeneratePseudolegalMoves() const { GetRookAttacks(source, our_pieces_ | their_pieces_) - our_pieces_; for (const auto& destination : attacked) { - result.emplace_back(source, destination); + result.emplace_back(Move::White(source, destination)); } } // Bishop (and queen) @@ -498,7 +503,7 @@ MoveList ChessBoard::GeneratePseudolegalMoves() const { GetBishopAttacks(source, our_pieces_ | their_pieces_) - our_pieces_; for (const auto& destination : attacked) { - result.emplace_back(source, destination); + result.emplace_back(Move::White(source, destination)); } } if (processed_piece) continue; @@ -506,24 +511,25 @@ MoveList ChessBoard::GeneratePseudolegalMoves() const { if ((pawns_ & kPawnMask).get(source)) { // Moves forward. { - const auto dst_row = source.row() + 1; - const auto dst_col = source.col(); - const BoardSquare destination(dst_row, dst_col); + const Rank dst_rank = source.rank() + 1; + const File dst_file = source.file(); + const Square destination(dst_file, dst_rank); if (!our_pieces_.get(destination) && !their_pieces_.get(destination)) { - if (dst_row != RANK_8) { - result.emplace_back(source, destination); - if (dst_row == RANK_3) { + if (dst_rank != kRank8) { + result.emplace_back(Move::White(source, destination)); + if (dst_rank == kRank3) { // Maybe it'll be possible to move two squares. - if (!our_pieces_.get(RANK_4, dst_col) && - !their_pieces_.get(RANK_4, dst_col)) { - result.emplace_back(source, BoardSquare(RANK_4, dst_col)); + const Square jump_dst(dst_file, kRank4); + if (!our_pieces_.get(jump_dst) && !their_pieces_.get(jump_dst)) { + result.emplace_back(Move::White(source, jump_dst)); } } } else { // Promotions for (auto promotion : kPromotions) { - result.emplace_back(source, destination, promotion); + result.emplace_back( + Move::WhitePromotion(source, destination, promotion)); } } } @@ -531,25 +537,27 @@ MoveList ChessBoard::GeneratePseudolegalMoves() const { // Captures. { for (auto direction : {-1, 1}) { - const auto dst_row = source.row() + 1; - const auto dst_col = source.col() + direction; - if (dst_col < 0 || dst_col >= 8) continue; - const BoardSquare destination(dst_row, dst_col); + const auto dst_rank = source.rank() + 1; + const auto dst_file = source.file() + direction; + if (!dst_file.IsValid()) continue; + const Square destination(dst_file, dst_rank); if (their_pieces_.get(destination)) { - if (dst_row == RANK_8) { + if (dst_rank == kRank8) { // Promotion. for (auto promotion : kPromotions) { - result.emplace_back(source, destination, promotion); + result.emplace_back( + Move::WhitePromotion(source, destination, promotion)); } } else { // Ordinary capture. - result.emplace_back(source, destination); + result.emplace_back(Move::White(source, destination)); } - } else if (dst_row == RANK_6 && pawns_.get(RANK_8, dst_col)) { + } else if (dst_rank == kRank6 && + pawns_.get(Square(dst_file, kRank8))) { // En passant. // "Pawn" on opponent's file 8 means that en passant is possible. // Those fake pawns are reset in ApplyMove. - result.emplace_back(source, destination); + result.emplace_back(Move::WhiteEnPassant(source, destination)); } } } @@ -558,58 +566,77 @@ MoveList ChessBoard::GeneratePseudolegalMoves() const { // Knight. { for (const auto destination : - kKnightAttacks[source.as_int()] - our_pieces_) { - result.emplace_back(source, destination); + kKnightAttacks[source.as_idx()] - our_pieces_) { + result.emplace_back(Move::White(source, destination)); } } } return result; } // namespace lczero +bool ChessBoard::IsValid() const { + const auto all = ours() | theirs(); + auto check = all | pawns() | bishops() | rooks() | queens() | kings(); + if (check != all || + (pawns() & bishops()).as_int() || + (pawns() & rooks()).as_int() || + (pawns() & queens()).as_int() || + (pawns() & kings()).as_int() || + (bishops() & rooks()).as_int() || + (bishops() & queens()).as_int() || + (bishops() & kings()).as_int() || + (rooks() & queens()).as_int() || + (rooks() & kings()).as_int() || + (queens() & kings()).as_int()) { + return false; + } + return true; +} + bool ChessBoard::ApplyMove(Move move) { - const auto& from = move.from(); - const auto& to = move.to(); - const auto from_row = from.row(); - const auto from_col = from.col(); - const auto to_row = to.row(); - const auto to_col = to.col(); + assert(our_pieces_.intersects(BitBoard::FromSquare(move.from()))); +#ifndef NDEBUG + absl::Cleanup validate = [&] { + if (!IsValid()) { + CERR << "Move " + move.ToString(true) + + " resulted in invalid board: " + DebugString(); + assert(false); + } + }; +#endif + const Square& from = move.from(); + const Square& to = move.to(); + const Rank from_rank = from.rank(); + const File from_file = from.file(); + const Rank to_rank = to.rank(); + const File to_file = to.file(); // Castlings. if (from == our_king_) { castlings_.reset_we_can_00(); castlings_.reset_we_can_000(); - auto do_castling = [this](int king_dst, int rook_src, int rook_dst) { + auto do_castling = [this](File king_dst, Square rook_src, File rook_dst) { // Remove en passant flags. pawns_ &= kPawnMask; our_pieces_.reset(our_king_); our_pieces_.reset(rook_src); rooks_.reset(rook_src); - our_pieces_.set(king_dst); - our_pieces_.set(rook_dst); - rooks_.set(rook_dst); - our_king_ = king_dst; + our_king_ = Square(king_dst, kRank1); + our_pieces_.set(our_king_); + Square rook_dst_sq(rook_dst, kRank1); + our_pieces_.set(rook_dst_sq); + rooks_.set(rook_dst_sq); }; - if (from_row == RANK_1 && to_row == RANK_1) { - const auto our_rooks = rooks() & our_pieces_; - if (our_rooks.get(to)) { - // Castling. - if (to_col > from_col) { - // Kingside. - do_castling(G1, to.as_int(), F1); - } else { - // Queenside. - do_castling(C1, to.as_int(), D1); - } - return false; - } else if (from_col == FILE_E && to_col == FILE_G) { - // Non FRC-style e1g1 castling (as opposed to e1h1). - do_castling(G1, H1, F1); - return false; - } else if (from_col == FILE_E && to_col == FILE_C) { - // Non FRC-style e1c1 castling (as opposed to e1a1). - do_castling(C1, A1, D1); - return false; + if (move.is_castling()) { + // Castling. + if (to_file > from_file) { + // Kingside. + do_castling(kFileG, to, kFileF); + } else { + // Queenside. + do_castling(kFileC, to, kFileD); } + return false; } } @@ -619,22 +646,24 @@ bool ChessBoard::ApplyMove(Move move) { // Remove captured piece. bool reset_50_moves = their_pieces_.get(to); - their_pieces_.reset(to); - rooks_.reset(to); - bishops_.reset(to); - pawns_.reset(to); - if (to.as_int() == A8 + castlings_.their_kingside_rook()) { - castlings_.reset_they_can_00(); - } - if (to.as_int() == A8 + castlings_.their_queenside_rook()) { - castlings_.reset_they_can_000(); + if (reset_50_moves) { + their_pieces_.reset(to); + rooks_.reset(to); + bishops_.reset(to); + pawns_.reset(to); + if (to == Square(castlings_.their_kingside_rook, kRank8)) { + castlings_.reset_they_can_00(); + } + if (to == Square(castlings_.their_queenside_rook, kRank8)) { + castlings_.reset_they_can_000(); + } } // En passant. - if (from_row == RANK_5 && pawns_.get(from) && from_col != to_col && - pawns_.get(RANK_8, to_col)) { - pawns_.reset(RANK_5, to_col); - their_pieces_.reset(RANK_5, to_col); + if (move.is_en_passant()) { + const Square ep_pawn(to_file, kRank5); + pawns_.reset(ep_pawn); + their_pieces_.reset(ep_pawn); } // Remove en passant flags. @@ -650,15 +679,15 @@ bool ChessBoard::ApplyMove(Move move) { } // Promotion. - if (to_row == RANK_8 && pawns_.get(from)) { - switch (move.promotion()) { - case Move::Promotion::Rook: + if (move.is_promotion()) { + switch (move.promotion().idx) { + case kRook.idx: rooks_.set(to); break; - case Move::Promotion::Bishop: + case kBishop.idx: bishops_.set(to); break; - case Move::Promotion::Queen: + case kQueen.idx: rooks_.set(to); bishops_.set(to); break; @@ -669,11 +698,11 @@ bool ChessBoard::ApplyMove(Move move) { } // Reset castling rights. - if (from_row == RANK_1 && rooks_.get(from)) { - if (from_col == castlings_.our_queenside_rook()) { + if (from_rank == kRank1 && rooks_.get(from)) { + if (from_file == castlings_.our_queenside_rook) { castlings_.reset_we_can_000(); } - if (from_col == castlings_.our_kingside_rook()) { + if (from_file == castlings_.our_kingside_rook) { castlings_.reset_we_can_00(); } } @@ -687,23 +716,23 @@ bool ChessBoard::ApplyMove(Move move) { pawns_.reset(from); // Set en passant flag. - if (to_row - from_row == 2 && pawns_.get(to)) { - BoardSquare ep_sq(to_row - 1, to_col); - if (kPawnAttacks[ep_sq.as_int()].intersects(their_pieces_ & pawns_)) { - pawns_.set(0, to_col); + if (to_rank - from_rank == 2 && pawns_.get(to)) { + Square ep_sq(to_file, to_rank - 1); + if (kPawnAttacks[ep_sq.as_idx()].intersects(their_pieces_ & pawns_)) { + pawns_.set(Square(to_file, kRank1)); } } return reset_50_moves; } -bool ChessBoard::IsUnderAttack(BoardSquare square) const { - const int row = square.row(); - const int col = square.col(); +bool ChessBoard::IsUnderAttack(Square square) const { + const Rank rank = square.rank(); + const File file = square.file(); // Check king. { - const int krow = their_king_.row(); - const int kcol = their_king_.col(); - if (std::abs(krow - row) <= 1 && std::abs(kcol - col) <= 1) return true; + const Rank krank = their_king_.rank(); + const File kfile = their_king_.file(); + if (std::abs(krank - rank) <= 1 && std::abs(kfile - file) <= 1) return true; } // Check rooks (and queens). if (GetRookAttacks(square, our_pieces_ | their_pieces_) @@ -716,12 +745,12 @@ bool ChessBoard::IsUnderAttack(BoardSquare square) const { return true; } // Check pawns. - if (kPawnAttacks[square.as_int()].intersects(their_pieces_ & pawns_)) { + if (kPawnAttacks[square.as_idx()].intersects(their_pieces_ & pawns_)) { return true; } // Check knights. { - if (kKnightAttacks[square.as_int()].intersects(their_pieces_ - their_king_ - + if (kKnightAttacks[square.as_idx()].intersects(their_pieces_ - their_king_ - rooks_ - bishops_ - (pawns_ & kPawnMask))) { return true; @@ -730,60 +759,29 @@ bool ChessBoard::IsUnderAttack(BoardSquare square) const { return false; } -bool ChessBoard::IsSameMove(Move move1, Move move2) const { - // If moves are equal, it's the same move. - if (move1 == move2) return true; - // Explicitly check all legacy castling moves. Need to check for king, for - // e.g. rook e1a1 and e1c1 are different moves. - if (move1.from() != move2.from() || move1.from() != E1 || - our_king_ != move1.from()) { - return false; - } - if (move1.to() == A1 && move2.to() == C1) return true; - if (move1.to() == C1 && move2.to() == A1) return true; - if (move1.to() == G1 && move2.to() == H1) return true; - if (move1.to() == H1 && move2.to() == G1) return true; - return false; -} - -Move ChessBoard::GetLegacyMove(Move move) const { - if (our_king_ != move.from() || !our_pieces_.get(move.to())) { - return move; - } - if (move == Move(E1, H1)) return Move(E1, G1); - if (move == Move(E1, A1)) return Move(E1, C1); - return move; -} - -Move ChessBoard::GetModernMove(Move move) const { - if (our_king_ != E1 || move.from() != E1) return move; - if (move == Move(E1, G1) && !our_pieces_.get(G1)) return Move(E1, H1); - if (move == Move(E1, C1) && !our_pieces_.get(C1)) return Move(E1, A1); - return move; -} - KingAttackInfo ChessBoard::GenerateKingAttackInfo() const { KingAttackInfo king_attack_info; // Number of attackers that give check (used for double check detection). unsigned num_king_attackers = 0; - const int row = our_king_.row(); - const int col = our_king_.col(); + const int row = our_king_.rank().idx; + const int col = our_king_.file().idx; // King checks are unnecessary, as kings cannot give check. // Check rooks (and queens). - if (kRookAttacks[our_king_.as_int()].intersects(their_pieces_ & rooks_)) { + if (kRookAttacks[our_king_.as_idx()].intersects(their_pieces_ & rooks_)) { for (const auto& direction : kRookDirections) { auto dst_row = row; auto dst_col = col; BitBoard attack_line(0); bool possible_pinned_piece_found = false; - BoardSquare possible_pinned_piece; + Square possible_pinned_piece; while (true) { dst_row += direction.first; dst_col += direction.second; - if (!BoardSquare::IsValid(dst_row, dst_col)) break; - const BoardSquare destination(dst_row, dst_col); + if (!IsOnBoard(dst_row, dst_col)) break; + const Square destination(File::FromIdx(dst_col), + Rank::FromIdx(dst_row)); if (our_pieces_.get(destination)) { if (possible_pinned_piece_found) { // No pieces pinned. @@ -815,18 +813,19 @@ KingAttackInfo ChessBoard::GenerateKingAttackInfo() const { } } // Check bishops. - if (kBishopAttacks[our_king_.as_int()].intersects(their_pieces_ & bishops_)) { + if (kBishopAttacks[our_king_.as_idx()].intersects(their_pieces_ & bishops_)) { for (const auto& direction : kBishopDirections) { auto dst_row = row; auto dst_col = col; BitBoard attack_line(0); bool possible_pinned_piece_found = false; - BoardSquare possible_pinned_piece; + Square possible_pinned_piece; while (true) { dst_row += direction.first; dst_col += direction.second; - if (!BoardSquare::IsValid(dst_row, dst_col)) break; - const BoardSquare destination(dst_row, dst_col); + if (!IsOnBoard(dst_row, dst_col)) break; + const Square destination(File::FromIdx(dst_col), + Rank::FromIdx(dst_row)); if (our_pieces_.get(destination)) { if (possible_pinned_piece_found) { // No pieces pinned. @@ -859,7 +858,7 @@ KingAttackInfo ChessBoard::GenerateKingAttackInfo() const { } // Check pawns. const BitBoard attacking_pawns = - kPawnAttacks[our_king_.as_int()] & their_pieces_ & pawns_; + kPawnAttacks[our_king_.as_idx()] & their_pieces_ & pawns_; king_attack_info.attack_lines_ = king_attack_info.attack_lines_ | attacking_pawns; @@ -870,7 +869,7 @@ KingAttackInfo ChessBoard::GenerateKingAttackInfo() const { // Check knights. const BitBoard attacking_knights = - kKnightAttacks[our_king_.as_int()] & + kKnightAttacks[our_king_.as_idx()] & (their_pieces_ - their_king_ - rooks_ - bishops_ - (pawns_ & kPawnMask)); king_attack_info.attack_lines_ = king_attack_info.attack_lines_ | attacking_knights; @@ -893,8 +892,7 @@ bool ChessBoard::IsLegalMove(Move move, // En passant. Complex but rare. Just apply // and check that we are not under check. - if (from.row() == 4 && pawns_.get(from) && from.col() != to.col() && - pawns_.get(7, to.col())) { + if (move.is_en_passant()) { ChessBoard board(*this); board.ApplyMove(move); return !board.IsUnderCheck(); @@ -930,8 +928,7 @@ bool ChessBoard::IsLegalMove(Move move, // King moves. if (from == our_king_) { - if (from.row() != 0 || to.row() != 0 || - (abs(from.col() - to.col()) == 1 && !our_pieces_.get(to))) { + if (!move.is_castling()) { // Non-castling move. Already checked during movegen. return true; } @@ -947,10 +944,10 @@ bool ChessBoard::IsLegalMove(Move move, // The piece is pinned. Now check that it stays on the same line w.r.t. the // king. - const int dx_from = from.col() - our_king_.col(); - const int dy_from = from.row() - our_king_.row(); - const int dx_to = to.col() - our_king_.col(); - const int dy_to = to.row() - our_king_.row(); + const int dx_from = from.file() - our_king_.file(); + const int dy_from = from.rank() - our_king_.rank(); + const int dx_to = to.file() - our_king_.file(); + const int dy_to = to.rank() - our_king_.rank(); if (dx_from == 0 || dx_to == 0) { return (dx_from == dx_to); @@ -969,158 +966,156 @@ MoveList ChessBoard::GenerateLegalMoves() const { return result; } -void ChessBoard::SetFromFen(std::string fen, int* rule50_ply, int* moves) { +void ChessBoard::PutPiece(Square square, PieceType piece, bool is_theirs) { + (is_theirs ? their_pieces_ : our_pieces_).set(square); + if (piece == kKing) (is_theirs ? their_king_ : our_king_) = square; + if (piece == kPawn) pawns_.set(square); + if (piece == kRook || piece == kQueen) rooks_.set(square); + if (piece == kBishop || piece == kQueen) bishops_.set(square); +} + +void ChessBoard::SetFromFen(std::string_view fen, int* rule50_ply, int* moves) { Clear(); - int row = 7; - int col = 0; - - // Remove any trailing whitespaces to detect eof after the last field. - fen.erase(std::find_if(fen.rbegin(), fen.rend(), - [](char c) { return !std::isspace(c); }) - .base(), - fen.end()); - - std::istringstream fen_str(fen); - std::string board; - fen_str >> board; - std::string who_to_move = "w"; - if (!fen_str.eof()) fen_str >> who_to_move; - // Assume no castling rights. Other engines, e.g., Stockfish, assume kings and - // rooks on their initial rows can each castle with the outer-most rook. Our - // implementation currently supports 960 castling where white and black rooks - // have matching columns, so it's unclear which rights to assume. - std::string castlings = "-"; - if (!fen_str.eof()) fen_str >> castlings; - std::string en_passant = "-"; - if (!fen_str.eof()) fen_str >> en_passant; - int rule50_halfmoves = 0; - if (!fen_str.eof()) fen_str >> rule50_halfmoves; - int total_moves = 1; - if (!fen_str.eof()) fen_str >> total_moves; - if (!fen_str) throw Exception("Bad fen string: " + fen); - - for (char c : board) { + if (rule50_ply) *rule50_ply = 0; + if (moves) *moves = 1; + Rank rank = kRank8; + File file = kFileA; + size_t pos = 0; + + auto complain = [&](std::string_view msg) { + throw Exception("Bad fen string (" + std::string(msg) + + "): " + std::string(fen)); + }; + auto skip_whitespace = [&](std::string_view where = {}) { + if (!where.empty() && pos < fen.size() && fen[pos] != ' ') { + complain("space expected " + std::string(where)); + } + while (pos < fen.size() && fen[pos] == ' ') ++pos; + return pos == fen.size(); + }; + + // Skip leading whitespaces. + skip_whitespace(); + + // Parse board position. + for (; pos < fen.size(); ++pos) { + const char c = fen[pos]; + if (c == ' ') break; if (c == '/') { - --row; - if (row < 0) throw Exception("Bad fen string (too many rows): " + fen); - col = 0; + if (rank == kRank1) complain("too many ranks"); + --rank; + file = kFileA; continue; } - if (std::isdigit(c)) { - col += c - '0'; + if (c >= '1' && c <= '8') { + file += c - '0'; + if (file > File::FromIdx(8)) complain("too many files"); continue; } - if (col >= 8) throw Exception("Bad fen string (too many columns): " + fen); - - if (std::isupper(c)) { - // White piece. - our_pieces_.set(row, col); - } else { - // Black piece. - their_pieces_.set(row, col); + PieceType piece = PieceType::Parse(c); + if (!piece.IsValid()) complain("invalid character as piece"); + if (!file.IsValid() || !rank.IsValid()) complain("piece out of board"); + if (piece == kPawn && (rank == kRank1 || rank == kRank8)) { + complain("pawn on back rank"); } + PutPiece(Square(file, rank), piece, std::islower(c)); + ++file; + } + if (skip_whitespace("after the board")) return; - if (c == 'K') { - our_king_.set(row, col); - } else if (c == 'k') { - their_king_.set(row, col); - } else if (c == 'R' || c == 'r') { - rooks_.set(row, col); - } else if (c == 'B' || c == 'b') { - bishops_.set(row, col); - } else if (c == 'Q' || c == 'q') { - rooks_.set(row, col); - bishops_.set(row, col); - } else if (c == 'P' || c == 'p') { - if (row == 7 || row == 0) { - throw Exception("Bad fen string (pawn in first/last row): " + fen); - } - pawns_.set(row, col); - } else if (c == 'N' || c == 'n') { - // Do nothing - } else { - throw Exception("Bad fen string: " + fen); - } - ++col; + // Parsing side to move. + const char side_to_move = std::tolower(fen[pos++]); + if (side_to_move == 'b') { + Mirror(); + } else if (side_to_move != 'w') { + complain("invalid side to move"); } + if (skip_whitespace("after side to move")) return; - if (castlings != "-") { - uint8_t our_left_rook = FILE_A; - uint8_t our_right_rook = FILE_H; - uint8_t their_left_rook = FILE_A; - uint8_t their_right_rook = FILE_H; - for (char c : castlings) { - const bool is_black = std::islower(c); - const int king_col = (is_black ? their_king_ : our_king_).col(); - const auto rooks = - (is_black ? their_pieces_ : our_pieces_) & ChessBoard::rooks(); - auto find_rook = [rooks, king_col, fen](bool forward, uint8_t rank) { - uint8_t rook; - for (rook = forward ? FILE_A : FILE_H; rook != king_col; - rook += 2 * forward - 1) { - if (rooks.get(rank, rook)) break; - } - if (rook == king_col) { - throw Exception("Bad fen string (missing rook): " + fen); - } - return rook; - }; - if (c == 'K') { - // Finding rightmost rook. - our_right_rook = find_rook(false, RANK_1); - castlings_.set_we_can_00(); - } else if (c == 'Q') { - // Finding leftmost rook. - our_left_rook = find_rook(true, RANK_1); - castlings_.set_we_can_000(); - } else if (c >= 'A' && c <= 'H') { - int rook_col = c - 'A'; - if (rook_col < king_col) { - our_left_rook = rook_col; - castlings_.set_we_can_000(); - } else { - our_right_rook = rook_col; - castlings_.set_we_can_00(); + // Parse castling rights. + if (fen[pos] == '-') { + ++pos; + } else { + auto find_rook = [&](bool theirs, bool kingside) -> File { + const Rank rank = theirs ? kRank8 : kRank1; + for (File file = kingside ? kFileH : kFileA; + file != (theirs ? their_king_.file() : our_king_.file()); + kingside ? --file : ++file) { + Square sq(file, rank); + if (!rooks().get(sq)) continue; + if (theirs ? their_pieces_.get(sq) : our_pieces_.get(sq)) { + return file; } - } else if (c == 'k') { - // Finding rightmost rook. - their_right_rook = find_rook(false, RANK_8); + } + complain("missing rook for castling"); + return kFileA; // Unreachable. + }; + for (; pos < fen.size(); ++pos) { + const char c = fen[pos]; + if (c == ' ') break; + const bool theirs = bool(std::isupper(c)) == flipped(); + bool kingside = false; + File file; + if (c == 'K' || c == 'Q' || c == 'k' || c == 'q') { + kingside = std::tolower(c) == 'k'; + file = find_rook(theirs, kingside); + } else { + file = File::Parse(c); + if (!file.IsValid()) complain("invalid character in castling"); + kingside = file > (theirs ? their_king_.file() : our_king_.file()); + } + if (kingside && theirs) { castlings_.set_they_can_00(); - } else if (c == 'q') { - // Finding leftmost rook. - their_left_rook = find_rook(true, RANK_8); + castlings_.their_kingside_rook = file; + } else if (kingside && !theirs) { + castlings_.set_we_can_00(); + castlings_.our_kingside_rook = file; + } else if (!kingside && theirs) { castlings_.set_they_can_000(); - } else if (c >= 'a' && c <= 'h') { - int rook_col = c - 'a'; - if (rook_col < king_col) { - their_left_rook = rook_col; - castlings_.set_they_can_000(); - } else { - their_right_rook = rook_col; - castlings_.set_they_can_00(); - } - } else { - throw Exception("Bad fen string (unexpected casting symbol): " + fen); + castlings_.their_queenside_rook = file; + } else if (!kingside && !theirs) { + castlings_.set_we_can_000(); + castlings_.our_queenside_rook = file; } } - castlings_.SetRookPositions(our_left_rook, our_right_rook, their_left_rook, - their_right_rook); - } - - if (en_passant != "-") { - auto square = BoardSquare(en_passant); - if (square.row() != RANK_3 && square.row() != RANK_6) - throw Exception("Bad fen string: " + fen + " wrong en passant rank"); - pawns_.set((square.row() == RANK_3) ? RANK_1 : RANK_8, square.col()); } + if (skip_whitespace("after castling")) return; - if (who_to_move == "b" || who_to_move == "B") { - Mirror(); - } else if (who_to_move != "w" && who_to_move != "W") { - throw Exception("Bad fen string (side to move): " + fen); + // Parse en passant square. + if (fen[pos] == '-') { + ++pos; + } else { + if (pos + 2 >= fen.size()) complain("en passant square expected"); + const File file = File::Parse(fen[pos]); + const Rank rank = Rank::Parse(fen[pos + 1]); + if (!file.IsValid() || !rank.IsValid()) complain("bad en passant square"); + if (rank != (flipped() ? kRank3 : kRank6)) complain("bad en passant rank"); + if ((ours() | theirs()).get(Square(file, kRank6))) { + complain("en passant square occupied"); + } + if (!(theirs() & pawns()).get(Square(file, kRank5))) { + complain("no pawn to capture en passant"); + } + pawns_.set(Square(file, kRank8)); + pos += 2; } - if (rule50_ply) *rule50_ply = rule50_halfmoves; - if (moves) *moves = total_moves; + if (skip_whitespace("after en passant")) return; + + // Parse rule 50 halfmoves. + auto parse_int = [&](int* into, std::string_view error_msg) { + const std::string_view num = fen.substr(pos, fen.find(' ', pos) - pos); + int tmp; + auto res = std::from_chars(num.data(), num.data() + num.size(), tmp); + if (res.ec != std::errc()) complain(error_msg); + if (into) *into = tmp; + pos += num.size(); + }; + parse_int(rule50_ply, "bad rule 50 halfmoves"); + if (skip_whitespace("after rule-50 clock")) return; + + // Parse total moves. + parse_int(moves, "bad total moves"); + if (!skip_whitespace("after total moves")) complain("extra characters"); } bool ChessBoard::HasMatingMaterial() const { @@ -1147,49 +1142,117 @@ bool ChessBoard::HasMatingMaterial() const { } std::string ChessBoard::DebugString() const { + auto fen = BoardToFen(*this); + std::replace(fen.begin(), fen.end(), ' ', '_'); + return "https://lc0.org/fen/" + fen; +} + +Move ChessBoard::ParseMove(std::string_view move_str) const { + auto complain = [&move_str](std::string_view reason) { + throw Exception("Invalid move (" + std::string(reason) + + "): " + std::string(move_str)); + }; + if (move_str.size() < 4 || move_str.size() > 5) complain("wrong move size"); + File from_file = File::Parse(move_str[0]); + Rank from_rank = Rank::Parse(move_str[1]); + File to_file = File::Parse(move_str[2]); + Rank to_rank = Rank::Parse(move_str[3]); + if (!from_file.IsValid() || !from_rank.IsValid() || !to_file.IsValid() || + !to_rank.IsValid()) { + complain("bad square"); + } + if (flipped_) { + from_rank.Flip(); + to_rank.Flip(); + } + Square from(from_file, from_rank); + Square to(to_file, to_rank); + if (!our_pieces_.get(from)) complain("no piece to move"); + + // Pawns at back ranks are used to encode en-passant, that's why we need to + // check that a piece doesn't go from there. + if (pawns_.get(from) && (from_rank != kRank1 && from_rank != kRank8) && + (to_rank == kRank1 || to_rank == kRank8)) { + // Promotion. + PieceType promotion = + move_str.size() > 4 ? PieceType::Parse(move_str[4]) : kKnight; + if (!promotion.CanPromoteInto()) complain("invalid promotion"); + return Move::WhitePromotion(from, to, promotion); + } + if (from == our_king_ && our_pieces_.get(to)) { + // FRC-style castling. + return Move::WhiteCastling(from.file(), to.file()); + } + if (from == our_king_ && from == kSquareE1 && to == kSquareG1) { + // Kingside castling. + return Move::WhiteCastling(from.file(), kFileH); + } + if (from == our_king_ && from == kSquareE1 && to == kSquareC1) { + // Qeenside castling. + return Move::WhiteCastling(from.file(), kFileA); + } + if (from.file() != to.file() && pawns().get(from) && !their_pieces_.get(to)) { + // En passant. + return Move::WhiteEnPassant(from, to); + } + return Move::White(from, to); +} + +namespace { +char GetPieceAt(const lczero::ChessBoard& board, Square square) { + char c = '\0'; + if (board.ours().get(square) || board.theirs().get(square)) { + if (board.pawns().get(square)) { + c = 'P'; + } else if (board.kings().get(square)) { + c = 'K'; + } else if (board.bishops().get(square)) { + c = 'B'; + } else if (board.queens().get(square)) { + c = 'Q'; + } else if (board.rooks().get(square)) { + c = 'R'; + } else { + c = 'N'; + } + if (board.theirs().get(square)) { + c = std::tolower(c); // Capitals are for white. + } + } + return c; +} + +} // namespace + +std::string BoardToFen(const ChessBoard& in_board) { + ChessBoard board(in_board); + const bool black_to_move = board.flipped(); + if (black_to_move) board.Mirror(); std::string result; - for (int i = 7; i >= 0; --i) { - for (int j = 0; j < 8; ++j) { - if (!our_pieces_.get(i, j) && !their_pieces_.get(i, j)) { - if (i == 2 && pawns_.get(0, j)) - result += '*'; - else if (i == 5 && pawns_.get(7, j)) - result += '*'; - else - result += '.'; - continue; - } - if (our_king_ == i * 8 + j) { - result += 'K'; - continue; - } - if (their_king_ == i * 8 + j) { - result += 'k'; - continue; - } - char c = '?'; - if ((pawns_ & kPawnMask).get(i, j)) { - c = 'p'; - } else if (bishops_.get(i, j)) { - if (rooks_.get(i, j)) - c = 'q'; - else - c = 'b'; - } else if (rooks_.get(i, j)) { - c = 'r'; + for (Rank rank = kRank8; rank.IsValid(); --rank) { + int empty = 0; + for (File file = kFileA; file <= kFileH; ++file) { + Square square(file, rank); + char piece = GetPieceAt(board, square); + if (piece) { + if (empty) result += std::to_string(empty); + empty = 0; + result += piece; } else { - c = 'n'; + ++empty; } - if (our_pieces_.get(i, j)) c = std::toupper(c); - result += c; } - if (i == 0) { - result += " " + castlings_.DebugString(); - result += flipped_ ? " (from black's eyes)" : " (from white's eyes)"; - result += " Hash: " + std::to_string(Hash()); - } - result += '\n'; + if (empty) result += std::to_string(empty); + if (rank != kRank1) result += '/'; + } + result += black_to_move ? " b" : " w"; + result += " " + board.castlings().as_string(); + std::string ep = "-"; + if (!board.en_passant().empty()) { + const Square sq = *board.en_passant().begin(); + ep = Square(sq.file(), black_to_move ? kRank3 : kRank6).ToString(false); } + result += " " + ep; return result; } diff --git a/src/chess/board.h b/src/chess/board.h index 40cbfc16b4..d455fcb69d 100644 --- a/src/chess/board.h +++ b/src/chess/board.h @@ -31,6 +31,7 @@ #include #include "chess/bitboard.h" +#include "chess/types.h" #include "utils/hashcat.h" namespace lczero { @@ -43,10 +44,10 @@ class KingAttackInfo { public: bool in_check() const { return attack_lines_.as_int(); } bool in_double_check() const { return double_check_; } - bool is_pinned(const BoardSquare square) const { + bool is_pinned(const Square square) const { return pinned_pieces_.get(square); } - bool is_on_attack_line(const BoardSquare square) const { + bool is_on_attack_line(const Square square) const { return attack_lines_.get(square); } @@ -73,7 +74,7 @@ class ChessBoard { // If @rule50_ply and @moves are not nullptr, they are filled with number // of moves without capture and number of full moves since the beginning of // the game. - void SetFromFen(std::string fen, int* rule50_ply = nullptr, + void SetFromFen(std::string_view fen, int* rule50_ply = nullptr, int* moves = nullptr); // Nullifies the whole structure. void Clear(); @@ -89,7 +90,7 @@ class ChessBoard { // counter should be removed. bool ApplyMove(Move move); // Checks if the square is under attack from "theirs" (black). - bool IsUnderAttack(BoardSquare square) const; + bool IsUnderAttack(Square square) const; // Generates the king attack info used for legal move detection. KingAttackInfo GenerateKingAttackInfo() const; // Checks if "our" (white) king is under check. @@ -101,18 +102,20 @@ class ChessBoard { MoveList GenerateLegalMoves() const; // Check whether pseudolegal move is legal. bool IsLegalMove(Move move, const KingAttackInfo& king_attack_info) const; - // Returns whether two moves are actually the same move in the position. - bool IsSameMove(Move move1, Move move2) const; - // Returns the same move but with castling encoded in legacy way. - Move GetLegacyMove(Move move) const; - // Returns the same move but with castling encoded in modern way. - Move GetModernMove(Move move) const; + + // Parses a move from move_str. + // The input string should be in the "normal" notation rather than from the + // player to move, i.e. "e7e5" for the black pawn move. + // Output is currently "from the player to move" perspective (i.e. from=E2, + // to=E4 for the same black move). This is temporary, plan is to change it + // soon. + Move ParseMove(std::string_view move_str) const; uint64_t Hash() const { return HashCat({our_pieces_.as_int(), their_pieces_.as_int(), rooks_.as_int(), bishops_.as_int(), pawns_.as_int(), - (static_cast(our_king_.as_int()) << 24) | - (static_cast(their_king_.as_int()) << 16) | + (static_cast(our_king_.as_idx()) << 24) | + (static_cast(their_king_.as_idx()) << 16) | (static_cast(castlings_.as_int()) << 8) | static_cast(flipped_)}); } @@ -120,10 +123,10 @@ class ChessBoard { class Castlings { public: Castlings() - : our_queenside_rook_(FILE_A), - their_queenside_rook_(FILE_A), - our_kingside_rook_(FILE_H), - their_kingside_rook_(FILE_H), + : our_queenside_rook(kFileA), + their_queenside_rook(kFileA), + our_kingside_rook(kFileH), + their_kingside_rook(kFileH), data_(0) {} void set_we_can_00() { data_ |= 1; } @@ -143,8 +146,8 @@ class ChessBoard { bool no_legal_castle() const { return data_ == 0; } void Mirror() { - std::swap(our_queenside_rook_, their_queenside_rook_); - std::swap(our_kingside_rook_, their_kingside_rook_); + std::swap(our_queenside_rook, their_queenside_rook); + std::swap(our_kingside_rook, their_kingside_rook); data_ = ((data_ & 0b11) << 2) + ((data_ & 0b1100) >> 2); } @@ -154,17 +157,17 @@ class ChessBoard { std::string as_string() const { if (data_ == 0) return "-"; std::string result; - if (our_queenside_rook() == FILE_A && our_kingside_rook() == FILE_H && - their_queenside_rook() == FILE_A && their_kingside_rook() == FILE_H) { + if (our_queenside_rook == kFileA && our_kingside_rook == kFileH && + their_queenside_rook == kFileA && their_kingside_rook == kFileH) { if (we_can_00()) result += 'K'; if (we_can_000()) result += 'Q'; if (they_can_00()) result += 'k'; if (they_can_000()) result += 'q'; } else { - if (we_can_00()) result += 'A' + our_kingside_rook(); - if (we_can_000()) result += 'A' + our_queenside_rook(); - if (they_can_00()) result += 'a' + their_kingside_rook(); - if (they_can_000()) result += 'a' + their_queenside_rook(); + if (we_can_00()) result += our_kingside_rook.ToString(true); + if (we_can_000()) result += our_queenside_rook.ToString(true); + if (they_can_00()) result += their_kingside_rook.ToString(false); + if (they_can_000()) result += their_queenside_rook.ToString(false); } return result; } @@ -177,44 +180,25 @@ class ChessBoard { if (they_can_00()) result += 'k'; if (they_can_000()) result += 'q'; result += '['; - result += 'A' + our_queenside_rook(); - result += 'A' + our_kingside_rook(); - result += 'a' + their_queenside_rook(); - result += 'a' + their_kingside_rook(); + result += our_queenside_rook.ToString(true); + result += our_kingside_rook.ToString(true); + result += their_queenside_rook.ToString(false); + result += their_kingside_rook.ToString(false); result += ']'; return result; } uint8_t as_int() const { return data_; } + bool operator==(const Castlings& other) const = default; - bool operator==(const Castlings& other) const { - assert(our_queenside_rook_ == other.our_queenside_rook_ && - our_kingside_rook_ == other.our_kingside_rook_ && - their_queenside_rook_ == other.their_queenside_rook_ && - their_kingside_rook_ == other.their_kingside_rook_); - return data_ == other.data_; - } - - uint8_t our_queenside_rook() const { return our_queenside_rook_; } - uint8_t our_kingside_rook() const { return our_kingside_rook_; } - uint8_t their_queenside_rook() const { return their_queenside_rook_; } - uint8_t their_kingside_rook() const { return their_kingside_rook_; } - void SetRookPositions(uint8_t our_left, uint8_t our_right, - uint8_t their_left, uint8_t their_right) { - our_queenside_rook_ = our_left; - our_kingside_rook_ = our_right; - their_queenside_rook_ = their_left; - their_kingside_rook_ = their_right; - } - - private: // Position of "left" (queenside) rook in starting game position. - uint8_t our_queenside_rook_; - uint8_t their_queenside_rook_; + File our_queenside_rook; + File their_queenside_rook; // Position of "right" (kingside) rook in starting position. - uint8_t our_kingside_rook_; - uint8_t their_kingside_rook_; + File our_kingside_rook; + File their_kingside_rook; + private: // - Bit 0 -- "our" side's kingside castle. // - Bit 1 -- "our" side's queenside castle. // - Bit 2 -- opponent's side's kingside castle. @@ -236,48 +220,20 @@ class ChessBoard { rooks_ - bishops_; } BitBoard kings() const { - return our_king_.as_board() | their_king_.as_board(); + return BitBoard::FromSquare(our_king_) | BitBoard::FromSquare(their_king_); } const Castlings& castlings() const { return castlings_; } bool flipped() const { return flipped_; } - bool operator==(const ChessBoard& other) const { - return (our_pieces_ == other.our_pieces_) && - (their_pieces_ == other.their_pieces_) && (rooks_ == other.rooks_) && - (bishops_ == other.bishops_) && (pawns_ == other.pawns_) && - (our_king_ == other.our_king_) && - (their_king_ == other.their_king_) && - (castlings_ == other.castlings_) && (flipped_ == other.flipped_); - } - - bool operator!=(const ChessBoard& other) const { return !operator==(other); } - - enum Square : uint8_t { - // clang-format off - A1 = 0, B1, C1, D1, E1, F1, G1, H1, - A2, B2, C2, D2, E2, F2, G2, H2, - A3, B3, C3, D3, E3, F3, G3, H3, - A4, B4, C4, D4, E4, F4, G4, H4, - A5, B5, C5, D5, E5, F5, G5, H5, - A6, B6, C6, D6, E6, F6, G6, H6, - A7, B7, C7, D7, E7, F7, G7, H7, - A8, B8, C8, D8, E8, F8, G8, H8, - // clang-format on - }; - - enum File : uint8_t { - // clang-format off - FILE_A = 0, FILE_B, FILE_C, FILE_D, FILE_E, FILE_F, FILE_G, FILE_H - // clang-format on - }; - - enum Rank : uint8_t { - // clang-format off - RANK_1 = 0, RANK_2, RANK_3, RANK_4, RANK_5, RANK_6, RANK_7, RANK_8 - // clang-format on - }; + bool operator==(const ChessBoard& other) const = default; + bool operator!=(const ChessBoard& other) const = default; private: + // Sets the piece on the square. + void PutPiece(Square square, PieceType piece, bool is_theirs); + // Check internal state is consistent after state transformations. + bool IsValid() const; + // All white pieces. BitBoard our_pieces_; // All black pieces. @@ -292,10 +248,13 @@ class ChessBoard { // same for black pawns. Those "fake" pawns are not present in our_pieces_ and // their_pieces_ bitboards. BitBoard pawns_; - BoardSquare our_king_; - BoardSquare their_king_; + Square our_king_; + Square their_king_; Castlings castlings_; bool flipped_ = false; // aka "Black to move". }; +// Converts the board to FEN string. +std::string BoardToFen(const ChessBoard& board); + } // namespace lczero diff --git a/src/chess/board_test.cc b/src/chess/board_test.cc index 123574482d..eef6247d1d 100644 --- a/src/chess/board_test.cc +++ b/src/chess/board_test.cc @@ -23,42 +23,15 @@ #include #include "chess/bitboard.h" - #include "utils/exception.h" namespace lczero { -TEST(BoardSquare, BoardSquare) { - { - auto x = BoardSquare(ChessBoard::C2); - EXPECT_EQ(x.row(), 1); - EXPECT_EQ(x.col(), 2); - } - - { - auto x = BoardSquare("c2"); - EXPECT_EQ(x.row(), 1); - EXPECT_EQ(x.col(), 2); - } - - { - auto x = BoardSquare(1, 2); - EXPECT_EQ(x.row(), 1); - EXPECT_EQ(x.col(), 2); - } - - { - auto x = BoardSquare(1, 2); - x.Mirror(); - EXPECT_EQ(x.row(), 6); - EXPECT_EQ(x.col(), 2); - } -} - TEST(ChessBoard, IllegalFirstRankPawns) { ChessBoard board; - EXPECT_THROW(board.SetFromFen("nqrbkrnr/bnnbnbnn/8/8/8/8/NNNBPNBN/QNRPKPQQ w - - 0 1");, - Exception); + EXPECT_THROW( + board.SetFromFen("nqrbkrnr/bnnbnbnn/8/8/8/8/NNNBPNBN/QNRPKPQQ w - - 0 1"); + , Exception); } TEST(ChessBoard, PseudolegalMovesStartingPos) { @@ -109,26 +82,26 @@ int Perft(const ChessBoard& board, int max_depth, bool dump = false, new_board.ApplyMove(move); if (new_board.IsUnderCheck()) { if (iter != legal_moves.end()) { - EXPECT_NE(iter->as_packed_int(), move.as_packed_int()) - << board.DebugString() << "legal:[" << iter->as_string() - << "]==pseudo:(" << move.as_string() << ") Under check:\n" + EXPECT_NE(*iter, move) + << board.DebugString() << "legal:[" << iter->ToString(true) + << "]==pseudo:(" << move.ToString(true) << ") Under check:\n" << new_board.DebugString(); } continue; } - EXPECT_EQ(iter->as_packed_int(), move.as_packed_int()) - << board.DebugString() << "legal:[" << iter->as_string() << "]pseudo:(" - << move.as_string() << ") after:\n" - << new_board.DebugString(); + EXPECT_EQ(*iter, move) << board.DebugString() << "legal:[" + << iter->ToString(true) << "]pseudo:(" + << move.ToString(true) << ") after:\n" + << new_board.DebugString(); new_board.Mirror(); ++iter; int count = Perft(new_board, max_depth, dump, depth + 1); if (dump && depth == 0) { Move m = move; - if (board.flipped()) m.Mirror(); - std::cerr << m.as_string() << ": " << count << '\n'; + if (board.flipped()) m.Flip(); + std::cerr << m.ToString(true) << ": " << count << '\n'; } total_count += count; } @@ -2235,26 +2208,6 @@ TEST(ChessBoard, HasMatingMaterialMultipleBishopsNotSameColor) { EXPECT_TRUE(board.HasMatingMaterial()); } -TEST(ChessBoard, CastlingIsSameMove) { - ChessBoard board; - board.SetFromFen( - "r3k2r/ppp1bppp/2np1n2/4p1B1/4P1b1/2NP1N2/PPP1BPPP/R3K2R w KQkq - 0 1"); - EXPECT_TRUE(board.IsSameMove("e1c1", "e1c1")); - EXPECT_TRUE(board.IsSameMove("e1a1", "e1a1")); - EXPECT_TRUE(board.IsSameMove("e1c1", "e1a1")); - EXPECT_FALSE(board.IsSameMove("e1c1", "e1b1")); - EXPECT_FALSE(board.IsSameMove("e1b1", "e1a1")); - EXPECT_FALSE(board.IsSameMove("e1c1", "e1g1")); - EXPECT_FALSE(board.IsSameMove("e1a1", "e1h1")); - EXPECT_FALSE(board.IsSameMove("e1c1", "e1h1")); - EXPECT_FALSE(board.IsSameMove("e1a1", "e1g1")); - EXPECT_FALSE(board.IsSameMove("e1f1", "e1g1")); - EXPECT_FALSE(board.IsSameMove("e1f1", "e1h1")); - EXPECT_TRUE(board.IsSameMove("e2c2", "e2c2")); - EXPECT_TRUE(board.IsSameMove("e2a2", "e2a2")); - EXPECT_FALSE(board.IsSameMove("e2c2", "e2a2")); -} - namespace { void TestInvalid(std::string fen) { ChessBoard board; @@ -2267,7 +2220,6 @@ void TestInvalid(std::string fen) { } } // namespace - TEST(ChessBoard, InvalidFEN) { TestInvalid("rnbqkbnr/ppppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"); TestInvalid("rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR/8 w KQkq - 0 1"); @@ -2280,10 +2232,24 @@ TEST(ChessBoard, InvalidFEN) { TEST(ChessBoard, InvalidEnPassantFromKnightPromotion) { ChessBoard board; board.SetFromFen("Q3b3/2P2pnk/3R3p/p7/1pp1p3/PnP1P2P/2B2PP1/5RK1 w - - 1 31"); - board.ApplyMove(Move("c7c8")); + board.ApplyMove(board.ParseMove("c7c8")); EXPECT_TRUE(board.en_passant().empty()); } +// Move from an en-passant flag square was mistakenly marked as en-passant. +TEST(ChessBoard, QueenMoveFromEnPassantFlagBug) { + ChessBoard board; + board.SetFromFen("1Qnkr3/1p1b4/p2P2p1/P1q5/1NP3pP/1KN5/8/3R4 b - - 0 32"); + board.ApplyMove(board.ParseMove("b7b5")); + board.Mirror(); + auto m = board.ParseMove("b8c7"); + EXPECT_FALSE(m.is_en_passant()); + board.ApplyMove(m); + board.Mirror(); + MoveList legal_moves = {board.ParseMove("c5c7")}; + EXPECT_EQ(board.GenerateLegalMoves(), legal_moves); +} + } // namespace lczero int main(int argc, char** argv) { diff --git a/src/chess/callbacks.h b/src/chess/callbacks.h index 7e28d271b1..4205e2441a 100644 --- a/src/chess/callbacks.h +++ b/src/chess/callbacks.h @@ -35,6 +35,7 @@ #include "chess/bitboard.h" #include "chess/position.h" +#include "utils/exception.h" namespace lczero { @@ -65,27 +66,29 @@ struct ThinkingInfo { int64_t nodes = -1; // Nodes per second. int nps = -1; + // Evaluations per second. + int eps = -1; // Hash fullness * 1000 int hashfull = -1; // Moves to mate. - std::optional mate; + std::optional mate = std::nullopt; // Win in centipawns. - std::optional score; + std::optional score = std::nullopt; // Win/Draw/Lose probability * 1000. struct WDL { int w; int d; int l; }; - std::optional wdl; + std::optional wdl = std::nullopt; // Number of successful TB probes (not the same as playouts ending in TB hit). int tb_hits = -1; // Best line found. Moves are from perspective of white player. - std::vector pv; + std::vector pv = {}; // Multipv index. int multipv = -1; // Freeform comment. - std::string comment; + std::string comment = ""; // Those are extensions and not really UCI protocol. // 1 if it's "player1", 2 if it's "player2" @@ -93,9 +96,9 @@ struct ThinkingInfo { // Index of the game in the tournament (0-based). int game_id = -1; // The color of the player, if known. - std::optional is_black; + std::optional is_black = std::nullopt; // Moves left - std::optional moves_left; + std::optional moves_left = std::nullopt; }; // Is sent when a single game is finished. @@ -144,6 +147,34 @@ class UciResponder { virtual void OutputThinkingInfo(std::vector* infos) = 0; }; +// The responder which forwards the output to another responder, with +// observer-like subscription model. +class UciResponderForwarder : public UciResponder { + public: + void OutputBestMove(BestMoveInfo* info) override { + if (wrapped_) wrapped_->OutputBestMove(info); + } + void OutputThinkingInfo(std::vector* infos) override { + if (wrapped_) wrapped_->OutputThinkingInfo(infos); + } + void Register(UciResponder* wrapped) { + if (wrapped_) { + throw Exception("UciResponderForwarder already has a wrapped responder"); + } + wrapped_ = wrapped; + } + void Unregister(UciResponder* wrapped) { + if (wrapped_ != wrapped) { + throw Exception( + "UciResponderForwarder doesn't have this wrapped responder"); + } + wrapped_ = nullptr; + } + + private: + UciResponder* wrapped_ = nullptr; +}; + // The responder which calls callbacks. Used for easier transition from old // code. class CallbackUciResponder : public UciResponder { @@ -202,49 +233,4 @@ class TransformingUciResponder : public UciResponder { std::unique_ptr parent_; }; -class WDLResponseFilter : public TransformingUciResponder { - using TransformingUciResponder::TransformingUciResponder; - void TransformThinkingInfo(std::vector* infos) override { - for (auto& info : *infos) info.wdl.reset(); - } -}; - -class MovesLeftResponseFilter : public TransformingUciResponder { - using TransformingUciResponder::TransformingUciResponder; - void TransformThinkingInfo(std::vector* infos) override { - for (auto& info : *infos) info.moves_left.reset(); - } -}; - -// Remaps FRC castling to legacy castling. -class Chess960Transformer : public TransformingUciResponder { - public: - Chess960Transformer(std::unique_ptr parent, - ChessBoard head_board) - : TransformingUciResponder(std::move(parent)), head_board_(head_board) {} - - private: - void TransformBestMove(BestMoveInfo* best_move) override { - std::vector moves({best_move->bestmove, best_move->ponder}); - ConvertToLegacyCastling(head_board_, &moves); - best_move->bestmove = moves[0]; - best_move->ponder = moves[1]; - } - void TransformThinkingInfo(std::vector* infos) override { - for (auto& x : *infos) ConvertToLegacyCastling(head_board_, &x.pv); - } - static void ConvertToLegacyCastling(ChessBoard pos, - std::vector* moves) { - for (auto& move : *moves) { - if (pos.flipped()) move.Mirror(); - move = pos.GetLegacyMove(move); - pos.ApplyMove(move); - if (pos.flipped()) move.Mirror(); - pos.Mirror(); - } - } - - const ChessBoard head_board_; -}; - } // namespace lczero diff --git a/src/chess/gamestate.cc b/src/chess/gamestate.cc new file mode 100644 index 0000000000..faaea02c7f --- /dev/null +++ b/src/chess/gamestate.cc @@ -0,0 +1,52 @@ +/* + This file is part of Leela Chess Zero. + Copyright (C) 2024 The LCZero Authors + + Leela Chess is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Leela Chess is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Leela Chess. If not, see . + + Additional permission under GNU GPL version 3 section 7 + + If you modify this Program, or any covered work, by linking or + combining it with NVIDIA Corporation's libraries from the NVIDIA CUDA + Toolkit and the NVIDIA CUDA Deep Neural Network library (or a + modified version of those libraries), containing parts covered by the + terms of the respective license agreement, the licensors of this + Program grant you additional permission to convey the resulting work. +*/ + +#include "chess/gamestate.h" + +#include +#include + +namespace lczero { + +Position GameState::CurrentPosition() const { + return std::accumulate( + moves.begin(), moves.end(), startpos, + [](const Position& pos, Move m) { return Position(pos, m); }); +} + +std::vector GameState::GetPositions() const { + std::vector positions; + positions.reserve(moves.size() + 1); + positions.push_back(startpos); + std::transform(moves.begin(), moves.end(), std::back_inserter(positions), + [&](Move m) { + return Position(positions.back(), m); + }); + return positions; +} + +} // namespace lczero diff --git a/src/chess/gamestate.h b/src/chess/gamestate.h new file mode 100644 index 0000000000..e584b2d30b --- /dev/null +++ b/src/chess/gamestate.h @@ -0,0 +1,48 @@ +/* + This file is part of Leela Chess Zero. + Copyright (C) 2024 The LCZero Authors + + Leela Chess is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Leela Chess is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Leela Chess. If not, see . + + Additional permission under GNU GPL version 3 section 7 + + If you modify this Program, or any covered work, by linking or + combining it with NVIDIA Corporation's libraries from the NVIDIA CUDA + Toolkit and the NVIDIA CUDA Deep Neural Network library (or a + modified version of those libraries), containing parts covered by the + terms of the respective license agreement, the licensors of this + Program grant you additional permission to convey the resulting work. +*/ + +#pragma once + +#include + +#include "chess/position.h" + +namespace lczero { + +// A structure that is passed to Search/SearchEnvironment to provide the game +// state. Somewhat mirrors usi `position moves ...` command. +struct GameState { + Position startpos; + std::vector moves; + + // Returns the position of the last move in the list. + Position CurrentPosition() const; + // Returns positions after all moves, including the starting and the last one. + std::vector GetPositions() const; +}; + +} // namespace lczero \ No newline at end of file diff --git a/src/chess/pgn.h b/src/chess/pgn.h index 512d4c673b..dd50ab9c98 100644 --- a/src/chess/pgn.h +++ b/src/chess/pgn.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "chess/bitboard.h" #include "chess/board.h" @@ -77,7 +78,7 @@ class PgnReader { while (GzGetLine(file, line)) { // Check if we have a UTF-8 BOM. If so, just ignore it. // Only supposed to exist in the first line, but should not matter. - if (line.substr(0,3) == "\xEF\xBB\xBF") line = line.substr(3); + if (line.substr(0, 3) == "\xEF\xBB\xBF") line = line.substr(3); if (!line.empty() && line.back() == '\r') line.pop_back(); // TODO: support line breaks in tags to ensure they are properly ignored. if (line.empty() || line[0] == '[') { @@ -151,9 +152,7 @@ class PgnReader { // Board ApplyMove wants mirrored for black, but outside code wants // normal, so mirror it back again. // Check equal to 0 since we've already added the position. - if ((cur_game_.size() % 2) == 0) { - cur_game_.back().Mirror(); - } + if ((cur_game_.size() % 2) == 0) cur_game_.back().Flip(); cur_board_.Mirror(); } } @@ -173,18 +172,18 @@ class PgnReader { cur_startpos_ = ChessBoard::kStartposFen; } - Move::Promotion PieceToPromotion(int p) { + static std::optional PieceToPieceType(int p) { switch (p) { case -1: - return Move::Promotion::None; + return std::nullopt; case 2: - return Move::Promotion::Queen; + return kQueen; case 3: - return Move::Promotion::Bishop; + return kBishop; case 4: - return Move::Promotion::Knight; + return kKnight; case 5: - return Move::Promotion::Rook; + return kRook; default: // 0 and 1 are pawn and king, which are not legal promotions, other // numbers don't correspond to a known piece type. @@ -206,18 +205,14 @@ class PgnReader { p = 4; } else if (san[0] == 'R') { p = 5; - } else if (san[0] == 'O' && san.size() > 2 && san[1] == '-' && - san[2] == 'O') { + } else if (san.substr(0, 3) == "O-O") { Move m; auto king_board = board.kings() & board.ours(); - BoardSquare king_sq(GetLowestBit(king_board.as_int())); - if (san.size() > 4 && san[3] == '-' && san[4] == 'O') { - m = Move(BoardSquare(0, king_sq.col()), - BoardSquare(0, board.castlings().our_queenside_rook())); - } else { - m = Move(BoardSquare(0, king_sq.col()), - BoardSquare(0, board.castlings().our_kingside_rook())); - } + Square king_sq(File::FromIdx(GetLowestBit(king_board.as_int())), kRank1); + m = Move::WhiteCastling(king_sq.file(), + san.substr(3, 2) == "-O" + ? board.castlings().our_queenside_rook + : board.castlings().our_kingside_rook); return m; } if (p != 0) idx++; @@ -285,20 +280,29 @@ class PgnReader { auto plm = board.GenerateLegalMoves(); int pr1 = -1; int pc1 = -1; - for (BoardSquare sq : searchBits) { - if (sr1 != -1 && sq.row() != sr1) continue; - if (c1 != -1 && sq.col() != c1) continue; - if (std::find(plm.begin(), plm.end(), - Move(sq, BoardSquare(sr2, c2), PieceToPromotion(p2))) == - plm.end()) { + for (Square sq : searchBits) { + if (sr1 != -1 && sq.rank().idx != sr1) continue; + if (c1 != -1 && sq.file().idx != c1) continue; + std::optional promotion = PieceToPieceType(p2); + std::optional enpassant = std::nullopt; + if (!board.en_passant().empty()) { + auto sq = *board.en_passant().begin(); + enpassant = Square(sq.file(), kRank6); + } + Square to(File::FromIdx(c2), Rank::FromIdx(sr2)); + Move move_to_find = promotion ? Move::WhitePromotion(sq, to, *promotion) + : enpassant && *enpassant == to + ? Move::WhiteEnPassant(sq, to) + : Move::White(sq, to); + if (std::find(plm.begin(), plm.end(), move_to_find) == plm.end()) { continue; } if (pc1 != -1) { CERR << "Ambiguous!!"; throw Exception("Opening book move seems ambiguous."); } - pr1 = sq.row(); - pc1 = sq.col(); + pr1 = sq.rank().idx; + pc1 = sq.file().idx; } if (pc1 == -1) { CERR << "No Match!!"; @@ -310,8 +314,19 @@ class PgnReader { r1 = 7 - r1; } } - Move m(BoardSquare(r1, c1), BoardSquare(r2, c2), PieceToPromotion(p2)); - if (board.flipped()) m.Mirror(); + std::optional promotion = PieceToPieceType(p2); + + std::optional enpassant = std::nullopt; + if (!board.en_passant().empty()) { + auto sq = *board.en_passant().begin(); + enpassant = Square(sq.file(), board.flipped() ? kRank3 : kRank6); + } + Square from(File::FromIdx(c1), Rank::FromIdx(r1)); + Square to(File::FromIdx(c2), Rank::FromIdx(r2)); + Move m = promotion ? Move::WhitePromotion(from, to, *promotion) + : enpassant && *enpassant == to ? Move::WhiteEnPassant(from, to) + : Move::White(from, to); + if (board.flipped()) m.Flip(); return m; } diff --git a/src/chess/position.cc b/src/chess/position.cc index ec085f1e37..fe6ed51678 100644 --- a/src/chess/position.cc +++ b/src/chess/position.cc @@ -27,45 +27,20 @@ #include "chess/position.h" +#include #include #include #include #include -namespace { -// GetPieceAt returns the piece found at row, col on board or the null-char '\0' -// in case no piece there. -char GetPieceAt(const lczero::ChessBoard& board, int row, int col) { - char c = '\0'; - if (board.ours().get(row, col) || board.theirs().get(row, col)) { - if (board.pawns().get(row, col)) { - c = 'P'; - } else if (board.kings().get(row, col)) { - c = 'K'; - } else if (board.bishops().get(row, col)) { - c = 'B'; - } else if (board.queens().get(row, col)) { - c = 'Q'; - } else if (board.rooks().get(row, col)) { - c = 'R'; - } else { - c = 'N'; - } - if (board.theirs().get(row, col)) { - c = std::tolower(c); // Capitals are for white. - } - } - return c; -} +#include "chess/types.h" -} // namespace namespace lczero { Position::Position(const Position& parent, Move m) : rule50_ply_(parent.rule50_ply_ + 1), ply_count_(parent.ply_count_ + 1) { - them_board_ = parent.us_board_; - const bool is_zeroing = them_board_.ApplyMove(m); - us_board_ = them_board_; + us_board_ = parent.us_board_; + const bool is_zeroing = us_board_.ApplyMove(m); us_board_.Mirror(); if (is_zeroing) rule50_ply_ = 0; } @@ -73,15 +48,23 @@ Position::Position(const Position& parent, Move m) Position::Position(const ChessBoard& board, int rule50_ply, int game_ply) : rule50_ply_(rule50_ply), repetitions_(0), ply_count_(game_ply) { us_board_ = board; - them_board_ = board; - them_board_.Mirror(); +} + +Position Position::FromFen(std::string_view fen) { + Position pos; + pos.us_board_.SetFromFen(std::string(fen), &pos.rule50_ply_, &pos.ply_count_); + return pos; } uint64_t Position::Hash() const { return HashCat({us_board_.Hash(), static_cast(repetitions_)}); } -std::string Position::DebugString() const { return us_board_.DebugString(); } +std::string Position::DebugString() const { + std::string fen = PositionToFen(*this); + std::replace(fen.begin(), fen.end(), ' ', '_'); + return "https://lc0.org/fen/" + fen; +} GameResult operator-(const GameResult& res) { return res == GameResult::BLACK_WON ? GameResult::WHITE_WON @@ -114,6 +97,11 @@ void PositionHistory::Reset(const ChessBoard& board, int rule50_ply, positions_.emplace_back(board, rule50_ply, game_ply); } +void PositionHistory::Reset(const Position& pos) { + positions_.clear(); + positions_.push_back(pos); +} + void PositionHistory::Append(Move m) { // TODO(mooskagh) That should be emplace_back(Last(), m), but MSVS STL // has a bug in implementation of emplace_back, when @@ -130,7 +118,7 @@ int PositionHistory::ComputeLastMoveRepetitions(int* cycle_length) const { // TODO(crem) implement hash/cache based solution. if (last.GetRule50Ply() < 4) return 0; - for (int idx = positions_.size() - 3; idx >= 0; idx -= 2) { + for (int idx = positions_.size() - 5; idx >= 0; idx -= 2) { const auto& pos = positions_[idx]; if (pos.GetBoard() == last.GetBoard()) { *cycle_length = positions_.size() - 1 - idx; @@ -160,34 +148,8 @@ uint64_t PositionHistory::HashLast(int positions) const { return HashCat(hash, Last().GetRule50Ply()); } -std::string GetFen(const Position& pos) { - std::string result; - const ChessBoard& board = pos.GetWhiteBoard(); - for (int row = 7; row >= 0; --row) { - int emptycounter = 0; - for (int col = 0; col < 8; ++col) { - char piece = GetPieceAt(board, row, col); - if (emptycounter > 0 && piece) { - result += std::to_string(emptycounter); - emptycounter = 0; - } - if (piece) { - result += piece; - } else { - emptycounter++; - } - } - if (emptycounter > 0) result += std::to_string(emptycounter); - if (row > 0) result += "/"; - } - std::string enpassant = "-"; - if (!board.en_passant().empty()) { - auto sq = *board.en_passant().begin(); - enpassant = BoardSquare(pos.IsBlackToMove() ? 2 : 5, sq.col()).as_string(); - } - result += pos.IsBlackToMove() ? " b" : " w"; - result += " " + board.castlings().as_string(); - result += " " + enpassant; +std::string PositionToFen(const Position& pos) { + std::string result = BoardToFen(pos.GetBoard()); result += " " + std::to_string(pos.GetRule50Ply()); result += " " + std::to_string( (pos.GetGamePly() + (pos.IsBlackToMove() ? 1 : 2)) / 2); diff --git a/src/chess/position.h b/src/chess/position.h index 69241467c4..fc28990c6b 100644 --- a/src/chess/position.h +++ b/src/chess/position.h @@ -27,7 +27,9 @@ #pragma once +#include #include +#include #include "chess/board.h" @@ -35,10 +37,13 @@ namespace lczero { class Position { public: + Position() = default; // From parent position and move. Position(const Position& parent, Move m); // From particular position. Position(const ChessBoard& board, int rule50_ply, int game_ply); + // From fen. + static Position FromFen(std::string_view fen); uint64_t Hash() const; bool IsBlackToMove() const { return us_board_.flipped(); } @@ -64,33 +69,28 @@ class Position { // Gets board from the point of view of player to move. const ChessBoard& GetBoard() const { return us_board_; } - // Gets board from the point of view of opponent. - const ChessBoard& GetThemBoard() const { return them_board_; } - // Gets board from the point of view of the white player. - const ChessBoard& GetWhiteBoard() const { - return us_board_.flipped() ? them_board_ : us_board_; - }; + + bool operator==(const Position&) const = default; + bool operator!=(const Position&) const = default; std::string DebugString() const; private: // The board from the point of view of the player to move. ChessBoard us_board_; - // The board from the point of view of opponent. - ChessBoard them_board_; // How many half-moves without capture or pawn move was there. int rule50_ply_ = 0; // How many repetitions this position had before. For new positions it's 0. - int repetitions_; + int repetitions_ = 0; // How many half-moves since the position was repeated or 0. - int cycle_length_; + int cycle_length_ = 0; // number of half-moves since beginning of the game. int ply_count_ = 0; }; // GetFen returns a FEN notation for the position. -std::string GetFen(const Position& pos); +std::string PositionToFen(const Position& pos); // These are ordered so max() prefers the best result. enum class GameResult : uint8_t { UNDECIDED, BLACK_WON, DRAW, WHITE_WON }; @@ -101,9 +101,11 @@ class PositionHistory { PositionHistory() = default; PositionHistory(const PositionHistory& other) = default; PositionHistory(PositionHistory&& other) = default; + PositionHistory(std::span positions) + : positions_(positions.begin(), positions.end()) {} PositionHistory& operator=(const PositionHistory& other) = default; - PositionHistory& operator=(PositionHistory&& other) = default; + PositionHistory& operator=(PositionHistory&& other) = default; // Returns first position of the game (or fen from which it was initialized). const Position& Starting() const { return positions_.front(); } @@ -128,6 +130,7 @@ class PositionHistory { // Resets the position to a given state. void Reset(const ChessBoard& board, int rule50_ply, int game_ply); + void Reset(const Position& pos); // Appends a position to history. void Append(Move m); @@ -147,6 +150,8 @@ class PositionHistory { // Checks for any repetitions since the last time 50 move rule was reset. bool DidRepeatSinceLastZeroingMove() const; + std::span GetPositions() const { return positions_; } + private: int ComputeLastMoveRepetitions(int* cycle_length) const; diff --git a/src/chess/position_test.cc b/src/chess/position_test.cc index 69951bd048..f6dd453986 100644 --- a/src/chess/position_test.cc +++ b/src/chess/position_test.cc @@ -51,7 +51,7 @@ TEST(Position, SetFenGetFen) { history.Reset(board, no_capture_ply, 2 * game_move - (board.flipped() ? 1 : 2)); Position pos = history.Last(); - std::string target_fen = GetFen(pos); + std::string target_fen = PositionToFen(pos); EXPECT_EQ(source_fens[i], target_fen); } } @@ -62,12 +62,12 @@ TEST(PositionHistory, ComputeLastMoveRepetitionsWithoutLegalEnPassant) { PositionHistory history; board.SetFromFen("3b4/rp1r1k2/8/1RP2p1p/p1KP4/P3P2P/5P2/1R2B3 b - - 2 30"); history.Reset(board, 2, 30); - history.Append(Move("f7f8", true)); - history.Append(Move("f2f4", false)); - history.Append(Move("d7h7", true)); - history.Append(Move("c4d3", false)); - history.Append(Move("h7d7", true)); - history.Append(Move("d3c4", false)); + history.Append(history.Last().GetBoard().ParseMove("f7f8")); + history.Append(history.Last().GetBoard().ParseMove("f2f4")); + history.Append(history.Last().GetBoard().ParseMove("d7h7")); + history.Append(history.Last().GetBoard().ParseMove("c4d3")); + history.Append(history.Last().GetBoard().ParseMove("h7d7")); + history.Append(history.Last().GetBoard().ParseMove("d3c4")); int history_idx = history.GetLength() - 1; const Position& repeated_position = history.GetPositionAt(history_idx); EXPECT_EQ(repeated_position.GetRepetitions(), 1); @@ -78,12 +78,12 @@ TEST(PositionHistory, ComputeLastMoveRepetitionsWithLegalEnPassant) { PositionHistory history; board.SetFromFen("3b4/rp1r1k2/8/1RP2p1p/p1KP2p1/P3P2P/5P2/1R2B3 b - - 2 30"); history.Reset(board, 2, 30); - history.Append(Move("f7f8", true)); - history.Append(Move("f2f4", false)); - history.Append(Move("d7h7", true)); - history.Append(Move("c4d3", false)); - history.Append(Move("h7d7", true)); - history.Append(Move("d3c4", false)); + history.Append(history.Last().GetBoard().ParseMove("f7f8")); + history.Append(history.Last().GetBoard().ParseMove("f2f4")); + history.Append(history.Last().GetBoard().ParseMove("d7h7")); + history.Append(history.Last().GetBoard().ParseMove("c4d3")); + history.Append(history.Last().GetBoard().ParseMove("h7d7")); + history.Append(history.Last().GetBoard().ParseMove("d3c4")); int history_idx = history.GetLength() - 1; const Position& repeated_position = history.GetPositionAt(history_idx); EXPECT_EQ(repeated_position.GetRepetitions(), 0); @@ -94,12 +94,12 @@ TEST(PositionHistory, DidRepeatSinceLastZeroingMoveCurent) { PositionHistory history; board.SetFromFen("3b4/rp1r1k2/8/1RP2p1p/p1KP4/P3P2P/5P2/1R2B3 b - - 2 30"); history.Reset(board, 2, 30); - history.Append(Move("f7f8", true)); - history.Append(Move("f2f4", false)); - history.Append(Move("d7h7", true)); - history.Append(Move("c4d3", false)); - history.Append(Move("h7d7", true)); - history.Append(Move("d3c4", false)); + history.Append(history.Last().GetBoard().ParseMove("f7f8")); + history.Append(history.Last().GetBoard().ParseMove("f2f4")); + history.Append(history.Last().GetBoard().ParseMove("d7h7")); + history.Append(history.Last().GetBoard().ParseMove("c4d3")); + history.Append(history.Last().GetBoard().ParseMove("h7d7")); + history.Append(history.Last().GetBoard().ParseMove("d3c4")); EXPECT_TRUE(history.DidRepeatSinceLastZeroingMove()); } @@ -108,13 +108,13 @@ TEST(PositionHistory, DidRepeatSinceLastZeroingMoveBefore) { PositionHistory history; board.SetFromFen("3b4/rp1r1k2/8/1RP2p1p/p1KP4/P3P2P/5P2/1R2B3 b - - 2 30"); history.Reset(board, 2, 30); - history.Append(Move("f7f8", true)); - history.Append(Move("f2f4", false)); - history.Append(Move("d7h7", true)); - history.Append(Move("c4d3", false)); - history.Append(Move("h7d7", true)); - history.Append(Move("d3c4", false)); - history.Append(Move("d7e7", true)); + history.Append(history.Last().GetBoard().ParseMove("f7f8")); + history.Append(history.Last().GetBoard().ParseMove("f2f4")); + history.Append(history.Last().GetBoard().ParseMove("d7h7")); + history.Append(history.Last().GetBoard().ParseMove("c4d3")); + history.Append(history.Last().GetBoard().ParseMove("h7d7")); + history.Append(history.Last().GetBoard().ParseMove("d3c4")); + history.Append(history.Last().GetBoard().ParseMove("d7e7")); EXPECT_TRUE(history.DidRepeatSinceLastZeroingMove()); } @@ -123,14 +123,14 @@ TEST(PositionHistory, DidRepeatSinceLastZeroingMoveOlder) { PositionHistory history; board.SetFromFen("3b4/rp1r1k2/8/1RP2p1p/p1KP4/P3P2P/5P2/1R2B3 b - - 2 30"); history.Reset(board, 2, 30); - history.Append(Move("f7f8", true)); - history.Append(Move("f2f4", false)); - history.Append(Move("d7h7", true)); - history.Append(Move("c4d3", false)); - history.Append(Move("h7d7", true)); - history.Append(Move("d3c4", false)); - history.Append(Move("d7e7", true)); - history.Append(Move("c4b4", false)); + history.Append(history.Last().GetBoard().ParseMove("f7f8")); + history.Append(history.Last().GetBoard().ParseMove("f2f4")); + history.Append(history.Last().GetBoard().ParseMove("d7h7")); + history.Append(history.Last().GetBoard().ParseMove("c4d3")); + history.Append(history.Last().GetBoard().ParseMove("h7d7")); + history.Append(history.Last().GetBoard().ParseMove("d3c4")); + history.Append(history.Last().GetBoard().ParseMove("d7e7")); + history.Append(history.Last().GetBoard().ParseMove("c4b4")); EXPECT_TRUE(history.DidRepeatSinceLastZeroingMove()); } @@ -139,15 +139,15 @@ TEST(PositionHistory, DidRepeatSinceLastZeroingMoveBeforeZero) { PositionHistory history; board.SetFromFen("3b4/rp1r1k2/8/1RP2p1p/p1KP4/P3P2P/5P2/1R2B3 b - - 2 30"); history.Reset(board, 2, 30); - history.Append(Move("f7f8", true)); - history.Append(Move("f2f4", false)); - history.Append(Move("d7h7", true)); - history.Append(Move("c4d3", false)); - history.Append(Move("h7d7", true)); - history.Append(Move("d3c4", false)); - history.Append(Move("d7e7", true)); - history.Append(Move("c4b4", false)); - history.Append(Move("h5h4", true)); + history.Append(history.Last().GetBoard().ParseMove("f7f8")); + history.Append(history.Last().GetBoard().ParseMove("f2f4")); + history.Append(history.Last().GetBoard().ParseMove("d7h7")); + history.Append(history.Last().GetBoard().ParseMove("c4d3")); + history.Append(history.Last().GetBoard().ParseMove("h7d7")); + history.Append(history.Last().GetBoard().ParseMove("d3c4")); + history.Append(history.Last().GetBoard().ParseMove("d7e7")); + history.Append(history.Last().GetBoard().ParseMove("c4b4")); + history.Append(history.Last().GetBoard().ParseMove("h5h4")); EXPECT_FALSE(history.DidRepeatSinceLastZeroingMove()); } @@ -156,8 +156,8 @@ TEST(PositionHistory, DidRepeatSinceLastZeroingMoveNeverRepeated) { PositionHistory history; board.SetFromFen("3b4/rp1r1k2/8/1RP2p1p/p1KP4/P3P2P/5P2/1R2B3 b - - 2 30"); history.Reset(board, 2, 30); - history.Append(Move("f7f8", true)); - history.Append(Move("f2f4", false)); + history.Append(history.Last().GetBoard().ParseMove("f7f8")); + history.Append(history.Last().GetBoard().ParseMove("f2f4")); EXPECT_FALSE(history.DidRepeatSinceLastZeroingMove()); } diff --git a/src/chess/types.h b/src/chess/types.h new file mode 100644 index 0000000000..2fce81a352 --- /dev/null +++ b/src/chess/types.h @@ -0,0 +1,242 @@ +/* + This file is part of Leela Chess Zero. + Copyright (C) 2025 The LCZero Authors + + Leela Chess is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Leela Chess is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Leela Chess. If not, see . + + Additional permission under GNU GPL version 3 section 7 + + If you modify this Program, or any covered work, by linking or + combining it with NVIDIA Corporation's libraries from the NVIDIA CUDA + Toolkit and the NVIDIA CUDA Deep Neural Network library (or a + modified version of those libraries), containing parts covered by the + terms of the respective license agreement, the licensors of this + Program grant you additional permission to convey the resulting work. +*/ + +#pragma once + +#include +#include +#include + +namespace lczero { + +struct PieceType { + uint8_t idx; + static constexpr PieceType FromIdx(uint8_t idx) { return PieceType{idx}; } + static PieceType Parse(char c); + std::string ToString(bool uppercase = false) const { + return std::string(1, "nqrbpk"[idx] + (uppercase ? 'A' - 'a' : 0)); + } + bool CanPromoteInto() const { return idx < 4; } + bool IsValid() const { return idx < 6; } + bool operator==(const PieceType& other) const = default; + bool operator!=(const PieceType& other) const = default; + + private: + constexpr explicit PieceType(uint8_t idx) : idx(idx) {} +}; + +constexpr PieceType kKnight = PieceType::FromIdx(0), + kQueen = PieceType::FromIdx(1), + kRook = PieceType::FromIdx(2), + kBishop = PieceType::FromIdx(3), + kPawn = PieceType::FromIdx(4), + kKing = PieceType::FromIdx(5); + +struct File { + uint8_t idx; + File() : idx(0x80) {} // Not on board. + constexpr bool IsValid() const { return idx < 8; } + static constexpr File FromIdx(uint8_t idx) { return File{idx}; } + static constexpr File Parse(char c) { return File(std::tolower(c) - 'a'); } + std::string ToString(bool uppercase = false) const { + return std::string(1, (uppercase ? 'A' : 'a') + idx); + } + void Flop() { idx ^= 0b111; } + auto operator<=>(const File& other) const = default; + void operator++() { ++idx; } + void operator--() { --idx; } + void operator+=(int delta) { idx += delta; } + File operator+(int delta) const { return File(idx + delta); } + File operator-(int delta) const { return File(idx - delta); } + + private: + constexpr explicit File(uint8_t idx) : idx(idx) {} +}; + +constexpr File kFileA = File::FromIdx(0), kFileB = File::FromIdx(1), + kFileC = File::FromIdx(2), kFileD = File::FromIdx(3), + kFileE = File::FromIdx(4), kFileF = File::FromIdx(5), + kFileG = File::FromIdx(6), kFileH = File::FromIdx(7); + +struct Rank { + uint8_t idx; + constexpr bool IsValid() const { return idx < 8; } + static constexpr Rank FromIdx(uint8_t idx) { return Rank{idx}; } + static constexpr Rank Parse(char c) { return Rank(c - '1'); } + void Flip() { idx ^= 0b111; } + std::string ToString() const { return std::string(1, '1' + idx); } + auto operator<=>(const Rank& other) const = default; + void operator--() { --idx; } + void operator++() { ++idx; } + void operator+=(int delta) { idx += delta; } + Rank operator+(int delta) const { return Rank(idx + delta); } + Rank operator-(int delta) const { return Rank(idx - delta); } + + private: + constexpr explicit Rank(uint8_t idx) : idx(idx) {} +}; + +constexpr Rank kRank1 = Rank::FromIdx(0), kRank2 = Rank::FromIdx(1), + kRank3 = Rank::FromIdx(2), kRank4 = Rank::FromIdx(3), + kRank5 = Rank::FromIdx(4), kRank6 = Rank::FromIdx(5), + kRank7 = Rank::FromIdx(6), kRank8 = Rank::FromIdx(7); + +// Stores a coordinates of a single square. +class Square { + public: + constexpr Square() = default; + constexpr Square(File file, Rank rank) : idx_(rank.idx * 8 + file.idx) {} + static constexpr Square FromIdx(uint8_t idx) { return Square{idx}; } + static constexpr Square Parse(std::string_view); + constexpr File file() const { return File::FromIdx(idx_ % 8); } + constexpr Rank rank() const { return Rank::FromIdx(idx_ / 8); } + // Flips the ranks. 1 becomes 8, 2 becomes 7, etc. Files remain the same. + void Flip() { idx_ ^= 0b111000; } + std::string ToString(bool uppercase = false) const { + return file().ToString(uppercase) + rank().ToString(); + } + constexpr bool operator==(const Square& other) const = default; + constexpr bool operator!=(const Square& other) const = default; + constexpr uint8_t as_idx() const { return idx_; } + + private: + explicit constexpr Square(uint8_t idx) : idx_(idx) {} + + // 0 is a1, 1 is b1, 8 is a2, 63 is h8. + uint8_t idx_; +}; + +constexpr Square kSquareA1 = Square(kFileA, kRank1), + kSquareC1 = Square(kFileC, kRank1), + kSquareE1 = Square(kFileE, kRank1), + kSquareG1 = Square(kFileG, kRank1), + kSquareH1 = Square(kFileH, kRank1); + +class Move { + public: + Move() = default; + static constexpr Move White(Square from, Square to) { + return Move((from.as_idx() << 6) | to.as_idx()); + } + static constexpr Move WhitePromotion(Square from, Square to, + PieceType promotion_piece) { + return Move((from.as_idx() << 6) | to.as_idx() | kPromotion | + (promotion_piece.idx << 12)); + } + static constexpr Move WhiteCastling(File king, File rook) { + return Move((king.idx << 6) | rook.idx | kCastling); + } + static constexpr Move WhiteEnPassant(Square from, Square to) { + return Move((from.as_idx() << 6) | to.as_idx() | kEnPassant); + } + + bool operator==(const Move& other) const = default; + bool operator!=(const Move& other) const = default; + + // Mirrors the ranks of the move. + void Flip() { data_ ^= kFlipMask; } + std::string ToString(bool is_chess960) const; + + Square from() const { return Square::FromIdx((data_ & kFromMask) >> 6); } + Square to() const { return Square::FromIdx(data_ & kToMask); } + bool is_promotion() const { return data_ & kPromotion; } + PieceType promotion() const { + return PieceType::FromIdx((data_ & kPieceMask) >> 12); + } + bool is_castling() const { return (data_ & kSpecialMask) == kCastling; } + bool is_en_passant() const { return (data_ & kSpecialMask) == kEnPassant; } + // TODO remove this once UciReponder starts using std::optional for ponder. + bool is_null() const { return data_ == 0; } + + uint16_t raw_data() const { return data_; } + + private: + explicit constexpr Move(uint16_t data) : data_(data) {} + + // Move encoding using 16 bits: + // - bits 0-5: "to" square (6 bits) + // - bits 6-11: "from" square (6 bits) + // - bits 12-13: if is_promotion: promotion piece type + // if !is_promotion: SpecialMove + // - bit 14: is_promotion flag + // - bit 15: reserved (potentially for side-to-move) + // Castling is always encoded as a "king takes rook" move. + uint16_t data_ = 0; + + enum Masks : uint16_t { + // clang-format off + kToMask = 0b0000000000111111, + kFromMask = 0b0000111111000000, + kSpecialMask = 0b0111000000000000, + kCastling = 0b0001000000000000, + kEnPassant = 0b0010000000000000, + kPromotion = 0b0100000000000000, + kPieceMask = 0b0011000000000000, + // If/when we have side-to-move bit, also flip it here. + kFlipMask = 0b0000111000111000, + // clang-format on + }; +}; + +inline int operator-(File a, File b) { return static_cast(a.idx) - b.idx; } +inline int operator-(Rank a, Rank b) { return static_cast(a.idx) - b.idx; } + +inline constexpr Square Square::Parse(std::string_view str) { + return Square(File::Parse(str[0]), Rank::Parse(str[1])); +} + +inline PieceType PieceType::Parse(char c) { + switch (tolower(c)) { + case 'n': + return kKnight; + case 'q': + return kQueen; + case 'r': + return kRook; + case 'b': + return kBishop; + case 'p': + return kPawn; + case 'k': + return kKing; + default: + return PieceType{6}; + } +} + +inline std::string Move::ToString(bool is_chess960) const { + if (is_castling() && !is_chess960) { + return from().ToString() + (to().file() > from().file() ? "g" : "c") + + to().rank().ToString(); + } + return from().ToString() + to().ToString() + + (is_promotion() ? promotion().ToString(false) : ""); +} + +using MoveList = std::vector; + +} // namespace lczero \ No newline at end of file diff --git a/src/chess/uciloop.cc b/src/chess/uciloop.cc index 4135895965..398a8bd7bd 100644 --- a/src/chess/uciloop.cc +++ b/src/chess/uciloop.cc @@ -43,24 +43,43 @@ #include "version.h" namespace lczero { - namespace { + +const OptionId kUciChess960{ + {.long_flag = "chess960", + .uci_option = "UCI_Chess960", + .help_text = "Castling moves are encoded as \"king takes rook\".", + .visibility = OptionId::kAlwaysVisible}}; +const OptionId kShowWDL{{.long_flag = "show-wdl", + .uci_option = "UCI_ShowWDL", + .help_text = "Show win, draw and lose probability.", + .visibility = OptionId::kAlwaysVisible}}; +const OptionId kShowEPS{ + {.long_flag = "show-eps", + .uci_option = "UCI_ShowEPS", + .help_text = "Show neural network evaluations per second.", + .visibility = OptionId::kAlwaysVisible}}; +const OptionId kShowMovesleft{{.long_flag = "show-movesleft", + .uci_option = "UCI_ShowMovesLeft", + .help_text = "Show estimated moves left.", + .visibility = OptionId::kAlwaysVisible}}; + const std::unordered_map> kKnownCommands = { {{"uci"}, {}}, {{"isready"}, {}}, - {{"setoption"}, {"context", "name", "value"}}, + {{"setoption"}, {"name", "value"}}, {{"ucinewgame"}, {}}, {{"position"}, {"fen", "startpos", "moves"}}, {{"go"}, {"infinite", "wtime", "btime", "winc", "binc", "movestogo", "depth", "mate", "nodes", "movetime", "searchmoves", "ponder"}}, - {{"start"}, {}}, {{"stop"}, {}}, {{"ponderhit"}, {}}, {{"quit"}, {}}, {{"xyzzy"}, {}}, {{"fen"}, {}}, + {{"wait"}, {}} }; std::pair> @@ -80,6 +99,26 @@ ParseCommand(const std::string& line) { throw Exception("Unknown command: " + line); } + // Special parsing for setoption to keep strings unmodified. + if (command->first == "setoption") { + iss >> token; + if (token != "name") { + throw Exception("setoption must be followed by name"); + } + int name_pos = iss.eof() ? line.length() : static_cast(iss.tellg()); + std::optional value_pos; + while (iss >> token) { + if (token == "value") { + value_pos = iss.eof() ? line.length() : static_cast(iss.tellg()); + params["value"] = Trim(line.substr(*value_pos)); + break; + } + } + params["name"] = Trim(line.substr( + name_pos, value_pos ? *value_pos - name_pos - 5 : std::string::npos)); + return {"setoption", params}; + } + std::string whitespace; while (iss >> token) { auto iter = command->second.find(token); @@ -125,45 +164,47 @@ int GetNumeric(const std::unordered_map& params, bool ContainsKey(const std::unordered_map& params, const std::string& key) { - return params.find(key) != params.end(); + return params.contains(key); } } // namespace -void UciLoop::RunLoop() { - std::cout.setf(std::ios::unitbuf); - std::string line; - while (std::getline(std::cin, line)) { - LOGFILE << ">> " << line; - try { - auto command = ParseCommand(line); - // Ignore empty line. - if (command.first.empty()) continue; - if (!DispatchCommand(command.first, command.second)) break; - } catch (Exception& ex) { - SendResponse(std::string("error ") + ex.what()); - } - } +UciLoop::UciLoop(StringUciResponder* uci_responder, OptionsParser* options, + EngineControllerBase* engine) + : uci_responder_(uci_responder), options_(options), engine_(engine) { + engine_->RegisterUciResponder(uci_responder_); } +UciLoop::~UciLoop() { engine_->UnregisterUciResponder(uci_responder_); } + bool UciLoop::DispatchCommand( const std::string& command, const std::unordered_map& params) { if (command == "uci") { - CmdUci(); + uci_responder_->SendId(); + for (const auto& option : options_->ListOptionsUci()) { + uci_responder_->SendRawResponse(option); + } + uci_responder_->SendRawResponse("uciok"); } else if (command == "isready") { - CmdIsReady(); + engine_->EnsureReady(); + uci_responder_->SendRawResponse("readyok"); } else if (command == "setoption") { - CmdSetOption(GetOrEmpty(params, "name"), GetOrEmpty(params, "value"), - GetOrEmpty(params, "context")); + if (GetOrEmpty(params, "name").empty()) { + throw Exception("setoption requires name"); + } else { + options_->SetUciOption(GetOrEmpty(params, "name"), + GetOrEmpty(params, "value")); + } } else if (command == "ucinewgame") { - CmdUciNewGame(); + engine_->NewGame(); } else if (command == "position") { if (ContainsKey(params, "fen") == ContainsKey(params, "startpos")) { throw Exception("Position requires either fen or startpos"); } const std::vector moves = StrSplitAtWhitespace(GetOrEmpty(params, "moves")); - CmdPosition(GetOrEmpty(params, "fen"), moves); + const std::string fen = GetOrEmpty(params, "fen"); + engine_->SetPosition(fen.empty() ? ChessBoard::kStartposFen : fen, moves); } else if (command == "go") { GoParams go_params; if (ContainsKey(params, "infinite")) { @@ -196,17 +237,15 @@ bool UciLoop::DispatchCommand( UCIGOOPTION(nodes); UCIGOOPTION(movetime); #undef UCIGOOPTION - CmdGo(go_params); + engine_->Go(go_params); + } else if (command == "wait") { + engine_->Wait(); } else if (command == "stop") { - CmdStop(); + engine_->Stop(); } else if (command == "ponderhit") { - CmdPonderHit(); - } else if (command == "start") { - CmdStart(); - } else if (command == "fen") { - CmdFen(); + engine_->PonderHit(); } else if (command == "xyzzy") { - SendResponse("Nothing happens."); + uci_responder_->SendRawResponse("Nothing happens."); } else if (command == "quit") { return false; } else { @@ -215,37 +254,49 @@ bool UciLoop::DispatchCommand( return true; } -void UciLoop::SendResponse(const std::string& response) { - SendResponses({response}); +bool UciLoop::ProcessLine(const std::string& line) { + auto command = ParseCommand(line); + // Ignore empty line. + if (command.first.empty()) return true; + return DispatchCommand(command.first, command.second); } -void UciLoop::SendResponses(const std::vector& responses) { - static std::mutex output_mutex; - std::lock_guard lock(output_mutex); - for (auto& response : responses) { - LOGFILE << "<< " << response; - std::cout << response << std::endl; - } +void StringUciResponder::PopulateParams(OptionsParser* options) { + options->Add(kUciChess960) = false; + options->Add(kShowWDL) = false; + options->Add(kShowEPS) = false; + options->Add(kShowMovesleft) = false; + options_ = &options->GetOptionsDict(); +} + +bool StringUciResponder::IsChess960() const { + return options_ ? options_->Get(kUciChess960) : false; +} + +void StringUciResponder::SendRawResponse(const std::string& response) { + SendRawResponses({response}); } -void UciLoop::SendId() { - SendResponse("id name Lc0 v" + GetVersionStr()); - SendResponse("id author The LCZero Authors."); +void StringUciResponder::SendId() { + SendRawResponse("id name Lc0 v" + GetVersionStr()); + SendRawResponse("id author The LCZero Authors."); } -void UciLoop::SendBestMove(const BestMoveInfo& move) { - std::string res = "bestmove " + move.bestmove.as_string(); - if (move.ponder) res += " ponder " + move.ponder.as_string(); - if (move.player != -1) res += " player " + std::to_string(move.player); - if (move.game_id != -1) res += " gameid " + std::to_string(move.game_id); - if (move.is_black) - res += " side " + std::string(*move.is_black ? "black" : "white"); - SendResponse(res); +void StringUciResponder::OutputBestMove(BestMoveInfo* info) { + const bool c960 = IsChess960(); + std::string res = "bestmove " + info->bestmove.ToString(c960); + if (!info->ponder.is_null()) res += " ponder " + info->ponder.ToString(c960); + if (info->player != -1) res += " player " + std::to_string(info->player); + if (info->game_id != -1) res += " gameid " + std::to_string(info->game_id); + if (info->is_black) + res += " side " + std::string(*info->is_black ? "black" : "white"); + SendRawResponse(res); } -void UciLoop::SendInfo(const std::vector& infos) { +void StringUciResponder::OutputThinkingInfo(std::vector* infos) { std::vector reses; - for (const auto& info : infos) { + const bool c960 = IsChess960(); + for (const auto& info : *infos) { std::string res = "info"; if (info.player != -1) res += " player " + std::to_string(info.player); if (info.game_id != -1) res += " gameid " + std::to_string(info.game_id); @@ -258,26 +309,39 @@ void UciLoop::SendInfo(const std::vector& infos) { if (info.nodes >= 0) res += " nodes " + std::to_string(info.nodes); if (info.mate) res += " score mate " + std::to_string(*info.mate); if (info.score) res += " score cp " + std::to_string(*info.score); - if (info.wdl) { + if (info.wdl && options_ && options_->Get(kShowWDL)) { res += " wdl " + std::to_string(info.wdl->w) + " " + std::to_string(info.wdl->d) + " " + std::to_string(info.wdl->l); } - if (info.moves_left) { + if (info.moves_left && options_ && options_->Get(kShowMovesleft)) { res += " movesleft " + std::to_string(*info.moves_left); } if (info.hashfull >= 0) res += " hashfull " + std::to_string(info.hashfull); if (info.nps >= 0) res += " nps " + std::to_string(info.nps); + if (info.eps >= 0 && options_ && options_->Get(kShowEPS)) { + res += " eps " + std::to_string(info.eps); + } if (info.tb_hits >= 0) res += " tbhits " + std::to_string(info.tb_hits); if (info.multipv >= 0) res += " multipv " + std::to_string(info.multipv); if (!info.pv.empty()) { res += " pv"; - for (const auto& move : info.pv) res += " " + move.as_string(); + for (const auto& move : info.pv) res += " " + move.ToString(c960); } if (!info.comment.empty()) res += " string " + info.comment; reses.push_back(std::move(res)); } - SendResponses(reses); + SendRawResponses(reses); +} + +void StdoutUciResponder::SendRawResponses( + const std::vector& responses) { + static std::mutex output_mutex; + std::lock_guard lock(output_mutex); + for (auto& response : responses) { + LOGFILE << "<< " << response; + std::cout << response << std::endl; + } } } // namespace lczero diff --git a/src/chess/uciloop.h b/src/chess/uciloop.h index be83088f16..4b8b2fd751 100644 --- a/src/chess/uciloop.h +++ b/src/chess/uciloop.h @@ -35,62 +35,93 @@ #include "chess/callbacks.h" #include "utils/exception.h" +#include "utils/optionsparser.h" namespace lczero { struct GoParams { - std::optional wtime; - std::optional btime; - std::optional winc; - std::optional binc; - std::optional movestogo; - std::optional depth; - std::optional mate; - std::optional nodes; - std::optional movetime; + std::optional wtime = std::nullopt; + std::optional btime = std::nullopt; + std::optional winc = std::nullopt; + std::optional binc = std::nullopt; + std::optional movestogo = std::nullopt; + std::optional depth = std::nullopt; + std::optional mate = std::nullopt; + std::optional nodes = std::nullopt; + std::optional movetime = std::nullopt; bool infinite = false; - std::vector searchmoves; + std::vector searchmoves = {}; bool ponder = false; }; -class UciLoop { +class StringUciResponder : public UciResponder { public: - virtual ~UciLoop() {} - virtual void RunLoop(); + void PopulateParams(OptionsParser* options); - // Sends response to host. - void SendResponse(const std::string& response); - // Sends responses to host ensuring they are received as a block. - virtual void SendResponses(const std::vector& responses); - void SendBestMove(const BestMoveInfo& move); - void SendInfo(const std::vector& infos); void SendId(); + void OutputBestMove(BestMoveInfo* info) override; + void OutputThinkingInfo(std::vector* infos) override; - // Command handlers. - virtual void CmdUci() { throw Exception("Not supported"); } - virtual void CmdIsReady() { throw Exception("Not supported"); } - virtual void CmdSetOption(const std::string& /*name*/, - const std::string& /*value*/, - const std::string& /*context*/) { - throw Exception("Not supported"); - } - virtual void CmdUciNewGame() { throw Exception("Not supported"); } - virtual void CmdPosition(const std::string& /*position*/, - const std::vector& /*moves*/) { - throw Exception("Not supported"); - } - virtual void CmdFen() { throw Exception("Not supported"); } - virtual void CmdGo(const GoParams& /*params*/) { - throw Exception("Not supported"); - } - virtual void CmdStop() { throw Exception("Not supported"); } - virtual void CmdPonderHit() { throw Exception("Not supported"); } - virtual void CmdStart() { throw Exception("Not supported"); } + // Sends response to host. + void SendRawResponse(const std::string& response); + // Sends responses to host ensuring they are received as a block. + virtual void SendRawResponses(const std::vector& responses) = 0; private: + bool IsChess960() const; + + const OptionsDict* options_ = nullptr; // absl_nullable +}; + +class EngineControllerBase { + public: + virtual ~EngineControllerBase() = default; + + // Blocks. + virtual void EnsureReady() = 0; + + // Must not block. + virtual void NewGame() = 0; + + // Blocks. + virtual void SetPosition(const std::string& fen, + const std::vector& moves) = 0; + + // Must not block. + virtual void Go(const GoParams& params) = 0; + virtual void PonderHit() = 0; + // Can block + virtual void Wait() = 0; + // Must not block. + virtual void Stop() = 0; + + // Register and unregister the UCI responder using observer pattern. + virtual void RegisterUciResponder(UciResponder*) = 0; + virtual void UnregisterUciResponder(UciResponder*) = 0; +}; + +class UciLoop { + public: + UciLoop(StringUciResponder* uci_responder, OptionsParser* options, + EngineControllerBase* engine); + virtual ~UciLoop(); + + // Returns false if the loop should stop. + bool ProcessLine(const std::string& line); + + protected: bool DispatchCommand( const std::string& command, const std::unordered_map& params); + + StringUciResponder* uci_responder_; // absl_nonnull + OptionsParser* options_; // absl_notnull + EngineControllerBase* engine_; // absl_notnull +}; + +class StdoutUciResponder : public StringUciResponder { + public: + void SendRawResponses(const std::vector& responses) override; }; } // namespace lczero diff --git a/src/engine.cc b/src/engine.cc index e4157657c6..c4c487c020 100644 --- a/src/engine.cc +++ b/src/engine.cc @@ -1,6 +1,6 @@ /* This file is part of Leela Chess Zero. - Copyright (C) 2018-2019 The LCZero Authors + Copyright (C) 2024 The LCZero Authors Leela Chess is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -28,447 +28,250 @@ #include "engine.h" #include -#include -#include -#include "mcts/search.h" -#include "mcts/stoppers/factory.h" -#include "utils/commandline.h" -#include "utils/configfile.h" -#include "utils/logging.h" +#include "chess/position.h" +#include "neural/backend.h" +#include "neural/memcache.h" +#include "neural/register.h" +#include "neural/shared_params.h" +#include "syzygy/syzygy.h" namespace lczero { namespace { -const OptionId kThreadsOptionId{ - "threads", "Threads", - "Number of (CPU) worker threads to use, 0 for the backend default.", 't'}; -const OptionId kLogFileId{"logfile", "LogFile", - "Write log to that file. Special value to " - "output the log to the console.", - 'l'}; const OptionId kSyzygyTablebaseId{ - "syzygy-paths", "SyzygyPath", - "List of Syzygy tablebase directories, list entries separated by system " - "separator (\";\" for Windows, \":\" for Linux).", - 's'}; -const OptionId kPonderId{"", "Ponder", - "This option is ignored. Here to please chess GUIs."}; -const OptionId kUciChess960{ - "chess960", "UCI_Chess960", - "Castling moves are encoded as \"king takes rook\"."}; -const OptionId kShowWDL{"show-wdl", "UCI_ShowWDL", - "Show win, draw and lose probability."}; -const OptionId kShowMovesleft{"show-movesleft", "UCI_ShowMovesLeft", - "Show estimated moves left."}; -const OptionId kStrictUciTiming{"strict-uci-timing", "StrictTiming", - "The UCI host compensates for lag, waits for " - "the 'readyok' reply before sending 'go' and " - "only then starts timing."}; + {.long_flag = "syzygy-paths", + .uci_option = "SyzygyPath", + .help_text = + "List of Syzygy tablebase directories, list entries separated by " + "system separator (\";\" for Windows, \":\" for Linux).", + .short_flag = 's', + .visibility = OptionId::kAlwaysVisible}}; +const OptionId kStrictUciTiming{ + {.long_flag = "strict-uci-timing", + .uci_option = "StrictTiming", + .help_text = "The UCI host compensates for lag, waits for the 'readyok' " + "reply before sending 'go' and only then starts timing.", + .visibility = OptionId::kProOnly}}; +const OptionId kPonderId{ + {.long_flag = "", + .uci_option = "Ponder", + .help_text = + "Indicates to the engine that it will be requested to ponder. This " + "postpones resetting the search tree until the search is started.", + .visibility = OptionId::kAlwaysVisible}}; + const OptionId kPreload{"preload", "", "Initialize backend and load net on engine startup."}; -const OptionId kValueOnly{ - "value-only", "ValueOnly", - "In value only mode all search parameters are ignored and the position is " - "evaluated by getting the valuation of every child position and choosing " - "the worst for the opponent."}; -const OptionId kClearTree{"", "ClearTree", - "Clear the tree before the next search."}; - -MoveList StringsToMovelist(const std::vector& moves, - const ChessBoard& board) { - MoveList result; - if (moves.size()) { - result.reserve(moves.size()); - const auto legal_moves = board.GenerateLegalMoves(); - const auto end = legal_moves.end(); - for (const auto& move : moves) { - const auto m = board.GetModernMove({move, board.flipped()}); - if (std::find(legal_moves.begin(), end, m) != end) result.emplace_back(m); - } - if (result.empty()) throw Exception("No legal searchmoves."); - } - return result; -} - } // namespace -EngineController::EngineController(std::unique_ptr uci_responder, - const OptionsDict& options) - : options_(options), - uci_responder_(std::move(uci_responder)), - current_position_{ChessBoard::kStartposFen, {}} {} - -void EngineController::PopulateOptions(OptionsParser* options) { - using namespace std::placeholders; - const bool is_simple = - CommandLine::BinaryName().find("simple") != std::string::npos; - NetworkFactory::PopulateOptions(options); - options->Add(kThreadsOptionId, 0, 128) = 0; - options->Add(kNNCacheSizeId, 0, 999999999) = 2000000; - SearchParams::Populate(options); - - ConfigFile::PopulateOptions(options); - if (is_simple) { - options->HideAllOptions(); - options->UnhideOption(kThreadsOptionId); - options->UnhideOption(NetworkFactory::kWeightsId); - options->UnhideOption(SearchParams::kContemptId); - options->UnhideOption(SearchParams::kMultiPvId); - } +void Engine::PopulateOptions(OptionsParser* options) { + options->Add(kPonderId) = false; options->Add(kSyzygyTablebaseId); - // Add "Ponder" option to signal to GUIs that we support pondering. - // This option is currently not used by lc0 in any way. - options->Add(kPonderId) = true; - options->Add(kUciChess960) = false; - options->Add(kShowWDL) = false; - options->Add(kShowMovesleft) = false; - - PopulateTimeManagementOptions(is_simple ? RunType::kSimpleUci : RunType::kUci, - options); - options->Add(kStrictUciTiming) = false; - options->HideOption(kStrictUciTiming); - options->Add(kPreload) = false; - options->Add(kValueOnly) = false; - options->Add(kClearTree); - options->HideOption(kClearTree); } -void EngineController::ResetMoveTimer() { - move_start_time_ = std::chrono::steady_clock::now(); +namespace { +GameState MakeGameState(const std::string& fen, + const std::vector& moves) { + GameState state; + state.startpos = Position::FromFen(fen); + ChessBoard cur_board = state.startpos.GetBoard(); + state.moves.reserve(moves.size()); + for (const auto& move : moves) { + Move m = cur_board.ParseMove(move); + state.moves.push_back(m); + cur_board.ApplyMove(m); + cur_board.Mirror(); + } + return state; } +} // namespace -// Updates values from Uci options. -void EngineController::UpdateFromUciOptions() { - SharedLock lock(busy_mutex_); +class Engine::UciPonderForwarder : public UciResponder { + public: + UciPonderForwarder(Engine* engine) : engine_(engine) {} - // Syzygy tablebases. - std::string tb_paths = options_.Get(kSyzygyTablebaseId); - if (!tb_paths.empty() && tb_paths != tb_paths_) { - syzygy_tb_ = std::make_unique(); - CERR << "Loading Syzygy tablebases from " << tb_paths; - if (!syzygy_tb_->init(tb_paths)) { - CERR << "Failed to load Syzygy tablebases!"; - syzygy_tb_ = nullptr; - } - tb_paths_ = tb_paths; - } else if (tb_paths.empty()) { - syzygy_tb_ = nullptr; - tb_paths_.clear(); + void OutputBestMove(BestMoveInfo* info) override { + if (!wrapped_) return; + wrapped_->OutputBestMove(info); } - - // Network. - const auto network_configuration = - NetworkFactory::BackendConfiguration(options_); - if (network_configuration_ != network_configuration) { - network_ = NetworkFactory::LoadNetwork(options_); - network_configuration_ = network_configuration; + void OutputThinkingInfo(std::vector* infos) override { + if (!wrapped_) return; + if (engine_->last_go_params_ && engine_->last_go_params_->ponder) { + assert(engine_->last_position_ && + !engine_->last_position_->moves.empty()); + const Move ponder_move_ = engine_->last_position_->moves.back(); + // Output all stats from main variation (not necessary the ponder move) + // but PV only from ponder move. + ThinkingInfo ponder_info; + for (const auto& info : *infos) { + if (info.multipv <= 1) { + ponder_info = info; + if (ponder_info.mate) ponder_info.mate = -*ponder_info.mate; + if (ponder_info.score) ponder_info.score = -*ponder_info.score; + if (ponder_info.depth > 1) ponder_info.depth--; + if (ponder_info.seldepth > 1) ponder_info.seldepth--; + if (ponder_info.wdl) + std::swap(ponder_info.wdl->w, ponder_info.wdl->l); + ponder_info.pv.clear(); + } + if (!info.pv.empty() && info.pv[0] == ponder_move_) { + ponder_info.pv.assign(info.pv.begin() + 1, info.pv.end()); + } + } + infos->clear(); + infos->push_back(ponder_info); + } + wrapped_->OutputThinkingInfo(infos); } - // Cache size. - cache_.SetCapacity(options_.Get(kNNCacheSizeId)); - - // Check whether we can update the move timer in "Go". - strict_uci_timing_ = options_.Get(kStrictUciTiming); -} - -void EngineController::EnsureReady() { - std::unique_lock lock(busy_mutex_); - // If a UCI host is waiting for our ready response, we can consider the move - // not started until we're done ensuring ready. - ResetMoveTimer(); -} - -void EngineController::NewGame() { - // In case anything relies upon defaulting to default position and just calls - // newgame and goes straight into go. - ResetMoveTimer(); - SharedLock lock(busy_mutex_); - cache_.Clear(); - search_.reset(); - tree_.reset(); - CreateFreshTimeManager(); - current_position_ = {ChessBoard::kStartposFen, {}}; - UpdateFromUciOptions(); -} - -void EngineController::SetPosition(const std::string& fen, - const std::vector& moves_str) { - // Some UCI hosts just call position then immediately call go, while starting - // the clock on calling 'position'. - ResetMoveTimer(); - SharedLock lock(busy_mutex_); - current_position_ = CurrentPosition{fen, moves_str}; - search_.reset(); -} - -Position EngineController::ApplyPositionMoves() { - ChessBoard board; - int no_capture_ply; - int game_move; - board.SetFromFen(current_position_.fen, &no_capture_ply, &game_move); - int game_ply = 2 * game_move - (board.flipped() ? 1 : 2); - Position pos(board, no_capture_ply, game_ply); - for (std::string move_str : current_position_.moves) { - Move move(move_str); - if (pos.IsBlackToMove()) move.Mirror(); - pos = Position(pos, move); + void Register(UciResponder* wrapped) { + if (wrapped_) { + throw Exception("UciPonderForwarder already has a wrapped responder"); + } + wrapped_ = wrapped; } - return pos; -} - -void EngineController::SetupPosition( - const std::string& fen, const std::vector& moves_str) { - SharedLock lock(busy_mutex_); - search_.reset(); - - UpdateFromUciOptions(); - - if (!tree_) tree_ = std::make_unique(); - - std::vector moves; - for (const auto& move : moves_str) moves.emplace_back(move); - const bool is_same_game = tree_->ResetToPosition(fen, moves); - if (!is_same_game) CreateFreshTimeManager(); -} - -void EngineController::CreateFreshTimeManager() { - time_manager_ = MakeTimeManager(options_); -} - -namespace { - -class PonderResponseTransformer : public TransformingUciResponder { - public: - PonderResponseTransformer(std::unique_ptr parent, - std::string ponder_move) - : TransformingUciResponder(std::move(parent)), - ponder_move_(std::move(ponder_move)) {} - - void TransformThinkingInfo(std::vector* infos) override { - // Output all stats from main variation (not necessary the ponder move) - // but PV only from ponder move. - ThinkingInfo ponder_info; - for (const auto& info : *infos) { - if (info.multipv <= 1) { - ponder_info = info; - if (ponder_info.mate) ponder_info.mate = -*ponder_info.mate; - if (ponder_info.score) ponder_info.score = -*ponder_info.score; - if (ponder_info.depth > 1) ponder_info.depth--; - if (ponder_info.seldepth > 1) ponder_info.seldepth--; - if (ponder_info.wdl) std::swap(ponder_info.wdl->w, ponder_info.wdl->l); - ponder_info.pv.clear(); - } - if (!info.pv.empty() && info.pv[0].as_string() == ponder_move_) { - ponder_info.pv.assign(info.pv.begin() + 1, info.pv.end()); - } + void Unregister(UciResponder* wrapped) { + if (wrapped_ != wrapped) { + throw Exception("UciPonderForwarder doesn't have this wrapped responder"); } - infos->clear(); - infos->push_back(ponder_info); + wrapped_ = nullptr; } private: - std::string ponder_move_; + UciResponder* wrapped_ = nullptr; + Engine* const engine_; }; -void ValueOnlyGo(NodeTree* tree, Network* network, const OptionsDict& options, - std::unique_ptr responder) { - auto input_format = network->GetCapabilities().input_format; - - const auto& board = tree->GetPositionHistory().Last().GetBoard(); - auto legal_moves = board.GenerateLegalMoves(); - tree->GetCurrentHead()->CreateEdges(legal_moves); - PositionHistory history = tree->GetPositionHistory(); - std::vector planes; - for (auto edge : tree->GetCurrentHead()->Edges()) { - history.Append(edge.GetMove()); - if (history.ComputeGameResult() == GameResult::UNDECIDED) { - planes.emplace_back(EncodePositionForNN( - input_format, history, 8, FillEmptyHistory::FEN_ONLY, nullptr)); - } - history.Pop(); +Engine::Engine(const SearchFactory& factory, const OptionsDict& opts) + : uci_forwarder_(std::make_unique(this)), + options_(opts), + search_(factory.CreateSearch(uci_forwarder_.get(), &options_)) { + if (options_.Get(kPreload)) { + UpdateBackendConfig(); + EnsureSyzygyTablebasesLoaded(); } +} - std::vector comp_q; - int batch_size = options.Get(SearchParams::kMiniBatchSizeId); - if (batch_size == 0) batch_size = network->GetMiniBatchSize(); - - for (size_t i = 0; i < planes.size(); i += batch_size) { - auto comp = network->NewComputation(); - for (int j = 0; j < batch_size; j++) { - comp->AddInput(std::move(planes[i + j])); - if (i + j + 1 == planes.size()) break; - } - comp->ComputeBlocking(); +Engine::~Engine() { EnsureSearchStopped(); } - for (int j = 0; j < batch_size; j++) comp_q.push_back(comp->GetQVal(j)); - } +void Engine::EnsureSearchStopped() { + search_->AbortSearch(); + search_->WaitSearch(); +} - Move best; - int comp_idx = 0; - float max_q = std::numeric_limits::lowest(); - for (auto edge : tree->GetCurrentHead()->Edges()) { - history.Append(edge.GetMove()); - auto result = history.ComputeGameResult(); - float q = -1; - if (result == GameResult::UNDECIDED) { - // NN eval is for side to move perspective - so if its good, its bad for - // us. - q = -comp_q[comp_idx]; - comp_idx++; - } else if (result == GameResult::DRAW) { - q = 0; - } else { - // A legal move to a non-drawn terminal without tablebases must be a - // win. - q = 1; - } - if (q >= max_q) { - max_q = q; - best = edge.GetMove(tree->GetPositionHistory().IsBlackToMove()); - } - history.Pop(); +void Engine::UpdateBackendConfig() { + LOGFILE << "Update backend configuration."; + const std::string backend_name = + options_.Get(SharedBackendParams::kBackendId); + if (!backend_ || backend_name != backend_name_ || + backend_->UpdateConfiguration(options_) == Backend::NEED_RESTART) { + backend_name_ = backend_name; + backend_ = CreateMemCache(BackendManager::Get()->CreateFromParams(options_), + options_); + search_->SetBackend(backend_.get()); + } else { + backend_->SetCacheSize( + options_.Get(SharedBackendParams::kNNCacheSizeId)); } - std::vector infos; - ThinkingInfo thinking; - thinking.depth = 1; - infos.push_back(thinking); - responder->OutputThinkingInfo(&infos); - BestMoveInfo info(best); - responder->OutputBestMove(&info); } -} // namespace +void Engine::EnsureSyzygyTablebasesLoaded() { + const std::string tb_paths = options_.Get(kSyzygyTablebaseId); + if (tb_paths == previous_tb_paths_) return; + previous_tb_paths_ = tb_paths; -void EngineController::Go(const GoParams& params) { - // TODO: should consecutive calls to go be considered to be a continuation and - // hence have the same start time like this behaves, or should we check start - // time hasn't changed since last call to go and capture the new start time - // now? - if (strict_uci_timing_ || !move_start_time_) ResetMoveTimer(); - go_params_ = params; - - std::unique_ptr responder = - std::make_unique(uci_responder_.get()); - - // Setting up current position, now that it's known whether it's ponder or - // not. - if (params.ponder && !current_position_.moves.empty()) { - std::vector moves(current_position_.moves); - std::string ponder_move = moves.back(); - moves.pop_back(); - SetupPosition(current_position_.fen, moves); - responder = std::make_unique( - std::move(responder), ponder_move); + if (tb_paths.empty()) { + LOGFILE << "Reset Syzygy tablebases."; + syzygy_tb_.reset(); } else { - SetupPosition(current_position_.fen, current_position_.moves); - } - - if (!options_.Get(kUciChess960)) { - // Remap FRC castling to legacy castling. - responder = std::make_unique( - std::move(responder), tree_->HeadPosition().GetBoard()); + syzygy_tb_ = std::make_unique(); + CERR << "Loading Syzygy tablebases from " << tb_paths; + if (!syzygy_tb_->init(tb_paths)) { + CERR << "Failed to load Syzygy tablebases!"; + syzygy_tb_.reset(); + } } - if (!options_.Get(kShowWDL)) { - // Strip WDL information from the response. - responder = std::make_unique(std::move(responder)); - } + search_->SetSyzygyTablebase(syzygy_tb_.get()); +} - if (!options_.Get(kShowMovesleft)) { - // Strip movesleft information from the response. - responder = std::make_unique(std::move(responder)); - } - if (options_.Get(kValueOnly)) { - ValueOnlyGo(tree_.get(), network_.get(), options_, std::move(responder)); +// Initializes the search with either the specified position for the normal +// search or the position one ply trimmed for the ponder search. +void Engine::InitializeSearchPosition(bool for_ponder) { + LOGFILE << "Setting a new search position."; + assert(last_position_); + if (!for_ponder) { + search_->SetPosition(*last_position_); return; } - - if (options_.Get