Merge pull request #285 from glinscott/next

Version 0.5

.gitignore

@@ -26,3 +26,7 @@ training/tf/leelaz-model-*
 training/tf/leelalogs
 training/tf/models/
 .idea/
+/.vs
+*.exe
+*.dll
+/go/src/client/settings.json

README.md

@@ -34,28 +34,33 @@ Of course, we also appreciate code reviews, pull requests and Windows testers!
   https://github.com/KhronosGroup/OpenCL-Headers/tree/master/opencl22/)
 * OpenCL ICD loader (ocl-icd-libopencl1 on Debian/Ubuntu, or reference implementation at https://github.com/KhronosGroup/OpenCL-ICD-Loader)
 * An OpenCL capable device, preferably a very, very fast GPU, with recent
-  drivers is strongly recommended (OpenCL 1.2 support should be enough, even
-  OpenCL 1.1 might work). If you do not have a GPU, modify config.h in the
-  source and remove the line that says `#define USE_OPENCL`.
+  drivers is strongly recommended but not required. (OpenCL 1.2 support should be enough, even
+  OpenCL 1.1 might work).
 * Tensorflow 1.4 or higher (for training)
 * The program has been tested on Linux.
 
+
 ## Example of compiling - Ubuntu 16.04
 
     # Install dependencies
-    sudo apt install libboost-all-dev libopenblas-dev opencl-headers ocl-icd-libopencl1 ocl-icd-opencl-dev zlib1g-dev
+    sudo apt install g++ git libboost-all-dev libopenblas-dev opencl-headers ocl-icd-libopencl1 ocl-icd-opencl-dev zlib1g-dev
 
     # Test for OpenCL support & compatibility
     sudo apt install clinfo && clinfo
 
-    # Clone github repo
-    git clone git@github.com:glinscott/leela-chess.git
+    # Clone git repo
+    git clone https://github.com/glinscott/leela-chess.git
     cd leela-chess
     git submodule update --init --recursive
     mkdir build && cd build
 
-    # Configure, build and run tests
+    # Configure
     cmake ..
+
+    # Or configure without GPU support
+    cmake -DFEATURE_USE_CPU_ONLY=1 ..
+
+    # Build and run tests
     make
     ./tests
 

go/src/client/main.go

@@ -45,6 +45,8 @@ func readSettings(path string) (string, string) {
 	if err != nil {
 		// File was not found
 		fmt.Printf("Please enter your username and password, an account will be automatically created.\n")
+		fmt.Printf("Note that this password will be stored in plain text, so avoid a password that is\n")
+		fmt.Printf("also used for sensitive applications. It also cannot be recovered.\n")
 		fmt.Printf("Enter username : ")
 		fmt.Scanf("%s\n", &settings.User)
 		fmt.Printf("Enter password : ")
@@ -76,15 +78,16 @@ func getExtraParams() map[string]string {
 	return map[string]string{
 		"user":     *USER,
 		"password": *PASSWORD,
-		"version":  "4",
+		"version":  "5",
 	}
 }
 
-func uploadGame(httpClient *http.Client, path string, pgn string, nextGame client.NextGameResponse, retryCount uint) error {
+func uploadGame(httpClient *http.Client, path string, pgn string, nextGame client.NextGameResponse, version string, retryCount uint) error {
 	extraParams := getExtraParams()
 	extraParams["training_id"] = strconv.Itoa(int(nextGame.TrainingId))
 	extraParams["network_id"] = strconv.Itoa(int(nextGame.NetworkId))
 	extraParams["pgn"] = pgn
+	extraParams["engineVersion"] = version
 	request, err := client.BuildUploadRequest(*HOSTNAME+"/upload_game", extraParams, "file", path)
 	if err != nil {
 		return err
@@ -99,7 +102,7 @@ func uploadGame(httpClient *http.Client, path string, pgn string, nextGame clien
 		log.Print(err)
 		log.Print("Error uploading, retrying...")
 		time.Sleep(time.Second * (2 << retryCount))
-		err = uploadGame(httpClient, path, pgn, nextGame, retryCount+1)
+		err = uploadGame(httpClient, path, pgn, nextGame, version, retryCount+1)
 		return err
 	}
 	resp.Body.Close()
@@ -131,6 +134,7 @@ type CmdWrapper struct {
 	Pgn      string
 	Input    io.WriteCloser
 	BestMove chan string
+	Version  string
 }
 
 func (c *CmdWrapper) openInput() {
@@ -179,6 +183,8 @@ func (c *CmdWrapper) launch(networkPath string, args []string, input bool) {
 				c.Pgn += line + "\n"
 			} else if strings.HasPrefix(line, "bestmove ") {
 				c.BestMove <- strings.Split(line, " ")[1]
+			} else if strings.HasPrefix(line, "id name lczero ") {
+				c.Version = strings.Split(line, " ")[3]
 			}
 		}
 	}()
@@ -200,7 +206,7 @@ func (c *CmdWrapper) launch(networkPath string, args []string, input bool) {
 	}
 }
 
-func playMatch(baselinePath string, candidatePath string, params []string, flip bool) (int, string, error) {
+func playMatch(baselinePath string, candidatePath string, params []string, flip bool) (int, string, string, error) {
 	baseline := CmdWrapper{}
 	baseline.launch(baselinePath, params, true)
 	defer baseline.Input.Close()
@@ -225,7 +231,7 @@ func playMatch(baselinePath string, candidatePath string, params []string, flip
 	move_history := ""
 	turn := 0
 	for {
-		if turn >= 450 || game.Outcome() != chess.NoOutcome {
+		if turn >= 450 || game.Outcome() != chess.NoOutcome || len(game.EligibleDraws()) > 1 {
 			if game.Outcome() == chess.WhiteWon {
 				result = 1
 			} else if game.Outcome() == chess.BlackWon {
@@ -255,7 +261,7 @@ func playMatch(baselinePath string, candidatePath string, params []string, flip
 			err := game.MoveStr(best_move)
 			if err != nil {
 				log.Println("Error decoding: " + best_move + " for game:\n" + game.String())
-				return 0, "", err
+				return 0, "", "", err
 			}
 			if len(move_history) == 0 {
 				move_history = " moves"
@@ -264,15 +270,15 @@ func playMatch(baselinePath string, candidatePath string, params []string, flip
 			turn += 1
 		case <-time.After(60 * time.Second):
 			log.Println("Bestmove has timed out, aborting match")
-			return 0, "", errors.New("timeout")
+			return 0, "", "", errors.New("timeout")
 		}
 	}
 
 	chess.UseNotation(chess.AlgebraicNotation{})(game)
-	return result, game.String(), nil
+	return result, game.String(), candidate.Version, nil
 }
 
-func train(networkPath string, count int, params []string) (string, string) {
+func train(networkPath string, count int, params []string) (string, string, string) {
 	// pid is intended for use in multi-threaded training
 	pid := os.Getpid()
 
@@ -297,7 +303,7 @@ func train(networkPath string, count int, params []string) (string, string) {
 		log.Fatal(err)
 	}
 
-	return path.Join(train_dir, "training.0.gz"), c.Pgn
+	return path.Join(train_dir, "training.0.gz"), c.Pgn, c.Version
 }
 
 func getNetwork(httpClient *http.Client, sha string, clearOld bool) (string, error) {
@@ -344,19 +350,21 @@ func nextGame(httpClient *http.Client, count int) error {
 		if err != nil {
 			return err
 		}
-		result, pgn, err := playMatch(networkPath, candidatePath, params, nextGame.Flip)
+		result, pgn, version, err := playMatch(networkPath, candidatePath, params, nextGame.Flip)
 		if err != nil {
 			return err
 		}
-		go client.UploadMatchResult(httpClient, *HOSTNAME, nextGame.MatchGameId, result, pgn, getExtraParams())
+		extraParams := getExtraParams()
+		extraParams["engineVersion"] = version
+		go client.UploadMatchResult(httpClient, *HOSTNAME, nextGame.MatchGameId, result, pgn, extraParams)
 		return nil
 	} else if nextGame.Type == "train" {
 		networkPath, err := getNetwork(httpClient, nextGame.Sha, true)
 		if err != nil {
 			return err
 		}
-		trainFile, pgn := train(networkPath, count, params)
-		go uploadGame(httpClient, trainFile, pgn, nextGame, 0)
+		trainFile, pgn, version := train(networkPath, count, params)
+		go uploadGame(httpClient, trainFile, pgn, nextGame, version, 0)
 		return nil
 	}
 
@@ -388,6 +396,6 @@ func main() {
 			continue
 		}
 		elapsed := time.Since(start)
-		log.Printf("Completed %d games in %s time", i + 1, elapsed)
+		log.Printf("Completed %d games in %s time", i+1, elapsed)
 	}
 }

go/src/server/db/models.go

@@ -80,6 +80,8 @@ type MatchGame struct {
 	Result  int
 	Done    bool
 	Flip    bool
+
+	EngineVersion string
 }
 
 type TrainingGame struct {
@@ -97,4 +99,6 @@ type TrainingGame struct {
 	Path      string
 	Pgn       string
 	Compacted bool
+
+	EngineVersion string
 }

go/src/server/main.go

@@ -307,6 +307,7 @@ func uploadGame(c *gin.Context) {
 		NetworkID:     network.ID,
 		Version:       uint(version),
 		Pgn:           c.PostForm("pgn"),
+		EngineVersion: c.PostForm("engineVersion"),
 	}
 	db.GetDB().Create(&game)
 	db.GetDB().Model(&game).Update("path", filepath.Join("games", fmt.Sprintf("run%d/training.%d.gz", training_run.ID, game.ID)))
@@ -429,10 +430,11 @@ func matchResult(c *gin.Context) {
 	}
 
 	err = db.GetDB().Model(&match_game).Updates(db.MatchGame{
-		Version: uint(version),
-		Result:  int(result),
-		Done:    true,
-		Pgn:     c.PostForm("pgn"),
+		Version:       uint(version),
+		Result:        int(result),
+		Done:          true,
+		Pgn:           c.PostForm("pgn"),
+		EngineVersion: c.PostForm("engineVersion"),
 	}).Error
 	if err != nil {
 		log.Println(err)

go/src/server/main_test.go

@@ -197,7 +197,7 @@ func uploadTestNetwork(s *StoreSuite, contentString string, networkId int) {
 	s.w = httptest.NewRecorder()
 	content := []byte(contentString)
 	var buf bytes.Buffer
-	zw := gzip.NewWriter(&buf)
+	zw := gzip.NewWriterLevel(&buf, BestCompression)
 	zw.Write(content)
 	zw.Close()
 

lc0/src/mcts/search.cc

@@ -228,7 +228,8 @@ void Search::SendUciInfo() {
   uci_info_.seldepth = root_node_->max_depth;
   uci_info_.time = GetTimeSinceStart();
   uci_info_.nodes = total_nodes_;
-  uci_info_.nps = uci_info_.nodes * 1000 / uci_info_.time;
+  uci_info_.nps =
+      uci_info_.time ? (uci_info_.nodes * 1000 / uci_info_.time) : 0;
   uci_info_.score = -91 * log(2 / (best_move_node_->q + 1) - 1);
   uci_info_.pv.clear();
 

lc0/src/utils/bititer.h

@@ -18,6 +18,9 @@
 
 #pragma once
 #include <cstdint>
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
 
 namespace lczero {
 
@@ -30,7 +33,15 @@ class BitIterator {
   bool operator!=(const BitIterator& other) { return value_ != other.value_; }
 
   void operator++() { value_ &= (value_ - 1); }
-  T operator*() const { return __builtin_ctzll(value_); }
+  T operator*() const {
+#ifdef _MSC_VER
+    unsigned long result;
+    _BitScanForward64(&result, value_);
+    return result;
+#else
+    return __builtin_ctzll(value_);
+#endif
+  }
 
  private:
   std::uint64_t value_;

lc0/src/utils/optional.h

@@ -0,0 +1,41 @@
+/*
+This file is part of Leela Chess Zero.
+Copyright (C) 2018 The LCZero Authors
+
+Leela Chess is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+Leela Chess is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with Leela Chess.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+namespace {
+
+// Very poor-man implementation of std::optional. It literally cannot do
+// anything, but it's enough for our use case.
+template <class T>
+class optional {
+ public:
+  operator bool() const { return has_value_; }
+  constexpr const T& operator*() const& { return value_; }
+  optional<T>& operator=(const T& value) {
+    value_ = value;
+    has_value_ = true;
+    return *this;
+  }
+
+ private:
+  T value_;
+  bool has_value_ = false;
+};
+
+}  // namespace

scripts/run.sh

@@ -3,13 +3,13 @@
 # Add -debug to see engine output
 WDR=$HOME/Workspace/chess/lczero-weights
 DIR=$PWD/../build
-NXT=lc_gen4
-CUR=lc_gen3
+NXT=ID55
+CUR=ID45
 
 cutechess-cli -rounds 100 -tournament gauntlet -concurrency 4 \
  -pgnout results.pgn \
- -engine name=$NXT cmd=$DIR/lczero arg="--threads=1" arg="--noise" arg="--weights=$WDR/gen4-64x6.txt" arg="--playouts=800" arg="--noponder" arg="--gpu=0" \
- -engine name=$CUR cmd=$DIR/lczero arg="--threads=1" arg="--noise" arg="--weights=$WDR/gen3-64x6.txt" arg="--playouts=800" arg="--noponder" arg="--gpu=1" \
+ -engine name=$NXT cmd=$DIR/lczero-new arg="--threads=1" arg="--noise" arg="--weights=$WDR/ID55" arg="--playouts=800" arg="--noponder" arg="--gpu=0" \
+ -engine name=$CUR cmd=$DIR/lczero arg="--threads=1" arg="--noise" arg="--weights=$WDR/ID45" arg="--playouts=800" arg="--noponder" arg="--gpu=1" \
  -each proto=uci tc=inf
 
 mv -v results.pgn "$NXT-vs-$CUR.pgn"

scripts/stats/netstats.py

@@ -61,12 +61,12 @@ def plot_stats(stats, name, cfg):
     max_plies = stats['white'].size
     fig=plt.figure(figsize=(12, 5), dpi=80)
     plt.xlim(0, max_plies)
-    plt.xlabel('plies')
+    plt.xlabel('ply (half-move)')
     plt.ylabel('games')
     games = int(np.sum(stats['plycount']))
     cutoff = (stats['plycount'][max_plies-1][0] / float(games)) * 100
     plt.title('{} games, (w {}%, b {}%, d {}%) - {:.2f}% cutoff [{}]'.format(games, w, b, d, cutoff, name))
-    
+
     for i, k in enumerate(['white', 'black', 'nomaterial', 'stalemate', '3-fold', '50-move']):
         stats[k][stats[k] == 0] = np.nan
         plt.plot(range(1, max_plies), stats[k][:max_plies-1], "{}".format(types[i]+colors[i]), label=k, markeredgecolor=edges[i])
@@ -90,7 +90,7 @@ def main(cfg):
         names = ['plycount', 'checkmate', 'stalemate', 'gameover', 'nomaterial', 'white', 'black', 'draw', '3-fold', '50-move']
         stats = {}
         failed = 0
-        
+
         for name in names:
             stats[name] = np.zeros((MAX_PLIES, 1), dtype=np.float32)
 
@@ -125,7 +125,7 @@ def main(cfg):
                         stats['black'][plies] += 1
                 else:
                     stats['draw'][plies] += 1
-                
+
                 if board.is_stalemate():
                     stats['stalemate'][plies] += 1
                 elif board.is_insufficient_material():
@@ -134,7 +134,7 @@ def main(cfg):
                     stats['3-fold'][plies] += 1
                 elif board.can_claim_fifty_moves():
                     stats['50-move'][plies] += 1
-                
+
                 if board.is_game_over():
                     stats['gameover'][plies] += 1
 
@@ -148,4 +148,3 @@ def main(cfg):
 if __name__ == "__main__":
     cfg = get_configuration()
     main(cfg)
-