Thing 39: Deriving Point Values for Chess Pieces (Part 3)

Part 1: here. Part 2: here.

In today's episode of deriving point values for chess pieces from endgame tablebases, we're gonna try to see how robust our numbers from before were. So, we'll try computing a few variants.

Variant 1: not starting with obviously promoted pieces (i.e. no triple knights, bishops or rooks or double queens on the same side.)

Not too much interesting to see here, seems to generally match the previous numbers, with the only notable thing being that the queen trends up instead of down starting at 3 pieces.

Variant 2: a different loss function (sum of squares on un-sigmoided values)

Oh boy, is this interesting? Seems like with this loss function the queen is worth waaaay less. But still trending upward. Pawns are also worth way more. Not sure what to think about that.

Variant 3: another different loss function (logarithm proper scoring rule)

Very interesting, this one is way closer to the same as the original values I got, but with the queen just slightly lower in value.

Code below. Download stats.json from the lichess tablebase to run the code.

import fs from "node:fs";

const stats = JSON.parse(fs.readFileSync("stats.json", "utf-8"));

type Data = {
	material: string;
	gamePoints: number;
}[];

const getData = (options?: {
	/**
	 * How much a cursed win counts as (cursed win = would be a win if not for the 50-move rule)
	 * Any value between 0 and 1 makes sense.
	 */
	curseFactor?: number;
	nonKingPieceCount?: number;
	tooManyPieces?: string[];
}): Data => {
	const {
		curseFactor = 0.25,
		nonKingPieceCount = 5,
		tooManyPieces = [],
	} = options ?? {};

	return Object.entries(stats)
		.map(([key, e]: [string, any]) => {
			const wins = e.histogram.white.wdl[2] + e.histogram.black.wdl[-2];
			const cursedWins =
				e.histogram.white.wdl[1] + e.histogram.black.wdl[-1];
			const draws = e.histogram.white.wdl[0] + e.histogram.black.wdl[0];
			const blessedLosses =
				e.histogram.white.wdl[-1] + e.histogram.black.wdl[1];
			const losses = e.histogram.white.wdl[-2] + e.histogram.black.wdl[2];
			const total = wins + cursedWins + draws + blessedLosses + losses;
			const gamePoints =
				(wins +
					curseFactor * cursedWins -
					curseFactor * blessedLosses -
					losses) /
				total;
			return {
				material: key,
				gamePoints,
			};
		})
		.filter(
			({ material }) =>
				material.length === "KvK".length + nonKingPieceCount &&
				tooManyPieces.every((substr) => !material.includes(substr)),
		);
};

type Piece = "P" | "N" | "B" | "R" | "Q";

type Points = Record<Piece, number>;

type LossFunc = (goal: number, prediction: number) => number;

const sigmoid = (x: number) => Math.tanh(x);

const sum = (arr: number[]) => arr.reduce((a, b) => a + b, 0);

const predictGame = (points: Points, material: [string, string]) => {
	const [mat1, mat2] = material;
	return sigmoid(
		sum([...mat1].map((p) => (p in points ? points[p as Piece] : 0))) -
			sum([...mat2].map((p) => (p in points ? points[p as Piece] : 0))),
	);
};

const getBaselineLoss = (data: Data, lossFunc: LossFunc) => {
	let loss = 0;
	for (const { gamePoints } of data) {
		const prediction = 0;
		loss += lossFunc(gamePoints, prediction);
	}
	return loss;
};

const computeLoss = (data: Data, points: Points, lossFunc: LossFunc) => {
	let loss = 0;
	for (const { material, gamePoints } of data) {
		const prediction = predictGame(
			points,
			material.split("v") as [string, string],
		);
		loss += lossFunc(gamePoints, prediction);
	}
	return loss / getBaselineLoss(data, lossFunc);
};

const optimizeInRange = (
	data: Data,
	constraints: Record<Piece, { min: number; max: number; step: number }>,
	lossFunc: LossFunc,
): Points => {
	let bestLoss = Infinity;
	let best: Points | null = null;
	let i = 0;
	for (
		let p = constraints.P.min;
		p <= constraints.P.max;
		p += constraints.P.step
	) {
		for (
			let n = constraints.N.min;
			n <= constraints.N.max;
			n += constraints.N.step
		) {
			for (
				let b = constraints.B.min;
				b <= constraints.B.max;
				b += constraints.B.step
			) {
				for (
					let r = constraints.R.min;
					r <= constraints.R.max;
					r += constraints.R.step
				) {
					for (
						let q = constraints.Q.min;
						q <= constraints.Q.max;
						q += constraints.Q.step
					) {
						i += 1;
						const pointValues = {
							P: p,
							N: n,
							B: b,
							R: r,
							Q: q,
						};
						const loss = computeLoss(data, pointValues, lossFunc);
						if (loss < bestLoss) {
							best = pointValues;
							bestLoss = loss;
						}
					}
				}
			}
		}
	}
	return best as Points;
};

const nudge = (
	data: Data,
	points: Points,
	epsilon: number,
	lossFunc: LossFunc,
) => {
	return optimizeInRange(
		data,
		Object.fromEntries(
			Object.entries(points).map(([key, value]) => {
				return [
					key,
					{
						min: value - epsilon,
						max: value + epsilon,
						step: epsilon,
					},
				];
			}),
		) as any,
		lossFunc,
	);
};

const optimizeForEpsilon = (
	data: Data,
	initialPoints: Points,
	epsilon: number,
	lossFunc: LossFunc,
) => {
	let prevLoss = 0;

	let points = { ...initialPoints };

	for (let i = 0; i < 1000; i += 1) {
		points = nudge(data, points, epsilon, lossFunc);
		const loss = computeLoss(data, points, lossFunc);
		if (loss === prevLoss) {
			break;
		}
		prevLoss = loss;
		if (i > 998) {
			console.log("TOOK TOO LONG");
		}
	}

	return points;
};

const getPointValues = (
	data: Data,
	lossFunc: LossFunc,
	initialValues = {
		P: 0,
		N: 0,
		B: 0,
		R: 0,
		Q: 0,
	},
) => {
	let pointValues = initialValues;

	pointValues = optimizeForEpsilon(data, pointValues, 1, lossFunc);
	pointValues = optimizeForEpsilon(data, pointValues, 0.1, lossFunc);
	pointValues = optimizeForEpsilon(data, pointValues, 0.01, lossFunc);
	pointValues = optimizeForEpsilon(data, pointValues, 0.001, lossFunc);

	return pointValues;
};

/**
 * Normalize a set of points so that the rook is 5 (and queen will usually be around 9).
 */
const normalize = (points: Points) => {
	const factor = 5 / points.R;
	return {
		P: points.P * factor,
		N: points.N * factor,
		B: points.B * factor,
		R: points.R * factor,
		Q: points.Q * factor,
	};
};

const formatTable = (i: number, points: Points) => {
	return `| ${points.P.toFixed(2)} | ${points.N.toFixed(2)} | ${points.B.toFixed(2)} | ${points.R.toFixed(2)} | ${points.Q.toFixed(2)} |`;
};

// Compute point values for pieces based on number of pieces on the board.

console.log("Variant 1: not starting with obviously promoted pieces");

for (let i = 1; i <= 5; i += 1) {
	const lossFunc: LossFunc = (a, b) => Math.pow(a - b, 2);
	const data = getData({
		nonKingPieceCount: i,
		tooManyPieces: ["BBB", "NNN", "RRR", "QQ"],
	});
	const pointValues = getPointValues(data, lossFunc);
	console.log(`${i} non-king pieces`);
	console.log(formatTable(i, normalize(pointValues)));
	console.log("loss", computeLoss(data, pointValues, lossFunc));
}

console.log("Variant 2: different loss function");

for (let i = 1; i <= 5; i += 1) {
	const lossFunc: LossFunc = (a, b) =>
		Math.pow(Math.atanh(a) - Math.atanh(b), 2);
	const data = getData({
		nonKingPieceCount: i,
	});
	const pointValues = getPointValues(data, lossFunc);
	console.log(`${i} non-king pieces`);
	console.log(formatTable(i, normalize(pointValues)));
	console.log("loss", computeLoss(data, pointValues, lossFunc));
}

console.log("Variant 3: another different loss function");

for (let i = 1; i <= 5; i += 1) {
	const lossFunc: LossFunc = (a, b) =>
		((a + 1) / 2) * Math.log((b + 1) / 2) +
		(1 - (a + 1) / 2) * Math.log(1 - (b + 1) / 2);
	const data = getData({
		nonKingPieceCount: i,
	});
	const pointValues = getPointValues(data, lossFunc);
	console.log(`${i} non-king pieces`);
	console.log(formatTable(i, normalize(pointValues)));
	console.log("loss", computeLoss(data, pointValues, lossFunc));
}