From fba3059adea6017dc1157ae935c578585478dc88 Mon Sep 17 00:00:00 2001 From: sscg13 Date: Mon, 23 Feb 2026 22:52:49 -0800 Subject: [PATCH 1/4] compress indices, match SF Co-authored-by: Viren6 <94880762+Viren6@users.noreply.github.com> --- model/modules/features/full_threats.py | 19 ++++++------ training_data_loader.cpp | 43 +++++++++++++++++--------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/model/modules/features/full_threats.py b/model/modules/features/full_threats.py index 664539ff..cd8f1847 100644 --- a/model/modules/features/full_threats.py +++ b/model/modules/features/full_threats.py @@ -8,9 +8,9 @@ class FullThreats(DoubleFeatureTransformer): - HASH = 0x8F234CB8 - FEATURE_NAME = "Full_Threats^" - INPUT_FEATURE_NAME = "Full_Threats" + HASH = 0x8F234CB9 + FEATURE_NAME = "Full_Threatsv2^" + INPUT_FEATURE_NAME = "Full_Threatsv2" MAX_ACTIVE_FEATURES = 128 + 32 NUM_SQ = 64 @@ -18,13 +18,13 @@ class FullThreats(DoubleFeatureTransformer): NUM_PLANES = NUM_SQ * NUM_PT # 768 NUM_BUCKETS = NUM_SQ // 2 # 32 - NUM_THREAT_FEATURES = 60144 + NUM_THREAT_FEATURES = 53564 NUM_PSQ_FEATURES = NUM_PLANES * NUM_BUCKETS # 24,576 - NUM_INPUTS = NUM_THREAT_FEATURES + NUM_PSQ_FEATURES # 84,720 + NUM_INPUTS = NUM_THREAT_FEATURES + NUM_PSQ_FEATURES # 78,140 NUM_INPUTS_VIRTUAL = NUM_PLANES # 768 # Export size: threats + 11-piece-type PSQ (704 * 32 = 22,528) - NUM_REAL_FEATURES = NUM_THREAT_FEATURES + 704 * 32 # 82,672 + NUM_REAL_FEATURES = NUM_THREAT_FEATURES + 704 * 32 # 76,092 def __init__(self, num_outputs: int): super().__init__(self.NUM_INPUTS, num_outputs) @@ -186,17 +186,18 @@ def halfka_psqts() -> list[int]: chess.QUEEN: 2538, } + num_threats = 53564 num_psq = 768 * 32 # 24,576 - num_total = 60_144 + num_psq + num_total = num_threats + num_psq values = [0] * num_total for ksq in range(64): for s in range(64): for pt, val in piece_values.items(): - idxw = 60_144 + _halfka_idx( + idxw = num_threats + _halfka_idx( True, ksq, s, chess.Piece(pt, chess.WHITE) ) - idxb = 60_144 + _halfka_idx( + idxb = num_threats + _halfka_idx( True, ksq, s, chess.Piece(pt, chess.BLACK) ) values[idxw] = val diff --git a/training_data_loader.cpp b/training_data_loader.cpp index 7090ee8d..a1b984b0 100644 --- a/training_data_loader.cpp +++ b/training_data_loader.cpp @@ -143,10 +143,10 @@ constexpr auto threatfeaturecalc = []() { constexpr ThreatOffsetTable threatoffsets = threatfeaturecalc.table; constexpr int threatfeatures = threatfeaturecalc.totalfeatures; -static_assert(threatfeatures == 60144); +static_assert(threatfeatures == 53564); -struct Full_Threats { - static constexpr std::string_view NAME = "Full_Threats"; +struct Full_Threatsv2 { + static constexpr std::string_view NAME = "Full_Threatsv2"; static constexpr int SQUARE_NB = 64; static constexpr int PIECE_NB = 12; @@ -175,11 +175,11 @@ struct Full_Threats { }; static constexpr int map[PIECE_TYPE_NB][PIECE_TYPE_NB] = { - {0, 1, -1, 2, -1, -1}, - {0, 1, 2, 3, 4, -1}, - {0, 1, 2, 3, -1, -1}, - {0, 1, 2, 3, -1, -1}, - {0, 1, 2, 3, 4, -1}, + { 0, 1, -1, 2, -1, -1}, + { 0, 1, 2, 3, 4, -1}, + { 0, 1, 2, 3, -1, -1}, + { 0, 1, 2, 3, -1, -1}, + { 0, 1, 2, 3, 4, -1}, {-1, -1, -1, -1, -1, -1} }; @@ -196,7 +196,7 @@ struct Full_Threats { // clang-format on static constexpr int NUM_SQ = 64; - static constexpr int NUM_PT = 12; + static constexpr int NUM_PT = 11; static constexpr int NUM_PLANES = NUM_SQ * NUM_PT; static constexpr int NUM_THREAT_FEATURES = threatfeatures; @@ -226,14 +226,29 @@ struct Full_Threats { { return -1; } + + if (enemy && (attkr.type() == attkd.type()) && (attkr.type() != PieceType::Pawn)) + { + attkd = attkr; + std::swap(from, to); + } + + int targetindex = int((attkr == attkd) ? Color::White : attkd.color()) + * ((1 + numvalidtargets[(int) attkr])/ 2) + map[(int) attkr.type()][(int) attkd.type()]; + + if ((8 * (int)attkd.color() + (int)attkd.type()) > (8 + (int)attkr.type()) + && attkr != Piece(PieceType::Pawn, Color::Black)) + { + targetindex--; + } + + Bitboard attacks = (attkr.type() == PieceType::Pawn) ? bb::pawnAttacks(Bitboard::square(Square(from)), attkr.color()) : bb::detail::pseudoAttacks()[attkr.type()][Square(from)]; - Bitboard upto = Bitboard::square(to); + return int(threatoffsets[(int) attkr][65] - + (int(attkd.color()) * (numvalidtargets[(int) attkr] / 2) - + map[(int) attkr.type()][(int) attkd.type()]) - * threatoffsets[(int) attkr][64] + + targetindex * threatoffsets[(int) attkr][64] + threatoffsets[(int) attkr][(int) from] + (Bitboard::fromBits((1ULL << (int) to) - 1) & attacks).count()); } @@ -357,7 +372,7 @@ auto find_feature(std::string_view name) { auto get_feature(std::string_view name) { return find_feature(name); } From 60b256e738eaf2fb787304eb453a2aaa2d09223c Mon Sep 17 00:00:00 2001 From: sscg13 Date: Tue, 24 Feb 2026 00:10:03 -0800 Subject: [PATCH 2/4] use right feature set in data loader bench --- training_data_loader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training_data_loader.cpp b/training_data_loader.cpp index a1b984b0..c0d20b43 100644 --- a/training_data_loader.cpp +++ b/training_data_loader.cpp @@ -1096,7 +1096,7 @@ int main(int argc, char** argv) { .pc_y2 = 2.0, .pc_y3 = 1.0}; const DataloaderDDPConfig ddp_config = {.rank = 0, .world_size = 1}; - auto stream = create_sparse_batch_stream("Full_Threats", concurrency, file_count, files, + auto stream = create_sparse_batch_stream("Full_Threatsv2", concurrency, file_count, files, batch_size, cyclic, config, ddp_config); auto t0 = std::chrono::high_resolution_clock::now(); From 6e396094eb990f4f95049cf6926f5c745a773a0e Mon Sep 17 00:00:00 2001 From: sscg13 Date: Wed, 25 Feb 2026 15:28:08 -0800 Subject: [PATCH 3/4] we hate resolving merge conflicts --- training_data_loader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training_data_loader.cpp b/training_data_loader.cpp index c0d20b43..6f723ba0 100644 --- a/training_data_loader.cpp +++ b/training_data_loader.cpp @@ -93,7 +93,7 @@ struct HalfKAv2_hm { } }; -constexpr int numvalidtargets[12] = {6, 6, 10, 10, 8, 8, 8, 8, 10, 10, 0, 0}; +constexpr int numvalidtargets[12] = {5, 6, 9, 9, 7, 7, 7, 7, 9, 9, 0, 0}; using ThreatOffsetTable = std::array, 12>; From 34aa5ca94d03994dbbf9949ab59feab4ae3a2be5 Mon Sep 17 00:00:00 2001 From: sscg13 Date: Sat, 28 Feb 2026 10:33:28 -0800 Subject: [PATCH 4/4] update modules --- model/modules/features/__init__.py | 4 ++-- model/modules/features/{full_threats.py => full_threatsv2.py} | 2 +- model/utils/serialize.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename model/modules/features/{full_threats.py => full_threatsv2.py} (99%) diff --git a/model/modules/features/__init__.py b/model/modules/features/__init__.py index ae0d916e..cdbdd8f9 100644 --- a/model/modules/features/__init__.py +++ b/model/modules/features/__init__.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from .halfka_v2_hm import HalfKav2Hm -from .full_threats import FullThreats +from .full_threatsv2 import FullThreatsv2 import tyro from typing import Annotated @@ -10,7 +10,7 @@ _FEATURES: dict[str, type] = { "HalfKAv2_hm^": HalfKav2Hm, - "Full_Threats^": FullThreats, + "Full_Threatsv2^": FullThreatsv2, } diff --git a/model/modules/features/full_threats.py b/model/modules/features/full_threatsv2.py similarity index 99% rename from model/modules/features/full_threats.py rename to model/modules/features/full_threatsv2.py index cd8f1847..fb3d1a98 100644 --- a/model/modules/features/full_threats.py +++ b/model/modules/features/full_threatsv2.py @@ -7,7 +7,7 @@ from .halfka_v2_hm import InverseKingBuckets, _halfka_idx -class FullThreats(DoubleFeatureTransformer): +class FullThreatsv2(DoubleFeatureTransformer): HASH = 0x8F234CB9 FEATURE_NAME = "Full_Threatsv2^" INPUT_FEATURE_NAME = "Full_Threatsv2" diff --git a/model/utils/serialize.py b/model/utils/serialize.py index 2e2be804..d4b2989b 100644 --- a/model/utils/serialize.py +++ b/model/utils/serialize.py @@ -12,7 +12,7 @@ from ..config import ModelConfig from ..model import NNUEModel from ..modules import BaseFeatureTransformer, get_feature_cls -from ..modules.features.full_threats import FullThreats +from ..modules.features.full_threatsv2 import FullThreatsv2 from ..quantize import QuantizationConfig