diff --git a/src/nnue/nnue_accumulator.cpp b/src/nnue/nnue_accumulator.cpp index 8325ee99bf3..8a37187e69a 100644 --- a/src/nnue/nnue_accumulator.cpp +++ b/src/nnue/nnue_accumulator.cpp @@ -180,6 +180,90 @@ void AccumulatorStack::backward_update_incremental(Color per namespace { +constexpr IndexType Dimensions = FeatureTransformer::OutputDimensions; + +#ifndef USE_RVV + +using Tiling = SIMDTiling; + +template +sf_always_inline inline void apply_psq_features(IndexType j, + vec_t acc[], + const PSQFeatureSet::IndexList& list, + const FeatureTransformer& ft) { + static_assert(sign == 1 || sign == -1); + + const usize tileOff = j * Tiling::TileHeight; + for (int i = 0; i < list.ssize(); ++i) + { + auto* column = reinterpret_cast( + &ft.weights[list[i] * Dimensions + tileOff]); + for (IndexType k = 0; k < Tiling::NumRegs; ++k) + if constexpr (sign == +1) + acc[k] = vec_add_16(acc[k], column[k]); + else + acc[k] = vec_sub_16(acc[k], column[k]); + } +} + +template +sf_always_inline inline void apply_threat_features(IndexType j, + vec_t acc[Tiling::NumRegs], + const ThreatFeatureSet::IndexList& list, + const FeatureTransformer& ft) { + static_assert(sign == 1 || sign == -1); + + const usize tileOff = j * Tiling::TileHeight; + for (int i = 0; i < list.ssize(); ++i) + { + auto* column = reinterpret_cast( + &ft.threatAndPpWeights[list[i] * Dimensions + tileOff]); +#ifdef USE_NEON + for (IndexType k = 0; k < Tiling::NumRegs; k += 2) + { + if constexpr (sign == +1) + { + acc[k] = vaddw_s8(acc[k], vget_low_s8(column[k / 2])); + acc[k + 1] = vaddw_high_s8(acc[k + 1], column[k / 2]); + } + else + { + acc[k] = vsubw_s8(acc[k], vget_low_s8(column[k / 2])); + acc[k + 1] = vsubw_high_s8(acc[k + 1], column[k / 2]); + } + } +#else + for (IndexType k = 0; k < Tiling::NumRegs; ++k) + if constexpr (sign == +1) + acc[k] = vec_add_16(acc[k], vec_convert_8_16(column[k])); + else + acc[k] = vec_sub_16(acc[k], vec_convert_8_16(column[k])); +#endif + } +} + +template +sf_always_inline inline void apply_psqt(IndexType j, + psqt_vec_t psqt[Tiling::NumPsqtRegs], + const ValueList& list, + const PSQTWeightType* weights) { + static_assert(sign == 1 || sign == -1); + + const usize psqtTileOff = j * Tiling::PsqtTileHeight; + for (int i = 0; i < list.ssize(); ++i) + { + auto* column = reinterpret_cast( + &weights[list[i] * PSQTBuckets + psqtTileOff]); + for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) + if constexpr (sign == +1) + psqt[k] = vec_add_psqt_32(psqt[k], column[k]); + else + psqt[k] = vec_sub_psqt_32(psqt[k], column[k]); + } +} + +#endif + void apply_combined(Color perspective, const FeatureTransformer& featureTransformer, const AccumulatorState& from, @@ -188,7 +272,6 @@ void apply_combined(Color perspective, const PSQFeatureSet::IndexList& psqRemoved, const ThreatFeatureSet::IndexList& thrAdded, const ThreatFeatureSet::IndexList& thrRemoved) { - constexpr IndexType Dimensions = FeatureTransformer::OutputDimensions; const auto& fromAcc = from.accumulation[perspective]; auto& toAcc = to.accumulation[perspective]; @@ -196,15 +279,11 @@ void apply_combined(Color perspective, const auto& fromPsqtAcc = from.psqtAccumulation[perspective]; auto& toPsqtAcc = to.psqtAccumulation[perspective]; -#ifdef VECTOR - using Tiling = SIMDTiling; +#ifndef USE_RVV vec_t acc[Tiling::NumRegs]; psqt_vec_t psqt[Tiling::NumPsqtRegs]; - const auto* psqWeights = &featureTransformer.weights[0]; - const auto* threatAndPpWeights = &featureTransformer.threatAndPpWeights[0]; - for (IndexType j = 0; j < Dimensions / Tiling::TileHeight; ++j) { const usize tileOff = j * Tiling::TileHeight; @@ -214,55 +293,11 @@ void apply_combined(Color perspective, for (IndexType k = 0; k < Tiling::NumRegs; ++k) acc[k] = fromTile[k]; - for (int i = 0; i < psqRemoved.ssize(); ++i) - { - auto* row = - reinterpret_cast(&psqWeights[psqRemoved[i] * Dimensions + tileOff]); - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_sub_16(acc[k], row[k]); - } - - for (int i = 0; i < psqAdded.ssize(); ++i) - { - auto* row = - reinterpret_cast(&psqWeights[psqAdded[i] * Dimensions + tileOff]); - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_add_16(acc[k], row[k]); - } - - for (int i = 0; i < thrRemoved.ssize(); ++i) - { - auto* column = reinterpret_cast( - &threatAndPpWeights[thrRemoved[i] * Dimensions + tileOff]); - - #ifdef USE_NEON - for (IndexType k = 0; k < Tiling::NumRegs; k += 2) - { - acc[k] = vsubw_s8(acc[k], vget_low_s8(column[k / 2])); - acc[k + 1] = vsubw_high_s8(acc[k + 1], column[k / 2]); - } - #else - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_sub_16(acc[k], vec_convert_8_16(column[k])); - #endif - } - - for (int i = 0; i < thrAdded.ssize(); ++i) - { - auto* column = reinterpret_cast( - &threatAndPpWeights[thrAdded[i] * Dimensions + tileOff]); + apply_psq_features<-1>(j, acc, psqRemoved, featureTransformer); + apply_psq_features<+1>(j, acc, psqAdded, featureTransformer); - #ifdef USE_NEON - for (IndexType k = 0; k < Tiling::NumRegs; k += 2) - { - acc[k] = vaddw_s8(acc[k], vget_low_s8(column[k / 2])); - acc[k + 1] = vaddw_high_s8(acc[k + 1], column[k / 2]); - } - #else - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_add_16(acc[k], vec_convert_8_16(column[k])); - #endif - } + apply_threat_features<-1>(j, acc, thrRemoved, featureTransformer); + apply_threat_features<+1>(j, acc, thrAdded, featureTransformer); for (IndexType k = 0; k < Tiling::NumRegs; k++) vec_store(&toTile[k], acc[k]); @@ -277,44 +312,17 @@ void apply_combined(Color perspective, for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) psqt[k] = fromTilePsqt[k]; - for (int i = 0; i < psqRemoved.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.psqtWeights[psqRemoved[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]); - } - - for (int i = 0; i < psqAdded.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.psqtWeights[psqAdded[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]); - } - - for (int i = 0; i < thrRemoved.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer - .threatAndPpPsqtWeights[thrRemoved[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]); - } + apply_psqt<-1>(j, psqt, psqRemoved, featureTransformer.psqtWeights.data()); + apply_psqt<+1>(j, psqt, psqAdded, featureTransformer.psqtWeights.data()); - for (int i = 0; i < thrAdded.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.threatAndPpPsqtWeights[thrAdded[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]); - } + apply_psqt<-1>(j, psqt, thrRemoved, featureTransformer.threatAndPpPsqtWeights.data()); + apply_psqt<+1>(j, psqt, thrAdded, featureTransformer.threatAndPpPsqtWeights.data()); for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) vec_store_psqt(&toTilePsqt[k], psqt[k]); } -#elif defined(USE_RVV) +#else usize tileOffset = 0; @@ -372,47 +380,6 @@ void apply_combined(Color perspective, tileOffset += vl; } -#else - - toAcc = fromAcc; - toPsqtAcc = fromPsqtAcc; - - for (const auto index : psqRemoved) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - toAcc[j] -= featureTransformer.weights[offset + j]; - for (usize k = 0; k < PSQTBuckets; ++k) - toPsqtAcc[k] -= featureTransformer.psqtWeights[index * PSQTBuckets + k]; - } - - for (const auto index : psqAdded) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - toAcc[j] += featureTransformer.weights[offset + j]; - for (usize k = 0; k < PSQTBuckets; ++k) - toPsqtAcc[k] += featureTransformer.psqtWeights[index * PSQTBuckets + k]; - } - - for (const auto index : thrRemoved) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - toAcc[j] -= featureTransformer.threatAndPpWeights[offset + j]; - for (usize k = 0; k < PSQTBuckets; ++k) - toPsqtAcc[k] -= featureTransformer.threatAndPpPsqtWeights[index * PSQTBuckets + k]; - } - - for (const auto index : thrAdded) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - toAcc[j] += featureTransformer.threatAndPpWeights[offset + j]; - for (usize k = 0; k < PSQTBuckets; ++k) - toPsqtAcc[k] += featureTransformer.threatAndPpPsqtWeights[index * PSQTBuckets + k]; - } - #endif } @@ -458,8 +425,8 @@ void update_accumulator_incremental(Color perspective, PSQFeatureSet::append_changed_indices(perspective, ksq, dirtyPiece, psqAdded, psqRemoved); } - apply_combined(perspective, featureTransformer, computed, target_state, psqAdded, psqRemoved, - thrAdded, thrRemoved); + apply_combined(perspective, featureTransformer, computed, target_state, psqAdded, + psqRemoved, thrAdded, thrRemoved); target_state.computed[perspective] = true; } @@ -662,23 +629,16 @@ void update_accumulator_hybrid(Color perspective, PairFeatureSet::append_changed_indices(perspective, newKsq, target.dirtyPawnPairs, thrRemoved, thrAdded, threatPpBase, pfStride); - constexpr IndexType Dimensions = FeatureTransformer::OutputDimensions; - const auto& fromAcc = computed.accumulation[perspective]; auto& toAcc = target.accumulation[perspective]; const auto& fromPsqtAcc = computed.psqtAccumulation[perspective]; auto& toPsqtAcc = target.psqtAccumulation[perspective]; -#ifdef VECTOR - using Tiling = SIMDTiling; - +#ifndef USE_RVV vec_t acc[Tiling::NumRegs]; psqt_vec_t psqt[Tiling::NumPsqtRegs]; - const auto* weights = &featureTransformer.weights[0]; - const auto* threatAndPpWeights = &featureTransformer.threatAndPpWeights[0]; - for (IndexType j = 0; j < Dimensions / Tiling::TileHeight; ++j) { const usize tileOff = j * Tiling::TileHeight; @@ -690,20 +650,8 @@ void update_accumulator_hybrid(Color perspective, for (IndexType k = 0; k < Tiling::NumRegs; ++k) acc[k] = newEntryTile[k]; - for (int i = 0; i < newRemove.ssize(); ++i) - { - auto* column = - reinterpret_cast(&weights[newRemove[i] * Dimensions + tileOff]); - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_sub_16(acc[k], column[k]); - } - for (int i = 0; i < newAdd.ssize(); ++i) - { - auto* column = - reinterpret_cast(&weights[newAdd[i] * Dimensions + tileOff]); - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_add_16(acc[k], column[k]); - } + apply_psq_features<-1>(j, acc, newRemove, featureTransformer); + apply_psq_features<+1>(j, acc, newAdd, featureTransformer); for (IndexType k = 0; k < Tiling::NumRegs; ++k) { @@ -717,54 +665,11 @@ void update_accumulator_hybrid(Color perspective, } // ... then we adjust - for (int i = 0; i < oldRemove.ssize(); ++i) - { - auto* column = - reinterpret_cast(&weights[oldRemove[i] * Dimensions + tileOff]); - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_add_16(acc[k], column[k]); - } - for (int i = 0; i < oldAdd.ssize(); ++i) - { - auto* column = - reinterpret_cast(&weights[oldAdd[i] * Dimensions + tileOff]); - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_sub_16(acc[k], column[k]); - } + apply_psq_features<+1>(j, acc, oldRemove, featureTransformer); + apply_psq_features<-1>(j, acc, oldAdd, featureTransformer); - for (int i = 0; i < thrRemoved.ssize(); ++i) - { - auto* column = reinterpret_cast( - &threatAndPpWeights[thrRemoved[i] * Dimensions + tileOff]); - - #ifdef USE_NEON - for (IndexType k = 0; k < Tiling::NumRegs; k += 2) - { - acc[k] = vsubw_s8(acc[k], vget_low_s8(column[k / 2])); - acc[k + 1] = vsubw_high_s8(acc[k + 1], column[k / 2]); - } - #else - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_sub_16(acc[k], vec_convert_8_16(column[k])); - #endif - } - - for (int i = 0; i < thrAdded.ssize(); ++i) - { - auto* column = reinterpret_cast( - &threatAndPpWeights[thrAdded[i] * Dimensions + tileOff]); - - #ifdef USE_NEON - for (IndexType k = 0; k < Tiling::NumRegs; k += 2) - { - acc[k] = vaddw_s8(acc[k], vget_low_s8(column[k / 2])); - acc[k + 1] = vaddw_high_s8(acc[k + 1], column[k / 2]); - } - #else - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_add_16(acc[k], vec_convert_8_16(column[k])); - #endif - } + apply_threat_features<-1>(j, acc, thrRemoved, featureTransformer); + apply_threat_features<+1>(j, acc, thrAdded, featureTransformer); for (IndexType k = 0; k < Tiling::NumRegs; k++) vec_store(&toTile[k], acc[k]); @@ -783,20 +688,8 @@ void update_accumulator_hybrid(Color perspective, for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) psqt[k] = newEntryTilePsqt[k]; - for (int i = 0; i < newRemove.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.psqtWeights[newRemove[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]); - } - for (int i = 0; i < newAdd.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.psqtWeights[newAdd[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]); - } + apply_psqt<-1>(j, psqt, newRemove, featureTransformer.psqtWeights.data()); + apply_psqt<+1>(j, psqt, newAdd, featureTransformer.psqtWeights.data()); for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) { @@ -805,112 +698,17 @@ void update_accumulator_hybrid(Color perspective, psqt[k] = vec_sub_psqt_32(psqt[k], oldEntryTilePsqt[k]); } - for (int i = 0; i < oldRemove.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.psqtWeights[oldRemove[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]); - } - for (int i = 0; i < oldAdd.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.psqtWeights[oldAdd[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]); - } + apply_psqt<+1>(j, psqt, oldRemove, featureTransformer.psqtWeights.data()); + apply_psqt<-1>(j, psqt, oldAdd, featureTransformer.psqtWeights.data()); - for (int i = 0; i < thrRemoved.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer - .threatAndPpPsqtWeights[thrRemoved[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]); - } - - for (int i = 0; i < thrAdded.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.threatAndPpPsqtWeights[thrAdded[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]); - } + apply_psqt<-1>(j, psqt, thrRemoved, featureTransformer.threatAndPpPsqtWeights.data()); + apply_psqt<+1>(j, psqt, thrAdded, featureTransformer.threatAndPpPsqtWeights.data()); for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) vec_store_psqt(&toTilePsqt[k], psqt[k]); } - #else - for (const auto index : newRemove) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - newEntry.accumulation[j] -= featureTransformer.weights[offset + j]; - - for (usize k = 0; k < PSQTBuckets; ++k) - newEntry.psqtAccumulation[k] -= featureTransformer.psqtWeights[index * PSQTBuckets + k]; - } - for (const auto index : newAdd) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - newEntry.accumulation[j] += featureTransformer.weights[offset + j]; - - for (usize k = 0; k < PSQTBuckets; ++k) - newEntry.psqtAccumulation[k] += featureTransformer.psqtWeights[index * PSQTBuckets + k]; - } - - toAcc = newEntry.accumulation; - toPsqtAcc = newEntry.psqtAccumulation; - - for (IndexType j = 0; j < Dimensions; ++j) - { - toAcc[j] += fromAcc[j]; - toAcc[j] -= oldEntry.accumulation[j]; - } - for (usize k = 0; k < PSQTBuckets; ++k) - { - toPsqtAcc[k] += fromPsqtAcc[k]; - toPsqtAcc[k] -= oldEntry.psqtAccumulation[k]; - } - - for (const auto index : oldRemove) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - toAcc[j] += featureTransformer.weights[offset + j]; - - for (usize k = 0; k < PSQTBuckets; ++k) - toPsqtAcc[k] += featureTransformer.psqtWeights[index * PSQTBuckets + k]; - } - for (const auto index : oldAdd) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - toAcc[j] -= featureTransformer.weights[offset + j]; - - for (usize k = 0; k < PSQTBuckets; ++k) - toPsqtAcc[k] -= featureTransformer.psqtWeights[index * PSQTBuckets + k]; - } - - for (const auto index : thrRemoved) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - toAcc[j] -= featureTransformer.threatAndPpWeights[offset + j]; - for (usize k = 0; k < PSQTBuckets; ++k) - toPsqtAcc[k] -= featureTransformer.threatAndPpPsqtWeights[index * PSQTBuckets + k]; - } - for (const auto index : thrAdded) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - toAcc[j] += featureTransformer.threatAndPpWeights[offset + j]; - for (usize k = 0; k < PSQTBuckets; ++k) - toPsqtAcc[k] += featureTransformer.threatAndPpPsqtWeights[index * PSQTBuckets + k]; - } - +// TODO #endif newEntry.pieces = currentPieces; @@ -926,9 +724,6 @@ void update_accumulator_refresh_cache(Color perspective, const Position& pos, AccumulatorState& accumulator, AccumulatorCaches& cache) { - constexpr auto Dimensions = FeatureTransformer::OutputDimensions; - - using Tiling [[maybe_unused]] = SIMDTiling; const Square ksq = pos.square(perspective); auto& entry = cache[ksq][perspective]; @@ -963,13 +758,10 @@ void update_accumulator_refresh_cache(Color perspective, accumulator.computed[perspective] = true; -#ifdef VECTOR +#ifndef USE_RVV vec_t acc[Tiling::NumRegs]; psqt_vec_t psqt[Tiling::NumPsqtRegs]; - const auto* weights = &featureTransformer.weights[0]; - const auto* threatAndPpWeights = &featureTransformer.threatAndPpWeights[0]; - for (IndexType j = 0; j < Dimensions / Tiling::TileHeight; ++j) { const usize tileOff = j * Tiling::TileHeight; @@ -979,40 +771,13 @@ void update_accumulator_refresh_cache(Color perspective, for (IndexType k = 0; k < Tiling::NumRegs; ++k) acc[k] = entryTile[k]; - for (int i = 0; i < removed.ssize(); ++i) - { - auto* column = - reinterpret_cast(&weights[removed[i] * Dimensions + tileOff]); - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_sub_16(acc[k], column[k]); - } - for (int i = 0; i < added.ssize(); ++i) - { - auto* column = - reinterpret_cast(&weights[added[i] * Dimensions + tileOff]); - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_add_16(acc[k], column[k]); - } + apply_psq_features<-1>(j, acc, removed, featureTransformer); + apply_psq_features<+1>(j, acc, added, featureTransformer); for (IndexType k = 0; k < Tiling::NumRegs; k++) vec_store(&entryTile[k], acc[k]); - for (int i = 0; i < active.ssize(); ++i) - { - auto* column = reinterpret_cast( - &threatAndPpWeights[active[i] * Dimensions + tileOff]); - - #ifdef USE_NEON - for (IndexType k = 0; k < Tiling::NumRegs; k += 2) - { - acc[k] = vaddw_s8(acc[k], vget_low_s8(column[k / 2])); - acc[k + 1] = vaddw_high_s8(acc[k + 1], column[k / 2]); - } - #else - for (IndexType k = 0; k < Tiling::NumRegs; ++k) - acc[k] = vec_add_16(acc[k], vec_convert_8_16(column[k])); - #endif - } + apply_threat_features<+1>(j, acc, active, featureTransformer); for (IndexType k = 0; k < Tiling::NumRegs; k++) vec_store(&accTile[k], acc[k]); @@ -1028,37 +793,19 @@ void update_accumulator_refresh_cache(Color perspective, for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) psqt[k] = entryTilePsqt[k]; - for (int i = 0; i < removed.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.psqtWeights[removed[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]); - } - for (int i = 0; i < added.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.psqtWeights[added[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]); - } + apply_psqt<-1>(j, psqt, removed, featureTransformer.psqtWeights.data()); + apply_psqt<+1>(j, psqt, added, featureTransformer.psqtWeights.data()); for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) vec_store_psqt(&entryTilePsqt[k], psqt[k]); - for (int i = 0; i < active.ssize(); ++i) - { - auto* columnPsqt = reinterpret_cast( - &featureTransformer.threatAndPpPsqtWeights[active[i] * PSQTBuckets + psqtTileOff]); - for (usize k = 0; k < Tiling::NumPsqtRegs; ++k) - psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]); - } + apply_psqt<+1>(j, psqt, active, featureTransformer.threatAndPpPsqtWeights.data()); for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) vec_store_psqt(&accTilePsqt[k], psqt[k]); } -#elif defined(USE_RVV) +#else const auto* weights = &featureTransformer.weights[0]; const auto* threatWeights = &featureTransformer.threatAndPpWeights[0]; @@ -1115,46 +862,6 @@ void update_accumulator_refresh_cache(Color perspective, tileOffset += vl; } - -#else - - for (const auto index : removed) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - entry.accumulation[j] -= featureTransformer.weights[offset + j]; - - for (usize k = 0; k < PSQTBuckets; ++k) - entry.psqtAccumulation[k] -= featureTransformer.psqtWeights[index * PSQTBuckets + k]; - } - for (const auto index : added) - { - const IndexType offset = Dimensions * index; - for (IndexType j = 0; j < Dimensions; ++j) - entry.accumulation[j] += featureTransformer.weights[offset + j]; - - for (usize k = 0; k < PSQTBuckets; ++k) - entry.psqtAccumulation[k] += featureTransformer.psqtWeights[index * PSQTBuckets + k]; - } - - // The accumulator of the refresh entry has been updated. - // Now copy its content to the actual accumulator we were refreshing. - accumulator.accumulation[perspective] = entry.accumulation; - accumulator.psqtAccumulation[perspective] = entry.psqtAccumulation; - - for (const auto index : active) - { - const IndexType offset = Dimensions * index; - - for (IndexType j = 0; j < Dimensions; ++j) - accumulator.accumulation[perspective][j] += - featureTransformer.threatAndPpWeights[offset + j]; - - for (usize k = 0; k < PSQTBuckets; ++k) - accumulator.psqtAccumulation[perspective][k] += - featureTransformer.threatAndPpPsqtWeights[index * PSQTBuckets + k]; - } - #endif } diff --git a/src/nnue/simd.h b/src/nnue/simd.h index a869f351f83..60745d3690e 100644 --- a/src/nnue/simd.h +++ b/src/nnue/simd.h @@ -340,6 +340,17 @@ inline __m128i vec_convert_8_16(u64 x) { #define MaxChunkSize 16 #else +using vec_t = WeightType; +using vec_i8_t = ThreatWeightType; +using psqt_vec_t = PSQTWeightType; + #define vec_store(a, b) *(a) = (b) + #define vec_convert_8_16(a) static_cast(a) + #define vec_add_16(a, b) ((a) + (b)) + #define vec_sub_16(a, b) ((a) - (b)) + #define vec_store_psqt(a, b) *(a) = (b) + #define vec_add_psqt_32(a, b) ((a) + (b)) + #define vec_sub_psqt_32(a, b) ((a) - (b)) + #undef VECTOR #endif @@ -523,12 +534,16 @@ class SIMDTiling { static constexpr int NumPsqtRegs = BestRegisterCount(); - static constexpr IndexType TileHeight = NumRegs * sizeof(vec_t) / 2; - static constexpr IndexType PsqtTileHeight = NumPsqtRegs * sizeof(psqt_vec_t) / 4; +#else + public: + static constexpr int NumRegs = HalfDimensions; + static constexpr int NumPsqtRegs = PSQTBuckets; +#endif + static constexpr IndexType TileHeight = NumRegs * sizeof(vec_t) / sizeof(WeightType); + static constexpr IndexType PsqtTileHeight = NumPsqtRegs * sizeof(psqt_vec_t) / sizeof(PSQTWeightType); static_assert(HalfDimensions % TileHeight == 0, "TileHeight must divide HalfDimensions"); static_assert(PSQTBuckets % PsqtTileHeight == 0, "PsqtTileHeight must divide PSQTBuckets"); -#endif }; }