n/a

BUILD

@@ -4,6 +4,7 @@ load("@rules_cc//cc:cc_test.bzl", "cc_test")
 load("@bazel_skylib//lib:selects.bzl", "selects")
 load("//:hwy_tests.bzl", "HWY_TESTS")
 load("@rules_license//rules:license.bzl", "license")
+load("//tools/build_defs/testing:bzl_library.bzl", "bzl_library")
 
 package(
     default_applicable_licenses = [":license"],
@@ -118,6 +119,27 @@ COPTS = select({
     ],
 })
 
+HWY_TEST_COPTS = select({
+    ":compiler_msvc": [],
+    "//conditions:default": [
+        # gTest triggers this warning (which is enabled by the
+        # extra-semi in COPTS), so we need to disable it here,
+        # but it's still enabled for :hwy.
+        "-Wno-c++98-compat-extra-semi",
+    ],
+})
+
+# Common to all tests.
+HWY_TEST_DEPS = [
+    ":hwy_test_util",
+    ":hwy",
+    ":nanobenchmark",
+    ":timer",
+] + select({
+    ":compiler_msvc": [],
+    "//conditions:default": ["@com_google_googletest//:gtest_main"],
+})
+
 DEFINES = select({
     ":compiler_msvc": ["HWY_SHARED_DEFINE"],
     ":compiler_clangcl": ["HWY_SHARED_DEFINE"],
@@ -344,9 +366,16 @@ cc_library(
     copts = COPTS,
     textual_hdrs = [
         "hwy/contrib/dot/dot-inl.h",
+        # copybara:strip_begin(internal)
+        "hwy/contrib/dot/one_to_many-inl.h",
+        "hwy/contrib/dot/one_to_many_kernels-inl.h",
+        # copybara:strip_end
     ],
     deps = [
         ":hwy",
+        # copybara:strip_begin(internal)
+        ":prefetch_pipeline",
+        # copybara:strip_end
     ],
 )
 
@@ -607,6 +636,25 @@ cc_library(
     ],
 )
 
+# copybara:strip_begin(internal)
+cc_library(
+    name = "prefetch_pipeline",
+    hdrs = [
+        "hwy/contrib/pipeline/prefetch_pipeline.h",
+        "hwy/contrib/pipeline/prefetch_pipeline_2d.h",
+        "hwy/contrib/pipeline/prefetch_pipeline_types.h",
+        "hwy/contrib/pipeline/prefetch_tuner.h",
+        "hwy/contrib/pipeline/prefetch_tuner_registry.h",
+    ],
+    compatible_with = [],
+    copts = COPTS,
+    deps = [
+        ":hwy",
+        ":timer",
+    ],
+)
+# copybara:strip_end
+
 cc_test(
     name = "list_targets",
     size = "small",
@@ -628,27 +676,6 @@ cc_test(
     ],
 )
 
-HWY_TEST_COPTS = select({
-    ":compiler_msvc": [],
-    "//conditions:default": [
-        # gTest triggers this warning (which is enabled by the
-        # extra-semi in COPTS), so we need to disable it here,
-        # but it's still enabled for :hwy.
-        "-Wno-c++98-compat-extra-semi",
-    ],
-})
-
-# Common to all tests.
-HWY_TEST_DEPS = [
-    ":hwy_test_util",
-    ":hwy",
-    ":nanobenchmark",
-    ":timer",
-] + select({
-    ":compiler_msvc": [],
-    "//conditions:default": ["@com_google_googletest//:gtest_main"],
-})
-
 [
     [
         cc_test(
@@ -714,3 +741,10 @@ test_suite(
     name = "hwy_ops_tests",
     tags = ["hwy_ops_test"],
 )
+
+bzl_library(
+    name = "hwy_tests_bzl",
+    srcs = ["hwy_tests.bzl"],
+    parse_tests = False,
+    visibility = ["//visibility:private"],
+)

hwy/cache_control.h

@@ -92,8 +92,10 @@ HWY_INLINE HWY_ATTR_CACHE void FlushStream() {
 #endif
 }
 
-// Optionally begins loading the cache line containing "p" to reduce latency of
-// subsequent actual loads.
+// Optionally begins loading the cache line containing "p" into all cache
+// levels, including L1, to reduce latency of subsequent actual loads. This
+// corresponds to the T0 temporal locality hint on x86, which is ideal when data
+// is about to be directly consumed.
 template <typename T>
 HWY_INLINE HWY_ATTR_CACHE void Prefetch(const T* p) {
   (void)p;
@@ -109,6 +111,38 @@ HWY_INLINE HWY_ATTR_CACHE void Prefetch(const T* p) {
 #endif  //  HWY_DISABLE_CACHE_CONTROL
 }
 
+// Begins loading the cache line containing "p" into the L1 cache only, passing
+// a Non-Temporal Access (NTA) hint. This minimizes pollution of outer memory
+// caches (L2/L3) and is ideal for data accessed exactly once.
+template <typename T>
+HWY_INLINE HWY_ATTR_CACHE void PrefetchForImmediateuse(const T* p) {
+  (void)p;
+#ifndef HWY_DISABLE_CACHE_CONTROL
+#if HWY_ARCH_X86 && !(HWY_COMPILER_CLANGCL && !defined(__MMX__))
+  _mm_prefetch(reinterpret_cast<const char*>(p), _MM_HINT_NTA);
+#elif HWY_COMPILER_GCC || HWY_COMPILER_CLANGCL  // includes clang
+  // Hint=0 specifically sets Non-Temporal local locality
+  __builtin_prefetch(p, /*write=*/0, /*hint=*/0);
+#endif
+#endif  //  HWY_DISABLE_CACHE_CONTROL
+}
+
+// Attempts to stage the cache line containing "p" into the L3/L2 outer caches
+// without aggressively staging it immediately into the L1. This restricts L1
+// and LFB thrashing on architectures like Intel when hiding massive DRAM delay.
+template <typename T>
+HWY_INLINE HWY_ATTR_CACHE void PrefetchForFutureUse(const T* p) {
+  (void)p;
+#ifndef HWY_DISABLE_CACHE_CONTROL
+#if HWY_ARCH_X86 && !(HWY_COMPILER_CLANGCL && !defined(__MMX__))
+  _mm_prefetch(reinterpret_cast<const char*>(p), _MM_HINT_T2);
+#elif HWY_COMPILER_GCC || HWY_COMPILER_CLANGCL  // includes clang
+  // Hint=1 requests Moderate degrees of temporal locality (L2/L3 bounds)
+  __builtin_prefetch(p, /*write=*/0, /*hint=*/1);
+#endif
+#endif  //  HWY_DISABLE_CACHE_CONTROL
+}
+
 // Invalidates and flushes the cache line containing "p", if possible.
 HWY_INLINE HWY_ATTR_CACHE void FlushCacheline(const void* p) {
 #if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL)