Special implementation for string hash with sizes in range [33, 64].

AES instructions are used, when available. We load 33 - 64 bytes of the string into 4 128-bit overlapping vectors and use two rounds of AES encrypt and decrypt to mix the bits.

```
name                                    INSTRUCTIONS/op  INSTRUCTIONS/op  vs base
BM_HASHING_Combine_contiguous_Fleet_hot  2.047 ± 0%        1.927 ± 0%  -5.86% (p=0.000 n=30)
BM_HASHING_Combine_contiguous_Fleet_cold 2.229 ± 0%        2.109 ± 0%  -5.38% (p=0.000 n=30)

name                                     CYCLES/op    CYCLES/op   vs base
BM_HASHING_Combine_contiguous_Fleet_hot   520.0m ± 0%   501.0m ± 1%  -3.65% (p=0.000 n=30)
BM_HASHING_Combine_contiguous_Fleet_cold  1.754 ± 1%    1.696 ± 2%   -3.33% (p=0.003 n=30)
BM_HASHING_Combine_contiguous_0_hot       557.5m ± 0%   542.5m ± 0%  -2.69% (p=0.000 n=30)
BM_HASHING_Combine_contiguous_0_cold      1.769 ± 2%    1.742 ± 2%       ~ (p=0.117 n=30)
BM_HASHING_Combine_contiguous_1_hot       389.0m ± 0%   371.0m ± 1%  -4.63% (p=0.000 n=30)
BM_HASHING_Combine_contiguous_1_cold      1.450 ± 1%    1.389 ± 2%   -4.24% (p=0.000 n=30)
BM_HASHING_Combine_contiguous_2_hot       555.0m ± 0%   547.0m ± 0%  -1.44% (p=0.000 n=30)
BM_HASHING_Combine_contiguous_2_cold      1.526 ± 2%    1.504 ± 2%   -1.41% (p=0.024 n=30)
BM_HASHING_Combine_contiguous_3_hot       507.0m ± 0%   470.5m ± 1%  -7.20% (p=0.000 n=30)
BM_HASHING_Combine_contiguous_3_cold      1.187 ± 1%    1.139 ± 1%   -4.05% (p=0.000 n=30)
geomean                                   873.5m        843.5m       -3.43%
```

PiperOrigin-RevId: 836234239
Change-Id: I4a2344d77becefd3d5e788790f913fbe3611e8b0

Author: goldvitaly

absl/hash/internal/hash.cc

@@ -26,6 +26,20 @@
 #include "absl/base/prefetch.h"
 #include "absl/hash/internal/city.h"
 
+
+#ifdef ABSL_AES_INTERNAL_HAVE_X86_SIMD
+#error ABSL_AES_INTERNAL_HAVE_X86_SIMD cannot be directly set
+#elif defined(__SSE4_2__) && defined(__AES__)
+#define ABSL_AES_INTERNAL_HAVE_X86_SIMD
+#endif
+
+
+#ifdef ABSL_AES_INTERNAL_HAVE_X86_SIMD
+#include <smmintrin.h>
+#include <wmmintrin.h>
+#include <xmmintrin.h>
+#endif  // ABSL_AES_INTERNAL_HAVE_X86_SIMD
+
 namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace hash_internal {
@@ -43,47 +57,86 @@ uint64_t Mix32Bytes(const uint8_t* ptr, uint64_t current_state) {
   return cs0 ^ cs1;
 }
 
-[[maybe_unused]] uint64_t LowLevelHashLenGt32(const void* data, size_t len,
-                                              uint64_t seed) {
+#ifdef ABSL_AES_INTERNAL_HAVE_X86_SIMD
+uint64_t LowLevelHash33To64(const uint8_t* ptr, size_t len, uint64_t seed) {
   assert(len > 32);
-  const uint8_t* ptr = static_cast<const uint8_t*>(data);
+  assert(len <= 64);
+  __m128i state =
+      _mm_set_epi64x(static_cast<int64_t>(seed), static_cast<int64_t>(len));
+  auto a = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr));
+  auto b = _mm_loadu_si128(reinterpret_cast<const __m128i*>(ptr + 16));
+  auto* last32_ptr = ptr + len - 32;
+  auto c = _mm_loadu_si128(reinterpret_cast<const __m128i*>(last32_ptr));
+  auto d = _mm_loadu_si128(reinterpret_cast<const __m128i*>(last32_ptr + 16));
+
+  // Bits of the second argument to _mm_aesdec_si128/_mm_aesenc_si128 are
+  // XORed with the state argument after encryption.
+  // We use each value as the first argument to shuffle all the bits around.
+  // We do not add any salt to the state or loaded data, instead we vary
+  // instructions used to mix bits _mm_aesdec_si128/_mm_aesenc_si128 and
+  // _mm_add_epi64/_mm_sub_epi64.
+  // _mm_add_epi64/_mm_sub_epi64 are combined to one instruction with data
+  // loading like `vpaddq  xmm1, xmm0, xmmword ptr [rdi]`.
+  auto na = _mm_aesdec_si128(_mm_add_epi64(state, a), state);
+  auto nb = _mm_aesdec_si128(_mm_sub_epi64(state, b), state);
+  auto nc = _mm_aesenc_si128(_mm_add_epi64(state, c), state);
+  auto nd = _mm_aesenc_si128(_mm_sub_epi64(state, d), state);
+
+  // We perform another round of encryption to mix bits between two halves of
+  // the input.
+  auto res128 = _mm_add_epi64(_mm_aesenc_si128(_mm_add_epi64(na, nc), nd),
+                              _mm_aesdec_si128(_mm_sub_epi64(nb, nd), na));
+  auto x64 = static_cast<uint64_t>(_mm_cvtsi128_si64(res128));
+  auto y64 = static_cast<uint64_t>(_mm_extract_epi64(res128, 1));
+  return x64 ^ y64;
+}
+#else
+uint64_t LowLevelHash33To64(const uint8_t* ptr, size_t len, uint64_t seed) {
+  assert(len > 32);
+  assert(len <= 64);
   uint64_t current_state = seed ^ kStaticRandomData[0] ^ len;
   const uint8_t* last_32_ptr = ptr + len - 32;
+  return Mix32Bytes(last_32_ptr, Mix32Bytes(ptr, current_state));
+}
+#endif  // ABSL_AES_INTERNAL_HAVE_X86_SIMD
 
-  if (len > 64) {
-    // If we have more than 64 bytes, we're going to handle chunks of 64
-    // bytes at a time. We're going to build up four separate hash states
-    // which we will then hash together. This avoids short dependency chains.
-    uint64_t duplicated_state0 = current_state;
-    uint64_t duplicated_state1 = current_state;
-    uint64_t duplicated_state2 = current_state;
-
-    do {
-      PrefetchToLocalCache(ptr + 5 * ABSL_CACHELINE_SIZE);
-
-      uint64_t a = absl::base_internal::UnalignedLoad64(ptr);
-      uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8);
-      uint64_t c = absl::base_internal::UnalignedLoad64(ptr + 16);
-      uint64_t d = absl::base_internal::UnalignedLoad64(ptr + 24);
-      uint64_t e = absl::base_internal::UnalignedLoad64(ptr + 32);
-      uint64_t f = absl::base_internal::UnalignedLoad64(ptr + 40);
-      uint64_t g = absl::base_internal::UnalignedLoad64(ptr + 48);
-      uint64_t h = absl::base_internal::UnalignedLoad64(ptr + 56);
-
-      current_state = Mix(a ^ kStaticRandomData[1], b ^ current_state);
-      duplicated_state0 = Mix(c ^ kStaticRandomData[2], d ^ duplicated_state0);
-
-      duplicated_state1 = Mix(e ^ kStaticRandomData[3], f ^ duplicated_state1);
-      duplicated_state2 = Mix(g ^ kStaticRandomData[4], h ^ duplicated_state2);
-
-      ptr += 64;
-      len -= 64;
-    } while (len > 64);
-
-    current_state = (current_state ^ duplicated_state0) ^
-                    (duplicated_state1 + duplicated_state2);
-  }
-
+[[maybe_unused]] ABSL_ATTRIBUTE_NOINLINE uint64_t
+LowLevelHashLenGt64(const void* data, size_t len, uint64_t seed) {
+  assert(len > 64);
+  const uint8_t* ptr = static_cast<const uint8_t*>(data);
+  uint64_t current_state = seed ^ kStaticRandomData[0] ^ len;
+  const uint8_t* last_32_ptr = ptr + len - 32;
+  // If we have more than 64 bytes, we're going to handle chunks of 64
+  // bytes at a time. We're going to build up four separate hash states
+  // which we will then hash together. This avoids short dependency chains.
+  uint64_t duplicated_state0 = current_state;
+  uint64_t duplicated_state1 = current_state;
+  uint64_t duplicated_state2 = current_state;
+
+  do {
+    PrefetchToLocalCache(ptr + 5 * ABSL_CACHELINE_SIZE);
+
+    uint64_t a = absl::base_internal::UnalignedLoad64(ptr);
+    uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8);
+    uint64_t c = absl::base_internal::UnalignedLoad64(ptr + 16);
+    uint64_t d = absl::base_internal::UnalignedLoad64(ptr + 24);
+    uint64_t e = absl::base_internal::UnalignedLoad64(ptr + 32);
+    uint64_t f = absl::base_internal::UnalignedLoad64(ptr + 40);
+    uint64_t g = absl::base_internal::UnalignedLoad64(ptr + 48);
+    uint64_t h = absl::base_internal::UnalignedLoad64(ptr + 56);
+
+    current_state = Mix(a ^ kStaticRandomData[1], b ^ current_state);
+    duplicated_state0 = Mix(c ^ kStaticRandomData[2], d ^ duplicated_state0);
+
+    duplicated_state1 = Mix(e ^ kStaticRandomData[3], f ^ duplicated_state1);
+    duplicated_state2 = Mix(g ^ kStaticRandomData[4], h ^ duplicated_state2);
+
+    ptr += 64;
+    len -= 64;
+  } while (len > 64);
+
+  current_state = (current_state ^ duplicated_state0) ^
+                  (duplicated_state1 + duplicated_state2);
   // We now have a data `ptr` with at most 64 bytes and the current state
   // of the hashing state machine stored in current_state.
   if (len > 32) {
@@ -96,6 +149,15 @@ uint64_t Mix32Bytes(const uint8_t* ptr, uint64_t current_state) {
   return Mix32Bytes(last_32_ptr, current_state);
 }
 
+[[maybe_unused]] uint64_t LowLevelHashLenGt32(const void* data, size_t len,
+                                              uint64_t seed) {
+  assert(len > 32);
+  if (ABSL_PREDICT_FALSE(len > 64)) {
+    return LowLevelHashLenGt64(data, len, seed);
+  }
+  return LowLevelHash33To64(static_cast<const uint8_t*>(data), len, seed);
+}
+
 ABSL_ATTRIBUTE_ALWAYS_INLINE inline uint64_t HashBlockOn32Bit(
     const unsigned char* data, size_t len, uint64_t state) {
   // TODO(b/417141985): expose and use CityHash32WithSeed.

absl/hash/internal/low_level_hash_test.cc

@@ -364,6 +364,41 @@ TEST(LowLevelHashTest, VerifyGolden) {
   GTEST_SKIP()
       << "We only maintain golden data for little endian 64 bit systems with "
          "128 bit intristics.";
+#elif defined(__SSE4_2__) && defined(__AES__)
+  constexpr uint64_t kGolden[kNumGoldenOutputs] = {
+      0xd6bdb2c9ba5e55f2, 0xffd3e23d4115a8ae, 0x2c3218ef486127de,
+      0x554fa7f3a262b886, 0x06304cbf82e312d3, 0x490b3fb5af80622c,
+      0x7398a90b8cc59c5d, 0x65fb3168b98030ab, 0xd4564363c53617bb,
+      0x0545c26351925fe7, 0xc30700723b634bf4, 0xfb23a140a76dbe94,
+      0x2fa1467fe218a47c, 0x92e05ec3a7b966eb, 0x6112b56e5624dd50,
+      0x8760801365f9d722, 0x41f7187b61db0e5e, 0x7fe9188a1f5f50ad,
+      0x25800bd4c2002ef1, 0x91fecd33a78ef0aa, 0x93986ad71e983613,
+      0xe4c78173c7ea537b, 0x0bbdc2bcabdb50b1, 0xd9aa134df2d87623,
+      0x6c4907c9477a9409, 0xc3e418a5dbda52e5, 0x4d24f3e9d0dda93a,
+      0xcdb565a363dbe45f, 0xa95f228c8ee57478, 0x6b8f00bab5130227,
+      0x2d05a0f44818b67a, 0xa64b55b071afbbea, 0xa205bfe6c724ce4d,
+      0x69dd26ca8ac21744, 0xef80e2ff2f6a9bc0, 0xde266c0baa202c20,
+      0xfa3463080ac74c50, 0x379d968a40125c2b, 0x4cbbd0a7b3c7d648,
+      0xc92afd93f4c665d2, 0x6e28f5adb7ae38dc, 0x7c689c9c237be35e,
+      0xaea41b29bd9d0f73, 0x832cef631d77e59f, 0x70cac8e87bc37dd3,
+      0x8e8c98bbde68e764, 0xd6117aeb3ddedded, 0xd796ab808e766240,
+      0x8953d0ea1a7d9814, 0xa212eba4281b391c, 0x21a555a8939ce597,
+      0x809d31660f6d81a8, 0x2356524b20ab400f, 0x5bc611e1e49d0478,
+      0xba9c065e2f385ce2, 0xb0a0fd12f4e83899, 0x14d076a35b1ff2ca,
+      0x8acd0bb8cf9a93c0, 0xe62e8ec094039ee4, 0x38a536a7072bdc61,
+      0xca256297602524f8, 0xfc62ebfb3530caeb, 0x8d8b0c05520569f6,
+      0xbbaca65cf154c59d, 0x3739b5ada7e338d3, 0xdb9ea31f47365340,
+      0x410b5c9c1da56755, 0x7e0abc03dbd10283, 0x136f87be70ed442e,
+      0x6b727d4feddbe1e9, 0x074ebb21183b01df, 0x3fe92185b1985484,
+      0xc5d8efd3c68305ca, 0xd9bada21b17e272e, 0x64d73133e1360f83,
+      0xeb8563aa993e21f9, 0xe5e8da50cceab28f, 0x7a6f92eb3223d2f3,
+      0xbdaf98370ea9b31b, 0x1682a84457f077bc, 0x4abd2d33b6e3be37,
+      0xb35bc81a7c9d4c04, 0x3e5bde3fb7cfe63d, 0xff3abe6e2ffec974,
+      0xb8116dd26cf6feec, 0x7a77a6e4ed0cf081, 0xb71eec2d5a184316,
+      0x6fa932f77b4da817, 0x795f79b33909b2c4, 0x1b8755ef6b5eb34e,
+      0x2255b72d7d6b2d79, 0xf2bdafafa90bd50a, 0x442a578f02cb1fc8,
+      0xc25aefe55ecf83db, 0x3114c056f9c5a676,
+  };
 #else
   constexpr uint64_t kGolden[kNumGoldenOutputs] = {
       0x669da02f8d009e0f, 0xceb19bf2255445cd, 0x0e746992d6d43a7c,