From 2d08b9dcdd8cc1365a0a5f2a185329954302d389 Mon Sep 17 00:00:00 2001 From: Badrish C Date: Wed, 25 Mar 2026 17:33:56 -0700 Subject: [PATCH 01/19] plan --- website/docs/dev/fast-parsing-plan.md | 316 ++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 website/docs/dev/fast-parsing-plan.md diff --git a/website/docs/dev/fast-parsing-plan.md b/website/docs/dev/fast-parsing-plan.md new file mode 100644 index 00000000000..b6c7ef2d9e4 --- /dev/null +++ b/website/docs/dev/fast-parsing-plan.md @@ -0,0 +1,316 @@ +# Optimize RESP Command Parsing (`ParseCommand`) + +## Problem Statement + +`ParseCommand` in `libs/server/Resp/Parser/RespCommand.cs` is the entry point for parsing every RESP command from the network buffer. It uses a multi-tier approach: + +| Tier | Method | Commands Covered | Complexity | +|------|--------|-----------------|------------| +| 1 | `FastParseCommand` | ~40 most common (GET, SET, PING, DEL, INCR, TTL...) | O(1) — matches RESP header+command in one 8-byte read | +| 2 | `FastParseArrayCommand` | ~100 more (HSET, ZADD, LPUSH, SUBSCRIBE, GEOADD...) | O(depth) — 950 lines of nested switch/if-else by length→first-char→ulong compare | +| 3 | `SlowParseCommand` | ~80 remaining (CLUSTER *, CONFIG *, ACL *, admin cmds...) | O(n) — sequential `SequenceEqual` comparisons | + +**The hot path (Tier 1) works well** — it reads `*N\r\n$L\r\nCMD\r\n` as a single ulong and matches in one comparison. However: + +- **Tier 1** still does sequential ulong comparisons per candidate when multiple commands share the same `(count << 4 | length)` bucket — e.g., 3 commands at `(1<<4)|3`: GET, DEL, TTL +- **Tier 2** walks through deeply nested switch→switch→if-else chains — ~5-20 comparisons per command +- **Tier 3** does sequential `SequenceEqual` through ~80 entries — worst case ~80 comparisons for unknown commands +- **MakeUpperCase** is called on every non-fast-path command, using byte-by-byte scanning + +Estimated cost: Tier 1 ≈ 5-15 cycles, Tier 2 ≈ 30-60 cycles, Tier 3 ≈ 100-300 cycles. + +## Proposed Solution: SIMD-Accelerated Parsing + Cache-Friendly Hash Table + +Two complementary optimizations: +1. **SIMD-based FastParseCommand** for the top ~20 commands using Vector128 full-pattern matching +2. **Cache-line-optimized hash table** with CRC32 intrinsic hashing and SIMD validation for all remaining ~170 commands + +### Architecture + +``` +ParseCommand + ├─ SimdFastParseCommand (REWRITE — Vector128 full 16-byte pattern match for top ~20 commands) + │ ├─ Load first 16 bytes as Vector128 + │ ├─ Group candidates by total encoded length (13-byte, 14-byte, 15-byte groups) + │ ├─ One AND (mask) per group + one Vector128.EqualsAll per candidate + │ └─ ~3 ops per candidate vs. current ~8-10 ops + │ + └─ On miss → ArrayParseCommand + ├─ MakeUpperCase (REWRITE — SIMD Vector128/256 bulk conversion) + ├─ SimdFastParseCommand retry (catches lowercase clients) + ├─ Parse RESP array header + └─ HashLookupCommand (NEW — replaces FastParseArrayCommand + SlowParseCommand) + ├─ Extract command name (byte* + length) + ├─ CRC32 hardware hash (single instruction via Sse42.Crc32 / Crc32.Arm) + ├─ Index into cache-line-aligned table (L1-resident, 8-16KB) + ├─ Validate via Vector128.EqualsAll (up to 16 bytes in one op) + ├─ Linear probe within same cache line for collisions + └─ If parent has subcommands → SubcommandHashLookup (same design) +``` + +### SIMD Design: FastParseCommand (Tier 1) + +**Core idea**: Replace the two-step "load header ulong + load lastWord ulong" approach with a single Vector128 load that matches the FULL RESP-encoded pattern in one comparison. + +For the top commands, the complete RESP encoding is: +``` +GET: *2\r\n$3\r\nGET\r\n → 13 bytes total +SET: *3\r\n$3\r\nSET\r\n → 13 bytes total +DEL: *2\r\n$3\r\nDEL\r\n → 13 bytes total +TTL: *2\r\n$3\r\nTTL\r\n → 13 bytes total +PING: *1\r\n$4\r\nPING\r\n → 14 bytes total +INCR: *2\r\n$4\r\nINCR\r\n → 14 bytes total +HGET: *3\r\n$4\r\nHGET\r\n → 14 bytes total +HSET: *4\r\n$4\r\nHSET\r\n → 14 bytes total +MGET: *2\r\n$4\r\nMGET\r\n → 14 bytes total +ZADD: *4\r\n$4\r\nZADD\r\n → 14 bytes total +``` + +**Algorithm**: +```csharp +if (remainingBytes >= 16) +{ + var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); + + // Group 1: 13-byte patterns (3-char command names) + // Mask out bytes 13-15 (they contain the next argument, not part of command) + var masked13 = Vector128.BitwiseAnd(input, Mask13); + + // Each comparison checks FULL RESP header + command name in ONE op + // Pattern includes *N\r\n$3\r\nXXX\r\n — the N distinguishes arg count + if (masked13 == PatternGET) { readHead += 13; count = 1; return RespCommand.GET; } + if (masked13 == PatternSET) { readHead += 13; count = 2; return RespCommand.SET; } + if (masked13 == PatternDEL) { readHead += 13; count = 1; return RespCommand.DEL; } + if (masked13 == PatternTTL) { readHead += 13; count = 1; return RespCommand.TTL; } + + // Group 2: 14-byte patterns (4-char command names) + var masked14 = Vector128.BitwiseAnd(input, Mask14); + if (masked14 == PatternPING) { readHead += 14; count = 0; return RespCommand.PING; } + if (masked14 == PatternINCR) { readHead += 14; count = 1; return RespCommand.INCR; } + if (masked14 == PatternHGET) { readHead += 14; count = 2; return RespCommand.HGET; } + if (masked14 == PatternHSET) { readHead += 14; count = 3; return RespCommand.HSET; } + // ... more 14-byte patterns + + // Group 3: 15-byte patterns (5-char command names) + var masked15 = Vector128.BitwiseAnd(input, Mask15); + if (masked15 == PatternLPUSH) { readHead += 15; count = 1; return RespCommand.LPUSH; } + // ... +} +``` + +**Why this is faster**: Each comparison verifies the ENTIRE command encoding (header + name + terminators) in a single `Vector128.EqualsAll` (~1 cycle), vs. the current approach which needs 2 ulong loads + 2 comparisons + count/length extraction (~8-10 ops). The mask is computed once per length group. Pattern vectors are `static readonly` fields, resolved at JIT time. + +**Limitation**: Only works for single-digit array length and single-digit command name length (covers commands with 1-9 args and 1-9 char names). This covers the top ~60 commands. Longer/rarer commands fall through to the hash table. + +### Cache-Friendly Hash Table Design (Tier 2+3 Replacement) + +**Key constraint**: Single-threaded access, so no concurrency overhead. Must be as fast as possible. + +**Entry structure** — 32 bytes, exactly half a cache line: +```csharp +[StructLayout(LayoutKind.Explicit, Size = 32)] +struct CommandEntry +{ + [FieldOffset(0)] public RespCommand Command; // 2 bytes (ushort) + [FieldOffset(2)] public byte NameLength; // 1 byte + [FieldOffset(3)] public byte Flags; // 1 byte (HasSubcommands, etc.) + [FieldOffset(4)] public int Reserved; // 4 bytes (alignment padding) + [FieldOffset(8)] public ulong NameWord0; // First 8 bytes of uppercase name + [FieldOffset(16)] public ulong NameWord1; // Bytes 8-15 (zero-padded) + [FieldOffset(24)] public ulong NameWord2; // Bytes 16-23 (zero-padded) +} +``` + +**Table layout**: +- Size: 256 entries (power of 2) for ~180 primary commands → ~70% load factor +- Total memory: 256 × 32 bytes = **8 KB** — fits entirely in L1 cache (typically 32-64KB) +- Two entries per 64-byte cache line → linear probing hits same cache line for 1st probe +- `GC.AllocateArray(256, pinned: true)` for zero-GC, pointer-stable access + +**Hash function** — Hardware CRC32 (single instruction): +```csharp +[MethodImpl(MethodImplOptions.AggressiveInlining)] +static uint ComputeHash(byte* name, int length) +{ + // Read first 8 bytes (zero-extended for short names) + ulong word0 = length >= 8 + ? *(ulong*)name + : ReadPartialWord(name, length); + + // Hardware CRC32 — single-cycle instruction on x86 (SSE4.2) and ARM + if (Sse42.IsSupported) + return Sse42.Crc32(Sse42.Crc32(0u, word0), (uint)length); + + if (System.Runtime.Intrinsics.Arm.Crc32.IsSupported) + return System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C( + System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C(0u, word0), (uint)length); + + // Software fallback: Fibonacci multiply-shift + return (uint)((word0 * 0x9E3779B97F4A7C15UL) >> 32) ^ (uint)(length * 2654435761U); +} +``` + +**Probe with SIMD validation**: +```csharp +[MethodImpl(MethodImplOptions.AggressiveInlining)] +static RespCommand Lookup(byte* name, int length) +{ + uint hash = ComputeHash(name, length); + int idx = (int)(hash & (TableSize - 1)); // Power-of-2 mask, no modulo + + // Linear probe — typically 1-2 iterations, within same cache line + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref table[idx]; + + // Empty slot → command not found + if (entry.NameLength == 0) return RespCommand.NONE; + + // Length mismatch → skip (single byte compare, very fast rejection) + if (entry.NameLength != (byte)length) { idx = (idx + 1) & (TableSize - 1); continue; } + + // SIMD validation: Compare up to 16 bytes of command name in ONE operation + if (length <= 8) + { + // Single ulong compare for short names (most commands) + ulong word0 = ReadPartialWord(name, length); + if (entry.NameWord0 == word0) return entry.Command; + } + else + { + // Vector128 compare for names 9-16 bytes (SUBSCRIBE, ZRANGEBYSCORE, etc.) + var input = Vector128.Create(*(ulong*)name, *(ulong*)(name + length - 8)); + var expected = Vector128.Create(entry.NameWord0, entry.NameWord1); + if (Vector128.EqualsAll(input.AsByte(), expected.AsByte())) + return entry.Command; + } + + idx = (idx + 1) & (TableSize - 1); + } + return RespCommand.NONE; +} +``` + +**Why this is fast**: +- **Hash**: 1 CRC32 instruction (~3 cycles on x86/ARM) +- **Index**: 1 AND instruction (~1 cycle) +- **Load entry**: 1 memory load from L1 (~4 cycles, cache-warm) +- **Length check**: 1 byte compare (~1 cycle, fast rejection for 90%+ of misses) +- **Name validate**: 1 ulong compare for names ≤8 bytes (most commands), or 1 Vector128 compare for longer names +- **Total**: ~10-12 cycles for a hit, ~8 cycles for a miss +- **Linear probe**: 2nd probe hits same cache line (32-byte entries, 64-byte cache line) + +### SIMD Case Conversion (MakeUpperCase) + +Replace the current byte-by-byte loop with bulk SIMD conversion: +```csharp +// Detect lowercase bytes: compare against 'a' and 'z' ranges +var lower_a = Vector128.Create((byte)('a' - 1)); // 0x60 +var upper_z = Vector128.Create((byte)'z'); // 0x7A +var caseBit = Vector128.Create((byte)0x20); + +var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); + +// Find bytes in range 'a'-'z' using saturating subtract technique +var aboveA = Vector128.GreaterThan(input, lower_a); +var belowZ = Vector128.LessThanOrEqual(input, upper_z); +var isLower = Vector128.BitwiseAnd(aboveA, belowZ); + +if (isLower != Vector128.Zero) +{ + // Clear bit 5 (0x20) for lowercase bytes only + var toSubtract = Vector128.BitwiseAnd(isLower, caseBit); + var result = Vector128.Subtract(input, toSubtract); + result.StoreUnsafe(ref Unsafe.AsRef(ptr)); + return true; // Modified +} +return false; // Already uppercase +``` + +This converts 16 bytes at once (32 with Vector256). Most RESP command headers are 12-20 bytes, so one or two SIMD operations cover the entire command. + +### Subcommand Hash Lookup + +Commands with subcommands (CLUSTER ~40, CLIENT 8, ACL 10, CONFIG 3, COMMAND 5, SCRIPT 3, LATENCY 3, SLOWLOG 4, MODULE 1, PUBSUB 3, MEMORY 1 — total ~80 subcommands across 12 parents) use the same hash table design but smaller: +- CLUSTER: 64-entry table (~40 subcommands, 62% load) +- Others: 16 or 32-entry tables +- Same CRC32 hash + linear probe, same 32-byte entries +- Each parent's `Flags` field indicates `HasSubcommands` + +### Expected Performance Impact + +| Command Type | Current Cost | New Cost | Speedup | +|-------------|-------------|---------|---------| +| GET, SET (SIMD fast tier) | ~8-10 ops | ~3-4 ops (Vector128 match) | 2-3x | +| PING, INCR (SIMD fast tier) | ~10-12 ops | ~5-6 ops (2nd group match) | 2x | +| HSET, ZADD, LPUSH (Tier 2) | ~30-60 cycles | ~10-12 cycles (hash) | 3-5x | +| CLUSTER INFO, ACL LIST (Tier 3) | ~100-300 cycles | ~15-20 cycles (2 hash lookups) | 7-15x | +| Unknown commands | ~300+ cycles | ~8 cycles (hash miss) | 40x+ | + +### Risk Mitigation + +- All ~2000 existing tests serve as regression suite +- The hash table is built at static init time and is read-only — no concurrency issues +- The SIMD patterns are statically verified at build time +- Fallback paths exist for all SIMD operations (non-SIMD hardware) +- The hash table construction can assert zero collisions at startup + +## Implementation Todos + +### Phase 1: Hash Table Infrastructure +1. **build-hash-table** — Create `RespCommandHashLookup` static class in `libs/server/Resp/Parser/RespCommandHashLookup.cs`: + - 32-byte `CommandEntry` struct with `StructLayout(Explicit)` + - Pinned GC array for zero-GC pointer-stable access + - CRC32 intrinsic hash with Sse42/Arm.Crc32/software fallback + - `Lookup(byte* name, int length)` with linear probing + SIMD validation + - Static constructor populates all ~180 primary commands + - Assert zero unresolved collisions at init (fail-fast if hash function degrades) + +2. **build-subcommand-tables** — Per-parent subcommand hash tables (same design, smaller): + - CLUSTER (64 entries), CLIENT/ACL/COMMAND (16-32 entries), others (16 entries) + - `LookupSubcommand(RespCommand parent, byte* name, int length)` method + - Preserve specific error messages for unknown subcommands + +### Phase 2: SIMD Fast Tier +3. **simd-fast-parse** — Rewrite `FastParseCommand` to use Vector128 pattern matching: + - Pre-build Vector128 patterns for top ~20-30 commands (static readonly fields) + - Group by total encoded byte length (13, 14, 15, 16+ bytes) + - One mask per group, one Vector128.EqualsAll per candidate + - Extract count and readHead advance from the pattern metadata + - Preserve inline command handling (PING/QUIT without array framing) + +### Phase 3: Integration +4. **replace-array-parse** — Modify `ArrayParseCommand` to use hash lookup: + - After parsing RESP array header, extract command name (bytes + length) + - Call `RespCommandHashLookup.Lookup()` replacing FastParseArrayCommand + SlowParseCommand + - For `HasSubcommands` results, extract subcommand and do second lookup + - Delete or gut `FastParseArrayCommand` (~950 lines) and simplify `SlowParseCommand` (~800 lines) + +5. **handle-special-commands** — Preserve special-case behavior: + - BITOP + pseudo-subcommands (AND/OR/XOR/NOT/DIFF): after hash identifies BITOP, parse the operator subcommand inline or via subcommand hash + - Custom commands via TryParseCustomCommand: checked before hash lookup, as currently done + - SET variants (SETEXNX etc.): keep in SIMD fast tier via separate patterns for different arg counts + +### Phase 4: SIMD Case Conversion +6. **simd-uppercase** — Replace `MakeUpperCase` with SIMD version: + - Vector128 (SSE2/AdvSimd): 16 bytes at a time + - Vector256 (AVX2): 32 bytes at a time when available + - Keep existing two-ulong fast-check as the first gate (already uppercase → skip) + - Falls back to scalar loop for remaining bytes + +### Phase 5: Validation +7. **run-tests** — Full regression testing: + - `dotnet test test/Garnet.test -f net10.0 -c Debug` + - Focus on `RespTests`, `RespCommandTests`, ACL tests, Cluster tests + - Verify command parsing identity for all commands (every enum value must be reachable) + +8. **benchmark** — Performance validation: + - Microbenchmark each tier: top commands, moderate commands, rare commands, unknown commands + - Compare cycle counts before/after using BenchmarkDotNet + - Profile branch misprediction rates and cache miss rates + +## Files to Modify + +- `libs/server/Resp/Parser/RespCommand.cs` — Rewrite FastParseCommand (SIMD), replace FastParseArrayCommand + SlowParseCommand with hash lookup calls +- `libs/server/Resp/RespServerSession.cs` — SIMD MakeUpperCase +- **New file**: `libs/server/Resp/Parser/RespCommandHashLookup.cs` — Hash table + subcommand tables From 2e95bbf7f262e9a3b7c5b40c54d334ddfb5e5cc9 Mon Sep 17 00:00:00 2001 From: Badrish C Date: Wed, 25 Mar 2026 19:00:14 -0700 Subject: [PATCH 02/19] Optimize ParseCommand with hash-based command lookup Add RespCommandHashLookup: a cache-friendly O(1) hash table for RESP command name resolution. Uses hardware CRC32 (SSE4.2/ARM) for hashing, 32-byte cache-line-aligned entries, and linear probing within L1 cache. Key changes: - New RespCommandHashLookup.cs: static hash table (512 entries, 16KB) mapping uppercase command name bytes to RespCommand enum values - Per-parent subcommand hash tables for CLUSTER, CONFIG, CLIENT, ACL, COMMAND, SCRIPT, LATENCY, SLOWLOG, MODULE, PUBSUB, MEMORY, BITOP - ArrayParseCommand now uses hash lookup for primary commands instead of the ~950-line FastParseArrayCommand nested switch/if-else chains - BITOP pseudo-subcommands (AND/OR/XOR/NOT/DIFF) handled inline via dedicated ParseBitopSubcommand method with hash-based subcommand lookup - Subcommand dispatch (CLUSTER, CONFIG, etc.) falls through to existing SlowParseCommand for full backward compatibility - FastParseCommand hot path (GET, SET, PING, DEL) is completely untouched Performance: O(1) hash lookup (~10-12 cycles) replaces O(n) sequential comparisons (~30-300 cycles) for the long tail of ~170+ commands. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- libs/server/Resp/Parser/RespCommand.cs | 99 ++- .../Resp/Parser/RespCommandHashLookup.cs | 820 ++++++++++++++++++ 2 files changed, 912 insertions(+), 7 deletions(-) create mode 100644 libs/server/Resp/Parser/RespCommandHashLookup.cs diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index cc81121b1df..d7312815cae 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -2816,13 +2816,8 @@ private RespCommand ArrayParseCommand(bool writeErrorOnFailure, ref int count, r // Move readHead to start of command payload readHead = (int)(ptr - recvBufferPtr); - // Try parsing the most important variable-length commands - cmd = FastParseArrayCommand(ref count, ref specificErrorMessage); - - if (cmd == RespCommand.NONE) - { - cmd = SlowParseCommand(ref count, ref specificErrorMessage, out success); - } + // Extract command name via GetUpperCaseCommand (reads $len\r\n header, uppercases, advances readHead) + cmd = HashLookupCommand(ref count, ref specificErrorMessage, out success); // Parsing for command name was successful, but the command is unknown if (writeErrorOnFailure && success && cmd == RespCommand.INVALID) @@ -2841,5 +2836,95 @@ private RespCommand ArrayParseCommand(bool writeErrorOnFailure, ref int count, r } return cmd; } + + /// + /// Hash-based command lookup. Extracts the command name from the RESP buffer, + /// looks it up in the static hash table, and handles subcommand dispatch. + /// Replaces the former FastParseArrayCommand + SlowParseCommand chain. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private RespCommand HashLookupCommand(ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) + { + // Extract the command name (reads $len\r\n...\r\n, advances readHead, uppercases in-place) + var command = GetUpperCaseCommand(out success); + if (!success) + { + return RespCommand.INVALID; + } + + // Account for the command name being taken off the read head + count -= 1; + + // Check for custom commands first (same order as original code) + if (TryParseCustomCommand(command, out var customCmd)) + { + return customCmd; + } + + // Hash table lookup for the primary command + fixed (byte* namePtr = command) + { + var cmd = RespCommandHashLookup.Lookup(namePtr, command.Length); + + if (cmd == RespCommand.NONE) + { + // Fall back to the old slow parser for commands not in the hash table + return SlowParseCommand(command, ref count, ref specificErrorMessage, out success); + } + + // BITOP has pseudo-subcommands (AND/OR/XOR/NOT/DIFF) that must be parsed inline + if (cmd == RespCommand.BITOP) + { + return ParseBitopSubcommand(ref count, ref specificErrorMessage, out success); + } + + // Check if this command has subcommands that need a second lookup + // Subcommand dispatch uses the existing SlowParseCommand which handles + // all edge cases (case-insensitive matching, specific error messages, etc.) + // These are admin commands (CLUSTER, CONFIG, ACL) — not the hot path. + if (RespCommandHashLookup.HasSubcommands(namePtr, command.Length)) + { + return SlowParseCommand(command, ref count, ref specificErrorMessage, out success); + } + + return cmd; + } + } + + /// + /// Parse the BITOP pseudo-subcommand (AND/OR/XOR/NOT/DIFF). + /// Called after the primary hash lookup identifies BITOP. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private RespCommand ParseBitopSubcommand(ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) + { + if (count == 0) + { + specificErrorMessage = CmdStrings.RESP_SYNTAX_ERROR; + success = true; + return RespCommand.INVALID; + } + + var subCommand = GetUpperCaseCommand(out success); + if (!success) + { + return RespCommand.NONE; + } + + count--; + + fixed (byte* subPtr = subCommand) + { + var subCmd = RespCommandHashLookup.LookupSubcommand(RespCommand.BITOP, subPtr, subCommand.Length); + if (subCmd != RespCommand.NONE) + { + return subCmd; + } + } + + // Unrecognized BITOP operation + specificErrorMessage = CmdStrings.RESP_SYNTAX_ERROR; + return RespCommand.INVALID; + } } } \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommandHashLookup.cs b/libs/server/Resp/Parser/RespCommandHashLookup.cs new file mode 100644 index 00000000000..a0b6884cc3c --- /dev/null +++ b/libs/server/Resp/Parser/RespCommandHashLookup.cs @@ -0,0 +1,820 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace Garnet.server +{ + /// + /// Cache-friendly O(1) hash table for RESP command name lookup. + /// Replaces the nested switch/if-else chains in FastParseArrayCommand and SlowParseCommand. + /// + /// Design: + /// - 32-byte entries (half a cache line) with open addressing and linear probing + /// - CRC32 hardware hash (single instruction) with multiply-shift software fallback + /// - Vector128 validation for command names > 8 bytes + /// - Table fits in L1 cache (~8-16KB) + /// - Single-threaded read-only access after static init + /// + internal static unsafe class RespCommandHashLookup + { + /// + /// Entry in the command hash table. Exactly 32 bytes = half a cache line. + /// Two entries per cache line gives excellent spatial locality during linear probing. + /// + [StructLayout(LayoutKind.Explicit, Size = 32)] + private struct CommandEntry + { + /// The command enum value. + [FieldOffset(0)] + public RespCommand Command; + + /// Length of the command name in bytes. + [FieldOffset(2)] + public byte NameLength; + + /// Flags (e.g., HasSubcommands). + [FieldOffset(3)] + public byte Flags; + + /// First 8 bytes of the uppercase command name. + [FieldOffset(8)] + public ulong NameWord0; + + /// Bytes 8-15 of the uppercase command name (zero-padded). + [FieldOffset(16)] + public ulong NameWord1; + + /// Bytes 16-23 of the uppercase command name (zero-padded). + [FieldOffset(24)] + public ulong NameWord2; + } + + /// Flag indicating the command has subcommands that require a second lookup. + internal const byte FlagHasSubcommands = 1; + + // Primary command table: 512 entries = 16KB, fits in L1 cache + private const int PrimaryTableBits = 9; + private const int PrimaryTableSize = 1 << PrimaryTableBits; + private const int PrimaryTableMask = PrimaryTableSize - 1; + private const int MaxProbes = 16; + + private static readonly CommandEntry[] primaryTable; + + // Subcommand tables (per parent command) + private static readonly CommandEntry[] clusterSubTable; + private static readonly int clusterSubTableMask; + + private static readonly CommandEntry[] clientSubTable; + private static readonly int clientSubTableMask; + + private static readonly CommandEntry[] aclSubTable; + private static readonly int aclSubTableMask; + + private static readonly CommandEntry[] commandSubTable; + private static readonly int commandSubTableMask; + + private static readonly CommandEntry[] configSubTable; + private static readonly int configSubTableMask; + + private static readonly CommandEntry[] scriptSubTable; + private static readonly int scriptSubTableMask; + + private static readonly CommandEntry[] latencySubTable; + private static readonly int latencySubTableMask; + + private static readonly CommandEntry[] slowlogSubTable; + private static readonly int slowlogSubTableMask; + + private static readonly CommandEntry[] moduleSubTable; + private static readonly int moduleSubTableMask; + + private static readonly CommandEntry[] pubsubSubTable; + private static readonly int pubsubSubTableMask; + + private static readonly CommandEntry[] memorySubTable; + private static readonly int memorySubTableMask; + + private static readonly CommandEntry[] bitopSubTable; + private static readonly int bitopSubTableMask; + + static RespCommandHashLookup() + { + // Build primary command table + primaryTable = GC.AllocateArray(PrimaryTableSize, pinned: true); + PopulatePrimaryTable(); + + // Build subcommand tables + clusterSubTable = BuildSubTable(ClusterSubcommands, out clusterSubTableMask); + clientSubTable = BuildSubTable(ClientSubcommands, out clientSubTableMask); + aclSubTable = BuildSubTable(AclSubcommands, out aclSubTableMask); + commandSubTable = BuildSubTable(CommandSubcommands, out commandSubTableMask); + configSubTable = BuildSubTable(ConfigSubcommands, out configSubTableMask); + scriptSubTable = BuildSubTable(ScriptSubcommands, out scriptSubTableMask); + latencySubTable = BuildSubTable(LatencySubcommands, out latencySubTableMask); + slowlogSubTable = BuildSubTable(SlowlogSubcommands, out slowlogSubTableMask); + moduleSubTable = BuildSubTable(ModuleSubcommands, out moduleSubTableMask); + pubsubSubTable = BuildSubTable(PubsubSubcommands, out pubsubSubTableMask); + memorySubTable = BuildSubTable(MemorySubcommands, out memorySubTableMask); + bitopSubTable = BuildSubTable(BitopSubcommands, out bitopSubTableMask); + } + + #region Public API + + /// + /// Look up a primary command name in the hash table. + /// + /// Pointer to the uppercase command name bytes. + /// Length of the command name. + /// The matching RespCommand, or RespCommand.NONE if not found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RespCommand Lookup(byte* name, int length) + { + return LookupInTable(primaryTable, PrimaryTableMask, name, length); + } + + /// + /// Check if the given command has subcommands. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool HasSubcommands(byte* name, int length) + { + uint hash = ComputeHash(name, length); + int idx = (int)(hash & (uint)PrimaryTableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref primaryTable[idx]; + if (entry.NameLength == 0) return false; + if (entry.NameLength == (byte)length && MatchName(ref entry, name, length)) + return (entry.Flags & FlagHasSubcommands) != 0; + idx = (idx + 1) & PrimaryTableMask; + } + return false; + } + + /// + /// Look up a subcommand for a given parent command. + /// + /// The parent command. + /// Pointer to the uppercase subcommand name bytes. + /// Length of the subcommand name. + /// The matching RespCommand, or RespCommand.NONE if not found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RespCommand LookupSubcommand(RespCommand parent, byte* name, int length) + { + var (table, mask) = parent switch + { + RespCommand.CLUSTER => (clusterSubTable, clusterSubTableMask), + RespCommand.CLIENT => (clientSubTable, clientSubTableMask), + RespCommand.ACL => (aclSubTable, aclSubTableMask), + RespCommand.COMMAND => (commandSubTable, commandSubTableMask), + RespCommand.CONFIG => (configSubTable, configSubTableMask), + RespCommand.SCRIPT => (scriptSubTable, scriptSubTableMask), + RespCommand.LATENCY => (latencySubTable, latencySubTableMask), + RespCommand.SLOWLOG => (slowlogSubTable, slowlogSubTableMask), + RespCommand.MODULE => (moduleSubTable, moduleSubTableMask), + RespCommand.PUBSUB => (pubsubSubTable, pubsubSubTableMask), + RespCommand.MEMORY => (memorySubTable, memorySubTableMask), + RespCommand.BITOP => (bitopSubTable, bitopSubTableMask), + _ => (null, 0) + }; + + if (table == null) return RespCommand.NONE; + return LookupInTable(table, mask, name, length); + } + + #endregion + + #region Hash and Match + + /// + /// Compute hash from command name bytes using hardware CRC32 when available. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint ComputeHash(byte* name, int length) + { + ulong word0 = length >= 8 ? *(ulong*)name : ReadPartialWord(name, length); + + if (Sse42.X64.IsSupported) + { + uint crc = (uint)Sse42.X64.Crc32(0UL, word0); + return Sse42.Crc32(crc, (uint)length); + } + + if (System.Runtime.Intrinsics.Arm.Crc32.Arm64.IsSupported) + { + uint crc = (uint)System.Runtime.Intrinsics.Arm.Crc32.Arm64.ComputeCrc32C(0U, word0); + return System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C(crc, (uint)length); + } + + // Software fallback: Fibonacci multiply-shift + return (uint)((word0 * 0x9E3779B97F4A7C15UL) >> 32) ^ (uint)(length * 2654435761U); + } + + /// + /// Compute hash from a ReadOnlySpan (used during table construction). + /// + private static uint ComputeHash(ReadOnlySpan name) + { + ulong word0 = GetWordFromSpan(name, 0); + + if (Sse42.X64.IsSupported) + { + uint crc = (uint)Sse42.X64.Crc32(0UL, word0); + return Sse42.Crc32(crc, (uint)name.Length); + } + + if (System.Runtime.Intrinsics.Arm.Crc32.Arm64.IsSupported) + { + uint crc = (uint)System.Runtime.Intrinsics.Arm.Crc32.Arm64.ComputeCrc32C(0U, word0); + return System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C(crc, (uint)name.Length); + } + + return (uint)((word0 * 0x9E3779B97F4A7C15UL) >> 32) ^ (uint)(name.Length * 2654435761U); + } + + /// + /// Read up to 8 bytes from a pointer, zero-extending short reads. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong ReadPartialWord(byte* p, int len) + { + // For len >= 8, just read the full ulong + // For len < 8, read and mask + Debug.Assert(len > 0 && len < 8); + + return len switch + { + 1 => *p, + 2 => *(ushort*)p, + 3 => *(ushort*)p | ((ulong)p[2] << 16), + 4 => *(uint*)p, + 5 => *(uint*)p | ((ulong)p[4] << 32), + 6 => *(uint*)p | ((ulong)*(ushort*)(p + 4) << 32), + 7 => *(uint*)p | ((ulong)*(ushort*)(p + 4) << 32) | ((ulong)p[6] << 48), + _ => 0 + }; + } + + /// + /// Compare entry name against input name bytes. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool MatchName(ref CommandEntry entry, byte* name, int length) + { + if (length <= 8) + { + ulong inputWord = length == 8 ? *(ulong*)name : ReadPartialWord(name, length); + return entry.NameWord0 == inputWord; + } + else if (length <= 16) + { + // Compare first 8 bytes and last 8 bytes (may overlap for 9-15 byte names) + return entry.NameWord0 == *(ulong*)name && + entry.NameWord1 == *(ulong*)(name + length - 8); + } + else + { + // Compare first 8, middle 8, and last 8 bytes + return entry.NameWord0 == *(ulong*)name && + entry.NameWord1 == *(ulong*)(name + 8) && + entry.NameWord2 == *(ulong*)(name + length - 8); + } + } + + /// + /// Core lookup in any hash table (primary or subcommand). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static RespCommand LookupInTable(CommandEntry[] table, int tableMask, byte* name, int length) + { + uint hash = ComputeHash(name, length); + int idx = (int)(hash & (uint)tableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref table[idx]; + + // Empty slot — command not found + if (entry.NameLength == 0) return RespCommand.NONE; + + // Fast rejection: length mismatch (single byte compare) + if (entry.NameLength == (byte)length && MatchName(ref entry, name, length)) + return entry.Command; + + idx = (idx + 1) & tableMask; + } + + return RespCommand.NONE; + } + + #endregion + + #region Table Construction + + /// + /// Read a ulong from a span at the given offset, zero-padding short reads. + /// + private static ulong GetWordFromSpan(ReadOnlySpan span, int offset) + { + if (offset >= span.Length) return 0; + int remaining = span.Length - offset; + if (remaining >= 8) return MemoryMarshal.Read(span.Slice(offset)); + + ulong word = 0; + for (int i = 0; i < remaining; i++) + word |= (ulong)span[offset + i] << (i * 8); + return word; + } + + /// + /// Insert a command into a hash table. + /// + private static void InsertIntoTable(CommandEntry[] table, int tableMask, ReadOnlySpan name, RespCommand command, byte flags = 0) + { + uint hash = ComputeHash(name); + int idx = (int)(hash & (uint)tableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref table[idx]; + if (entry.NameLength == 0) + { + entry.Command = command; + entry.NameLength = (byte)name.Length; + entry.Flags = flags; + entry.NameWord0 = GetWordFromSpan(name, 0); + entry.NameWord1 = name.Length > 8 ? GetWordFromSpan(name, name.Length - 8) : 0; + entry.NameWord2 = name.Length > 16 ? GetWordFromSpan(name, name.Length - 8) : 0; + + // For names > 16 bytes, store bytes 8-15 exactly and last 8 bytes in word2 + if (name.Length > 16) + { + entry.NameWord1 = GetWordFromSpan(name, 8); + entry.NameWord2 = GetWordFromSpan(name, name.Length - 8); + } + else if (name.Length > 8) + { + entry.NameWord1 = GetWordFromSpan(name, name.Length - 8); + } + + return; + } + idx = (idx + 1) & tableMask; + } + + throw new InvalidOperationException( + $"Hash table overflow: could not insert command '{System.Text.Encoding.ASCII.GetString(name)}' after {MaxProbes} probes. Increase table size."); + } + + /// + /// Build a subcommand hash table from a list of (name, command) pairs. + /// + private static CommandEntry[] BuildSubTable(ReadOnlySpan<(string Name, RespCommand Command)> subcommands, out int mask) + { + // Find next power of 2 that gives at most ~70% load factor + int size = 16; + while (size * 7 / 10 < subcommands.Length) size <<= 1; + mask = size - 1; + + var table = GC.AllocateArray(size, pinned: true); + foreach (var (name, command) in subcommands) + { + InsertIntoTable(table, mask, System.Text.Encoding.ASCII.GetBytes(name), command); + } + return table; + } + + #endregion + + #region Command Definitions + + private static void PopulatePrimaryTable() + { + // Helper to insert with optional subcommand flag + void Add(string name, RespCommand cmd, bool hasSub = false) + { + InsertIntoTable(primaryTable, PrimaryTableMask, + System.Text.Encoding.ASCII.GetBytes(name), cmd, + hasSub ? FlagHasSubcommands : (byte)0); + } + + // ===== Data commands (read + write) ===== + + // String commands + Add("GET", RespCommand.GET); + Add("SET", RespCommand.SET); + Add("DEL", RespCommand.DEL); + Add("INCR", RespCommand.INCR); + Add("DECR", RespCommand.DECR); + Add("INCRBY", RespCommand.INCRBY); + Add("DECRBY", RespCommand.DECRBY); + Add("INCRBYFLOAT", RespCommand.INCRBYFLOAT); + Add("APPEND", RespCommand.APPEND); + Add("GETSET", RespCommand.GETSET); + Add("GETDEL", RespCommand.GETDEL); + Add("GETEX", RespCommand.GETEX); + Add("GETRANGE", RespCommand.GETRANGE); + Add("SETRANGE", RespCommand.SETRANGE); + Add("STRLEN", RespCommand.STRLEN); + Add("SUBSTR", RespCommand.SUBSTR); + Add("SETNX", RespCommand.SETNX); + Add("SETEX", RespCommand.SETEX); + Add("PSETEX", RespCommand.PSETEX); + Add("MGET", RespCommand.MGET); + Add("MSET", RespCommand.MSET); + Add("MSETNX", RespCommand.MSETNX); + Add("DUMP", RespCommand.DUMP); + Add("RESTORE", RespCommand.RESTORE); + Add("GETBIT", RespCommand.GETBIT); + Add("SETBIT", RespCommand.SETBIT); + Add("GETWITHETAG", RespCommand.GETWITHETAG); + Add("GETIFNOTMATCH", RespCommand.GETIFNOTMATCH); + Add("SETIFMATCH", RespCommand.SETIFMATCH); + Add("SETIFGREATER", RespCommand.SETIFGREATER); + Add("DELIFGREATER", RespCommand.DELIFGREATER); + Add("LCS", RespCommand.LCS); + + // Key commands + Add("EXISTS", RespCommand.EXISTS); + Add("TTL", RespCommand.TTL); + Add("PTTL", RespCommand.PTTL); + Add("EXPIRE", RespCommand.EXPIRE); + Add("PEXPIRE", RespCommand.PEXPIRE); + Add("EXPIREAT", RespCommand.EXPIREAT); + Add("PEXPIREAT", RespCommand.PEXPIREAT); + Add("EXPIRETIME", RespCommand.EXPIRETIME); + Add("PEXPIRETIME", RespCommand.PEXPIRETIME); + Add("PERSIST", RespCommand.PERSIST); + Add("TYPE", RespCommand.TYPE); + Add("RENAME", RespCommand.RENAME); + Add("RENAMENX", RespCommand.RENAMENX); + Add("UNLINK", RespCommand.UNLINK); + Add("KEYS", RespCommand.KEYS); + Add("SCAN", RespCommand.SCAN); + Add("DBSIZE", RespCommand.DBSIZE); + Add("SELECT", RespCommand.SELECT); + Add("SWAPDB", RespCommand.SWAPDB); + Add("MIGRATE", RespCommand.MIGRATE); + + // Bitmap commands + Add("BITCOUNT", RespCommand.BITCOUNT); + Add("BITPOS", RespCommand.BITPOS); + Add("BITFIELD", RespCommand.BITFIELD); + Add("BITFIELD_RO", RespCommand.BITFIELD_RO); + Add("BITOP", RespCommand.BITOP); + + // HyperLogLog commands + Add("PFADD", RespCommand.PFADD); + Add("PFCOUNT", RespCommand.PFCOUNT); + Add("PFMERGE", RespCommand.PFMERGE); + + // Hash commands + Add("HSET", RespCommand.HSET); + Add("HGET", RespCommand.HGET); + Add("HDEL", RespCommand.HDEL); + Add("HLEN", RespCommand.HLEN); + Add("HEXISTS", RespCommand.HEXISTS); + Add("HGETALL", RespCommand.HGETALL); + Add("HKEYS", RespCommand.HKEYS); + Add("HVALS", RespCommand.HVALS); + Add("HMSET", RespCommand.HMSET); + Add("HMGET", RespCommand.HMGET); + Add("HSETNX", RespCommand.HSETNX); + Add("HINCRBY", RespCommand.HINCRBY); + Add("HINCRBYFLOAT", RespCommand.HINCRBYFLOAT); + Add("HRANDFIELD", RespCommand.HRANDFIELD); + Add("HSCAN", RespCommand.HSCAN); + Add("HSTRLEN", RespCommand.HSTRLEN); + Add("HTTL", RespCommand.HTTL); + Add("HPTTL", RespCommand.HPTTL); + Add("HEXPIRE", RespCommand.HEXPIRE); + Add("HPEXPIRE", RespCommand.HPEXPIRE); + Add("HEXPIREAT", RespCommand.HEXPIREAT); + Add("HPEXPIREAT", RespCommand.HPEXPIREAT); + Add("HEXPIRETIME", RespCommand.HEXPIRETIME); + Add("HPEXPIRETIME", RespCommand.HPEXPIRETIME); + Add("HPERSIST", RespCommand.HPERSIST); + Add("HCOLLECT", RespCommand.HCOLLECT); + + // List commands + Add("LPUSH", RespCommand.LPUSH); + Add("RPUSH", RespCommand.RPUSH); + Add("LPUSHX", RespCommand.LPUSHX); + Add("RPUSHX", RespCommand.RPUSHX); + Add("LPOP", RespCommand.LPOP); + Add("RPOP", RespCommand.RPOP); + Add("LLEN", RespCommand.LLEN); + Add("LINDEX", RespCommand.LINDEX); + Add("LINSERT", RespCommand.LINSERT); + Add("LRANGE", RespCommand.LRANGE); + Add("LREM", RespCommand.LREM); + Add("LSET", RespCommand.LSET); + Add("LTRIM", RespCommand.LTRIM); + Add("LPOS", RespCommand.LPOS); + Add("LMOVE", RespCommand.LMOVE); + Add("LMPOP", RespCommand.LMPOP); + Add("RPOPLPUSH", RespCommand.RPOPLPUSH); + Add("BLPOP", RespCommand.BLPOP); + Add("BRPOP", RespCommand.BRPOP); + Add("BLMOVE", RespCommand.BLMOVE); + Add("BRPOPLPUSH", RespCommand.BRPOPLPUSH); + Add("BLMPOP", RespCommand.BLMPOP); + + // Set commands + Add("SADD", RespCommand.SADD); + Add("SREM", RespCommand.SREM); + Add("SPOP", RespCommand.SPOP); + Add("SCARD", RespCommand.SCARD); + Add("SMEMBERS", RespCommand.SMEMBERS); + Add("SISMEMBER", RespCommand.SISMEMBER); + Add("SMISMEMBER", RespCommand.SMISMEMBER); + Add("SRANDMEMBER", RespCommand.SRANDMEMBER); + Add("SMOVE", RespCommand.SMOVE); + Add("SSCAN", RespCommand.SSCAN); + Add("SDIFF", RespCommand.SDIFF); + Add("SDIFFSTORE", RespCommand.SDIFFSTORE); + Add("SINTER", RespCommand.SINTER); + Add("SINTERCARD", RespCommand.SINTERCARD); + Add("SINTERSTORE", RespCommand.SINTERSTORE); + Add("SUNION", RespCommand.SUNION); + Add("SUNIONSTORE", RespCommand.SUNIONSTORE); + + // Sorted set commands + Add("ZADD", RespCommand.ZADD); + Add("ZREM", RespCommand.ZREM); + Add("ZCARD", RespCommand.ZCARD); + Add("ZSCORE", RespCommand.ZSCORE); + Add("ZMSCORE", RespCommand.ZMSCORE); + Add("ZRANK", RespCommand.ZRANK); + Add("ZREVRANK", RespCommand.ZREVRANK); + Add("ZCOUNT", RespCommand.ZCOUNT); + Add("ZLEXCOUNT", RespCommand.ZLEXCOUNT); + Add("ZRANGE", RespCommand.ZRANGE); + Add("ZRANGEBYLEX", RespCommand.ZRANGEBYLEX); + Add("ZRANGEBYSCORE", RespCommand.ZRANGEBYSCORE); + Add("ZRANGESTORE", RespCommand.ZRANGESTORE); + Add("ZREVRANGE", RespCommand.ZREVRANGE); + Add("ZREVRANGEBYLEX", RespCommand.ZREVRANGEBYLEX); + Add("ZREVRANGEBYSCORE", RespCommand.ZREVRANGEBYSCORE); + Add("ZPOPMIN", RespCommand.ZPOPMIN); + Add("ZPOPMAX", RespCommand.ZPOPMAX); + Add("ZRANDMEMBER", RespCommand.ZRANDMEMBER); + Add("ZSCAN", RespCommand.ZSCAN); + Add("ZINCRBY", RespCommand.ZINCRBY); + Add("ZDIFF", RespCommand.ZDIFF); + Add("ZDIFFSTORE", RespCommand.ZDIFFSTORE); + Add("ZINTER", RespCommand.ZINTER); + Add("ZINTERCARD", RespCommand.ZINTERCARD); + Add("ZINTERSTORE", RespCommand.ZINTERSTORE); + Add("ZUNION", RespCommand.ZUNION); + Add("ZUNIONSTORE", RespCommand.ZUNIONSTORE); + Add("ZMPOP", RespCommand.ZMPOP); + Add("BZMPOP", RespCommand.BZMPOP); + Add("BZPOPMAX", RespCommand.BZPOPMAX); + Add("BZPOPMIN", RespCommand.BZPOPMIN); + Add("ZREMRANGEBYLEX", RespCommand.ZREMRANGEBYLEX); + Add("ZREMRANGEBYRANK", RespCommand.ZREMRANGEBYRANK); + Add("ZREMRANGEBYSCORE", RespCommand.ZREMRANGEBYSCORE); + Add("ZTTL", RespCommand.ZTTL); + Add("ZPTTL", RespCommand.ZPTTL); + Add("ZEXPIRE", RespCommand.ZEXPIRE); + Add("ZPEXPIRE", RespCommand.ZPEXPIRE); + Add("ZEXPIREAT", RespCommand.ZEXPIREAT); + Add("ZPEXPIREAT", RespCommand.ZPEXPIREAT); + Add("ZEXPIRETIME", RespCommand.ZEXPIRETIME); + Add("ZPEXPIRETIME", RespCommand.ZPEXPIRETIME); + Add("ZPERSIST", RespCommand.ZPERSIST); + Add("ZCOLLECT", RespCommand.ZCOLLECT); + + // Geo commands + Add("GEOADD", RespCommand.GEOADD); + Add("GEOPOS", RespCommand.GEOPOS); + Add("GEOHASH", RespCommand.GEOHASH); + Add("GEODIST", RespCommand.GEODIST); + Add("GEOSEARCH", RespCommand.GEOSEARCH); + Add("GEOSEARCHSTORE", RespCommand.GEOSEARCHSTORE); + Add("GEORADIUS", RespCommand.GEORADIUS); + Add("GEORADIUS_RO", RespCommand.GEORADIUS_RO); + Add("GEORADIUSBYMEMBER", RespCommand.GEORADIUSBYMEMBER); + Add("GEORADIUSBYMEMBER_RO", RespCommand.GEORADIUSBYMEMBER_RO); + + // Scripting + Add("EVAL", RespCommand.EVAL); + Add("EVALSHA", RespCommand.EVALSHA); + + // Pub/Sub + Add("PUBLISH", RespCommand.PUBLISH); + Add("SUBSCRIBE", RespCommand.SUBSCRIBE); + Add("PSUBSCRIBE", RespCommand.PSUBSCRIBE); + Add("UNSUBSCRIBE", RespCommand.UNSUBSCRIBE); + Add("PUNSUBSCRIBE", RespCommand.PUNSUBSCRIBE); + Add("SPUBLISH", RespCommand.SPUBLISH); + Add("SSUBSCRIBE", RespCommand.SSUBSCRIBE); + + // Custom object scan + Add("CUSTOMOBJECTSCAN", RespCommand.COSCAN); + + // ===== Control / admin commands ===== + Add("PING", RespCommand.PING); + Add("ECHO", RespCommand.ECHO); + Add("QUIT", RespCommand.QUIT); + Add("AUTH", RespCommand.AUTH); + Add("HELLO", RespCommand.HELLO); + Add("INFO", RespCommand.INFO); + Add("TIME", RespCommand.TIME); + Add("ROLE", RespCommand.ROLE); + Add("SAVE", RespCommand.SAVE); + Add("LASTSAVE", RespCommand.LASTSAVE); + Add("BGSAVE", RespCommand.BGSAVE); + Add("COMMITAOF", RespCommand.COMMITAOF); + Add("FLUSHALL", RespCommand.FLUSHALL); + Add("FLUSHDB", RespCommand.FLUSHDB); + Add("FORCEGC", RespCommand.FORCEGC); + Add("PURGEBP", RespCommand.PURGEBP); + Add("FAILOVER", RespCommand.FAILOVER); + Add("MONITOR", RespCommand.MONITOR); + Add("REGISTERCS", RespCommand.REGISTERCS); + Add("ASYNC", RespCommand.ASYNC); + Add("DEBUG", RespCommand.DEBUG); + Add("EXPDELSCAN", RespCommand.EXPDELSCAN); + Add("WATCH", RespCommand.WATCH); + Add("WATCHMS", RespCommand.WATCHMS); + Add("WATCHOS", RespCommand.WATCHOS); + Add("MULTI", RespCommand.MULTI); + Add("EXEC", RespCommand.EXEC); + Add("DISCARD", RespCommand.DISCARD); + Add("UNWATCH", RespCommand.UNWATCH); + Add("RUNTXP", RespCommand.RUNTXP); + Add("ASKING", RespCommand.ASKING); + Add("READONLY", RespCommand.READONLY); + Add("READWRITE", RespCommand.READWRITE); + Add("REPLICAOF", RespCommand.REPLICAOF); + Add("SECONDARYOF", RespCommand.SECONDARYOF); + Add("SLAVEOF", RespCommand.SECONDARYOF); + + // Parent commands with subcommands + Add("SCRIPT", RespCommand.SCRIPT, hasSub: true); + Add("CONFIG", RespCommand.CONFIG, hasSub: true); + Add("CLIENT", RespCommand.CLIENT, hasSub: true); + Add("CLUSTER", RespCommand.CLUSTER, hasSub: true); + Add("ACL", RespCommand.ACL, hasSub: true); + Add("COMMAND", RespCommand.COMMAND, hasSub: true); + Add("LATENCY", RespCommand.LATENCY, hasSub: true); + Add("SLOWLOG", RespCommand.SLOWLOG, hasSub: true); + Add("MODULE", RespCommand.MODULE, hasSub: true); + Add("PUBSUB", RespCommand.PUBSUB, hasSub: true); + Add("MEMORY", RespCommand.MEMORY, hasSub: true); + } + + #endregion + + #region Subcommand Definitions + + private static readonly (string Name, RespCommand Command)[] ClusterSubcommands = + [ + ("ADDSLOTS", RespCommand.CLUSTER_ADDSLOTS), + ("ADDSLOTSRANGE", RespCommand.CLUSTER_ADDSLOTSRANGE), + ("AOFSYNC", RespCommand.CLUSTER_AOFSYNC), + ("APPENDLOG", RespCommand.CLUSTER_APPENDLOG), + ("ATTACH_SYNC", RespCommand.CLUSTER_ATTACH_SYNC), + ("BANLIST", RespCommand.CLUSTER_BANLIST), + ("BEGIN_REPLICA_RECOVER", RespCommand.CLUSTER_BEGIN_REPLICA_RECOVER), + ("BUMPEPOCH", RespCommand.CLUSTER_BUMPEPOCH), + ("COUNTKEYSINSLOT", RespCommand.CLUSTER_COUNTKEYSINSLOT), + ("DELKEYSINSLOT", RespCommand.CLUSTER_DELKEYSINSLOT), + ("DELKEYSINSLOTRANGE", RespCommand.CLUSTER_DELKEYSINSLOTRANGE), + ("DELSLOTS", RespCommand.CLUSTER_DELSLOTS), + ("DELSLOTSRANGE", RespCommand.CLUSTER_DELSLOTSRANGE), + ("ENDPOINT", RespCommand.CLUSTER_ENDPOINT), + ("FAILOVER", RespCommand.CLUSTER_FAILOVER), + ("FAILREPLICATIONOFFSET", RespCommand.CLUSTER_FAILREPLICATIONOFFSET), + ("FAILSTOPWRITES", RespCommand.CLUSTER_FAILSTOPWRITES), + ("FLUSHALL", RespCommand.CLUSTER_FLUSHALL), + ("FORGET", RespCommand.CLUSTER_FORGET), + ("GETKEYSINSLOT", RespCommand.CLUSTER_GETKEYSINSLOT), + ("GOSSIP", RespCommand.CLUSTER_GOSSIP), + ("HELP", RespCommand.CLUSTER_HELP), + ("INFO", RespCommand.CLUSTER_INFO), + ("INITIATE_REPLICA_SYNC", RespCommand.CLUSTER_INITIATE_REPLICA_SYNC), + ("KEYSLOT", RespCommand.CLUSTER_KEYSLOT), + ("MEET", RespCommand.CLUSTER_MEET), + ("MIGRATE", RespCommand.CLUSTER_MIGRATE), + ("MTASKS", RespCommand.CLUSTER_MTASKS), + ("MYID", RespCommand.CLUSTER_MYID), + ("MYPARENTID", RespCommand.CLUSTER_MYPARENTID), + ("NODES", RespCommand.CLUSTER_NODES), + ("PUBLISH", RespCommand.CLUSTER_PUBLISH), + ("SPUBLISH", RespCommand.CLUSTER_SPUBLISH), + ("REPLICAS", RespCommand.CLUSTER_REPLICAS), + ("REPLICATE", RespCommand.CLUSTER_REPLICATE), + ("RESET", RespCommand.CLUSTER_RESET), + ("SEND_CKPT_FILE_SEGMENT", RespCommand.CLUSTER_SEND_CKPT_FILE_SEGMENT), + ("SEND_CKPT_METADATA", RespCommand.CLUSTER_SEND_CKPT_METADATA), + ("SETCONFIGEPOCH", RespCommand.CLUSTER_SETCONFIGEPOCH), + ("SETSLOT", RespCommand.CLUSTER_SETSLOT), + ("SETSLOTSRANGE", RespCommand.CLUSTER_SETSLOTSRANGE), + ("SHARDS", RespCommand.CLUSTER_SHARDS), + ("SLOTS", RespCommand.CLUSTER_SLOTS), + ("SLOTSTATE", RespCommand.CLUSTER_SLOTSTATE), + ("SYNC", RespCommand.CLUSTER_SYNC), + ]; + + private static readonly (string Name, RespCommand Command)[] ClientSubcommands = + [ + ("ID", RespCommand.CLIENT_ID), + ("INFO", RespCommand.CLIENT_INFO), + ("LIST", RespCommand.CLIENT_LIST), + ("KILL", RespCommand.CLIENT_KILL), + ("GETNAME", RespCommand.CLIENT_GETNAME), + ("SETNAME", RespCommand.CLIENT_SETNAME), + ("SETINFO", RespCommand.CLIENT_SETINFO), + ("UNBLOCK", RespCommand.CLIENT_UNBLOCK), + ]; + + private static readonly (string Name, RespCommand Command)[] AclSubcommands = + [ + ("CAT", RespCommand.ACL_CAT), + ("DELUSER", RespCommand.ACL_DELUSER), + ("GENPASS", RespCommand.ACL_GENPASS), + ("GETUSER", RespCommand.ACL_GETUSER), + ("LIST", RespCommand.ACL_LIST), + ("LOAD", RespCommand.ACL_LOAD), + ("SAVE", RespCommand.ACL_SAVE), + ("SETUSER", RespCommand.ACL_SETUSER), + ("USERS", RespCommand.ACL_USERS), + ("WHOAMI", RespCommand.ACL_WHOAMI), + ]; + + private static readonly (string Name, RespCommand Command)[] CommandSubcommands = + [ + ("COUNT", RespCommand.COMMAND_COUNT), + ("DOCS", RespCommand.COMMAND_DOCS), + ("INFO", RespCommand.COMMAND_INFO), + ("GETKEYS", RespCommand.COMMAND_GETKEYS), + ("GETKEYSANDFLAGS", RespCommand.COMMAND_GETKEYSANDFLAGS), + ]; + + private static readonly (string Name, RespCommand Command)[] ConfigSubcommands = + [ + ("GET", RespCommand.CONFIG_GET), + ("REWRITE", RespCommand.CONFIG_REWRITE), + ("SET", RespCommand.CONFIG_SET), + ]; + + private static readonly (string Name, RespCommand Command)[] ScriptSubcommands = + [ + ("LOAD", RespCommand.SCRIPT_LOAD), + ("FLUSH", RespCommand.SCRIPT_FLUSH), + ("EXISTS", RespCommand.SCRIPT_EXISTS), + ]; + + private static readonly (string Name, RespCommand Command)[] LatencySubcommands = + [ + ("HELP", RespCommand.LATENCY_HELP), + ("HISTOGRAM", RespCommand.LATENCY_HISTOGRAM), + ("RESET", RespCommand.LATENCY_RESET), + ]; + + private static readonly (string Name, RespCommand Command)[] SlowlogSubcommands = + [ + ("HELP", RespCommand.SLOWLOG_HELP), + ("GET", RespCommand.SLOWLOG_GET), + ("LEN", RespCommand.SLOWLOG_LEN), + ("RESET", RespCommand.SLOWLOG_RESET), + ]; + + private static readonly (string Name, RespCommand Command)[] ModuleSubcommands = + [ + ("LOADCS", RespCommand.MODULE_LOADCS), + ]; + + private static readonly (string Name, RespCommand Command)[] PubsubSubcommands = + [ + ("CHANNELS", RespCommand.PUBSUB_CHANNELS), + ("NUMSUB", RespCommand.PUBSUB_NUMSUB), + ("NUMPAT", RespCommand.PUBSUB_NUMPAT), + ]; + + private static readonly (string Name, RespCommand Command)[] MemorySubcommands = + [ + ("USAGE", RespCommand.MEMORY_USAGE), + ]; + + private static readonly (string Name, RespCommand Command)[] BitopSubcommands = + [ + ("AND", RespCommand.BITOP_AND), + ("OR", RespCommand.BITOP_OR), + ("XOR", RespCommand.BITOP_XOR), + ("NOT", RespCommand.BITOP_NOT), + ("DIFF", RespCommand.BITOP_DIFF), + ]; + + #endregion + } +} \ No newline at end of file From 7ce5f19fb6f17ffe1a18905a52af228728512b20 Mon Sep 17 00:00:00 2001 From: Badrish C Date: Wed, 25 Mar 2026 19:00:14 -0700 Subject: [PATCH 03/19] Optimize ParseCommand with SIMD fast path and hash-based command lookup Add three optimization tiers to RESP command parsing: Tier 1 - SIMD Vector128 FastParseCommand: - 30 static Vector128 patterns matching full RESP encoding (*N\r\n$L\r\nCMD\r\n) - Single 16-byte load + masked comparison validates header + command in one op - Covers top commands: GET, SET, DEL, TTL, PING, INCR, DECR, EXISTS, etc. - Falls through to existing scalar ulong switch for variable-arg commands Tier 2 - CRC32 hash table (RespCommandHashLookup): - 512-entry cache-line-aligned table (16KB, L1-resident) with hardware CRC32 hash - O(1) lookup for ~200 primary commands + 12 subcommand tables - Replaces ~950-line FastParseArrayCommand nested switch/if-else chains - BITOP pseudo-subcommands handled via dedicated ParseBitopSubcommand Tier 3 - SlowParseCommand (existing): - Subcommand dispatch for admin commands (CLUSTER, CONFIG, ACL, etc.) Additional optimizations: - HashLookupCommand uses GetCommand instead of GetUpperCaseCommand (MakeUpperCase already uppercased the buffer, avoiding redundant work) - TryParseCustomCommand moved after hash lookup (built-in commands are far more common than custom extensions) - FastParseCommand hot path preserved as scalar fallback for edge cases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- libs/server/Resp/Parser/RespCommand.cs | 199 ++++- .../Resp/Parser/RespCommandHashLookup.cs | 820 ++++++++++++++++++ 2 files changed, 1010 insertions(+), 9 deletions(-) create mode 100644 libs/server/Resp/Parser/RespCommandHashLookup.cs diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index cc81121b1df..e934390340c 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Text; using Garnet.common; using Microsoft.Extensions.Logging; @@ -642,6 +643,51 @@ enum RespCommandOption : byte /// internal sealed unsafe partial class RespServerSession : ServerSessionBase { + // SIMD Vector128 patterns for FastParseCommand. + // Each encodes the full RESP header + command: *N\r\n$L\r\nCMD\r\n + // Masks zero out trailing bytes for patterns shorter than 16 bytes. + private static readonly Vector128 s_mask13 = Vector128.Create( + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00).AsByte(); + private static readonly Vector128 s_mask14 = Vector128.Create( + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00).AsByte(); + private static readonly Vector128 s_mask15 = Vector128.Create( + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00).AsByte(); + + // 13-byte: *N\r\n$3\r\nXXX\r\n + private static readonly Vector128 s_GET = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'3', (byte)'\r', (byte)'\n', (byte)'G', (byte)'E', (byte)'T', (byte)'\r', (byte)'\n', 0, 0, 0); + private static readonly Vector128 s_SET = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'3', (byte)'\r', (byte)'\n', (byte)'S', (byte)'E', (byte)'T', (byte)'\r', (byte)'\n', 0, 0, 0); + private static readonly Vector128 s_DEL = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'3', (byte)'\r', (byte)'\n', (byte)'D', (byte)'E', (byte)'L', (byte)'\r', (byte)'\n', 0, 0, 0); + private static readonly Vector128 s_TTL = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'3', (byte)'\r', (byte)'\n', (byte)'T', (byte)'T', (byte)'L', (byte)'\r', (byte)'\n', 0, 0, 0); + + // 14-byte: *N\r\n$4\r\nXXXX\r\n + private static readonly Vector128 s_PING = Vector128.Create((byte)'*', (byte)'1', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'P', (byte)'I', (byte)'N', (byte)'G', (byte)'\r', (byte)'\n', 0, 0); + private static readonly Vector128 s_INCR = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'I', (byte)'N', (byte)'C', (byte)'R', (byte)'\r', (byte)'\n', 0, 0); + private static readonly Vector128 s_DECR = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'D', (byte)'E', (byte)'C', (byte)'R', (byte)'\r', (byte)'\n', 0, 0); + private static readonly Vector128 s_EXEC = Vector128.Create((byte)'*', (byte)'1', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'E', (byte)'X', (byte)'E', (byte)'C', (byte)'\r', (byte)'\n', 0, 0); + private static readonly Vector128 s_PTTL = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'P', (byte)'T', (byte)'T', (byte)'L', (byte)'\r', (byte)'\n', 0, 0); + private static readonly Vector128 s_DUMP = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'D', (byte)'U', (byte)'M', (byte)'P', (byte)'\r', (byte)'\n', 0, 0); + + // 15-byte: *N\r\n$5\r\nXXXXX\r\n + private static readonly Vector128 s_MULTI = Vector128.Create((byte)'*', (byte)'1', (byte)'\r', (byte)'\n', (byte)'$', (byte)'5', (byte)'\r', (byte)'\n', (byte)'M', (byte)'U', (byte)'L', (byte)'T', (byte)'I', (byte)'\r', (byte)'\n', 0); + private static readonly Vector128 s_PFADD = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'5', (byte)'\r', (byte)'\n', (byte)'P', (byte)'F', (byte)'A', (byte)'D', (byte)'D', (byte)'\r', (byte)'\n', 0); + private static readonly Vector128 s_SETNX = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'5', (byte)'\r', (byte)'\n', (byte)'S', (byte)'E', (byte)'T', (byte)'N', (byte)'X', (byte)'\r', (byte)'\n', 0); + private static readonly Vector128 s_SETEX = Vector128.Create((byte)'*', (byte)'4', (byte)'\r', (byte)'\n', (byte)'$', (byte)'5', (byte)'\r', (byte)'\n', (byte)'S', (byte)'E', (byte)'T', (byte)'E', (byte)'X', (byte)'\r', (byte)'\n', 0); + + // 16-byte: *N\r\n$6\r\nXXXXXX\r\n (no mask needed) + private static readonly Vector128 s_EXISTS = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'E', (byte)'X', (byte)'I', (byte)'S', (byte)'T', (byte)'S', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_GETDEL = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'G', (byte)'E', (byte)'T', (byte)'D', (byte)'E', (byte)'L', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_APPEND = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'A', (byte)'P', (byte)'P', (byte)'E', (byte)'N', (byte)'D', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_INCRBY = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'I', (byte)'N', (byte)'C', (byte)'R', (byte)'B', (byte)'Y', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_DECRBY = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'D', (byte)'E', (byte)'C', (byte)'R', (byte)'B', (byte)'Y', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_GETBIT = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'G', (byte)'E', (byte)'T', (byte)'B', (byte)'I', (byte)'T', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_SETBIT = Vector128.Create((byte)'*', (byte)'4', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'S', (byte)'E', (byte)'T', (byte)'B', (byte)'I', (byte)'T', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_GETSET = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'G', (byte)'E', (byte)'T', (byte)'S', (byte)'E', (byte)'T', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_ASKING = Vector128.Create((byte)'*', (byte)'1', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'A', (byte)'S', (byte)'K', (byte)'I', (byte)'N', (byte)'G', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_PSETEX = Vector128.Create((byte)'*', (byte)'4', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'P', (byte)'S', (byte)'E', (byte)'T', (byte)'E', (byte)'X', (byte)'\r', (byte)'\n'); + private static readonly Vector128 s_SUBSTR = Vector128.Create((byte)'*', (byte)'4', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'S', (byte)'U', (byte)'B', (byte)'S', (byte)'T', (byte)'R', (byte)'\r', (byte)'\n'); /// /// Fast-parses command type for inline RESP commands, starting at the current read head in the receive buffer /// and advances read head. @@ -680,8 +726,8 @@ private RespCommand FastParseInlineCommand(out int count) } /// - /// Fast-parses for command type, starting at the current read head in the receive buffer - /// and advances the read head to the position after the parsed command. + /// Fast-parses for command type using SIMD Vector128 matching for the most common commands, + /// falling back to scalar ulong matching for the rest. /// /// Outputs the number of arguments stored with the command /// RespCommand that was parsed or RespCommand.NONE, if no command was matched in this pass. @@ -691,6 +737,53 @@ private RespCommand FastParseCommand(out int count) var ptr = recvBufferPtr + readHead; var remainingBytes = bytesRead - readHead; + // SIMD fast path: match the full RESP-encoded pattern (*N\r\n$L\r\nCMD\r\n) in a + // single Vector128 comparison. Each check validates the header AND command name + // simultaneously: 1 load + 1 AND (mask) + 1 EqualsAll = 3 ops per candidate. + if (Vector128.IsHardwareAccelerated && remainingBytes >= 16) + { + var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); + + // 13-byte patterns: 3-char commands (GET, SET, DEL, TTL) + var m13 = Vector128.BitwiseAnd(input, s_mask13); + if (Vector128.EqualsAll(m13, s_GET)) { readHead += 13; count = 1; return RespCommand.GET; } + if (Vector128.EqualsAll(m13, s_SET)) { readHead += 13; count = 2; return RespCommand.SET; } + if (Vector128.EqualsAll(m13, s_DEL)) { readHead += 13; count = 1; return RespCommand.DEL; } + if (Vector128.EqualsAll(m13, s_TTL)) { readHead += 13; count = 1; return RespCommand.TTL; } + + // 14-byte patterns: 4-char commands (PING, INCR, DECR, EXEC, PTTL, DUMP) + var m14 = Vector128.BitwiseAnd(input, s_mask14); + if (Vector128.EqualsAll(m14, s_PING)) { readHead += 14; count = 0; return RespCommand.PING; } + if (Vector128.EqualsAll(m14, s_INCR)) { readHead += 14; count = 1; return RespCommand.INCR; } + if (Vector128.EqualsAll(m14, s_DECR)) { readHead += 14; count = 1; return RespCommand.DECR; } + if (Vector128.EqualsAll(m14, s_EXEC)) { readHead += 14; count = 0; return RespCommand.EXEC; } + if (Vector128.EqualsAll(m14, s_PTTL)) { readHead += 14; count = 1; return RespCommand.PTTL; } + if (Vector128.EqualsAll(m14, s_DUMP)) { readHead += 14; count = 1; return RespCommand.DUMP; } + + // 15-byte patterns: 5-char commands (MULTI, PFADD, SETNX, SETEX) + var m15 = Vector128.BitwiseAnd(input, s_mask15); + if (Vector128.EqualsAll(m15, s_MULTI)) { readHead += 15; count = 0; return RespCommand.MULTI; } + if (Vector128.EqualsAll(m15, s_PFADD)) { readHead += 15; count = 2; return RespCommand.PFADD; } + if (Vector128.EqualsAll(m15, s_SETNX)) { readHead += 15; count = 2; return RespCommand.SETNX; } + if (Vector128.EqualsAll(m15, s_SETEX)) { readHead += 15; count = 3; return RespCommand.SETEX; } + + // 16-byte patterns: 6-char commands (no mask — exact 16-byte match) + if (Vector128.EqualsAll(input, s_EXISTS)) { readHead += 16; count = 1; return RespCommand.EXISTS; } + if (Vector128.EqualsAll(input, s_GETDEL)) { readHead += 16; count = 1; return RespCommand.GETDEL; } + if (Vector128.EqualsAll(input, s_APPEND)) { readHead += 16; count = 2; return RespCommand.APPEND; } + if (Vector128.EqualsAll(input, s_INCRBY)) { readHead += 16; count = 2; return RespCommand.INCRBY; } + if (Vector128.EqualsAll(input, s_DECRBY)) { readHead += 16; count = 2; return RespCommand.DECRBY; } + if (Vector128.EqualsAll(input, s_GETBIT)) { readHead += 16; count = 2; return RespCommand.GETBIT; } + if (Vector128.EqualsAll(input, s_SETBIT)) { readHead += 16; count = 3; return RespCommand.SETBIT; } + if (Vector128.EqualsAll(input, s_GETSET)) { readHead += 16; count = 2; return RespCommand.GETSET; } + if (Vector128.EqualsAll(input, s_ASKING)) { readHead += 16; count = 0; return RespCommand.ASKING; } + if (Vector128.EqualsAll(input, s_PSETEX)) { readHead += 16; count = 3; return RespCommand.PSETEX; } + if (Vector128.EqualsAll(input, s_SUBSTR)) { readHead += 16; count = 3; return RespCommand.SUBSTR; } + } + + // Scalar fallback: handles variable-arg commands, inline commands, + // commands with names > 6 chars, and non-SIMD hardware. + // Check if the package starts with "*_\r\n$_\r\n" (_ = masked out), // i.e. an array with a single-digit length and single-digit first string length. if ((remainingBytes >= 8) && (*(ulong*)ptr & 0xFFFF00FFFFFF00FF) == MemoryMarshal.Read("*\0\r\n$\0\r\n"u8)) @@ -2816,13 +2909,8 @@ private RespCommand ArrayParseCommand(bool writeErrorOnFailure, ref int count, r // Move readHead to start of command payload readHead = (int)(ptr - recvBufferPtr); - // Try parsing the most important variable-length commands - cmd = FastParseArrayCommand(ref count, ref specificErrorMessage); - - if (cmd == RespCommand.NONE) - { - cmd = SlowParseCommand(ref count, ref specificErrorMessage, out success); - } + // Extract command name via GetUpperCaseCommand (reads $len\r\n header, uppercases, advances readHead) + cmd = HashLookupCommand(ref count, ref specificErrorMessage, out success); // Parsing for command name was successful, but the command is unknown if (writeErrorOnFailure && success && cmd == RespCommand.INVALID) @@ -2841,5 +2929,98 @@ private RespCommand ArrayParseCommand(bool writeErrorOnFailure, ref int count, r } return cmd; } + + /// + /// Hash-based command lookup. Extracts the command name from the RESP buffer, + /// looks it up in the static hash table, and handles subcommand dispatch. + /// Replaces the former FastParseArrayCommand + SlowParseCommand chain. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private RespCommand HashLookupCommand(ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) + { + // Extract the command name (reads $len\r\n...\r\n, advances readHead). + // MakeUpperCase has already uppercased the command name in the buffer, + // so we use GetCommand (no redundant ToUpperInPlace call). + var command = GetCommand(out success); + if (!success) + { + return RespCommand.INVALID; + } + + // Account for the command name being taken off the read head + count -= 1; + + // Hash table lookup for the primary command (checked before custom commands + // since built-in commands are far more common) + fixed (byte* namePtr = command) + { + var cmd = RespCommandHashLookup.Lookup(namePtr, command.Length); + + if (cmd == RespCommand.NONE) + { + // Not a built-in command — check custom commands before falling to slow path + if (TryParseCustomCommand(command, out var customCmd)) + { + return customCmd; + } + + // Fall back to the old slow parser for commands not in the hash table + return SlowParseCommand(command, ref count, ref specificErrorMessage, out success); + } + + // BITOP has pseudo-subcommands (AND/OR/XOR/NOT/DIFF) that must be parsed inline + if (cmd == RespCommand.BITOP) + { + return ParseBitopSubcommand(ref count, ref specificErrorMessage, out success); + } + + // Check if this command has subcommands that need a second lookup + // Subcommand dispatch uses the existing SlowParseCommand which handles + // all edge cases (case-insensitive matching, specific error messages, etc.) + // These are admin commands (CLUSTER, CONFIG, ACL) — not the hot path. + if (RespCommandHashLookup.HasSubcommands(namePtr, command.Length)) + { + return SlowParseCommand(command, ref count, ref specificErrorMessage, out success); + } + + return cmd; + } + } + + /// + /// Parse the BITOP pseudo-subcommand (AND/OR/XOR/NOT/DIFF). + /// Called after the primary hash lookup identifies BITOP. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private RespCommand ParseBitopSubcommand(ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) + { + if (count == 0) + { + specificErrorMessage = CmdStrings.RESP_SYNTAX_ERROR; + success = true; + return RespCommand.INVALID; + } + + var subCommand = GetUpperCaseCommand(out success); + if (!success) + { + return RespCommand.NONE; + } + + count--; + + fixed (byte* subPtr = subCommand) + { + var subCmd = RespCommandHashLookup.LookupSubcommand(RespCommand.BITOP, subPtr, subCommand.Length); + if (subCmd != RespCommand.NONE) + { + return subCmd; + } + } + + // Unrecognized BITOP operation + specificErrorMessage = CmdStrings.RESP_SYNTAX_ERROR; + return RespCommand.INVALID; + } } } \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommandHashLookup.cs b/libs/server/Resp/Parser/RespCommandHashLookup.cs new file mode 100644 index 00000000000..a0b6884cc3c --- /dev/null +++ b/libs/server/Resp/Parser/RespCommandHashLookup.cs @@ -0,0 +1,820 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace Garnet.server +{ + /// + /// Cache-friendly O(1) hash table for RESP command name lookup. + /// Replaces the nested switch/if-else chains in FastParseArrayCommand and SlowParseCommand. + /// + /// Design: + /// - 32-byte entries (half a cache line) with open addressing and linear probing + /// - CRC32 hardware hash (single instruction) with multiply-shift software fallback + /// - Vector128 validation for command names > 8 bytes + /// - Table fits in L1 cache (~8-16KB) + /// - Single-threaded read-only access after static init + /// + internal static unsafe class RespCommandHashLookup + { + /// + /// Entry in the command hash table. Exactly 32 bytes = half a cache line. + /// Two entries per cache line gives excellent spatial locality during linear probing. + /// + [StructLayout(LayoutKind.Explicit, Size = 32)] + private struct CommandEntry + { + /// The command enum value. + [FieldOffset(0)] + public RespCommand Command; + + /// Length of the command name in bytes. + [FieldOffset(2)] + public byte NameLength; + + /// Flags (e.g., HasSubcommands). + [FieldOffset(3)] + public byte Flags; + + /// First 8 bytes of the uppercase command name. + [FieldOffset(8)] + public ulong NameWord0; + + /// Bytes 8-15 of the uppercase command name (zero-padded). + [FieldOffset(16)] + public ulong NameWord1; + + /// Bytes 16-23 of the uppercase command name (zero-padded). + [FieldOffset(24)] + public ulong NameWord2; + } + + /// Flag indicating the command has subcommands that require a second lookup. + internal const byte FlagHasSubcommands = 1; + + // Primary command table: 512 entries = 16KB, fits in L1 cache + private const int PrimaryTableBits = 9; + private const int PrimaryTableSize = 1 << PrimaryTableBits; + private const int PrimaryTableMask = PrimaryTableSize - 1; + private const int MaxProbes = 16; + + private static readonly CommandEntry[] primaryTable; + + // Subcommand tables (per parent command) + private static readonly CommandEntry[] clusterSubTable; + private static readonly int clusterSubTableMask; + + private static readonly CommandEntry[] clientSubTable; + private static readonly int clientSubTableMask; + + private static readonly CommandEntry[] aclSubTable; + private static readonly int aclSubTableMask; + + private static readonly CommandEntry[] commandSubTable; + private static readonly int commandSubTableMask; + + private static readonly CommandEntry[] configSubTable; + private static readonly int configSubTableMask; + + private static readonly CommandEntry[] scriptSubTable; + private static readonly int scriptSubTableMask; + + private static readonly CommandEntry[] latencySubTable; + private static readonly int latencySubTableMask; + + private static readonly CommandEntry[] slowlogSubTable; + private static readonly int slowlogSubTableMask; + + private static readonly CommandEntry[] moduleSubTable; + private static readonly int moduleSubTableMask; + + private static readonly CommandEntry[] pubsubSubTable; + private static readonly int pubsubSubTableMask; + + private static readonly CommandEntry[] memorySubTable; + private static readonly int memorySubTableMask; + + private static readonly CommandEntry[] bitopSubTable; + private static readonly int bitopSubTableMask; + + static RespCommandHashLookup() + { + // Build primary command table + primaryTable = GC.AllocateArray(PrimaryTableSize, pinned: true); + PopulatePrimaryTable(); + + // Build subcommand tables + clusterSubTable = BuildSubTable(ClusterSubcommands, out clusterSubTableMask); + clientSubTable = BuildSubTable(ClientSubcommands, out clientSubTableMask); + aclSubTable = BuildSubTable(AclSubcommands, out aclSubTableMask); + commandSubTable = BuildSubTable(CommandSubcommands, out commandSubTableMask); + configSubTable = BuildSubTable(ConfigSubcommands, out configSubTableMask); + scriptSubTable = BuildSubTable(ScriptSubcommands, out scriptSubTableMask); + latencySubTable = BuildSubTable(LatencySubcommands, out latencySubTableMask); + slowlogSubTable = BuildSubTable(SlowlogSubcommands, out slowlogSubTableMask); + moduleSubTable = BuildSubTable(ModuleSubcommands, out moduleSubTableMask); + pubsubSubTable = BuildSubTable(PubsubSubcommands, out pubsubSubTableMask); + memorySubTable = BuildSubTable(MemorySubcommands, out memorySubTableMask); + bitopSubTable = BuildSubTable(BitopSubcommands, out bitopSubTableMask); + } + + #region Public API + + /// + /// Look up a primary command name in the hash table. + /// + /// Pointer to the uppercase command name bytes. + /// Length of the command name. + /// The matching RespCommand, or RespCommand.NONE if not found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RespCommand Lookup(byte* name, int length) + { + return LookupInTable(primaryTable, PrimaryTableMask, name, length); + } + + /// + /// Check if the given command has subcommands. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool HasSubcommands(byte* name, int length) + { + uint hash = ComputeHash(name, length); + int idx = (int)(hash & (uint)PrimaryTableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref primaryTable[idx]; + if (entry.NameLength == 0) return false; + if (entry.NameLength == (byte)length && MatchName(ref entry, name, length)) + return (entry.Flags & FlagHasSubcommands) != 0; + idx = (idx + 1) & PrimaryTableMask; + } + return false; + } + + /// + /// Look up a subcommand for a given parent command. + /// + /// The parent command. + /// Pointer to the uppercase subcommand name bytes. + /// Length of the subcommand name. + /// The matching RespCommand, or RespCommand.NONE if not found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RespCommand LookupSubcommand(RespCommand parent, byte* name, int length) + { + var (table, mask) = parent switch + { + RespCommand.CLUSTER => (clusterSubTable, clusterSubTableMask), + RespCommand.CLIENT => (clientSubTable, clientSubTableMask), + RespCommand.ACL => (aclSubTable, aclSubTableMask), + RespCommand.COMMAND => (commandSubTable, commandSubTableMask), + RespCommand.CONFIG => (configSubTable, configSubTableMask), + RespCommand.SCRIPT => (scriptSubTable, scriptSubTableMask), + RespCommand.LATENCY => (latencySubTable, latencySubTableMask), + RespCommand.SLOWLOG => (slowlogSubTable, slowlogSubTableMask), + RespCommand.MODULE => (moduleSubTable, moduleSubTableMask), + RespCommand.PUBSUB => (pubsubSubTable, pubsubSubTableMask), + RespCommand.MEMORY => (memorySubTable, memorySubTableMask), + RespCommand.BITOP => (bitopSubTable, bitopSubTableMask), + _ => (null, 0) + }; + + if (table == null) return RespCommand.NONE; + return LookupInTable(table, mask, name, length); + } + + #endregion + + #region Hash and Match + + /// + /// Compute hash from command name bytes using hardware CRC32 when available. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint ComputeHash(byte* name, int length) + { + ulong word0 = length >= 8 ? *(ulong*)name : ReadPartialWord(name, length); + + if (Sse42.X64.IsSupported) + { + uint crc = (uint)Sse42.X64.Crc32(0UL, word0); + return Sse42.Crc32(crc, (uint)length); + } + + if (System.Runtime.Intrinsics.Arm.Crc32.Arm64.IsSupported) + { + uint crc = (uint)System.Runtime.Intrinsics.Arm.Crc32.Arm64.ComputeCrc32C(0U, word0); + return System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C(crc, (uint)length); + } + + // Software fallback: Fibonacci multiply-shift + return (uint)((word0 * 0x9E3779B97F4A7C15UL) >> 32) ^ (uint)(length * 2654435761U); + } + + /// + /// Compute hash from a ReadOnlySpan (used during table construction). + /// + private static uint ComputeHash(ReadOnlySpan name) + { + ulong word0 = GetWordFromSpan(name, 0); + + if (Sse42.X64.IsSupported) + { + uint crc = (uint)Sse42.X64.Crc32(0UL, word0); + return Sse42.Crc32(crc, (uint)name.Length); + } + + if (System.Runtime.Intrinsics.Arm.Crc32.Arm64.IsSupported) + { + uint crc = (uint)System.Runtime.Intrinsics.Arm.Crc32.Arm64.ComputeCrc32C(0U, word0); + return System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C(crc, (uint)name.Length); + } + + return (uint)((word0 * 0x9E3779B97F4A7C15UL) >> 32) ^ (uint)(name.Length * 2654435761U); + } + + /// + /// Read up to 8 bytes from a pointer, zero-extending short reads. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong ReadPartialWord(byte* p, int len) + { + // For len >= 8, just read the full ulong + // For len < 8, read and mask + Debug.Assert(len > 0 && len < 8); + + return len switch + { + 1 => *p, + 2 => *(ushort*)p, + 3 => *(ushort*)p | ((ulong)p[2] << 16), + 4 => *(uint*)p, + 5 => *(uint*)p | ((ulong)p[4] << 32), + 6 => *(uint*)p | ((ulong)*(ushort*)(p + 4) << 32), + 7 => *(uint*)p | ((ulong)*(ushort*)(p + 4) << 32) | ((ulong)p[6] << 48), + _ => 0 + }; + } + + /// + /// Compare entry name against input name bytes. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool MatchName(ref CommandEntry entry, byte* name, int length) + { + if (length <= 8) + { + ulong inputWord = length == 8 ? *(ulong*)name : ReadPartialWord(name, length); + return entry.NameWord0 == inputWord; + } + else if (length <= 16) + { + // Compare first 8 bytes and last 8 bytes (may overlap for 9-15 byte names) + return entry.NameWord0 == *(ulong*)name && + entry.NameWord1 == *(ulong*)(name + length - 8); + } + else + { + // Compare first 8, middle 8, and last 8 bytes + return entry.NameWord0 == *(ulong*)name && + entry.NameWord1 == *(ulong*)(name + 8) && + entry.NameWord2 == *(ulong*)(name + length - 8); + } + } + + /// + /// Core lookup in any hash table (primary or subcommand). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static RespCommand LookupInTable(CommandEntry[] table, int tableMask, byte* name, int length) + { + uint hash = ComputeHash(name, length); + int idx = (int)(hash & (uint)tableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref table[idx]; + + // Empty slot — command not found + if (entry.NameLength == 0) return RespCommand.NONE; + + // Fast rejection: length mismatch (single byte compare) + if (entry.NameLength == (byte)length && MatchName(ref entry, name, length)) + return entry.Command; + + idx = (idx + 1) & tableMask; + } + + return RespCommand.NONE; + } + + #endregion + + #region Table Construction + + /// + /// Read a ulong from a span at the given offset, zero-padding short reads. + /// + private static ulong GetWordFromSpan(ReadOnlySpan span, int offset) + { + if (offset >= span.Length) return 0; + int remaining = span.Length - offset; + if (remaining >= 8) return MemoryMarshal.Read(span.Slice(offset)); + + ulong word = 0; + for (int i = 0; i < remaining; i++) + word |= (ulong)span[offset + i] << (i * 8); + return word; + } + + /// + /// Insert a command into a hash table. + /// + private static void InsertIntoTable(CommandEntry[] table, int tableMask, ReadOnlySpan name, RespCommand command, byte flags = 0) + { + uint hash = ComputeHash(name); + int idx = (int)(hash & (uint)tableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref table[idx]; + if (entry.NameLength == 0) + { + entry.Command = command; + entry.NameLength = (byte)name.Length; + entry.Flags = flags; + entry.NameWord0 = GetWordFromSpan(name, 0); + entry.NameWord1 = name.Length > 8 ? GetWordFromSpan(name, name.Length - 8) : 0; + entry.NameWord2 = name.Length > 16 ? GetWordFromSpan(name, name.Length - 8) : 0; + + // For names > 16 bytes, store bytes 8-15 exactly and last 8 bytes in word2 + if (name.Length > 16) + { + entry.NameWord1 = GetWordFromSpan(name, 8); + entry.NameWord2 = GetWordFromSpan(name, name.Length - 8); + } + else if (name.Length > 8) + { + entry.NameWord1 = GetWordFromSpan(name, name.Length - 8); + } + + return; + } + idx = (idx + 1) & tableMask; + } + + throw new InvalidOperationException( + $"Hash table overflow: could not insert command '{System.Text.Encoding.ASCII.GetString(name)}' after {MaxProbes} probes. Increase table size."); + } + + /// + /// Build a subcommand hash table from a list of (name, command) pairs. + /// + private static CommandEntry[] BuildSubTable(ReadOnlySpan<(string Name, RespCommand Command)> subcommands, out int mask) + { + // Find next power of 2 that gives at most ~70% load factor + int size = 16; + while (size * 7 / 10 < subcommands.Length) size <<= 1; + mask = size - 1; + + var table = GC.AllocateArray(size, pinned: true); + foreach (var (name, command) in subcommands) + { + InsertIntoTable(table, mask, System.Text.Encoding.ASCII.GetBytes(name), command); + } + return table; + } + + #endregion + + #region Command Definitions + + private static void PopulatePrimaryTable() + { + // Helper to insert with optional subcommand flag + void Add(string name, RespCommand cmd, bool hasSub = false) + { + InsertIntoTable(primaryTable, PrimaryTableMask, + System.Text.Encoding.ASCII.GetBytes(name), cmd, + hasSub ? FlagHasSubcommands : (byte)0); + } + + // ===== Data commands (read + write) ===== + + // String commands + Add("GET", RespCommand.GET); + Add("SET", RespCommand.SET); + Add("DEL", RespCommand.DEL); + Add("INCR", RespCommand.INCR); + Add("DECR", RespCommand.DECR); + Add("INCRBY", RespCommand.INCRBY); + Add("DECRBY", RespCommand.DECRBY); + Add("INCRBYFLOAT", RespCommand.INCRBYFLOAT); + Add("APPEND", RespCommand.APPEND); + Add("GETSET", RespCommand.GETSET); + Add("GETDEL", RespCommand.GETDEL); + Add("GETEX", RespCommand.GETEX); + Add("GETRANGE", RespCommand.GETRANGE); + Add("SETRANGE", RespCommand.SETRANGE); + Add("STRLEN", RespCommand.STRLEN); + Add("SUBSTR", RespCommand.SUBSTR); + Add("SETNX", RespCommand.SETNX); + Add("SETEX", RespCommand.SETEX); + Add("PSETEX", RespCommand.PSETEX); + Add("MGET", RespCommand.MGET); + Add("MSET", RespCommand.MSET); + Add("MSETNX", RespCommand.MSETNX); + Add("DUMP", RespCommand.DUMP); + Add("RESTORE", RespCommand.RESTORE); + Add("GETBIT", RespCommand.GETBIT); + Add("SETBIT", RespCommand.SETBIT); + Add("GETWITHETAG", RespCommand.GETWITHETAG); + Add("GETIFNOTMATCH", RespCommand.GETIFNOTMATCH); + Add("SETIFMATCH", RespCommand.SETIFMATCH); + Add("SETIFGREATER", RespCommand.SETIFGREATER); + Add("DELIFGREATER", RespCommand.DELIFGREATER); + Add("LCS", RespCommand.LCS); + + // Key commands + Add("EXISTS", RespCommand.EXISTS); + Add("TTL", RespCommand.TTL); + Add("PTTL", RespCommand.PTTL); + Add("EXPIRE", RespCommand.EXPIRE); + Add("PEXPIRE", RespCommand.PEXPIRE); + Add("EXPIREAT", RespCommand.EXPIREAT); + Add("PEXPIREAT", RespCommand.PEXPIREAT); + Add("EXPIRETIME", RespCommand.EXPIRETIME); + Add("PEXPIRETIME", RespCommand.PEXPIRETIME); + Add("PERSIST", RespCommand.PERSIST); + Add("TYPE", RespCommand.TYPE); + Add("RENAME", RespCommand.RENAME); + Add("RENAMENX", RespCommand.RENAMENX); + Add("UNLINK", RespCommand.UNLINK); + Add("KEYS", RespCommand.KEYS); + Add("SCAN", RespCommand.SCAN); + Add("DBSIZE", RespCommand.DBSIZE); + Add("SELECT", RespCommand.SELECT); + Add("SWAPDB", RespCommand.SWAPDB); + Add("MIGRATE", RespCommand.MIGRATE); + + // Bitmap commands + Add("BITCOUNT", RespCommand.BITCOUNT); + Add("BITPOS", RespCommand.BITPOS); + Add("BITFIELD", RespCommand.BITFIELD); + Add("BITFIELD_RO", RespCommand.BITFIELD_RO); + Add("BITOP", RespCommand.BITOP); + + // HyperLogLog commands + Add("PFADD", RespCommand.PFADD); + Add("PFCOUNT", RespCommand.PFCOUNT); + Add("PFMERGE", RespCommand.PFMERGE); + + // Hash commands + Add("HSET", RespCommand.HSET); + Add("HGET", RespCommand.HGET); + Add("HDEL", RespCommand.HDEL); + Add("HLEN", RespCommand.HLEN); + Add("HEXISTS", RespCommand.HEXISTS); + Add("HGETALL", RespCommand.HGETALL); + Add("HKEYS", RespCommand.HKEYS); + Add("HVALS", RespCommand.HVALS); + Add("HMSET", RespCommand.HMSET); + Add("HMGET", RespCommand.HMGET); + Add("HSETNX", RespCommand.HSETNX); + Add("HINCRBY", RespCommand.HINCRBY); + Add("HINCRBYFLOAT", RespCommand.HINCRBYFLOAT); + Add("HRANDFIELD", RespCommand.HRANDFIELD); + Add("HSCAN", RespCommand.HSCAN); + Add("HSTRLEN", RespCommand.HSTRLEN); + Add("HTTL", RespCommand.HTTL); + Add("HPTTL", RespCommand.HPTTL); + Add("HEXPIRE", RespCommand.HEXPIRE); + Add("HPEXPIRE", RespCommand.HPEXPIRE); + Add("HEXPIREAT", RespCommand.HEXPIREAT); + Add("HPEXPIREAT", RespCommand.HPEXPIREAT); + Add("HEXPIRETIME", RespCommand.HEXPIRETIME); + Add("HPEXPIRETIME", RespCommand.HPEXPIRETIME); + Add("HPERSIST", RespCommand.HPERSIST); + Add("HCOLLECT", RespCommand.HCOLLECT); + + // List commands + Add("LPUSH", RespCommand.LPUSH); + Add("RPUSH", RespCommand.RPUSH); + Add("LPUSHX", RespCommand.LPUSHX); + Add("RPUSHX", RespCommand.RPUSHX); + Add("LPOP", RespCommand.LPOP); + Add("RPOP", RespCommand.RPOP); + Add("LLEN", RespCommand.LLEN); + Add("LINDEX", RespCommand.LINDEX); + Add("LINSERT", RespCommand.LINSERT); + Add("LRANGE", RespCommand.LRANGE); + Add("LREM", RespCommand.LREM); + Add("LSET", RespCommand.LSET); + Add("LTRIM", RespCommand.LTRIM); + Add("LPOS", RespCommand.LPOS); + Add("LMOVE", RespCommand.LMOVE); + Add("LMPOP", RespCommand.LMPOP); + Add("RPOPLPUSH", RespCommand.RPOPLPUSH); + Add("BLPOP", RespCommand.BLPOP); + Add("BRPOP", RespCommand.BRPOP); + Add("BLMOVE", RespCommand.BLMOVE); + Add("BRPOPLPUSH", RespCommand.BRPOPLPUSH); + Add("BLMPOP", RespCommand.BLMPOP); + + // Set commands + Add("SADD", RespCommand.SADD); + Add("SREM", RespCommand.SREM); + Add("SPOP", RespCommand.SPOP); + Add("SCARD", RespCommand.SCARD); + Add("SMEMBERS", RespCommand.SMEMBERS); + Add("SISMEMBER", RespCommand.SISMEMBER); + Add("SMISMEMBER", RespCommand.SMISMEMBER); + Add("SRANDMEMBER", RespCommand.SRANDMEMBER); + Add("SMOVE", RespCommand.SMOVE); + Add("SSCAN", RespCommand.SSCAN); + Add("SDIFF", RespCommand.SDIFF); + Add("SDIFFSTORE", RespCommand.SDIFFSTORE); + Add("SINTER", RespCommand.SINTER); + Add("SINTERCARD", RespCommand.SINTERCARD); + Add("SINTERSTORE", RespCommand.SINTERSTORE); + Add("SUNION", RespCommand.SUNION); + Add("SUNIONSTORE", RespCommand.SUNIONSTORE); + + // Sorted set commands + Add("ZADD", RespCommand.ZADD); + Add("ZREM", RespCommand.ZREM); + Add("ZCARD", RespCommand.ZCARD); + Add("ZSCORE", RespCommand.ZSCORE); + Add("ZMSCORE", RespCommand.ZMSCORE); + Add("ZRANK", RespCommand.ZRANK); + Add("ZREVRANK", RespCommand.ZREVRANK); + Add("ZCOUNT", RespCommand.ZCOUNT); + Add("ZLEXCOUNT", RespCommand.ZLEXCOUNT); + Add("ZRANGE", RespCommand.ZRANGE); + Add("ZRANGEBYLEX", RespCommand.ZRANGEBYLEX); + Add("ZRANGEBYSCORE", RespCommand.ZRANGEBYSCORE); + Add("ZRANGESTORE", RespCommand.ZRANGESTORE); + Add("ZREVRANGE", RespCommand.ZREVRANGE); + Add("ZREVRANGEBYLEX", RespCommand.ZREVRANGEBYLEX); + Add("ZREVRANGEBYSCORE", RespCommand.ZREVRANGEBYSCORE); + Add("ZPOPMIN", RespCommand.ZPOPMIN); + Add("ZPOPMAX", RespCommand.ZPOPMAX); + Add("ZRANDMEMBER", RespCommand.ZRANDMEMBER); + Add("ZSCAN", RespCommand.ZSCAN); + Add("ZINCRBY", RespCommand.ZINCRBY); + Add("ZDIFF", RespCommand.ZDIFF); + Add("ZDIFFSTORE", RespCommand.ZDIFFSTORE); + Add("ZINTER", RespCommand.ZINTER); + Add("ZINTERCARD", RespCommand.ZINTERCARD); + Add("ZINTERSTORE", RespCommand.ZINTERSTORE); + Add("ZUNION", RespCommand.ZUNION); + Add("ZUNIONSTORE", RespCommand.ZUNIONSTORE); + Add("ZMPOP", RespCommand.ZMPOP); + Add("BZMPOP", RespCommand.BZMPOP); + Add("BZPOPMAX", RespCommand.BZPOPMAX); + Add("BZPOPMIN", RespCommand.BZPOPMIN); + Add("ZREMRANGEBYLEX", RespCommand.ZREMRANGEBYLEX); + Add("ZREMRANGEBYRANK", RespCommand.ZREMRANGEBYRANK); + Add("ZREMRANGEBYSCORE", RespCommand.ZREMRANGEBYSCORE); + Add("ZTTL", RespCommand.ZTTL); + Add("ZPTTL", RespCommand.ZPTTL); + Add("ZEXPIRE", RespCommand.ZEXPIRE); + Add("ZPEXPIRE", RespCommand.ZPEXPIRE); + Add("ZEXPIREAT", RespCommand.ZEXPIREAT); + Add("ZPEXPIREAT", RespCommand.ZPEXPIREAT); + Add("ZEXPIRETIME", RespCommand.ZEXPIRETIME); + Add("ZPEXPIRETIME", RespCommand.ZPEXPIRETIME); + Add("ZPERSIST", RespCommand.ZPERSIST); + Add("ZCOLLECT", RespCommand.ZCOLLECT); + + // Geo commands + Add("GEOADD", RespCommand.GEOADD); + Add("GEOPOS", RespCommand.GEOPOS); + Add("GEOHASH", RespCommand.GEOHASH); + Add("GEODIST", RespCommand.GEODIST); + Add("GEOSEARCH", RespCommand.GEOSEARCH); + Add("GEOSEARCHSTORE", RespCommand.GEOSEARCHSTORE); + Add("GEORADIUS", RespCommand.GEORADIUS); + Add("GEORADIUS_RO", RespCommand.GEORADIUS_RO); + Add("GEORADIUSBYMEMBER", RespCommand.GEORADIUSBYMEMBER); + Add("GEORADIUSBYMEMBER_RO", RespCommand.GEORADIUSBYMEMBER_RO); + + // Scripting + Add("EVAL", RespCommand.EVAL); + Add("EVALSHA", RespCommand.EVALSHA); + + // Pub/Sub + Add("PUBLISH", RespCommand.PUBLISH); + Add("SUBSCRIBE", RespCommand.SUBSCRIBE); + Add("PSUBSCRIBE", RespCommand.PSUBSCRIBE); + Add("UNSUBSCRIBE", RespCommand.UNSUBSCRIBE); + Add("PUNSUBSCRIBE", RespCommand.PUNSUBSCRIBE); + Add("SPUBLISH", RespCommand.SPUBLISH); + Add("SSUBSCRIBE", RespCommand.SSUBSCRIBE); + + // Custom object scan + Add("CUSTOMOBJECTSCAN", RespCommand.COSCAN); + + // ===== Control / admin commands ===== + Add("PING", RespCommand.PING); + Add("ECHO", RespCommand.ECHO); + Add("QUIT", RespCommand.QUIT); + Add("AUTH", RespCommand.AUTH); + Add("HELLO", RespCommand.HELLO); + Add("INFO", RespCommand.INFO); + Add("TIME", RespCommand.TIME); + Add("ROLE", RespCommand.ROLE); + Add("SAVE", RespCommand.SAVE); + Add("LASTSAVE", RespCommand.LASTSAVE); + Add("BGSAVE", RespCommand.BGSAVE); + Add("COMMITAOF", RespCommand.COMMITAOF); + Add("FLUSHALL", RespCommand.FLUSHALL); + Add("FLUSHDB", RespCommand.FLUSHDB); + Add("FORCEGC", RespCommand.FORCEGC); + Add("PURGEBP", RespCommand.PURGEBP); + Add("FAILOVER", RespCommand.FAILOVER); + Add("MONITOR", RespCommand.MONITOR); + Add("REGISTERCS", RespCommand.REGISTERCS); + Add("ASYNC", RespCommand.ASYNC); + Add("DEBUG", RespCommand.DEBUG); + Add("EXPDELSCAN", RespCommand.EXPDELSCAN); + Add("WATCH", RespCommand.WATCH); + Add("WATCHMS", RespCommand.WATCHMS); + Add("WATCHOS", RespCommand.WATCHOS); + Add("MULTI", RespCommand.MULTI); + Add("EXEC", RespCommand.EXEC); + Add("DISCARD", RespCommand.DISCARD); + Add("UNWATCH", RespCommand.UNWATCH); + Add("RUNTXP", RespCommand.RUNTXP); + Add("ASKING", RespCommand.ASKING); + Add("READONLY", RespCommand.READONLY); + Add("READWRITE", RespCommand.READWRITE); + Add("REPLICAOF", RespCommand.REPLICAOF); + Add("SECONDARYOF", RespCommand.SECONDARYOF); + Add("SLAVEOF", RespCommand.SECONDARYOF); + + // Parent commands with subcommands + Add("SCRIPT", RespCommand.SCRIPT, hasSub: true); + Add("CONFIG", RespCommand.CONFIG, hasSub: true); + Add("CLIENT", RespCommand.CLIENT, hasSub: true); + Add("CLUSTER", RespCommand.CLUSTER, hasSub: true); + Add("ACL", RespCommand.ACL, hasSub: true); + Add("COMMAND", RespCommand.COMMAND, hasSub: true); + Add("LATENCY", RespCommand.LATENCY, hasSub: true); + Add("SLOWLOG", RespCommand.SLOWLOG, hasSub: true); + Add("MODULE", RespCommand.MODULE, hasSub: true); + Add("PUBSUB", RespCommand.PUBSUB, hasSub: true); + Add("MEMORY", RespCommand.MEMORY, hasSub: true); + } + + #endregion + + #region Subcommand Definitions + + private static readonly (string Name, RespCommand Command)[] ClusterSubcommands = + [ + ("ADDSLOTS", RespCommand.CLUSTER_ADDSLOTS), + ("ADDSLOTSRANGE", RespCommand.CLUSTER_ADDSLOTSRANGE), + ("AOFSYNC", RespCommand.CLUSTER_AOFSYNC), + ("APPENDLOG", RespCommand.CLUSTER_APPENDLOG), + ("ATTACH_SYNC", RespCommand.CLUSTER_ATTACH_SYNC), + ("BANLIST", RespCommand.CLUSTER_BANLIST), + ("BEGIN_REPLICA_RECOVER", RespCommand.CLUSTER_BEGIN_REPLICA_RECOVER), + ("BUMPEPOCH", RespCommand.CLUSTER_BUMPEPOCH), + ("COUNTKEYSINSLOT", RespCommand.CLUSTER_COUNTKEYSINSLOT), + ("DELKEYSINSLOT", RespCommand.CLUSTER_DELKEYSINSLOT), + ("DELKEYSINSLOTRANGE", RespCommand.CLUSTER_DELKEYSINSLOTRANGE), + ("DELSLOTS", RespCommand.CLUSTER_DELSLOTS), + ("DELSLOTSRANGE", RespCommand.CLUSTER_DELSLOTSRANGE), + ("ENDPOINT", RespCommand.CLUSTER_ENDPOINT), + ("FAILOVER", RespCommand.CLUSTER_FAILOVER), + ("FAILREPLICATIONOFFSET", RespCommand.CLUSTER_FAILREPLICATIONOFFSET), + ("FAILSTOPWRITES", RespCommand.CLUSTER_FAILSTOPWRITES), + ("FLUSHALL", RespCommand.CLUSTER_FLUSHALL), + ("FORGET", RespCommand.CLUSTER_FORGET), + ("GETKEYSINSLOT", RespCommand.CLUSTER_GETKEYSINSLOT), + ("GOSSIP", RespCommand.CLUSTER_GOSSIP), + ("HELP", RespCommand.CLUSTER_HELP), + ("INFO", RespCommand.CLUSTER_INFO), + ("INITIATE_REPLICA_SYNC", RespCommand.CLUSTER_INITIATE_REPLICA_SYNC), + ("KEYSLOT", RespCommand.CLUSTER_KEYSLOT), + ("MEET", RespCommand.CLUSTER_MEET), + ("MIGRATE", RespCommand.CLUSTER_MIGRATE), + ("MTASKS", RespCommand.CLUSTER_MTASKS), + ("MYID", RespCommand.CLUSTER_MYID), + ("MYPARENTID", RespCommand.CLUSTER_MYPARENTID), + ("NODES", RespCommand.CLUSTER_NODES), + ("PUBLISH", RespCommand.CLUSTER_PUBLISH), + ("SPUBLISH", RespCommand.CLUSTER_SPUBLISH), + ("REPLICAS", RespCommand.CLUSTER_REPLICAS), + ("REPLICATE", RespCommand.CLUSTER_REPLICATE), + ("RESET", RespCommand.CLUSTER_RESET), + ("SEND_CKPT_FILE_SEGMENT", RespCommand.CLUSTER_SEND_CKPT_FILE_SEGMENT), + ("SEND_CKPT_METADATA", RespCommand.CLUSTER_SEND_CKPT_METADATA), + ("SETCONFIGEPOCH", RespCommand.CLUSTER_SETCONFIGEPOCH), + ("SETSLOT", RespCommand.CLUSTER_SETSLOT), + ("SETSLOTSRANGE", RespCommand.CLUSTER_SETSLOTSRANGE), + ("SHARDS", RespCommand.CLUSTER_SHARDS), + ("SLOTS", RespCommand.CLUSTER_SLOTS), + ("SLOTSTATE", RespCommand.CLUSTER_SLOTSTATE), + ("SYNC", RespCommand.CLUSTER_SYNC), + ]; + + private static readonly (string Name, RespCommand Command)[] ClientSubcommands = + [ + ("ID", RespCommand.CLIENT_ID), + ("INFO", RespCommand.CLIENT_INFO), + ("LIST", RespCommand.CLIENT_LIST), + ("KILL", RespCommand.CLIENT_KILL), + ("GETNAME", RespCommand.CLIENT_GETNAME), + ("SETNAME", RespCommand.CLIENT_SETNAME), + ("SETINFO", RespCommand.CLIENT_SETINFO), + ("UNBLOCK", RespCommand.CLIENT_UNBLOCK), + ]; + + private static readonly (string Name, RespCommand Command)[] AclSubcommands = + [ + ("CAT", RespCommand.ACL_CAT), + ("DELUSER", RespCommand.ACL_DELUSER), + ("GENPASS", RespCommand.ACL_GENPASS), + ("GETUSER", RespCommand.ACL_GETUSER), + ("LIST", RespCommand.ACL_LIST), + ("LOAD", RespCommand.ACL_LOAD), + ("SAVE", RespCommand.ACL_SAVE), + ("SETUSER", RespCommand.ACL_SETUSER), + ("USERS", RespCommand.ACL_USERS), + ("WHOAMI", RespCommand.ACL_WHOAMI), + ]; + + private static readonly (string Name, RespCommand Command)[] CommandSubcommands = + [ + ("COUNT", RespCommand.COMMAND_COUNT), + ("DOCS", RespCommand.COMMAND_DOCS), + ("INFO", RespCommand.COMMAND_INFO), + ("GETKEYS", RespCommand.COMMAND_GETKEYS), + ("GETKEYSANDFLAGS", RespCommand.COMMAND_GETKEYSANDFLAGS), + ]; + + private static readonly (string Name, RespCommand Command)[] ConfigSubcommands = + [ + ("GET", RespCommand.CONFIG_GET), + ("REWRITE", RespCommand.CONFIG_REWRITE), + ("SET", RespCommand.CONFIG_SET), + ]; + + private static readonly (string Name, RespCommand Command)[] ScriptSubcommands = + [ + ("LOAD", RespCommand.SCRIPT_LOAD), + ("FLUSH", RespCommand.SCRIPT_FLUSH), + ("EXISTS", RespCommand.SCRIPT_EXISTS), + ]; + + private static readonly (string Name, RespCommand Command)[] LatencySubcommands = + [ + ("HELP", RespCommand.LATENCY_HELP), + ("HISTOGRAM", RespCommand.LATENCY_HISTOGRAM), + ("RESET", RespCommand.LATENCY_RESET), + ]; + + private static readonly (string Name, RespCommand Command)[] SlowlogSubcommands = + [ + ("HELP", RespCommand.SLOWLOG_HELP), + ("GET", RespCommand.SLOWLOG_GET), + ("LEN", RespCommand.SLOWLOG_LEN), + ("RESET", RespCommand.SLOWLOG_RESET), + ]; + + private static readonly (string Name, RespCommand Command)[] ModuleSubcommands = + [ + ("LOADCS", RespCommand.MODULE_LOADCS), + ]; + + private static readonly (string Name, RespCommand Command)[] PubsubSubcommands = + [ + ("CHANNELS", RespCommand.PUBSUB_CHANNELS), + ("NUMSUB", RespCommand.PUBSUB_NUMSUB), + ("NUMPAT", RespCommand.PUBSUB_NUMPAT), + ]; + + private static readonly (string Name, RespCommand Command)[] MemorySubcommands = + [ + ("USAGE", RespCommand.MEMORY_USAGE), + ]; + + private static readonly (string Name, RespCommand Command)[] BitopSubcommands = + [ + ("AND", RespCommand.BITOP_AND), + ("OR", RespCommand.BITOP_OR), + ("XOR", RespCommand.BITOP_XOR), + ("NOT", RespCommand.BITOP_NOT), + ("DIFF", RespCommand.BITOP_DIFF), + ]; + + #endregion + } +} \ No newline at end of file From 1a24e0065ff725ade216dcc65c44bd3c88f625e7 Mon Sep 17 00:00:00 2001 From: Badrish C Date: Thu, 26 Mar 2026 14:54:56 -0700 Subject: [PATCH 04/19] update --- .../Operations/CommandParsingBenchmark.cs | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs diff --git a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs new file mode 100644 index 00000000000..fbd11cf39c6 --- /dev/null +++ b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using BenchmarkDotNet.Attributes; +using Garnet.server; + +namespace BDN.benchmark.Operations +{ + /// + /// Benchmark for RESP command parsing only (no storage operations). + /// Calls ParseRespCommandBuffer directly to measure pure parsing throughput + /// across all optimization tiers. + /// + [MemoryDiagnoser] + public unsafe class CommandParsingBenchmark : OperationsBase + { + // Tier 1a: SIMD Vector128 fast path (3-6 char commands with fixed arg counts) + static ReadOnlySpan CMD_GET => "*2\r\n$3\r\nGET\r\n$1\r\na\r\n"u8; + static ReadOnlySpan CMD_SET => "*3\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\nb\r\n"u8; + static ReadOnlySpan CMD_INCR => "*2\r\n$4\r\nINCR\r\n$1\r\ni\r\n"u8; + static ReadOnlySpan CMD_EXISTS => "*2\r\n$6\r\nEXISTS\r\n$1\r\na\r\n"u8; + + // Tier 1b: Scalar ulong switch (variable-arg commands) + static ReadOnlySpan CMD_SETEX => "*4\r\n$5\r\nSETEX\r\n$1\r\na\r\n$2\r\n60\r\n$1\r\nb\r\n"u8; + static ReadOnlySpan CMD_EXPIRE => "*3\r\n$6\r\nEXPIRE\r\n$1\r\na\r\n$2\r\n60\r\n"u8; + + // Tier 2: Hash table lookup (commands not in FastParseCommand) + static ReadOnlySpan CMD_HSET => "*4\r\n$4\r\nHSET\r\n$1\r\nh\r\n$1\r\nf\r\n$1\r\nv\r\n"u8; + static ReadOnlySpan CMD_LPUSH => "*3\r\n$5\r\nLPUSH\r\n$1\r\nl\r\n$1\r\nv\r\n"u8; + static ReadOnlySpan CMD_ZADD => "*4\r\n$4\r\nZADD\r\n$1\r\nz\r\n$1\r\n1\r\n$1\r\nm\r\n"u8; + static ReadOnlySpan CMD_SUBSCRIBE => "*2\r\n$9\r\nSUBSCRIBE\r\n$2\r\nch\r\n"u8; + + // Pre-allocated buffers (pinned for pointer stability) + byte[] bufGet, bufSet, bufIncr, bufExists, bufSetex, bufExpire, bufHset, bufLpush, bufZadd, bufSubscribe; + + public override void GlobalSetup() + { + base.GlobalSetup(); + + // Pre-seed a key so GET/EXISTS don't return NOT_FOUND + SlowConsumeMessage("*3\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\nb\r\n"u8); + + bufGet = GC.AllocateArray(CMD_GET.Length, pinned: true); + CMD_GET.CopyTo(bufGet); + bufSet = GC.AllocateArray(CMD_SET.Length, pinned: true); + CMD_SET.CopyTo(bufSet); + bufIncr = GC.AllocateArray(CMD_INCR.Length, pinned: true); + CMD_INCR.CopyTo(bufIncr); + bufExists = GC.AllocateArray(CMD_EXISTS.Length, pinned: true); + CMD_EXISTS.CopyTo(bufExists); + bufSetex = GC.AllocateArray(CMD_SETEX.Length, pinned: true); + CMD_SETEX.CopyTo(bufSetex); + bufExpire = GC.AllocateArray(CMD_EXPIRE.Length, pinned: true); + CMD_EXPIRE.CopyTo(bufExpire); + bufHset = GC.AllocateArray(CMD_HSET.Length, pinned: true); + CMD_HSET.CopyTo(bufHset); + bufLpush = GC.AllocateArray(CMD_LPUSH.Length, pinned: true); + CMD_LPUSH.CopyTo(bufLpush); + bufZadd = GC.AllocateArray(CMD_ZADD.Length, pinned: true); + CMD_ZADD.CopyTo(bufZadd); + bufSubscribe = GC.AllocateArray(CMD_SUBSCRIBE.Length, pinned: true); + CMD_SUBSCRIBE.CopyTo(bufSubscribe); + } + + // === Tier 1a: SIMD Vector128 fast path === + + [Benchmark] + public RespCommand ParseGET() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufGet); + return result; + } + + [Benchmark] + public RespCommand ParseSET() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufSet); + return result; + } + + [Benchmark] + public RespCommand ParseINCR() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufIncr); + return result; + } + + [Benchmark] + public RespCommand ParseEXISTS() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufExists); + return result; + } + + // === Tier 1b: Scalar ulong switch === + + [Benchmark] + public RespCommand ParseSETEX() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufSetex); + return result; + } + + [Benchmark] + public RespCommand ParseEXPIRE() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufExpire); + return result; + } + + // === Tier 2: Hash table lookup === + + [Benchmark] + public RespCommand ParseHSET() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufHset); + return result; + } + + [Benchmark] + public RespCommand ParseLPUSH() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufLpush); + return result; + } + + [Benchmark] + public RespCommand ParseZADD() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufZadd); + return result; + } + + [Benchmark] + public RespCommand ParseSUBSCRIBE() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufSubscribe); + return result; + } + } +} + +// To test long-tail commands, add this temporarily and rebuild From aeaed64c12a8d9ed073c416d3ea8cefd228effae Mon Sep 17 00:00:00 2001 From: Badrish C Date: Thu, 26 Mar 2026 15:41:41 -0700 Subject: [PATCH 05/19] Add CommandParsingBenchmark with 16 commands across all parser tiers Benchmarks ParseRespCommandBuffer directly to measure pure parsing throughput. Commands categorized by their position in the OLD parser: - Tier 1a SIMD: PING, GET, SET, INCR, EXISTS - Tier 1b Scalar: SETEX, EXPIRE - FastParseArrayCommand top: HSET, LPUSH, ZADD - FastParseArrayCommand deep: ZRANGEBYSCORE, ZREMRANGEBYSCORE, HINCRBYFLOAT - SlowParseCommand: SUBSCRIBE, GEORADIUS, SETIFMATCH Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Operations/CommandParsingBenchmark.cs | 75 ++++++++++++++++++- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs index d1d6645e956..1641a4c21ae 100644 --- a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs +++ b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs @@ -25,14 +25,24 @@ public unsafe class CommandParsingBenchmark : OperationsBase static ReadOnlySpan CMD_SETEX => "*4\r\n$5\r\nSETEX\r\n$1\r\na\r\n$2\r\n60\r\n$1\r\nb\r\n"u8; static ReadOnlySpan CMD_EXPIRE => "*3\r\n$6\r\nEXPIRE\r\n$1\r\na\r\n$2\r\n60\r\n"u8; - // Tier 2: Hash table lookup (commands not in FastParseCommand) + // Old Tier 2 (FastParseArrayCommand): near top of switch chains (short names, common first chars) static ReadOnlySpan CMD_HSET => "*4\r\n$4\r\nHSET\r\n$1\r\nh\r\n$1\r\nf\r\n$1\r\nv\r\n"u8; static ReadOnlySpan CMD_LPUSH => "*3\r\n$5\r\nLPUSH\r\n$1\r\nl\r\n$1\r\nv\r\n"u8; static ReadOnlySpan CMD_ZADD => "*4\r\n$4\r\nZADD\r\n$1\r\nz\r\n$1\r\n1\r\n$1\r\nm\r\n"u8; + + // Old Tier 2 (FastParseArrayCommand): deep in switch chains (long names, double-digit $ header) + static ReadOnlySpan CMD_ZRANGEBYSCORE => "*4\r\n$13\r\nZRANGEBYSCORE\r\n$1\r\nz\r\n$1\r\n0\r\n$2\r\n10\r\n"u8; + static ReadOnlySpan CMD_ZREMRANGEBYSCORE => "*4\r\n$16\r\nZREMRANGEBYSCORE\r\n$1\r\nz\r\n$1\r\n0\r\n$2\r\n10\r\n"u8; + static ReadOnlySpan CMD_HINCRBYFLOAT => "*4\r\n$12\r\nHINCRBYFLOAT\r\n$1\r\nh\r\n$1\r\nf\r\n$3\r\n1.5\r\n"u8; + + // Old Tier 3 (SlowParseCommand): sequential SequenceEqual scan static ReadOnlySpan CMD_SUBSCRIBE => "*2\r\n$9\r\nSUBSCRIBE\r\n$2\r\nch\r\n"u8; + static ReadOnlySpan CMD_GEORADIUS => "*6\r\n$9\r\nGEORADIUS\r\n$1\r\ng\r\n$1\r\n0\r\n$1\r\n0\r\n$3\r\n100\r\n$2\r\nkm\r\n"u8; + static ReadOnlySpan CMD_SETIFMATCH => "*4\r\n$10\r\nSETIFMATCH\r\n$1\r\na\r\n$1\r\nb\r\n$1\r\n0\r\n"u8; // Pre-allocated buffers (pinned for pointer stability) byte[] bufPing, bufGet, bufSet, bufIncr, bufExists, bufSetex, bufExpire, bufHset, bufLpush, bufZadd, bufSubscribe; + byte[] bufZrangebyscore, bufZremrangebyscore, bufHincrbyfloat, bufGeoradius, bufSetifmatch; public override void GlobalSetup() { @@ -63,6 +73,16 @@ public override void GlobalSetup() CMD_ZADD.CopyTo(bufZadd); bufSubscribe = GC.AllocateArray(CMD_SUBSCRIBE.Length, pinned: true); CMD_SUBSCRIBE.CopyTo(bufSubscribe); + bufZrangebyscore = GC.AllocateArray(CMD_ZRANGEBYSCORE.Length, pinned: true); + CMD_ZRANGEBYSCORE.CopyTo(bufZrangebyscore); + bufZremrangebyscore = GC.AllocateArray(CMD_ZREMRANGEBYSCORE.Length, pinned: true); + CMD_ZREMRANGEBYSCORE.CopyTo(bufZremrangebyscore); + bufHincrbyfloat = GC.AllocateArray(CMD_HINCRBYFLOAT.Length, pinned: true); + CMD_HINCRBYFLOAT.CopyTo(bufHincrbyfloat); + bufGeoradius = GC.AllocateArray(CMD_GEORADIUS.Length, pinned: true); + CMD_GEORADIUS.CopyTo(bufGeoradius); + bufSetifmatch = GC.AllocateArray(CMD_SETIFMATCH.Length, pinned: true); + CMD_SETIFMATCH.CopyTo(bufSetifmatch); } // === Tier 1a: SIMD Vector128 fast path === @@ -132,7 +152,7 @@ public RespCommand ParseEXPIRE() return result; } - // === Tier 2: Hash table lookup === + // === Old Tier 2 (FastParseArrayCommand): near top of switch === [Benchmark] public RespCommand ParseHSET() @@ -161,6 +181,37 @@ public RespCommand ParseZADD() return result; } + // === Old Tier 2 (FastParseArrayCommand): deep in switch (long names) === + + [Benchmark] + public RespCommand ParseZRANGEBYSCORE() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufZrangebyscore); + return result; + } + + [Benchmark] + public RespCommand ParseZREMRANGEBYSCORE() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufZremrangebyscore); + return result; + } + + [Benchmark] + public RespCommand ParseHINCRBYFLOAT() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufHincrbyfloat); + return result; + } + + // === Old Tier 3 (SlowParseCommand): sequential SequenceEqual scan === + [Benchmark] public RespCommand ParseSUBSCRIBE() { @@ -169,7 +220,23 @@ public RespCommand ParseSUBSCRIBE() result = session.ParseRespCommandBuffer(bufSubscribe); return result; } + + [Benchmark] + public RespCommand ParseGEORADIUS() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufGeoradius); + return result; + } + + [Benchmark] + public RespCommand ParseSETIFMATCH() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufSetifmatch); + return result; + } } } - -// To test long-tail commands, add this temporarily and rebuild From 0d01f9fa1364a7fd91adbb14f3ced1b1bef10fbd Mon Sep 17 00:00:00 2001 From: Badrish C Date: Thu, 26 Mar 2026 16:13:57 -0700 Subject: [PATCH 06/19] Add per-session MRU command cache for repeated command optimization 2-entry MRU cache sits after SIMD patterns but before scalar switch in FastParseCommand. Caches the last 2 matched command patterns as Vector128 + mask, enabling 3-op cache hits for repeated Tier 1b/2 commands (HSET, LPUSH, ZADD etc.) that would otherwise fall through to the scalar switch or hash table. Cache is populated on successful ArrayParseCommand resolution and excludes: synthetic ParseRespCommandBuffer calls (ACL checks), subcommand results, and custom commands. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Operations/CommandParsingBenchmark.cs | 2 +- libs/server/Resp/Parser/RespCommand.cs | 113 ++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs index 1641a4c21ae..21b067369f5 100644 --- a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs +++ b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs @@ -239,4 +239,4 @@ public RespCommand ParseSETIFMATCH() return result; } } -} +} \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index e934390340c..bc596872e2f 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -643,6 +643,16 @@ enum RespCommandOption : byte /// internal sealed unsafe partial class RespServerSession : ServerSessionBase { + // Per-session MRU command cache: 2 entries caching the last matched command patterns. + // Sits after SIMD patterns but before scalar switch — catches repeated Tier 1b/2 commands + // (HSET, LPUSH, ZADD etc.) in 3 ops instead of falling through to the hash table. + private Vector128 _cachedPattern0, _cachedMask0; + private RespCommand _cachedCmd0; + private byte _cachedLen0, _cachedCount0; + + private Vector128 _cachedPattern1, _cachedMask1; + private RespCommand _cachedCmd1; + private byte _cachedLen1, _cachedCount1; // SIMD Vector128 patterns for FastParseCommand. // Each encodes the full RESP header + command: *N\r\n$L\r\nCMD\r\n // Masks zero out trailing bytes for patterns shorter than 16 bytes. @@ -779,6 +789,32 @@ private RespCommand FastParseCommand(out int count) if (Vector128.EqualsAll(input, s_ASKING)) { readHead += 16; count = 0; return RespCommand.ASKING; } if (Vector128.EqualsAll(input, s_PSETEX)) { readHead += 16; count = 3; return RespCommand.PSETEX; } if (Vector128.EqualsAll(input, s_SUBSTR)) { readHead += 16; count = 3; return RespCommand.SUBSTR; } + + // MRU cache check: catches repeated commands that aren't in the SIMD pattern table + // (e.g., HSET, LPUSH, ZADD, ZRANGEBYSCORE). Same 3-op cost as one SIMD pattern check. + if (_cachedCmd0 != RespCommand.NONE) + { + if (Vector128.EqualsAll(Vector128.BitwiseAnd(input, _cachedMask0), _cachedPattern0)) + { + readHead += _cachedLen0; + count = _cachedCount0; + return _cachedCmd0; + } + + if (_cachedCmd1 != RespCommand.NONE && + Vector128.EqualsAll(Vector128.BitwiseAnd(input, _cachedMask1), _cachedPattern1)) + { + readHead += _cachedLen1; + count = _cachedCount1; + // Promote slot 1 → slot 0 (swap) + (_cachedPattern0, _cachedPattern1) = (_cachedPattern1, _cachedPattern0); + (_cachedMask0, _cachedMask1) = (_cachedMask1, _cachedMask0); + (_cachedCmd0, _cachedCmd1) = (_cachedCmd1, _cachedCmd0); + (_cachedLen0, _cachedLen1) = (_cachedLen1, _cachedLen0); + (_cachedCount0, _cachedCount1) = (_cachedCount1, _cachedCount0); + return _cachedCmd0; + } + } } // Scalar fallback: handles variable-arg commands, inline commands, @@ -2831,8 +2867,22 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) // If we have not found a command, continue parsing on slow path if (cmd == RespCommand.NONE) { + var cmdStartOffset = readHead; // Save position before ArrayParseCommand advances it cmd = ArrayParseCommand(writeErrorOnFailure, ref count, ref success); if (!success) return cmd; + + // Update MRU cache for commands resolved by scalar switch or hash table. + // Only for real command processing (writeErrorOnFailure=true), not synthetic + // ParseRespCommandBuffer calls (ACL checks etc.) which use temporary buffers. + // Exclude custom commands — they have runtime-registered names that can't be + // reliably cached alongside built-in commands. + if (writeErrorOnFailure && Vector128.IsHardwareAccelerated && + cmd != RespCommand.INVALID && cmd != RespCommand.NONE && + cmd != RespCommand.CustomTxn && cmd != RespCommand.CustomProcedure && + cmd != RespCommand.CustomRawStringCmd && cmd != RespCommand.CustomObjCmd) + { + UpdateCommandCache(cmdStartOffset, cmd, count); + } } // Set up parse state @@ -2854,6 +2904,69 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) return cmd; } + /// + /// Update the MRU command cache with a newly matched command from the scalar switch or hash table. + /// Captures the first 16 bytes of the RESP encoding and the appropriate mask so that + /// the cache check in FastParseCommand can match it via Vector128.EqualsAll. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private void UpdateCommandCache(int cmdStartOffset, RespCommand cmd, int argCount) + { + var ptr = recvBufferPtr + cmdStartOffset; + var availableBytes = bytesRead - cmdStartOffset; + + // Only cache commands where we have at least 16 bytes to load a full Vector128 + if (availableBytes < 16) return; + + // Determine total encoded command length: *N\r\n$L\r\n + name + \r\n + // Parse from the RESP header at cmdStartOffset + if (*ptr != '*') return; + + // Need single-digit array length and single-digit string length for cache + if (availableBytes < 8) return; + if ((*(ulong*)ptr & 0xFFFF00FFFFFF00FF) != MemoryMarshal.Read("*\0\r\n$\0\r\n"u8)) return; + + var nameLen = ptr[5] - '0'; + if (nameLen < 1 || nameLen > 9) return; + + var totalLen = 8 + nameLen + 2; // *N\r\n$L\r\n (8) + name (nameLen) + \r\n (2) + if (totalLen > 16) return; // Only cache commands that fit in 16 bytes + + // If readHead advanced past the parent command encoding, subcommand parsing happened + // (e.g., CLIENT LIST, CONFIG GET). Don't cache — pattern can't distinguish subcommands. + if (readHead > cmdStartOffset + totalLen) return; + + var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); + + // Select the appropriate mask based on total encoded length + Vector128 mask; + if (totalLen == 16) + mask = Vector128.AllBitsSet; + else if (totalLen == 15) + mask = s_mask15; + else if (totalLen == 14) + mask = s_mask14; + else if (totalLen == 13) + mask = s_mask13; + else + return; // Too short (name < 3 chars) — not worth caching + + var pattern = Vector128.BitwiseAnd(input, mask); + + // Demote slot 0 → slot 1, promote new match → slot 0 + _cachedPattern1 = _cachedPattern0; + _cachedMask1 = _cachedMask0; + _cachedCmd1 = _cachedCmd0; + _cachedLen1 = _cachedLen0; + _cachedCount1 = _cachedCount0; + + _cachedPattern0 = pattern; + _cachedMask0 = mask; + _cachedCmd0 = cmd; + _cachedLen0 = (byte)totalLen; + _cachedCount0 = (byte)argCount; + } + [MethodImpl(MethodImplOptions.NoInlining)] private void HandleAofCommitMode(RespCommand cmd) { From c38d1e6f31898785ef7ec23eefe60c967e520b1d Mon Sep 17 00:00:00 2001 From: Badrish C Date: Thu, 26 Mar 2026 17:12:10 -0700 Subject: [PATCH 07/19] update --- libs/server/Resp/Parser/RespCommand.cs | 52 +++++++++----------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index bc596872e2f..37a140d1c2d 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -2872,11 +2872,8 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) if (!success) return cmd; // Update MRU cache for commands resolved by scalar switch or hash table. - // Only for real command processing (writeErrorOnFailure=true), not synthetic - // ParseRespCommandBuffer calls (ACL checks etc.) which use temporary buffers. - // Exclude custom commands — they have runtime-registered names that can't be - // reliably cached alongside built-in commands. - if (writeErrorOnFailure && Vector128.IsHardwareAccelerated && + // Exclude custom commands — they have runtime-registered names. + if (Vector128.IsHardwareAccelerated && cmd != RespCommand.INVALID && cmd != RespCommand.NONE && cmd != RespCommand.CustomTxn && cmd != RespCommand.CustomProcedure && cmd != RespCommand.CustomRawStringCmd && cmd != RespCommand.CustomObjCmd) @@ -2918,38 +2915,25 @@ private void UpdateCommandCache(int cmdStartOffset, RespCommand cmd, int argCoun // Only cache commands where we have at least 16 bytes to load a full Vector128 if (availableBytes < 16) return; - // Determine total encoded command length: *N\r\n$L\r\n + name + \r\n - // Parse from the RESP header at cmdStartOffset - if (*ptr != '*') return; + // Compute how many bytes the parse actually consumed (command + subcommand if any) + var consumedBytes = readHead - cmdStartOffset; - // Need single-digit array length and single-digit string length for cache - if (availableBytes < 8) return; - if ((*(ulong*)ptr & 0xFFFF00FFFFFF00FF) != MemoryMarshal.Read("*\0\r\n$\0\r\n"u8)) return; - - var nameLen = ptr[5] - '0'; - if (nameLen < 1 || nameLen > 9) return; - - var totalLen = 8 + nameLen + 2; // *N\r\n$L\r\n (8) + name (nameLen) + \r\n (2) - if (totalLen > 16) return; // Only cache commands that fit in 16 bytes - - // If readHead advanced past the parent command encoding, subcommand parsing happened - // (e.g., CLIENT LIST, CONFIG GET). Don't cache — pattern can't distinguish subcommands. - if (readHead > cmdStartOffset + totalLen) return; + // Only cache if the full parse fits within 16 bytes (one Vector128) + if (consumedBytes < 13 || consumedBytes > 16) return; var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); - // Select the appropriate mask based on total encoded length - Vector128 mask; - if (totalLen == 16) - mask = Vector128.AllBitsSet; - else if (totalLen == 15) - mask = s_mask15; - else if (totalLen == 14) - mask = s_mask14; - else if (totalLen == 13) - mask = s_mask13; - else - return; // Too short (name < 3 chars) — not worth caching + // Select the appropriate mask — covers exactly the consumed bytes + Vector128 mask = consumedBytes switch + { + 16 => Vector128.AllBitsSet, + 15 => s_mask15, + 14 => s_mask14, + 13 => s_mask13, + _ => Vector128.Zero + }; + + if (mask == Vector128.Zero) return; var pattern = Vector128.BitwiseAnd(input, mask); @@ -2963,7 +2947,7 @@ private void UpdateCommandCache(int cmdStartOffset, RespCommand cmd, int argCoun _cachedPattern0 = pattern; _cachedMask0 = mask; _cachedCmd0 = cmd; - _cachedLen0 = (byte)totalLen; + _cachedLen0 = (byte)consumedBytes; _cachedCount0 = (byte)argCount; } From 80875e0b5013f12aae6a7e9e12913b901b869a3c Mon Sep 17 00:00:00 2001 From: Badrish C Date: Thu, 26 Mar 2026 17:12:10 -0700 Subject: [PATCH 08/19] update --- libs/server/Resp/Parser/RespCommand.cs | 52 +++++++++----------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index bc596872e2f..37a140d1c2d 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -2872,11 +2872,8 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) if (!success) return cmd; // Update MRU cache for commands resolved by scalar switch or hash table. - // Only for real command processing (writeErrorOnFailure=true), not synthetic - // ParseRespCommandBuffer calls (ACL checks etc.) which use temporary buffers. - // Exclude custom commands — they have runtime-registered names that can't be - // reliably cached alongside built-in commands. - if (writeErrorOnFailure && Vector128.IsHardwareAccelerated && + // Exclude custom commands — they have runtime-registered names. + if (Vector128.IsHardwareAccelerated && cmd != RespCommand.INVALID && cmd != RespCommand.NONE && cmd != RespCommand.CustomTxn && cmd != RespCommand.CustomProcedure && cmd != RespCommand.CustomRawStringCmd && cmd != RespCommand.CustomObjCmd) @@ -2918,38 +2915,25 @@ private void UpdateCommandCache(int cmdStartOffset, RespCommand cmd, int argCoun // Only cache commands where we have at least 16 bytes to load a full Vector128 if (availableBytes < 16) return; - // Determine total encoded command length: *N\r\n$L\r\n + name + \r\n - // Parse from the RESP header at cmdStartOffset - if (*ptr != '*') return; + // Compute how many bytes the parse actually consumed (command + subcommand if any) + var consumedBytes = readHead - cmdStartOffset; - // Need single-digit array length and single-digit string length for cache - if (availableBytes < 8) return; - if ((*(ulong*)ptr & 0xFFFF00FFFFFF00FF) != MemoryMarshal.Read("*\0\r\n$\0\r\n"u8)) return; - - var nameLen = ptr[5] - '0'; - if (nameLen < 1 || nameLen > 9) return; - - var totalLen = 8 + nameLen + 2; // *N\r\n$L\r\n (8) + name (nameLen) + \r\n (2) - if (totalLen > 16) return; // Only cache commands that fit in 16 bytes - - // If readHead advanced past the parent command encoding, subcommand parsing happened - // (e.g., CLIENT LIST, CONFIG GET). Don't cache — pattern can't distinguish subcommands. - if (readHead > cmdStartOffset + totalLen) return; + // Only cache if the full parse fits within 16 bytes (one Vector128) + if (consumedBytes < 13 || consumedBytes > 16) return; var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); - // Select the appropriate mask based on total encoded length - Vector128 mask; - if (totalLen == 16) - mask = Vector128.AllBitsSet; - else if (totalLen == 15) - mask = s_mask15; - else if (totalLen == 14) - mask = s_mask14; - else if (totalLen == 13) - mask = s_mask13; - else - return; // Too short (name < 3 chars) — not worth caching + // Select the appropriate mask — covers exactly the consumed bytes + Vector128 mask = consumedBytes switch + { + 16 => Vector128.AllBitsSet, + 15 => s_mask15, + 14 => s_mask14, + 13 => s_mask13, + _ => Vector128.Zero + }; + + if (mask == Vector128.Zero) return; var pattern = Vector128.BitwiseAnd(input, mask); @@ -2963,7 +2947,7 @@ private void UpdateCommandCache(int cmdStartOffset, RespCommand cmd, int argCoun _cachedPattern0 = pattern; _cachedMask0 = mask; _cachedCmd0 = cmd; - _cachedLen0 = (byte)totalLen; + _cachedLen0 = (byte)consumedBytes; _cachedCount0 = (byte)argCount; } From c2b50da73e3d7f146a70d6a53edc8c4f85892771 Mon Sep 17 00:00:00 2001 From: Badrish C Date: Thu, 26 Mar 2026 18:28:51 -0700 Subject: [PATCH 09/19] Replace SlowParseCommand with hash-based subcommand dispatch Eliminate SlowParseCommand for all subcommand routing. HandleSubcommandLookup now uses per-parent hash tables for CLUSTER, CLIENT, ACL, CONFIG, COMMAND, SCRIPT, LATENCY, SLOWLOG, MODULE, PUBSUB, MEMORY subcommands. Key fixes: - Fix CLUSTER SET-CONFIG-EPOCH hash entry (was SETCONFIGEPOCH, missing hyphens) - Handle edge cases: COMMAND with 0 args, case-insensitive GETKEYS/USAGE - Error message formatting: GenericErrUnknownSubCommand for CLUSTER/LATENCY, GenericErrUnknownSubCommandNoHelp for others - Remove writeErrorOnFailure guard from MRU cache (unnecessary) - Use consumedBytes (readHead - cmdStartOffset) for cache entry sizing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- libs/server/Resp/Parser/RespCommand.cs | 78 +++++++++++++++++-- .../Resp/Parser/RespCommandHashLookup.cs | 2 +- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 37a140d1c2d..cbdd4a44ccd 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -3071,19 +3071,87 @@ private RespCommand HashLookupCommand(ref int count, ref ReadOnlySpan spec return ParseBitopSubcommand(ref count, ref specificErrorMessage, out success); } - // Check if this command has subcommands that need a second lookup - // Subcommand dispatch uses the existing SlowParseCommand which handles - // all edge cases (case-insensitive matching, specific error messages, etc.) - // These are admin commands (CLUSTER, CONFIG, ACL) — not the hot path. + // Commands with subcommands — dispatch via subcommand hash table if (RespCommandHashLookup.HasSubcommands(namePtr, command.Length)) { - return SlowParseCommand(command, ref count, ref specificErrorMessage, out success); + return HandleSubcommandLookup(cmd, ref count, ref specificErrorMessage, out success); } return cmd; } } + /// + /// Handles subcommand dispatch for parent commands (CLUSTER, CONFIG, CLIENT, etc.) + /// using per-parent hash tables. Replaces the former SlowParseCommand subcommand chains. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private RespCommand HandleSubcommandLookup(RespCommand parentCmd, ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) + { + success = true; + + // COMMAND with no args returns RespCommand.COMMAND (lists all commands) + if (parentCmd == RespCommand.COMMAND && count == 0) + { + return RespCommand.COMMAND; + } + + // Most parent commands require at least one subcommand argument + if (count == 0) + { + specificErrorMessage = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, + parentCmd.ToString())); + return RespCommand.INVALID; + } + + // Extract and uppercase the subcommand name + var subCommand = GetUpperCaseCommand(out var gotSubCommand); + if (!gotSubCommand) + { + success = false; + return RespCommand.NONE; + } + + count--; + + // Hash table lookup for the subcommand + fixed (byte* subNamePtr = subCommand) + { + var subCmd = RespCommandHashLookup.LookupSubcommand(parentCmd, subNamePtr, subCommand.Length); + if (subCmd != RespCommand.NONE) + { + return subCmd; + } + } + + // Hash miss — try case-insensitive fallbacks for specific commands + if (parentCmd == RespCommand.COMMAND) + { + if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.GETKEYS)) + return RespCommand.COMMAND_GETKEYS; + if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.GETKEYSANDFLAGS)) + return RespCommand.COMMAND_GETKEYSANDFLAGS; + } + else if (parentCmd == RespCommand.MEMORY) + { + if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.USAGE)) + return RespCommand.MEMORY_USAGE; + } + + // Generate error message for unknown subcommand + string errMsg = parentCmd switch + { + RespCommand.CLUSTER or RespCommand.LATENCY => + string.Format(CmdStrings.GenericErrUnknownSubCommand, + Encoding.UTF8.GetString(subCommand), parentCmd.ToString()), + _ => + string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, + Encoding.UTF8.GetString(subCommand), parentCmd.ToString()), + }; + specificErrorMessage = Encoding.UTF8.GetBytes(errMsg); + return RespCommand.INVALID; + } + /// /// Parse the BITOP pseudo-subcommand (AND/OR/XOR/NOT/DIFF). /// Called after the primary hash lookup identifies BITOP. diff --git a/libs/server/Resp/Parser/RespCommandHashLookup.cs b/libs/server/Resp/Parser/RespCommandHashLookup.cs index a0b6884cc3c..e6952c37147 100644 --- a/libs/server/Resp/Parser/RespCommandHashLookup.cs +++ b/libs/server/Resp/Parser/RespCommandHashLookup.cs @@ -716,7 +716,7 @@ private static readonly (string Name, RespCommand Command)[] ClusterSubcommands ("RESET", RespCommand.CLUSTER_RESET), ("SEND_CKPT_FILE_SEGMENT", RespCommand.CLUSTER_SEND_CKPT_FILE_SEGMENT), ("SEND_CKPT_METADATA", RespCommand.CLUSTER_SEND_CKPT_METADATA), - ("SETCONFIGEPOCH", RespCommand.CLUSTER_SETCONFIGEPOCH), + ("SET-CONFIG-EPOCH", RespCommand.CLUSTER_SETCONFIGEPOCH), ("SETSLOT", RespCommand.CLUSTER_SETSLOT), ("SETSLOTSRANGE", RespCommand.CLUSTER_SETSLOTSRANGE), ("SHARDS", RespCommand.CLUSTER_SHARDS), From e09968696c6adf1951bf9264e79ff0524e5feb23 Mon Sep 17 00:00:00 2001 From: Badrish C Date: Thu, 26 Mar 2026 18:35:40 -0700 Subject: [PATCH 10/19] Update add-garnet-command skill and copilot instructions for hash table parsing Replace references to FastParseArrayCommand/SlowParseCommand with hash table instructions. New commands now just need one Add() call in PopulatePrimaryTable(). Document subcommand table wiring and warn about wire-protocol spelling (hyphens etc.). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 2 +- .github/skills/add-garnet-command/SKILL.md | 63 ++++++++++++---------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index ddd3947c55c..0f11b86116c 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -92,7 +92,7 @@ Full guide: https://microsoft.github.io/garnet/docs/dev/garnet-api ### Steps for a new built-in command: 1. **Define the command**: Add enum value to `RespCommand` in `libs/server/Resp/Parser/RespCommand.cs`. For object commands (List, SortedSet, Hash, Set), also add a value to the `[ObjectName]Operation` enum in `libs/server/Objects/[ObjectName]/[ObjectName]Object.cs`. -2. **Add parsing logic**: In `libs/server/Resp/Parser/RespCommand.cs`, add to `FastParseCommand` (fixed arg count) or `FastParseArrayCommand` (variable args). +2. **Add parsing logic**: In `libs/server/Resp/Parser/RespCommandHashLookup.cs`, add an entry to `PopulatePrimaryTable()` (e.g., `Add("MYNEWCMD", RespCommand.MYNEWCMD)`). For commands with subcommands, set `hasSub: true` and add a subcommand table. The hash table provides O(1) lookup for all command name lengths. 3. **Declare the API method**: Add method signature to `IGarnetReadApi` (read-only) or `IGarnetApi` (read-write) in `libs/server/API/IGarnetApi.cs`. 4. **Implement the network handler**: Add a method to `RespServerSession` (the class is split across ~22 partial `.cs` files — object commands go in `libs/server/Resp/Objects/[ObjectName]Commands.cs`, others in `libs/server/Resp/BasicCommands.cs`, `ArrayCommands.cs`, `AdminCommands.cs`, `KeyAdminCommands.cs`, etc.). The handler parses arguments from the network buffer via `parseState.GetArgSliceByRef(i)` (returns `ref PinnedSpanByte`), calls the storage API, and writes the RESP response using `RespWriteUtils` helper methods, then calls `SendAndReset()` to flush the response buffer. 5. **Add dispatch route**: In `libs/server/Resp/RespServerSession.cs`, add a case to `ProcessBasicCommands` or `ProcessArrayCommands` calling the handler from step 4. diff --git a/.github/skills/add-garnet-command/SKILL.md b/.github/skills/add-garnet-command/SKILL.md index faeec7c6192..f1c16ced757 100644 --- a/.github/skills/add-garnet-command/SKILL.md +++ b/.github/skills/add-garnet-command/SKILL.md @@ -82,17 +82,39 @@ EVALSHA, // Note: Update LastDataCommand if adding new data commands after this **File:** `libs/server/Resp/Parser/RespCommand.cs` -Two parsing paths exist: +Two parsing tiers exist: -### Fast path: `FastParseCommand()` / `FastParseArrayCommand()` -Two fast-path methods exist with different constraints: -- **`FastParseCommand()`**: For commands with a fixed number of arguments and command names up to **9 characters**. Uses `ulong` pointer comparisons on `(count << 4) | length` patterns. -- **`FastParseArrayCommand()`**: For commands with a variable number of arguments and command names up to **16 characters**. Uses similar `ulong` comparison patterns but accommodates longer names. +### Hash table path: `RespCommandHashLookup` (primary path for most commands) +The hash table in `libs/server/Resp/Parser/RespCommandHashLookup.cs` provides O(1) lookup for all built-in commands. This is the **recommended path for all new commands**. -Only add here if the command name is a simple word (no dots or special characters). +**To add a new primary command**, add one line to `PopulatePrimaryTable()`: +```csharp +Add("DELIFGREATER", RespCommand.DELIFGREATER); +``` -### Slow path: `SlowParseCommand()` -For longer names, dot-prefixed names (like `RI.CREATE`), or names that don't fit the fast-path pattern. +**To add a command with subcommands** (e.g., `MYPARENT SUBCMD`): +1. Add the parent with `hasSub: true`: + ```csharp + Add("MYPARENT", RespCommand.MYPARENT, hasSub: true); + ``` +2. Define the subcommand array: + ```csharp + private static readonly (string Name, RespCommand Command)[] MyparentSubcommands = + [ + ("SUBCMD1", RespCommand.MYPARENT_SUBCMD1), + ("SUBCMD2", RespCommand.MYPARENT_SUBCMD2), + ]; + ``` +3. Build the table in the static constructor: + ```csharp + myparentSubTable = BuildSubTable(MyparentSubcommands, out myparentSubTableMask); + ``` +4. Wire it into `LookupSubcommand()`: + ```csharp + RespCommand.MYPARENT => (myparentSubTable, myparentSubTableMask), + ``` + +**⚠️ Important:** Use the exact wire-protocol spelling for the hash table name string. Some commands use hyphens (e.g., `"SET-CONFIG-EPOCH"` not `"SETCONFIGEPOCH"`). Check `CmdStrings.cs` for the canonical spelling. **⚠️ Convention:** Define the command name string in **`libs/server/Resp/CmdStrings.cs`** and reference it from the parser, rather than using inline `"..."u8` literals. This keeps command name strings centralized and reusable (e.g., for error messages). @@ -101,26 +123,13 @@ For longer names, dot-prefixed names (like `RI.CREATE`), or names that don't fit public static ReadOnlySpan DELIFGREATER => "DELIFGREATER"u8; ``` -**Pattern for slow-path commands:** -```csharp -else if (command.SequenceEqual(CmdStrings.DELIFGREATER)) -{ - return RespCommand.DELIFGREATER; -} -``` - -**Pattern for dot-prefixed commands (e.g., `RI.CREATE`):** -```csharp -else if (command.SequenceEqual(CmdStrings.RICREATE)) -{ - return RespCommand.RICREATE; -} -``` - -Add this before the final `return RespCommand.INVALID;` at the end of `SlowParseCommand`. +### SIMD fast path: `FastParseCommand()` (optional, for hottest commands only) +Static `Vector128` patterns that match the full RESP encoding (`*N\r\n$L\r\nCMD\r\n`) in a single 16-byte comparison. Only needed for the most performance-critical commands with: +- Fixed argument count +- Command names of 3-6 characters +- No dots or special characters -**⚠️ Caveat: Dot-prefixed commands and ACL** -If your command uses a dot (e.g., `RI.CREATE`), you must also update **`libs/server/ACL/ACLParser.cs`** so that the ACL system can map the dotted wire name to the enum name. Search for how existing dot-handling works (look for `Replace(".", "")` or similar normalization). +Most new commands should **NOT** be added here — the hash table + MRU cache provide excellent performance for all commands. Only add SIMD patterns if benchmarking shows the command is a bottleneck. --- From 5c670638b5736771b226ea7abcd98f7245215e19 Mon Sep 17 00:00:00 2001 From: Badrish C Date: Thu, 26 Mar 2026 20:33:34 -0700 Subject: [PATCH 11/19] Production hardening: debug asserts, startup validation, code review fixes - Add Debug.Assert for command name length/positivity in hash table ops - Add startup ValidateSubTable: verifies every subcommand entry round-trips correctly through the hash table (catches typos like SET-CONFIG-EPOCH) - Clean up InsertIntoTable: remove redundant double-assignment of NameWord1/2, add explicit zero-init and clear comments on word layout contract - Fix comment in HashLookupCommand: document that MakeUpperCase only uppercases the first token, subcommands need GetUpperCaseCommand - Add comment documenting MRU cache zero-initialization safety Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- libs/server/Resp/Parser/RespCommand.cs | 6 ++- .../Resp/Parser/RespCommandHashLookup.cs | 54 +++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index cbdd4a44ccd..ca89f4d4082 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -646,6 +646,8 @@ internal sealed unsafe partial class RespServerSession : ServerSessionBase // Per-session MRU command cache: 2 entries caching the last matched command patterns. // Sits after SIMD patterns but before scalar switch — catches repeated Tier 1b/2 commands // (HSET, LPUSH, ZADD etc.) in 3 ops instead of falling through to the hash table. + // NOTE: All fields default to zero (RespCommand.NONE = 0x00, Vector128 = all zeros), + // so the cache starts empty and the _cachedCmd0 != RespCommand.NONE check is safe. private Vector128 _cachedPattern0, _cachedMask0; private RespCommand _cachedCmd0; private byte _cachedLen0, _cachedCount0; @@ -3036,8 +3038,10 @@ private RespCommand ArrayParseCommand(bool writeErrorOnFailure, ref int count, r private RespCommand HashLookupCommand(ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) { // Extract the command name (reads $len\r\n...\r\n, advances readHead). - // MakeUpperCase has already uppercased the command name in the buffer, + // MakeUpperCase has already uppercased the first command name token in the buffer, // so we use GetCommand (no redundant ToUpperInPlace call). + // NOTE: MakeUpperCase only uppercases the first token — subcommand names are + // uppercased separately via GetUpperCaseCommand in HandleSubcommandLookup. var command = GetCommand(out success); if (!success) { diff --git a/libs/server/Resp/Parser/RespCommandHashLookup.cs b/libs/server/Resp/Parser/RespCommandHashLookup.cs index e6952c37147..b7639034b5a 100644 --- a/libs/server/Resp/Parser/RespCommandHashLookup.cs +++ b/libs/server/Resp/Parser/RespCommandHashLookup.cs @@ -121,6 +121,42 @@ static RespCommandHashLookup() pubsubSubTable = BuildSubTable(PubsubSubcommands, out pubsubSubTableMask); memorySubTable = BuildSubTable(MemorySubcommands, out memorySubTableMask); bitopSubTable = BuildSubTable(BitopSubcommands, out bitopSubTableMask); + + // Validate all subcommand tables are round-trip correct + ValidateSubTable(RespCommand.CLUSTER, ClusterSubcommands, clusterSubTable, clusterSubTableMask); + ValidateSubTable(RespCommand.CLIENT, ClientSubcommands, clientSubTable, clientSubTableMask); + ValidateSubTable(RespCommand.ACL, AclSubcommands, aclSubTable, aclSubTableMask); + ValidateSubTable(RespCommand.COMMAND, CommandSubcommands, commandSubTable, commandSubTableMask); + ValidateSubTable(RespCommand.CONFIG, ConfigSubcommands, configSubTable, configSubTableMask); + ValidateSubTable(RespCommand.SCRIPT, ScriptSubcommands, scriptSubTable, scriptSubTableMask); + ValidateSubTable(RespCommand.LATENCY, LatencySubcommands, latencySubTable, latencySubTableMask); + ValidateSubTable(RespCommand.SLOWLOG, SlowlogSubcommands, slowlogSubTable, slowlogSubTableMask); + ValidateSubTable(RespCommand.MODULE, ModuleSubcommands, moduleSubTable, moduleSubTableMask); + ValidateSubTable(RespCommand.PUBSUB, PubsubSubcommands, pubsubSubTable, pubsubSubTableMask); + ValidateSubTable(RespCommand.MEMORY, MemorySubcommands, memorySubTable, memorySubTableMask); + ValidateSubTable(RespCommand.BITOP, BitopSubcommands, bitopSubTable, bitopSubTableMask); + } + + /// + /// Validate that every entry in a subcommand definition array can be looked up in the hash table. + /// Called once during static init; throws on any mismatch to catch typos early. + /// + private static void ValidateSubTable(RespCommand parent, ReadOnlySpan<(string Name, RespCommand Command)> subcommands, + CommandEntry[] table, int mask) + { + foreach (var (name, expectedCmd) in subcommands) + { + var nameBytes = System.Text.Encoding.ASCII.GetBytes(name); + fixed (byte* p = nameBytes) + { + var found = LookupInTable(table, mask, p, nameBytes.Length); + if (found != expectedCmd) + { + throw new InvalidOperationException( + $"Hash table validation failed: {parent} subcommand '{name}' expected {expectedCmd} but got {found}"); + } + } + } } #region Public API @@ -267,6 +303,8 @@ private static ulong ReadPartialWord(byte* p, int len) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool MatchName(ref CommandEntry entry, byte* name, int length) { + Debug.Assert(length > 0, "MatchName: length must be positive"); + if (length <= 8) { ulong inputWord = length == 8 ? *(ulong*)name : ReadPartialWord(name, length); @@ -293,6 +331,9 @@ private static bool MatchName(ref CommandEntry entry, byte* name, int length) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static RespCommand LookupInTable(CommandEntry[] table, int tableMask, byte* name, int length) { + Debug.Assert(length > 0, "Command name length must be positive"); + Debug.Assert(table != null, "Hash table must not be null"); + uint hash = ComputeHash(name, length); int idx = (int)(hash & (uint)tableMask); @@ -337,6 +378,9 @@ private static ulong GetWordFromSpan(ReadOnlySpan span, int offset) /// private static void InsertIntoTable(CommandEntry[] table, int tableMask, ReadOnlySpan name, RespCommand command, byte flags = 0) { + Debug.Assert(name.Length > 0, "Command name must not be empty"); + Debug.Assert(name.Length <= 24, $"Command name too long for hash table entry: {System.Text.Encoding.ASCII.GetString(name)}"); + uint hash = ComputeHash(name); int idx = (int)(hash & (uint)tableMask); @@ -348,11 +392,15 @@ private static void InsertIntoTable(CommandEntry[] table, int tableMask, ReadOnl entry.Command = command; entry.NameLength = (byte)name.Length; entry.Flags = flags; + + // Store name words to match the layout expected by MatchName: + // length <= 8: Word0 = bytes 0..len-1 (zero-padded) + // length 9-16: Word0 = bytes 0..7, Word1 = bytes len-8..len-1 (overlapping) + // length 17-24: Word0 = bytes 0..7, Word1 = bytes 8..15, Word2 = bytes len-8..len-1 entry.NameWord0 = GetWordFromSpan(name, 0); - entry.NameWord1 = name.Length > 8 ? GetWordFromSpan(name, name.Length - 8) : 0; - entry.NameWord2 = name.Length > 16 ? GetWordFromSpan(name, name.Length - 8) : 0; + entry.NameWord1 = 0; + entry.NameWord2 = 0; - // For names > 16 bytes, store bytes 8-15 exactly and last 8 bytes in word2 if (name.Length > 16) { entry.NameWord1 = GetWordFromSpan(name, 8); From 4ead4a0b68684d5c5791d8e98d57e87b63126757 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Tue, 31 Mar 2026 18:02:47 -0700 Subject: [PATCH 12/19] updates --- libs/server/Resp/Parser/RespCommand.cs | 1869 +---------------- .../Resp/Parser/RespCommandHashLookup.cs | 2 +- 2 files changed, 22 insertions(+), 1849 deletions(-) diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index ca89f4d4082..48a33a078c6 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -819,8 +819,9 @@ private RespCommand FastParseCommand(out int count) } } - // Scalar fallback: handles variable-arg commands, inline commands, - // commands with names > 6 chars, and non-SIMD hardware. + // Scalar fast path: uses ulong comparisons to match common commands. + // On SIMD hardware, this catches commands when remainingBytes < 16 (SIMD needs 16). + // On non-SIMD hardware, this is the primary fast path. // Check if the package starts with "*_\r\n$_\r\n" (_ = masked out), // i.e. an array with a single-digit length and single-digit first string length. @@ -927,971 +928,10 @@ static RespCommand MatchedNone(RespServerSession session, int oldReadHead) return FastParseInlineCommand(out count); } - // Couldn't find a matching command in this pass count = -1; return RespCommand.NONE; } - /// - /// Fast parsing function for common command names. - /// Parses the receive buffer starting from the current read head and advances it to the end of - /// the parsed command/subcommand name. - /// - /// Reference to the number of remaining tokens in the packet. Will be reduced to number of command arguments. - /// The parsed command name. - private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan specificErrorMessage) - { - // Bytes remaining in the read buffer - int remainingBytes = bytesRead - readHead; - - // The current read head to continue reading from - byte* ptr = recvBufferPtr + readHead; - - // - // Fast-path parsing by (1) command string length, (2) First character of command name (optional) and (3) priority (manual order) - // - - // NOTE: A valid RESP string is at a minimum 7 characters long "$_\r\n_\r\n" - if (remainingBytes >= 7) - { - var oldReadHead = readHead; - - // Check if this is a string with a single-digit length ("$_\r\n" -> _ omitted) - if ((*(uint*)ptr & 0xFFFF00FF) == MemoryMarshal.Read("$\0\r\n"u8)) - { - // Extract length from string header - var length = ptr[1] - '0'; - - // Ensure that the complete command string is contained in the package. Otherwise exit early. - // Include 6 bytes to account for command string header and name terminator. - // 6 bytes = "$_\r\n" (4 bytes) + "\r\n" (2 bytes) at end of command name - if (remainingBytes >= length + 6) - { - // Optimistically increase read head and decrease the number of remaining elements - readHead += length + 6; - count -= 1; - - switch (length) - { - case 3: - if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("3\r\nDEL\r\n"u8)) - { - return RespCommand.DEL; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("3\r\nLCS\r\n"u8)) - { - return RespCommand.LCS; - } - - break; - - case 4: - switch ((ushort)ptr[4]) - { - case 'E': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nEVAL\r\n"u8)) - { - return RespCommand.EVAL; - } - break; - - case 'H': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHSET\r\n"u8)) - { - return RespCommand.HSET; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHGET\r\n"u8)) - { - return RespCommand.HGET; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHDEL\r\n"u8)) - { - return RespCommand.HDEL; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHLEN\r\n"u8)) - { - return RespCommand.HLEN; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHTTL\r\n"u8)) - { - return RespCommand.HTTL; - } - break; - - case 'K': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nKEYS\r\n"u8)) - { - return RespCommand.KEYS; - } - break; - - case 'L': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLPOP\r\n"u8)) - { - return RespCommand.LPOP; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLLEN\r\n"u8)) - { - return RespCommand.LLEN; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLREM\r\n"u8)) - { - return RespCommand.LREM; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLSET\r\n"u8)) - { - return RespCommand.LSET; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLPOS\r\n"u8)) - { - return RespCommand.LPOS; - } - break; - - case 'M': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nMGET\r\n"u8)) - { - return RespCommand.MGET; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nMSET\r\n"u8)) - { - return RespCommand.MSET; - } - break; - - case 'R': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nRPOP\r\n"u8)) - { - return RespCommand.RPOP; - } - break; - - case 'S': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nSCAN\r\n"u8)) - { - return RespCommand.SCAN; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nSADD\r\n"u8)) - { - return RespCommand.SADD; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nSREM\r\n"u8)) - { - return RespCommand.SREM; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nSPOP\r\n"u8)) - { - return RespCommand.SPOP; - } - break; - - case 'T': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nTYPE\r\n"u8)) - { - return RespCommand.TYPE; - } - break; - - case 'Z': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nZADD\r\n"u8)) - { - return RespCommand.ZADD; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nZREM\r\n"u8)) - { - return RespCommand.ZREM; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nZTTL\r\n"u8)) - { - return RespCommand.ZTTL; - } - break; - } - break; - - case 5: - switch ((ushort)ptr[4]) - { - case 'B': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nBITOP\r\n"u8)) - { - // Check for matching bit-operation - if (remainingBytes > length + 6 + 8) - { - // TODO: AND|OR|XOR|NOT|DIFF may not correctly handle mixed cases? - - var tag64 = *(ulong*)(ptr + 11); - var tag32 = (uint)tag64; - - if (tag32 == MemoryMarshal.Read("$2\r\n"u8)) - { - if (tag64 == MemoryMarshal.Read("$2\r\nOR\r\n"u8) || tag64 == MemoryMarshal.Read("$2\r\nor\r\n"u8)) - { - readHead += 8; // "$2\r\n" + "OR" + "\r\n" - count -= 1; - return RespCommand.BITOP_OR; - } - } - else if (tag32 == MemoryMarshal.Read("$3\r\n"u8) && remainingBytes > length + 6 + 9) - { - // Optimistically adjust - readHead += 9; // "$3\r\n" + AND|XOR|NOT + "\r\n" - count -= 1; - - tag64 = *(ulong*)(ptr + 12); - - if (tag64 == MemoryMarshal.Read("3\r\nAND\r\n"u8) || tag64 == MemoryMarshal.Read("3\r\nand\r\n"u8)) - { - return RespCommand.BITOP_AND; - } - else if (tag64 == MemoryMarshal.Read("3\r\nXOR\r\n"u8) || tag64 == MemoryMarshal.Read("3\r\nxor\r\n"u8)) - { - return RespCommand.BITOP_XOR; - } - else if (tag64 == MemoryMarshal.Read("3\r\nNOT\r\n"u8) || tag64 == MemoryMarshal.Read("3\r\nnot\r\n"u8)) - { - return RespCommand.BITOP_NOT; - } - - // Reset if no match - readHead -= 9; - count += 1; - } - else if (tag32 == MemoryMarshal.Read("$4\r\n"u8) && remainingBytes > length + 6 + 10) - { - // Optimistically adjust - readHead += 10; // "$4\r\nDIFF\r\n" - count -= 1; - - tag64 = *(ulong*)(ptr + 12); - - // Compare first 8 bytes then the trailing '\n' for "4\r\nDIFF\r\n" - if ((*(ulong*)(ptr + 12) == MemoryMarshal.Read("4\r\nDIFF\r"u8) || - *(ulong*)(ptr + 12) == MemoryMarshal.Read("4\r\ndiff\r"u8)) && - *(ptr + 20) == (byte)'\n') - { - return RespCommand.BITOP_DIFF; - } - - // Reset if no match - readHead -= 10; - count += 1; - } - - // Although we recognize BITOP, the pseudo-subcommand isn't recognized so fail early - specificErrorMessage = CmdStrings.RESP_SYNTAX_ERROR; - return RespCommand.NONE; - } - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nBRPOP\r\n"u8)) - { - return RespCommand.BRPOP; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nBLPOP\r\n"u8)) - { - return RespCommand.BLPOP; - } - break; - - case 'H': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHMSET\r\n"u8)) - { - return RespCommand.HMSET; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHMGET\r\n"u8)) - { - return RespCommand.HMGET; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHKEYS\r\n"u8)) - { - return RespCommand.HKEYS; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHVALS\r\n"u8)) - { - return RespCommand.HVALS; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHSCAN\r\n"u8)) - { - return RespCommand.HSCAN; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHPTTL\r\n"u8)) - { - return RespCommand.HPTTL; - } - break; - - case 'L': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nLPUSH\r\n"u8)) - { - return RespCommand.LPUSH; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nLTRIM\r\n"u8)) - { - return RespCommand.LTRIM; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nLMOVE\r\n"u8)) - { - return RespCommand.LMOVE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nLMPOP\r\n"u8)) - { - return RespCommand.LMPOP; - } - break; - - case 'P': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nPFADD\r\n"u8)) - { - return RespCommand.PFADD; - } - break; - - case 'R': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nRPUSH\r\n"u8)) - { - return RespCommand.RPUSH; - } - break; - - case 'S': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nSCARD\r\n"u8)) - { - return RespCommand.SCARD; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nSSCAN\r\n"u8)) - { - return RespCommand.SSCAN; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nSMOVE\r\n"u8)) - { - return RespCommand.SMOVE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nSDIFF\r\n"u8)) - { - return RespCommand.SDIFF; - } - break; - - case 'W': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nWATCH\r\n"u8)) - { - return RespCommand.WATCH; - } - break; - - case 'Z': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZCARD\r\n"u8)) - { - return RespCommand.ZCARD; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZRANK\r\n"u8)) - { - return RespCommand.ZRANK; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZDIFF\r\n"u8)) - { - return RespCommand.ZDIFF; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZSCAN\r\n"u8)) - { - return RespCommand.ZSCAN; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZMPOP\r\n"u8)) - { - return RespCommand.ZMPOP; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZPTTL\r\n"u8)) - { - return RespCommand.ZPTTL; - } - break; - } - break; - - case 6: - switch ((ushort)ptr[4]) - { - case 'B': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BLMOVE\r\n"u8)) - { - return RespCommand.BLMOVE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BLMPOP\r\n"u8)) - { - return RespCommand.BLMPOP; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BZMPOP\r\n"u8)) - { - return RespCommand.BZMPOP; - } - break; - case 'D': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("DBSIZE\r\n"u8)) - { - return RespCommand.DBSIZE; - } - break; - - case 'E': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("EXISTS\r\n"u8)) - { - return RespCommand.EXISTS; - } - break; - - case 'G': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEOADD\r\n"u8)) - { - return RespCommand.GEOADD; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEOPOS\r\n"u8)) - { - return RespCommand.GEOPOS; - } - break; - - case 'H': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HSETNX\r\n"u8)) - { - return RespCommand.HSETNX; - } - break; - - case 'L': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("LPUSHX\r\n"u8)) - { - return RespCommand.LPUSHX; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("LRANGE\r\n"u8)) - { - return RespCommand.LRANGE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("LINDEX\r\n"u8)) - { - return RespCommand.LINDEX; - } - break; - - case 'M': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("MSETNX\r\n"u8)) - { - return RespCommand.MSETNX; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("MEMORY\r\n"u8)) - { - // MEMORY USAGE - // 11 = "$5\r\nUSAGE\r\n".Length - if (remainingBytes >= length + 11) - { - if (*(ulong*)(ptr + 12) == MemoryMarshal.Read("$5\r\nUSAG"u8) && *(ulong*)(ptr + 15) == MemoryMarshal.Read("\nUSAGE\r\n"u8)) - { - count--; - readHead += 11; - return RespCommand.MEMORY_USAGE; - } - } - } - break; - - case 'R': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("RPUSHX\r\n"u8)) - { - return RespCommand.RPUSHX; - } - break; - - case 'S': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SELECT\r\n"u8)) - { - return RespCommand.SELECT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("STRLEN\r\n"u8)) - { - return RespCommand.STRLEN; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SUNION\r\n"u8)) - { - return RespCommand.SUNION; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SINTER\r\n"u8)) - { - return RespCommand.SINTER; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SCRIPT\r\n"u8)) - { - // SCRIPT EXISTS => "$6\r\nEXISTS\r\n".Length == 12 - // SCRIPT FLUSH => "$5\r\nFLUSH\r\n".Length == 11 - // SCRIPT LOAD => "$4\r\nLOAD\r\n".Length == 10 - - if (remainingBytes >= length + 10) - { - if (*(ulong*)(ptr + 4 + 8) == MemoryMarshal.Read("$4\r\nLOAD"u8) && *(ulong*)(ptr + 4 + 8 + 2) == MemoryMarshal.Read("\r\nLOAD\r\n"u8)) - { - count--; - readHead += 10; - return RespCommand.SCRIPT_LOAD; - } - - if (remainingBytes >= length + 11) - { - if (*(ulong*)(ptr + 4 + 8) == MemoryMarshal.Read("$5\r\nFLUS"u8) && *(ulong*)(ptr + 4 + 8 + 3) == MemoryMarshal.Read("\nFLUSH\r\n"u8)) - { - count--; - readHead += 11; - return RespCommand.SCRIPT_FLUSH; - } - - if (remainingBytes >= length + 12) - { - if (*(ulong*)(ptr + 4 + 8) == MemoryMarshal.Read("$6\r\nEXIS"u8) && *(ulong*)(ptr + 4 + 8 + 4) == MemoryMarshal.Read("EXISTS\r\n"u8)) - { - count--; - readHead += 12; - return RespCommand.SCRIPT_EXISTS; - } - } - } - } - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SWAPDB\r\n"u8)) - { - return RespCommand.SWAPDB; - } - break; - - case 'U': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("UNLINK\r\n"u8)) - { - return RespCommand.UNLINK; - } - break; - - case 'Z': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZCOUNT\r\n"u8)) - { - return RespCommand.ZCOUNT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZRANGE\r\n"u8)) - { - return RespCommand.ZRANGE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZUNION\r\n"u8)) - { - return RespCommand.ZUNION; - } - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZSCORE\r\n"u8)) - { - return RespCommand.ZSCORE; - } - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZINTER\r\n"u8)) - { - return RespCommand.ZINTER; - } - break; - } - - break; - case 7: - switch ((ushort)ptr[4]) - { - case 'E': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nEVALSHA\r\n"u8)) - { - return RespCommand.EVALSHA; - } - break; - - case 'G': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEOHASH\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.GEOHASH; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEODIST\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.GEODIST; - } - break; - - case 'H': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HGETALL\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HGETALL; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HEXISTS\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HEXISTS; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HEXPIRE\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HEXPIRE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HINCRBY\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HINCRBY; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HSTRLEN\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HSTRLEN; - } - break; - - case 'L': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("LINSERT\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.LINSERT; - } - break; - - case 'M': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("MONITOR\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.MONITOR; - } - break; - - case 'P': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("PFCOUNT\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.PFCOUNT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("PFMERGE\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.PFMERGE; - } - break; - case 'W': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("WATCHMS\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.WATCHMS; - } - - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("WATCHOS\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.WATCHOS; - } - - break; - - case 'Z': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPOPMIN\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZPOPMIN; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZEXPIRE\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZEXPIRE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPOPMAX\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZPOPMAX; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZINCRBY\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZINCRBY; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZMSCORE\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZMSCORE; - } - break; - } - break; - case 8: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZREVRANK"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZREVRANK; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SMEMBERS"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.SMEMBERS; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BITFIELD"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.BITFIELD; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("EXPIREAT"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.EXPIREAT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HPEXPIRE"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.HPEXPIRE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HPERSIST"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.HPERSIST; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPEXPIRE"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZPEXPIRE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPERSIST"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZPERSIST; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BZPOPMAX"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.BZPOPMAX; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BZPOPMIN"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.BZPOPMIN; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SPUBLISH"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.SPUBLISH; - } - break; - case 9: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SUBSCRIB"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("BE\r\n"u8)) - { - return RespCommand.SUBSCRIBE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SISMEMBE"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("ER\r\n"u8)) - { - return RespCommand.SISMEMBER; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZLEXCOUN"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("NT\r\n"u8)) - { - return RespCommand.ZLEXCOUNT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEOSEARC"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("CH\r\n"u8)) - { - return RespCommand.GEOSEARCH; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZREVRANG"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("GE\r\n"u8)) - { - return RespCommand.ZREVRANGE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("RPOPLPUS"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("SH\r\n"u8)) - { - return RespCommand.RPOPLPUSH; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("PEXPIREA"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("AT\r\n"u8)) - { - return RespCommand.PEXPIREAT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HEXPIREA"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("AT\r\n"u8)) - { - return RespCommand.HEXPIREAT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZEXPIREA"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("AT\r\n"u8)) - { - return RespCommand.ZEXPIREAT; - } - break; - case 10: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SSUBSCRI"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("BE\r\n"u8)) - { - return RespCommand.SSUBSCRIBE; - } - break; - } - - // Reset optimistically changed state, if no matching command was found - count += 1; - readHead = oldReadHead; - } - } - // Check if this is a string with a double-digit length ("$__\r" -> _ omitted) - else if ((*(uint*)ptr & 0xFF0000FF) == MemoryMarshal.Read("$\0\0\r"u8)) - { - // Extract length from string header - var length = ptr[2] - '0' + 10; - - // Ensure that the complete command string is contained in the package. Otherwise exit early. - // Include 7 bytes to account for command string header and name terminator. - // 7 bytes = "$__\r\n" (5 bytes) + "\r\n" (2 bytes) at end of command name - if (remainingBytes >= length + 7) - { - // Optimistically increase read head and decrease the number of remaining elements - readHead += length + 7; - count -= 1; - - // Match remaining character by length - // NOTE: Check should include the remaining array length terminator '\n' - switch (length) - { - case 10: - if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nPSUB"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("SCRIBE\r\n"u8)) - { - return RespCommand.PSUBSCRIBE; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nHRAN"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("DFIELD\r\n"u8)) - { - return RespCommand.HRANDFIELD; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nSDIF"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("FSTORE\r\n"u8)) - { - return RespCommand.SDIFFSTORE; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nEXPI"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.EXPIRETIME; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nSMIS"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("MEMBER\r\n"u8)) - { - return RespCommand.SMISMEMBER; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nSINT"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("ERCARD\r\n"u8)) - { - return RespCommand.SINTERCARD; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nZDIF"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("FSTORE\r\n"u8)) - { - return RespCommand.ZDIFFSTORE; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nBRPO"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("PLPUSH\r\n"u8)) - { - return RespCommand.BRPOPLPUSH; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nZINT"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("ERCARD\r\n"u8)) - { - return RespCommand.ZINTERCARD; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nHPEX"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("PIREAT\r\n"u8)) - { - return RespCommand.HPEXPIREAT; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nZPEX"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("PIREAT\r\n"u8)) - { - return RespCommand.ZPEXPIREAT; - } - break; - case 11: - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nUNSUB"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("SCRIBE\r\n"u8)) - { - return RespCommand.UNSUBSCRIBE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZRAND"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("MEMBER\r\n"u8)) - { - return RespCommand.ZRANDMEMBER; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nBITFI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("ELD_RO\r\n"u8)) - { - return RespCommand.BITFIELD_RO; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nSRAND"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("MEMBER\r\n"u8)) - { - return RespCommand.SRANDMEMBER; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nSUNIO"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("NSTORE\r\n"u8)) - { - return RespCommand.SUNIONSTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nSINTE"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RSTORE\r\n"u8)) - { - return RespCommand.SINTERSTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nPEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.PEXPIRETIME; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nHEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.HEXPIRETIME; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nINCRB"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("YFLOAT\r\n"u8)) - { - return RespCommand.INCRBYFLOAT; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZRANG"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("ESTORE\r\n"u8)) - { - return RespCommand.ZRANGESTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZRANG"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("EBYLEX\r\n"u8)) - { - return RespCommand.ZRANGEBYLEX; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZINTE"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RSTORE\r\n"u8)) - { - return RespCommand.ZINTERSTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZUNIO"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("NSTORE\r\n"u8)) - { - return RespCommand.ZUNIONSTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.ZEXPIRETIME; - } - break; - - case 12: - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nPUNSUB"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("SCRIBE\r\n"u8)) - { - return RespCommand.PUNSUBSCRIBE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nHINCRB"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("YFLOAT\r\n"u8)) - { - return RespCommand.HINCRBYFLOAT; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nHPEXPI"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.HPEXPIRETIME; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZPEXPI"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.ZPEXPIRETIME; - } - break; - - case 13: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("\nZRANGEB"u8) && *(ulong*)(ptr + 12) == MemoryMarshal.Read("YSCORE\r\n"u8)) - { - return RespCommand.ZRANGEBYSCORE; - } - break; - - case 14: - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZREMRA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("NGEBYLEX"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZREMRANGEBYLEX; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nGEOSEA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("RCHSTORE"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.GEOSEARCHSTORE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZREVRA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("NGEBYLEX"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZREVRANGEBYLEX; - } - break; - - case 15: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("\nZREMRAN"u8) && *(ulong*)(ptr + 12) == MemoryMarshal.Read("GEBYRANK"u8) && *(ushort*)(ptr + 20) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZREMRANGEBYRANK; - } - break; - - case 16: - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nCUSTOM"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("OBJECTSC"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("AN\r\n"u8)) - { - return RespCommand.COSCAN; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZREMRA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("NGEBYSCO"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("RE\r\n"u8)) - { - return RespCommand.ZREMRANGEBYSCORE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZREVRA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("NGEBYSCO"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("RE\r\n"u8)) - { - return RespCommand.ZREVRANGEBYSCORE; - } - break; - } - - // Reset optimistically changed state, if no matching command was found - count += 1; - readHead = oldReadHead; - } - } - } - - // No matching command name found in this pass - return RespCommand.NONE; - } - private bool TryParseCustomCommand(ReadOnlySpan command, out RespCommand cmd) { if (customCommandManagerSession.Match(command, out currentCustomTransaction)) @@ -1918,821 +958,6 @@ private bool TryParseCustomCommand(ReadOnlySpan command, out RespCommand c return false; } - /// - /// Parses the receive buffer, starting from the current read head, for all command names that are - /// not covered by FastParseArrayCommand() and advances the read head to the end of the command name. - /// - /// NOTE: Assumes the input command names have already been converted to upper-case. - /// - /// Reference to the number of remaining tokens in the packet. Will be reduced to number of command arguments. - /// If the command could not be parsed, will be non-empty if a specific error message should be returned. - /// True if the input RESP string was completely included in the buffer, false if we couldn't read the full command name. - /// The parsed command name. - private RespCommand SlowParseCommand(ref int count, ref ReadOnlySpan specificErrorMsg, out bool success) - { - // Try to extract the current string from the front of the read head - var command = GetCommand(out success); - - if (!success) - { - return RespCommand.INVALID; - } - - // Account for the command name being taken off the read head - count -= 1; - - if (TryParseCustomCommand(command, out var cmd)) - { - return cmd; - } - else - { - return SlowParseCommand(command, ref count, ref specificErrorMsg, out success); - } - } - - private RespCommand SlowParseCommand(ReadOnlySpan command, ref int count, ref ReadOnlySpan specificErrorMsg, out bool success) - { - success = true; - if (command.SequenceEqual(CmdStrings.SUBSCRIBE)) - { - return RespCommand.SUBSCRIBE; - } - else if (command.SequenceEqual(CmdStrings.SSUBSCRIBE)) - { - return RespCommand.SSUBSCRIBE; - } - else if (command.SequenceEqual(CmdStrings.RUNTXP)) - { - return RespCommand.RUNTXP; - } - else if (command.SequenceEqual(CmdStrings.SCRIPT)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.SCRIPT))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.LOAD)) - { - return RespCommand.SCRIPT_LOAD; - } - - if (subCommand.SequenceEqual(CmdStrings.FLUSH)) - { - return RespCommand.SCRIPT_FLUSH; - } - - if (subCommand.SequenceEqual(CmdStrings.EXISTS)) - { - return RespCommand.SCRIPT_EXISTS; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.SCRIPT)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.ECHO)) - { - return RespCommand.ECHO; - } - else if (command.SequenceEqual(CmdStrings.GEORADIUS)) - { - return RespCommand.GEORADIUS; - } - else if (command.SequenceEqual(CmdStrings.GEORADIUS_RO)) - { - return RespCommand.GEORADIUS_RO; - } - else if (command.SequenceEqual(CmdStrings.GEORADIUSBYMEMBER)) - { - return RespCommand.GEORADIUSBYMEMBER; - } - else if (command.SequenceEqual(CmdStrings.GEORADIUSBYMEMBER_RO)) - { - return RespCommand.GEORADIUSBYMEMBER_RO; - } - else if (command.SequenceEqual(CmdStrings.REPLICAOF)) - { - return RespCommand.REPLICAOF; - } - else if (command.SequenceEqual(CmdStrings.SECONDARYOF) || command.SequenceEqual(CmdStrings.SLAVEOF)) - { - return RespCommand.SECONDARYOF; - } - else if (command.SequenceEqual(CmdStrings.CONFIG)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.CONFIG))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.GET)) - { - return RespCommand.CONFIG_GET; - } - else if (subCommand.SequenceEqual(CmdStrings.REWRITE)) - { - return RespCommand.CONFIG_REWRITE; - } - else if (subCommand.SequenceEqual(CmdStrings.SET)) - { - return RespCommand.CONFIG_SET; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.CONFIG)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.CLIENT)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.CLIENT))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.ID)) - { - return RespCommand.CLIENT_ID; - } - else if (subCommand.SequenceEqual(CmdStrings.INFO)) - { - return RespCommand.CLIENT_INFO; - } - else if (subCommand.SequenceEqual(CmdStrings.LIST)) - { - return RespCommand.CLIENT_LIST; - } - else if (subCommand.SequenceEqual(CmdStrings.KILL)) - { - return RespCommand.CLIENT_KILL; - } - else if (subCommand.SequenceEqual(CmdStrings.GETNAME)) - { - return RespCommand.CLIENT_GETNAME; - } - else if (subCommand.SequenceEqual(CmdStrings.SETNAME)) - { - return RespCommand.CLIENT_SETNAME; - } - else if (subCommand.SequenceEqual(CmdStrings.SETINFO)) - { - return RespCommand.CLIENT_SETINFO; - } - else if (subCommand.SequenceEqual(CmdStrings.UNBLOCK)) - { - return RespCommand.CLIENT_UNBLOCK; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.CLIENT)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.AUTH)) - { - return RespCommand.AUTH; - } - else if (command.SequenceEqual(CmdStrings.INFO)) - { - return RespCommand.INFO; - } - else if (command.SequenceEqual(CmdStrings.ROLE)) - { - return RespCommand.ROLE; - } - else if (command.SequenceEqual(CmdStrings.COMMAND)) - { - if (count == 0) - { - return RespCommand.COMMAND; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.COUNT)) - { - return RespCommand.COMMAND_COUNT; - } - - if (subCommand.SequenceEqual(CmdStrings.INFO)) - { - return RespCommand.COMMAND_INFO; - } - - if (subCommand.SequenceEqual(CmdStrings.DOCS)) - { - return RespCommand.COMMAND_DOCS; - } - - if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.GETKEYS)) - { - return RespCommand.COMMAND_GETKEYS; - } - - if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.GETKEYSANDFLAGS)) - { - return RespCommand.COMMAND_GETKEYSANDFLAGS; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.COMMAND)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.PING)) - { - return RespCommand.PING; - } - else if (command.SequenceEqual(CmdStrings.HELLO)) - { - return RespCommand.HELLO; - } - else if (command.SequenceEqual(CmdStrings.CLUSTER)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.CLUSTER))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.BUMPEPOCH)) - { - return RespCommand.CLUSTER_BUMPEPOCH; - } - else if (subCommand.SequenceEqual(CmdStrings.FORGET)) - { - return RespCommand.CLUSTER_FORGET; - } - else if (subCommand.SequenceEqual(CmdStrings.gossip)) - { - return RespCommand.CLUSTER_GOSSIP; - } - else if (subCommand.SequenceEqual(CmdStrings.INFO)) - { - return RespCommand.CLUSTER_INFO; - } - else if (subCommand.SequenceEqual(CmdStrings.MEET)) - { - return RespCommand.CLUSTER_MEET; - } - else if (subCommand.SequenceEqual(CmdStrings.MYID)) - { - return RespCommand.CLUSTER_MYID; - } - else if (subCommand.SequenceEqual(CmdStrings.myparentid)) - { - return RespCommand.CLUSTER_MYPARENTID; - } - else if (subCommand.SequenceEqual(CmdStrings.NODES)) - { - return RespCommand.CLUSTER_NODES; - } - else if (subCommand.SequenceEqual(CmdStrings.SHARDS)) - { - return RespCommand.CLUSTER_SHARDS; - } - else if (subCommand.SequenceEqual(CmdStrings.RESET)) - { - return RespCommand.CLUSTER_RESET; - } - else if (subCommand.SequenceEqual(CmdStrings.FAILOVER)) - { - return RespCommand.CLUSTER_FAILOVER; - } - else if (subCommand.SequenceEqual(CmdStrings.ADDSLOTS)) - { - return RespCommand.CLUSTER_ADDSLOTS; - } - else if (subCommand.SequenceEqual(CmdStrings.ADDSLOTSRANGE)) - { - return RespCommand.CLUSTER_ADDSLOTSRANGE; - } - else if (subCommand.SequenceEqual(CmdStrings.COUNTKEYSINSLOT)) - { - return RespCommand.CLUSTER_COUNTKEYSINSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.DELSLOTS)) - { - return RespCommand.CLUSTER_DELSLOTS; - } - else if (subCommand.SequenceEqual(CmdStrings.DELSLOTSRANGE)) - { - return RespCommand.CLUSTER_DELSLOTSRANGE; - } - else if (subCommand.SequenceEqual(CmdStrings.GETKEYSINSLOT)) - { - return RespCommand.CLUSTER_GETKEYSINSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.HELP)) - { - return RespCommand.CLUSTER_HELP; - } - else if (subCommand.SequenceEqual(CmdStrings.KEYSLOT)) - { - return RespCommand.CLUSTER_KEYSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.SETSLOT)) - { - return RespCommand.CLUSTER_SETSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.SLOTS)) - { - return RespCommand.CLUSTER_SLOTS; - } - else if (subCommand.SequenceEqual(CmdStrings.REPLICAS)) - { - return RespCommand.CLUSTER_REPLICAS; - } - else if (subCommand.SequenceEqual(CmdStrings.REPLICATE)) - { - return RespCommand.CLUSTER_REPLICATE; - } - else if (subCommand.SequenceEqual(CmdStrings.delkeysinslot)) - { - return RespCommand.CLUSTER_DELKEYSINSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.delkeysinslotrange)) - { - return RespCommand.CLUSTER_DELKEYSINSLOTRANGE; - } - else if (subCommand.SequenceEqual(CmdStrings.setslotsrange)) - { - return RespCommand.CLUSTER_SETSLOTSRANGE; - } - else if (subCommand.SequenceEqual(CmdStrings.slotstate)) - { - return RespCommand.CLUSTER_SLOTSTATE; - } - else if (subCommand.SequenceEqual(CmdStrings.publish)) - { - return RespCommand.CLUSTER_PUBLISH; - } - else if (subCommand.SequenceEqual(CmdStrings.spublish)) - { - return RespCommand.CLUSTER_SPUBLISH; - } - else if (subCommand.SequenceEqual(CmdStrings.MIGRATE)) - { - return RespCommand.CLUSTER_MIGRATE; - } - else if (subCommand.SequenceEqual(CmdStrings.mtasks)) - { - return RespCommand.CLUSTER_MTASKS; - } - else if (subCommand.SequenceEqual(CmdStrings.aofsync)) - { - return RespCommand.CLUSTER_AOFSYNC; - } - else if (subCommand.SequenceEqual(CmdStrings.appendlog)) - { - return RespCommand.CLUSTER_APPENDLOG; - } - else if (subCommand.SequenceEqual(CmdStrings.attach_sync)) - { - return RespCommand.CLUSTER_ATTACH_SYNC; - } - else if (subCommand.SequenceEqual(CmdStrings.banlist)) - { - return RespCommand.CLUSTER_BANLIST; - } - else if (subCommand.SequenceEqual(CmdStrings.begin_replica_recover)) - { - return RespCommand.CLUSTER_BEGIN_REPLICA_RECOVER; - } - else if (subCommand.SequenceEqual(CmdStrings.endpoint)) - { - return RespCommand.CLUSTER_ENDPOINT; - } - else if (subCommand.SequenceEqual(CmdStrings.failreplicationoffset)) - { - return RespCommand.CLUSTER_FAILREPLICATIONOFFSET; - } - else if (subCommand.SequenceEqual(CmdStrings.failstopwrites)) - { - return RespCommand.CLUSTER_FAILSTOPWRITES; - } - else if (subCommand.SequenceEqual(CmdStrings.FLUSHALL)) - { - return RespCommand.CLUSTER_FLUSHALL; - } - else if (subCommand.SequenceEqual(CmdStrings.SETCONFIGEPOCH)) - { - return RespCommand.CLUSTER_SETCONFIGEPOCH; - } - else if (subCommand.SequenceEqual(CmdStrings.initiate_replica_sync)) - { - return RespCommand.CLUSTER_INITIATE_REPLICA_SYNC; - } - else if (subCommand.SequenceEqual(CmdStrings.send_ckpt_file_segment)) - { - return RespCommand.CLUSTER_SEND_CKPT_FILE_SEGMENT; - } - else if (subCommand.SequenceEqual(CmdStrings.send_ckpt_metadata)) - { - return RespCommand.CLUSTER_SEND_CKPT_METADATA; - } - else if (subCommand.SequenceEqual(CmdStrings.cluster_sync)) - { - return RespCommand.CLUSTER_SYNC; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommand, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.CLUSTER)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.LATENCY)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.LATENCY))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.HELP)) - { - return RespCommand.LATENCY_HELP; - } - else if (subCommand.SequenceEqual(CmdStrings.HISTOGRAM)) - { - return RespCommand.LATENCY_HISTOGRAM; - } - else if (subCommand.SequenceEqual(CmdStrings.RESET)) - { - return RespCommand.LATENCY_RESET; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommand, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.LATENCY)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.SLOWLOG)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.SLOWLOG))); - } - else if (count >= 1) - { - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.HELP)) - { - return RespCommand.SLOWLOG_HELP; - } - else if (subCommand.SequenceEqual(CmdStrings.GET)) - { - return RespCommand.SLOWLOG_GET; - } - else if (subCommand.SequenceEqual(CmdStrings.LEN)) - { - return RespCommand.SLOWLOG_LEN; - } - else if (subCommand.SequenceEqual(CmdStrings.RESET)) - { - return RespCommand.SLOWLOG_RESET; - } - } - } - else if (command.SequenceEqual(CmdStrings.TIME)) - { - return RespCommand.TIME; - } - else if (command.SequenceEqual(CmdStrings.QUIT)) - { - return RespCommand.QUIT; - } - else if (command.SequenceEqual(CmdStrings.SAVE)) - { - return RespCommand.SAVE; - } - else if (command.SequenceEqual(CmdStrings.EXPDELSCAN)) - { - return RespCommand.EXPDELSCAN; - } - else if (command.SequenceEqual(CmdStrings.LASTSAVE)) - { - return RespCommand.LASTSAVE; - } - else if (command.SequenceEqual(CmdStrings.BGSAVE)) - { - return RespCommand.BGSAVE; - } - else if (command.SequenceEqual(CmdStrings.COMMITAOF)) - { - return RespCommand.COMMITAOF; - } - else if (command.SequenceEqual(CmdStrings.FLUSHALL)) - { - return RespCommand.FLUSHALL; - } - else if (command.SequenceEqual(CmdStrings.FLUSHDB)) - { - return RespCommand.FLUSHDB; - } - else if (command.SequenceEqual(CmdStrings.FORCEGC)) - { - return RespCommand.FORCEGC; - } - else if (command.SequenceEqual(CmdStrings.MIGRATE)) - { - return RespCommand.MIGRATE; - } - else if (command.SequenceEqual(CmdStrings.PURGEBP)) - { - return RespCommand.PURGEBP; - } - else if (command.SequenceEqual(CmdStrings.FAILOVER)) - { - return RespCommand.FAILOVER; - } - else if (command.SequenceEqual(CmdStrings.MEMORY)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.MEMORY))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.USAGE)) - { - return RespCommand.MEMORY_USAGE; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.MEMORY)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.MONITOR)) - { - return RespCommand.MONITOR; - } - else if (command.SequenceEqual(CmdStrings.ACL)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.ACL))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.CAT)) - { - return RespCommand.ACL_CAT; - } - else if (subCommand.SequenceEqual(CmdStrings.DELUSER)) - { - return RespCommand.ACL_DELUSER; - } - else if (subCommand.SequenceEqual(CmdStrings.GENPASS)) - { - return RespCommand.ACL_GENPASS; - } - else if (subCommand.SequenceEqual(CmdStrings.GETUSER)) - { - return RespCommand.ACL_GETUSER; - } - else if (subCommand.SequenceEqual(CmdStrings.LIST)) - { - return RespCommand.ACL_LIST; - } - else if (subCommand.SequenceEqual(CmdStrings.LOAD)) - { - return RespCommand.ACL_LOAD; - } - else if (subCommand.SequenceEqual(CmdStrings.SAVE)) - { - return RespCommand.ACL_SAVE; - } - else if (subCommand.SequenceEqual(CmdStrings.SETUSER)) - { - return RespCommand.ACL_SETUSER; - } - else if (subCommand.SequenceEqual(CmdStrings.USERS)) - { - return RespCommand.ACL_USERS; - } - else if (subCommand.SequenceEqual(CmdStrings.WHOAMI)) - { - return RespCommand.ACL_WHOAMI; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.ACL)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.REGISTERCS)) - { - return RespCommand.REGISTERCS; - } - else if (command.SequenceEqual(CmdStrings.ASYNC)) - { - return RespCommand.ASYNC; - } - else if (command.SequenceEqual(CmdStrings.MODULE)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.MODULE))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.LOADCS)) - { - return RespCommand.MODULE_LOADCS; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.MODULE)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.PUBSUB)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.PUBSUB))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.CHANNELS)) - { - return RespCommand.PUBSUB_CHANNELS; - } - else if (subCommand.SequenceEqual(CmdStrings.NUMSUB)) - { - return RespCommand.PUBSUB_NUMSUB; - } - else if (subCommand.SequenceEqual(CmdStrings.NUMPAT)) - { - return RespCommand.PUBSUB_NUMPAT; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.PUBSUB)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.HCOLLECT)) - { - return RespCommand.HCOLLECT; - } - else if (command.SequenceEqual(CmdStrings.DEBUG)) - { - return RespCommand.DEBUG; - } - else if (command.SequenceEqual(CmdStrings.ZCOLLECT)) - { - return RespCommand.ZCOLLECT; - } - // Note: The commands below are not slow path commands, so they should probably move to earlier. - else if (command.SequenceEqual(CmdStrings.SETIFMATCH)) - { - return RespCommand.SETIFMATCH; - } - else if (command.SequenceEqual(CmdStrings.SETIFGREATER)) - { - return RespCommand.SETIFGREATER; - } - else if (command.SequenceEqual(CmdStrings.GETWITHETAG)) - { - return RespCommand.GETWITHETAG; - } - else if (command.SequenceEqual(CmdStrings.GETIFNOTMATCH)) - { - return RespCommand.GETIFNOTMATCH; - } - else if (command.SequenceEqual(CmdStrings.DELIFGREATER)) - { - return RespCommand.DELIFGREATER; - } - - // If this command name was not known to the slow pass, we are out of options and the command is unknown. - return RespCommand.INVALID; - } /// /// Attempts to skip to the end of the line ("\r\n") under the current read head. @@ -2873,7 +1098,7 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) cmd = ArrayParseCommand(writeErrorOnFailure, ref count, ref success); if (!success) return cmd; - // Update MRU cache for commands resolved by scalar switch or hash table. + // Update MRU cache for commands resolved by hash table. // Exclude custom commands — they have runtime-registered names. if (Vector128.IsHardwareAccelerated && cmd != RespCommand.INVALID && cmd != RespCommand.NONE && @@ -2884,6 +1109,9 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) } } + Debug.Assert(cmd != RespCommand.NONE, "ParseCommand should never return NONE - expected INVALID for unrecognized commands"); + Debug.Assert(count >= 0 || cmd == RespCommand.INVALID, "Argument count must be non-negative for valid commands"); + // Set up parse state parseState.Initialize(count); var ptr = recvBufferPtr + readHead; @@ -2904,7 +1132,7 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) } /// - /// Update the MRU command cache with a newly matched command from the scalar switch or hash table. + /// Update the MRU command cache with a newly matched command from the hash table. /// Captures the first 16 bytes of the RESP encoding and the appropriate mask so that /// the cache check in FastParseCommand can match it via Vector128.EqualsAll. /// @@ -3032,7 +1260,6 @@ private RespCommand ArrayParseCommand(bool writeErrorOnFailure, ref int count, r /// /// Hash-based command lookup. Extracts the command name from the RESP buffer, /// looks it up in the static hash table, and handles subcommand dispatch. - /// Replaces the former FastParseArrayCommand + SlowParseCommand chain. /// [MethodImpl(MethodImplOptions.NoInlining)] private RespCommand HashLookupCommand(ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) @@ -3065,14 +1292,9 @@ private RespCommand HashLookupCommand(ref int count, ref ReadOnlySpan spec return customCmd; } - // Fall back to the old slow parser for commands not in the hash table - return SlowParseCommand(command, ref count, ref specificErrorMessage, out success); - } - - // BITOP has pseudo-subcommands (AND/OR/XOR/NOT/DIFF) that must be parsed inline - if (cmd == RespCommand.BITOP) - { - return ParseBitopSubcommand(ref count, ref specificErrorMessage, out success); + // Not a built-in or custom command + Debug.Assert(cmd == RespCommand.NONE, "Hash table lookup returned unexpected value"); + return RespCommand.INVALID; } // Commands with subcommands — dispatch via subcommand hash table @@ -3087,7 +1309,7 @@ private RespCommand HashLookupCommand(ref int count, ref ReadOnlySpan spec /// /// Handles subcommand dispatch for parent commands (CLUSTER, CONFIG, CLIENT, etc.) - /// using per-parent hash tables. Replaces the former SlowParseCommand subcommand chains. + /// using per-parent hash tables. /// [MethodImpl(MethodImplOptions.NoInlining)] private RespCommand HandleSubcommandLookup(RespCommand parentCmd, ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) @@ -3103,8 +1325,9 @@ private RespCommand HandleSubcommandLookup(RespCommand parentCmd, ref int count, // Most parent commands require at least one subcommand argument if (count == 0) { - specificErrorMessage = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - parentCmd.ToString())); + specificErrorMessage = parentCmd == RespCommand.BITOP + ? CmdStrings.RESP_SYNTAX_ERROR + : Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, parentCmd.ToString())); return RespCommand.INVALID; } @@ -3128,67 +1351,17 @@ private RespCommand HandleSubcommandLookup(RespCommand parentCmd, ref int count, } } - // Hash miss — try case-insensitive fallbacks for specific commands - if (parentCmd == RespCommand.COMMAND) - { - if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.GETKEYS)) - return RespCommand.COMMAND_GETKEYS; - if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.GETKEYSANDFLAGS)) - return RespCommand.COMMAND_GETKEYSANDFLAGS; - } - else if (parentCmd == RespCommand.MEMORY) - { - if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.USAGE)) - return RespCommand.MEMORY_USAGE; - } - // Generate error message for unknown subcommand - string errMsg = parentCmd switch + specificErrorMessage = parentCmd switch { + RespCommand.BITOP => CmdStrings.RESP_SYNTAX_ERROR, RespCommand.CLUSTER or RespCommand.LATENCY => - string.Format(CmdStrings.GenericErrUnknownSubCommand, - Encoding.UTF8.GetString(subCommand), parentCmd.ToString()), + Encoding.UTF8.GetBytes(string.Format(CmdStrings.GenericErrUnknownSubCommand, + Encoding.UTF8.GetString(subCommand), parentCmd.ToString())), _ => - string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), parentCmd.ToString()), + Encoding.UTF8.GetBytes(string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, + Encoding.UTF8.GetString(subCommand), parentCmd.ToString())), }; - specificErrorMessage = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - - /// - /// Parse the BITOP pseudo-subcommand (AND/OR/XOR/NOT/DIFF). - /// Called after the primary hash lookup identifies BITOP. - /// - [MethodImpl(MethodImplOptions.NoInlining)] - private RespCommand ParseBitopSubcommand(ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) - { - if (count == 0) - { - specificErrorMessage = CmdStrings.RESP_SYNTAX_ERROR; - success = true; - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out success); - if (!success) - { - return RespCommand.NONE; - } - - count--; - - fixed (byte* subPtr = subCommand) - { - var subCmd = RespCommandHashLookup.LookupSubcommand(RespCommand.BITOP, subPtr, subCommand.Length); - if (subCmd != RespCommand.NONE) - { - return subCmd; - } - } - - // Unrecognized BITOP operation - specificErrorMessage = CmdStrings.RESP_SYNTAX_ERROR; return RespCommand.INVALID; } } diff --git a/libs/server/Resp/Parser/RespCommandHashLookup.cs b/libs/server/Resp/Parser/RespCommandHashLookup.cs index b7639034b5a..96bd2296a8b 100644 --- a/libs/server/Resp/Parser/RespCommandHashLookup.cs +++ b/libs/server/Resp/Parser/RespCommandHashLookup.cs @@ -515,7 +515,7 @@ void Add(string name, RespCommand cmd, bool hasSub = false) Add("BITPOS", RespCommand.BITPOS); Add("BITFIELD", RespCommand.BITFIELD); Add("BITFIELD_RO", RespCommand.BITFIELD_RO); - Add("BITOP", RespCommand.BITOP); + Add("BITOP", RespCommand.BITOP, hasSub: true); // HyperLogLog commands Add("PFADD", RespCommand.PFADD); From 56d8f3bfc2f5e07991376cc0bc309dcd232e8229 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Wed, 1 Apr 2026 09:26:17 -0700 Subject: [PATCH 13/19] code review updates --- libs/server/Lua/LuaTimeoutManager.cs | 2 +- libs/server/Resp/Parser/RespCommand.cs | 12 +- .../Resp/Parser/RespCommandHashLookup.cs | 88 ++++- .../cs/src/core/Allocator/AllocatorScan.cs | 2 +- .../Implementation/ContinuePending.cs | 2 +- playground/Bitmap/BitOp.cs | 12 +- website/docs/dev/fast-parsing-plan.md | 316 ------------------ 7 files changed, 101 insertions(+), 333 deletions(-) delete mode 100644 website/docs/dev/fast-parsing-plan.md diff --git a/libs/server/Lua/LuaTimeoutManager.cs b/libs/server/Lua/LuaTimeoutManager.cs index d7f8542498d..bccb2983fba 100644 --- a/libs/server/Lua/LuaTimeoutManager.cs +++ b/libs/server/Lua/LuaTimeoutManager.cs @@ -263,7 +263,7 @@ internal Registration RegisterForTimeout(SessionScriptCache cache) goto tryAgain; } - // Other threads might update registrations, so check that before returning + // Other threads might update registrations, so check that before returning checkUnmodified: if ((updatedRegistrations = Interlocked.CompareExchange(ref registrations, curRegistrations, curRegistrations)) != curRegistrations) { diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 48a33a078c6..f8824f5c568 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -1236,7 +1236,7 @@ private RespCommand ArrayParseCommand(bool writeErrorOnFailure, ref int count, r // Move readHead to start of command payload readHead = (int)(ptr - recvBufferPtr); - // Extract command name via GetUpperCaseCommand (reads $len\r\n header, uppercases, advances readHead) + // Extract command name via HashLookupCommand (reads $len\r\n header via GetCommand, relies on prior MakeUpperCase() pass, and advances readHead) cmd = HashLookupCommand(ref count, ref specificErrorMessage, out success); // Parsing for command name was successful, but the command is unknown @@ -1279,26 +1279,26 @@ private RespCommand HashLookupCommand(ref int count, ref ReadOnlySpan spec count -= 1; // Hash table lookup for the primary command (checked before custom commands - // since built-in commands are far more common) + // since built-in commands are far more common). + // Single probe returns both the command and whether it has subcommands. fixed (byte* namePtr = command) { - var cmd = RespCommandHashLookup.Lookup(namePtr, command.Length); + var cmd = RespCommandHashLookup.Lookup(namePtr, command.Length, out var hasSubcommands); if (cmd == RespCommand.NONE) { - // Not a built-in command — check custom commands before falling to slow path + // Not a built-in command — check custom commands if (TryParseCustomCommand(command, out var customCmd)) { return customCmd; } // Not a built-in or custom command - Debug.Assert(cmd == RespCommand.NONE, "Hash table lookup returned unexpected value"); return RespCommand.INVALID; } // Commands with subcommands — dispatch via subcommand hash table - if (RespCommandHashLookup.HasSubcommands(namePtr, command.Length)) + if (hasSubcommands) { return HandleSubcommandLookup(cmd, ref count, ref specificErrorMessage, out success); } diff --git a/libs/server/Resp/Parser/RespCommandHashLookup.cs b/libs/server/Resp/Parser/RespCommandHashLookup.cs index 96bd2296a8b..0f819e99bda 100644 --- a/libs/server/Resp/Parser/RespCommandHashLookup.cs +++ b/libs/server/Resp/Parser/RespCommandHashLookup.cs @@ -135,6 +135,60 @@ static RespCommandHashLookup() ValidateSubTable(RespCommand.PUBSUB, PubsubSubcommands, pubsubSubTable, pubsubSubTableMask); ValidateSubTable(RespCommand.MEMORY, MemorySubcommands, memorySubTable, memorySubTableMask); ValidateSubTable(RespCommand.BITOP, BitopSubcommands, bitopSubTable, bitopSubTableMask); + + // Validate primary table: every inserted command must round-trip via Lookup + ValidatePrimaryTable(); + } + + /// + /// Validate that every command inserted into the primary table can be looked up. + /// Called once during static init; throws on any mismatch to catch registration bugs early. + /// + private static void ValidatePrimaryTable() + { + // Scan all occupied slots and verify each round-trips via Lookup + Span word0Bytes = stackalloc byte[8]; + Span word1Bytes = stackalloc byte[8]; + Span word2Bytes = stackalloc byte[8]; + + for (int i = 0; i < primaryTable.Length; i++) + { + ref CommandEntry entry = ref primaryTable[i]; + if (entry.NameLength == 0) continue; + + // Reconstruct the name from the stored words + var nameBytes = new byte[entry.NameLength]; + MemoryMarshal.Write(word0Bytes, in entry.NameWord0); + MemoryMarshal.Write(word1Bytes, in entry.NameWord1); + MemoryMarshal.Write(word2Bytes, in entry.NameWord2); + + if (entry.NameLength <= 8) + { + word0Bytes.Slice(0, entry.NameLength).CopyTo(nameBytes); + } + else if (entry.NameLength <= 16) + { + word0Bytes.CopyTo(nameBytes); + // Word1 stores last 8 bytes (overlapping for lengths 9-16) + word1Bytes.CopyTo(nameBytes.AsSpan(entry.NameLength - 8)); + } + else + { + word0Bytes.CopyTo(nameBytes); + word1Bytes.CopyTo(nameBytes.AsSpan(8)); + word2Bytes.CopyTo(nameBytes.AsSpan(entry.NameLength - 8)); + } + + fixed (byte* p = nameBytes) + { + var found = Lookup(p, nameBytes.Length); + if (found != entry.Command) + { + throw new InvalidOperationException( + $"Primary hash table validation failed: '{System.Text.Encoding.ASCII.GetString(nameBytes)}' expected {entry.Command} but Lookup returned {found}"); + } + } + } } /// @@ -173,6 +227,33 @@ public static RespCommand Lookup(byte* name, int length) return LookupInTable(primaryTable, PrimaryTableMask, name, length); } + /// + /// Look up a primary command name and return whether it has subcommands, in a single probe. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RespCommand Lookup(byte* name, int length, out bool hasSubcommands) + { + hasSubcommands = false; + if ((uint)length > 24) + return RespCommand.NONE; + + uint hash = ComputeHash(name, length); + int idx = (int)(hash & (uint)PrimaryTableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref primaryTable[idx]; + if (entry.NameLength == 0) return RespCommand.NONE; + if (entry.NameLength == (byte)length && MatchName(ref entry, name, length)) + { + hasSubcommands = (entry.Flags & FlagHasSubcommands) != 0; + return entry.Command; + } + idx = (idx + 1) & PrimaryTableMask; + } + return RespCommand.NONE; + } + /// /// Check if the given command has subcommands. /// @@ -334,6 +415,9 @@ private static RespCommand LookupInTable(CommandEntry[] table, int tableMask, by Debug.Assert(length > 0, "Command name length must be positive"); Debug.Assert(table != null, "Hash table must not be null"); + // CommandEntry stores at most 24 bytes of name; longer names can never match. + if ((uint)length > 24) return RespCommand.NONE; + uint hash = ComputeHash(name, length); int idx = (int)(hash & (uint)tableMask); @@ -378,8 +462,8 @@ private static ulong GetWordFromSpan(ReadOnlySpan span, int offset) /// private static void InsertIntoTable(CommandEntry[] table, int tableMask, ReadOnlySpan name, RespCommand command, byte flags = 0) { - Debug.Assert(name.Length > 0, "Command name must not be empty"); - Debug.Assert(name.Length <= 24, $"Command name too long for hash table entry: {System.Text.Encoding.ASCII.GetString(name)}"); + if (name.Length == 0 || name.Length > 24) + throw new ArgumentException($"Command name must be 1-24 bytes, got {name.Length}: {System.Text.Encoding.ASCII.GetString(name)}"); uint hash = ComputeHash(name); int idx = (int)(hash & (uint)tableMask); diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs index 222f14b0ed5..8afa56ddcf5 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs @@ -268,7 +268,7 @@ private protected bool ScanLookup 0) _ = bContext.CompletePending(wait: true); - IterationComplete: + IterationComplete: if (resetCursor) cursor = 0; scanFunctions.OnStop(false, scanCursorState.acceptedCount); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ContinuePending.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ContinuePending.cs index 37d9280f6f0..62452efc62b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ContinuePending.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ContinuePending.cs @@ -246,7 +246,7 @@ internal OperationStatus ContinuePendingRMW(sessionFunctions, ref stackCtx); } - // Must do this *after* Unlocking. + // Must do this *after* Unlocking. CheckRetry: if (!HandleImmediateRetryStatus(status, sessionFunctions, ref pendingContext)) return status; diff --git a/playground/Bitmap/BitOp.cs b/playground/Bitmap/BitOp.cs index 0d8878f0849..893e619e778 100644 --- a/playground/Bitmap/BitOp.cs +++ b/playground/Bitmap/BitOp.cs @@ -360,7 +360,7 @@ private static void __bitop_simdX128_and(byte* dstBitmap, long dstLen, byte* src #endregion #region fillDstTail - fillTail: + fillTail: if (dstLen > srcLen) { batchSize = 8 * 4; @@ -530,7 +530,7 @@ private static void __bitop_simdX128_and_long(byte* dstBitmap, long dstLen, byte #endregion #region fillDstTail - fillTail: + fillTail: if (dstLen > srcLen) { batchSize = 8 * 4; @@ -729,7 +729,7 @@ private static void __bitop_simdX256_and(byte* dstBitmap, long dstLen, byte* src #endregion #region fillDstTail - fillTail: + fillTail: if (dstLen > srcLen) { batchSize = 8 * 4; @@ -828,7 +828,7 @@ private static void __bitop_multikey_scalar_and(byte* dstPtr, int dstLen, byte** *(long*)dstCurr = d00; dstCurr += batchSize; } - #endregion + #endregion fillTail: #region scalar_1x1 @@ -1046,7 +1046,7 @@ private static void __bitop_multikey_simdX128_and(byte* dstPtr, int dstLen, byte *(long*)dstCurr = d00; dstCurr += batchSize; } - #endregion + #endregion fillTail: #region scalar_1x1 @@ -1266,7 +1266,7 @@ private static void __bitop_multikey_simdX256_and(byte* dstPtr, int dstLen, byte *(long*)dstCurr = d00; dstCurr += batchSize; } - #endregion + #endregion fillTail: #region scalar_1x1 diff --git a/website/docs/dev/fast-parsing-plan.md b/website/docs/dev/fast-parsing-plan.md deleted file mode 100644 index b6c7ef2d9e4..00000000000 --- a/website/docs/dev/fast-parsing-plan.md +++ /dev/null @@ -1,316 +0,0 @@ -# Optimize RESP Command Parsing (`ParseCommand`) - -## Problem Statement - -`ParseCommand` in `libs/server/Resp/Parser/RespCommand.cs` is the entry point for parsing every RESP command from the network buffer. It uses a multi-tier approach: - -| Tier | Method | Commands Covered | Complexity | -|------|--------|-----------------|------------| -| 1 | `FastParseCommand` | ~40 most common (GET, SET, PING, DEL, INCR, TTL...) | O(1) — matches RESP header+command in one 8-byte read | -| 2 | `FastParseArrayCommand` | ~100 more (HSET, ZADD, LPUSH, SUBSCRIBE, GEOADD...) | O(depth) — 950 lines of nested switch/if-else by length→first-char→ulong compare | -| 3 | `SlowParseCommand` | ~80 remaining (CLUSTER *, CONFIG *, ACL *, admin cmds...) | O(n) — sequential `SequenceEqual` comparisons | - -**The hot path (Tier 1) works well** — it reads `*N\r\n$L\r\nCMD\r\n` as a single ulong and matches in one comparison. However: - -- **Tier 1** still does sequential ulong comparisons per candidate when multiple commands share the same `(count << 4 | length)` bucket — e.g., 3 commands at `(1<<4)|3`: GET, DEL, TTL -- **Tier 2** walks through deeply nested switch→switch→if-else chains — ~5-20 comparisons per command -- **Tier 3** does sequential `SequenceEqual` through ~80 entries — worst case ~80 comparisons for unknown commands -- **MakeUpperCase** is called on every non-fast-path command, using byte-by-byte scanning - -Estimated cost: Tier 1 ≈ 5-15 cycles, Tier 2 ≈ 30-60 cycles, Tier 3 ≈ 100-300 cycles. - -## Proposed Solution: SIMD-Accelerated Parsing + Cache-Friendly Hash Table - -Two complementary optimizations: -1. **SIMD-based FastParseCommand** for the top ~20 commands using Vector128 full-pattern matching -2. **Cache-line-optimized hash table** with CRC32 intrinsic hashing and SIMD validation for all remaining ~170 commands - -### Architecture - -``` -ParseCommand - ├─ SimdFastParseCommand (REWRITE — Vector128 full 16-byte pattern match for top ~20 commands) - │ ├─ Load first 16 bytes as Vector128 - │ ├─ Group candidates by total encoded length (13-byte, 14-byte, 15-byte groups) - │ ├─ One AND (mask) per group + one Vector128.EqualsAll per candidate - │ └─ ~3 ops per candidate vs. current ~8-10 ops - │ - └─ On miss → ArrayParseCommand - ├─ MakeUpperCase (REWRITE — SIMD Vector128/256 bulk conversion) - ├─ SimdFastParseCommand retry (catches lowercase clients) - ├─ Parse RESP array header - └─ HashLookupCommand (NEW — replaces FastParseArrayCommand + SlowParseCommand) - ├─ Extract command name (byte* + length) - ├─ CRC32 hardware hash (single instruction via Sse42.Crc32 / Crc32.Arm) - ├─ Index into cache-line-aligned table (L1-resident, 8-16KB) - ├─ Validate via Vector128.EqualsAll (up to 16 bytes in one op) - ├─ Linear probe within same cache line for collisions - └─ If parent has subcommands → SubcommandHashLookup (same design) -``` - -### SIMD Design: FastParseCommand (Tier 1) - -**Core idea**: Replace the two-step "load header ulong + load lastWord ulong" approach with a single Vector128 load that matches the FULL RESP-encoded pattern in one comparison. - -For the top commands, the complete RESP encoding is: -``` -GET: *2\r\n$3\r\nGET\r\n → 13 bytes total -SET: *3\r\n$3\r\nSET\r\n → 13 bytes total -DEL: *2\r\n$3\r\nDEL\r\n → 13 bytes total -TTL: *2\r\n$3\r\nTTL\r\n → 13 bytes total -PING: *1\r\n$4\r\nPING\r\n → 14 bytes total -INCR: *2\r\n$4\r\nINCR\r\n → 14 bytes total -HGET: *3\r\n$4\r\nHGET\r\n → 14 bytes total -HSET: *4\r\n$4\r\nHSET\r\n → 14 bytes total -MGET: *2\r\n$4\r\nMGET\r\n → 14 bytes total -ZADD: *4\r\n$4\r\nZADD\r\n → 14 bytes total -``` - -**Algorithm**: -```csharp -if (remainingBytes >= 16) -{ - var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); - - // Group 1: 13-byte patterns (3-char command names) - // Mask out bytes 13-15 (they contain the next argument, not part of command) - var masked13 = Vector128.BitwiseAnd(input, Mask13); - - // Each comparison checks FULL RESP header + command name in ONE op - // Pattern includes *N\r\n$3\r\nXXX\r\n — the N distinguishes arg count - if (masked13 == PatternGET) { readHead += 13; count = 1; return RespCommand.GET; } - if (masked13 == PatternSET) { readHead += 13; count = 2; return RespCommand.SET; } - if (masked13 == PatternDEL) { readHead += 13; count = 1; return RespCommand.DEL; } - if (masked13 == PatternTTL) { readHead += 13; count = 1; return RespCommand.TTL; } - - // Group 2: 14-byte patterns (4-char command names) - var masked14 = Vector128.BitwiseAnd(input, Mask14); - if (masked14 == PatternPING) { readHead += 14; count = 0; return RespCommand.PING; } - if (masked14 == PatternINCR) { readHead += 14; count = 1; return RespCommand.INCR; } - if (masked14 == PatternHGET) { readHead += 14; count = 2; return RespCommand.HGET; } - if (masked14 == PatternHSET) { readHead += 14; count = 3; return RespCommand.HSET; } - // ... more 14-byte patterns - - // Group 3: 15-byte patterns (5-char command names) - var masked15 = Vector128.BitwiseAnd(input, Mask15); - if (masked15 == PatternLPUSH) { readHead += 15; count = 1; return RespCommand.LPUSH; } - // ... -} -``` - -**Why this is faster**: Each comparison verifies the ENTIRE command encoding (header + name + terminators) in a single `Vector128.EqualsAll` (~1 cycle), vs. the current approach which needs 2 ulong loads + 2 comparisons + count/length extraction (~8-10 ops). The mask is computed once per length group. Pattern vectors are `static readonly` fields, resolved at JIT time. - -**Limitation**: Only works for single-digit array length and single-digit command name length (covers commands with 1-9 args and 1-9 char names). This covers the top ~60 commands. Longer/rarer commands fall through to the hash table. - -### Cache-Friendly Hash Table Design (Tier 2+3 Replacement) - -**Key constraint**: Single-threaded access, so no concurrency overhead. Must be as fast as possible. - -**Entry structure** — 32 bytes, exactly half a cache line: -```csharp -[StructLayout(LayoutKind.Explicit, Size = 32)] -struct CommandEntry -{ - [FieldOffset(0)] public RespCommand Command; // 2 bytes (ushort) - [FieldOffset(2)] public byte NameLength; // 1 byte - [FieldOffset(3)] public byte Flags; // 1 byte (HasSubcommands, etc.) - [FieldOffset(4)] public int Reserved; // 4 bytes (alignment padding) - [FieldOffset(8)] public ulong NameWord0; // First 8 bytes of uppercase name - [FieldOffset(16)] public ulong NameWord1; // Bytes 8-15 (zero-padded) - [FieldOffset(24)] public ulong NameWord2; // Bytes 16-23 (zero-padded) -} -``` - -**Table layout**: -- Size: 256 entries (power of 2) for ~180 primary commands → ~70% load factor -- Total memory: 256 × 32 bytes = **8 KB** — fits entirely in L1 cache (typically 32-64KB) -- Two entries per 64-byte cache line → linear probing hits same cache line for 1st probe -- `GC.AllocateArray(256, pinned: true)` for zero-GC, pointer-stable access - -**Hash function** — Hardware CRC32 (single instruction): -```csharp -[MethodImpl(MethodImplOptions.AggressiveInlining)] -static uint ComputeHash(byte* name, int length) -{ - // Read first 8 bytes (zero-extended for short names) - ulong word0 = length >= 8 - ? *(ulong*)name - : ReadPartialWord(name, length); - - // Hardware CRC32 — single-cycle instruction on x86 (SSE4.2) and ARM - if (Sse42.IsSupported) - return Sse42.Crc32(Sse42.Crc32(0u, word0), (uint)length); - - if (System.Runtime.Intrinsics.Arm.Crc32.IsSupported) - return System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C( - System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C(0u, word0), (uint)length); - - // Software fallback: Fibonacci multiply-shift - return (uint)((word0 * 0x9E3779B97F4A7C15UL) >> 32) ^ (uint)(length * 2654435761U); -} -``` - -**Probe with SIMD validation**: -```csharp -[MethodImpl(MethodImplOptions.AggressiveInlining)] -static RespCommand Lookup(byte* name, int length) -{ - uint hash = ComputeHash(name, length); - int idx = (int)(hash & (TableSize - 1)); // Power-of-2 mask, no modulo - - // Linear probe — typically 1-2 iterations, within same cache line - for (int probe = 0; probe < MaxProbes; probe++) - { - ref CommandEntry entry = ref table[idx]; - - // Empty slot → command not found - if (entry.NameLength == 0) return RespCommand.NONE; - - // Length mismatch → skip (single byte compare, very fast rejection) - if (entry.NameLength != (byte)length) { idx = (idx + 1) & (TableSize - 1); continue; } - - // SIMD validation: Compare up to 16 bytes of command name in ONE operation - if (length <= 8) - { - // Single ulong compare for short names (most commands) - ulong word0 = ReadPartialWord(name, length); - if (entry.NameWord0 == word0) return entry.Command; - } - else - { - // Vector128 compare for names 9-16 bytes (SUBSCRIBE, ZRANGEBYSCORE, etc.) - var input = Vector128.Create(*(ulong*)name, *(ulong*)(name + length - 8)); - var expected = Vector128.Create(entry.NameWord0, entry.NameWord1); - if (Vector128.EqualsAll(input.AsByte(), expected.AsByte())) - return entry.Command; - } - - idx = (idx + 1) & (TableSize - 1); - } - return RespCommand.NONE; -} -``` - -**Why this is fast**: -- **Hash**: 1 CRC32 instruction (~3 cycles on x86/ARM) -- **Index**: 1 AND instruction (~1 cycle) -- **Load entry**: 1 memory load from L1 (~4 cycles, cache-warm) -- **Length check**: 1 byte compare (~1 cycle, fast rejection for 90%+ of misses) -- **Name validate**: 1 ulong compare for names ≤8 bytes (most commands), or 1 Vector128 compare for longer names -- **Total**: ~10-12 cycles for a hit, ~8 cycles for a miss -- **Linear probe**: 2nd probe hits same cache line (32-byte entries, 64-byte cache line) - -### SIMD Case Conversion (MakeUpperCase) - -Replace the current byte-by-byte loop with bulk SIMD conversion: -```csharp -// Detect lowercase bytes: compare against 'a' and 'z' ranges -var lower_a = Vector128.Create((byte)('a' - 1)); // 0x60 -var upper_z = Vector128.Create((byte)'z'); // 0x7A -var caseBit = Vector128.Create((byte)0x20); - -var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); - -// Find bytes in range 'a'-'z' using saturating subtract technique -var aboveA = Vector128.GreaterThan(input, lower_a); -var belowZ = Vector128.LessThanOrEqual(input, upper_z); -var isLower = Vector128.BitwiseAnd(aboveA, belowZ); - -if (isLower != Vector128.Zero) -{ - // Clear bit 5 (0x20) for lowercase bytes only - var toSubtract = Vector128.BitwiseAnd(isLower, caseBit); - var result = Vector128.Subtract(input, toSubtract); - result.StoreUnsafe(ref Unsafe.AsRef(ptr)); - return true; // Modified -} -return false; // Already uppercase -``` - -This converts 16 bytes at once (32 with Vector256). Most RESP command headers are 12-20 bytes, so one or two SIMD operations cover the entire command. - -### Subcommand Hash Lookup - -Commands with subcommands (CLUSTER ~40, CLIENT 8, ACL 10, CONFIG 3, COMMAND 5, SCRIPT 3, LATENCY 3, SLOWLOG 4, MODULE 1, PUBSUB 3, MEMORY 1 — total ~80 subcommands across 12 parents) use the same hash table design but smaller: -- CLUSTER: 64-entry table (~40 subcommands, 62% load) -- Others: 16 or 32-entry tables -- Same CRC32 hash + linear probe, same 32-byte entries -- Each parent's `Flags` field indicates `HasSubcommands` - -### Expected Performance Impact - -| Command Type | Current Cost | New Cost | Speedup | -|-------------|-------------|---------|---------| -| GET, SET (SIMD fast tier) | ~8-10 ops | ~3-4 ops (Vector128 match) | 2-3x | -| PING, INCR (SIMD fast tier) | ~10-12 ops | ~5-6 ops (2nd group match) | 2x | -| HSET, ZADD, LPUSH (Tier 2) | ~30-60 cycles | ~10-12 cycles (hash) | 3-5x | -| CLUSTER INFO, ACL LIST (Tier 3) | ~100-300 cycles | ~15-20 cycles (2 hash lookups) | 7-15x | -| Unknown commands | ~300+ cycles | ~8 cycles (hash miss) | 40x+ | - -### Risk Mitigation - -- All ~2000 existing tests serve as regression suite -- The hash table is built at static init time and is read-only — no concurrency issues -- The SIMD patterns are statically verified at build time -- Fallback paths exist for all SIMD operations (non-SIMD hardware) -- The hash table construction can assert zero collisions at startup - -## Implementation Todos - -### Phase 1: Hash Table Infrastructure -1. **build-hash-table** — Create `RespCommandHashLookup` static class in `libs/server/Resp/Parser/RespCommandHashLookup.cs`: - - 32-byte `CommandEntry` struct with `StructLayout(Explicit)` - - Pinned GC array for zero-GC pointer-stable access - - CRC32 intrinsic hash with Sse42/Arm.Crc32/software fallback - - `Lookup(byte* name, int length)` with linear probing + SIMD validation - - Static constructor populates all ~180 primary commands - - Assert zero unresolved collisions at init (fail-fast if hash function degrades) - -2. **build-subcommand-tables** — Per-parent subcommand hash tables (same design, smaller): - - CLUSTER (64 entries), CLIENT/ACL/COMMAND (16-32 entries), others (16 entries) - - `LookupSubcommand(RespCommand parent, byte* name, int length)` method - - Preserve specific error messages for unknown subcommands - -### Phase 2: SIMD Fast Tier -3. **simd-fast-parse** — Rewrite `FastParseCommand` to use Vector128 pattern matching: - - Pre-build Vector128 patterns for top ~20-30 commands (static readonly fields) - - Group by total encoded byte length (13, 14, 15, 16+ bytes) - - One mask per group, one Vector128.EqualsAll per candidate - - Extract count and readHead advance from the pattern metadata - - Preserve inline command handling (PING/QUIT without array framing) - -### Phase 3: Integration -4. **replace-array-parse** — Modify `ArrayParseCommand` to use hash lookup: - - After parsing RESP array header, extract command name (bytes + length) - - Call `RespCommandHashLookup.Lookup()` replacing FastParseArrayCommand + SlowParseCommand - - For `HasSubcommands` results, extract subcommand and do second lookup - - Delete or gut `FastParseArrayCommand` (~950 lines) and simplify `SlowParseCommand` (~800 lines) - -5. **handle-special-commands** — Preserve special-case behavior: - - BITOP + pseudo-subcommands (AND/OR/XOR/NOT/DIFF): after hash identifies BITOP, parse the operator subcommand inline or via subcommand hash - - Custom commands via TryParseCustomCommand: checked before hash lookup, as currently done - - SET variants (SETEXNX etc.): keep in SIMD fast tier via separate patterns for different arg counts - -### Phase 4: SIMD Case Conversion -6. **simd-uppercase** — Replace `MakeUpperCase` with SIMD version: - - Vector128 (SSE2/AdvSimd): 16 bytes at a time - - Vector256 (AVX2): 32 bytes at a time when available - - Keep existing two-ulong fast-check as the first gate (already uppercase → skip) - - Falls back to scalar loop for remaining bytes - -### Phase 5: Validation -7. **run-tests** — Full regression testing: - - `dotnet test test/Garnet.test -f net10.0 -c Debug` - - Focus on `RespTests`, `RespCommandTests`, ACL tests, Cluster tests - - Verify command parsing identity for all commands (every enum value must be reachable) - -8. **benchmark** — Performance validation: - - Microbenchmark each tier: top commands, moderate commands, rare commands, unknown commands - - Compare cycle counts before/after using BenchmarkDotNet - - Profile branch misprediction rates and cache miss rates - -## Files to Modify - -- `libs/server/Resp/Parser/RespCommand.cs` — Rewrite FastParseCommand (SIMD), replace FastParseArrayCommand + SlowParseCommand with hash lookup calls -- `libs/server/Resp/RespServerSession.cs` — SIMD MakeUpperCase -- **New file**: `libs/server/Resp/Parser/RespCommandHashLookup.cs` — Hash table + subcommand tables From 0b119f13e83412c4d7472320e7b22a6a5b60bb37 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Wed, 1 Apr 2026 11:17:35 -0700 Subject: [PATCH 14/19] nits --- .../Operations/CommandParsingBenchmark.cs | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs index 21b067369f5..2130ec67831 100644 --- a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs +++ b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs @@ -14,28 +14,28 @@ namespace BDN.benchmark.Operations [MemoryDiagnoser] public unsafe class CommandParsingBenchmark : OperationsBase { - // Tier 1a: SIMD Vector128 fast path (3-6 char commands with fixed arg counts) + // Tier 0a: SIMD Vector128 fast path (3-6 char commands with fixed arg counts) static ReadOnlySpan CMD_PING => "*1\r\n$4\r\nPING\r\n"u8; static ReadOnlySpan CMD_GET => "*2\r\n$3\r\nGET\r\n$1\r\na\r\n"u8; static ReadOnlySpan CMD_SET => "*3\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\nb\r\n"u8; static ReadOnlySpan CMD_INCR => "*2\r\n$4\r\nINCR\r\n$1\r\ni\r\n"u8; static ReadOnlySpan CMD_EXISTS => "*2\r\n$6\r\nEXISTS\r\n$1\r\na\r\n"u8; - - // Tier 1b: Scalar ulong switch (variable-arg commands) static ReadOnlySpan CMD_SETEX => "*4\r\n$5\r\nSETEX\r\n$1\r\na\r\n$2\r\n60\r\n$1\r\nb\r\n"u8; + + // Tier 0b: Scalar ulong switch (variable-arg commands, or when remainingBytes < 16) static ReadOnlySpan CMD_EXPIRE => "*3\r\n$6\r\nEXPIRE\r\n$1\r\na\r\n$2\r\n60\r\n"u8; - // Old Tier 2 (FastParseArrayCommand): near top of switch chains (short names, common first chars) + // Tier 1: Hash table lookup via ArrayParseCommand → HashLookupCommand (+ MRU cache on 2nd+ call) static ReadOnlySpan CMD_HSET => "*4\r\n$4\r\nHSET\r\n$1\r\nh\r\n$1\r\nf\r\n$1\r\nv\r\n"u8; static ReadOnlySpan CMD_LPUSH => "*3\r\n$5\r\nLPUSH\r\n$1\r\nl\r\n$1\r\nv\r\n"u8; static ReadOnlySpan CMD_ZADD => "*4\r\n$4\r\nZADD\r\n$1\r\nz\r\n$1\r\n1\r\n$1\r\nm\r\n"u8; - // Old Tier 2 (FastParseArrayCommand): deep in switch chains (long names, double-digit $ header) + // Tier 1: Hash table lookup (long command names, double-digit $ header) static ReadOnlySpan CMD_ZRANGEBYSCORE => "*4\r\n$13\r\nZRANGEBYSCORE\r\n$1\r\nz\r\n$1\r\n0\r\n$2\r\n10\r\n"u8; static ReadOnlySpan CMD_ZREMRANGEBYSCORE => "*4\r\n$16\r\nZREMRANGEBYSCORE\r\n$1\r\nz\r\n$1\r\n0\r\n$2\r\n10\r\n"u8; static ReadOnlySpan CMD_HINCRBYFLOAT => "*4\r\n$12\r\nHINCRBYFLOAT\r\n$1\r\nh\r\n$1\r\nf\r\n$3\r\n1.5\r\n"u8; - // Old Tier 3 (SlowParseCommand): sequential SequenceEqual scan + // Tier 1: Hash table lookup (commands formerly in SlowParseCommand) static ReadOnlySpan CMD_SUBSCRIBE => "*2\r\n$9\r\nSUBSCRIBE\r\n$2\r\nch\r\n"u8; static ReadOnlySpan CMD_GEORADIUS => "*6\r\n$9\r\nGEORADIUS\r\n$1\r\ng\r\n$1\r\n0\r\n$1\r\n0\r\n$3\r\n100\r\n$2\r\nkm\r\n"u8; static ReadOnlySpan CMD_SETIFMATCH => "*4\r\n$10\r\nSETIFMATCH\r\n$1\r\na\r\n$1\r\nb\r\n$1\r\n0\r\n"u8; @@ -85,7 +85,7 @@ public override void GlobalSetup() CMD_SETIFMATCH.CopyTo(bufSetifmatch); } - // === Tier 1a: SIMD Vector128 fast path === + // === Tier 0a: SIMD Vector128 fast path === [Benchmark] public RespCommand ParsePING() @@ -132,7 +132,8 @@ public RespCommand ParseEXISTS() return result; } - // === Tier 1b: Scalar ulong switch === + // === Tier 0a: SIMD fast path (SETEX is a 15-byte SIMD pattern) === + // === Tier 0b: Scalar ulong switch === [Benchmark] public RespCommand ParseSETEX() @@ -152,7 +153,7 @@ public RespCommand ParseEXPIRE() return result; } - // === Old Tier 2 (FastParseArrayCommand): near top of switch === + // === Tier 1: Hash table lookup (short names, MRU cache on repeat) === [Benchmark] public RespCommand ParseHSET() @@ -181,7 +182,7 @@ public RespCommand ParseZADD() return result; } - // === Old Tier 2 (FastParseArrayCommand): deep in switch (long names) === + // === Tier 1: Hash table lookup (long names) === [Benchmark] public RespCommand ParseZRANGEBYSCORE() @@ -210,7 +211,7 @@ public RespCommand ParseHINCRBYFLOAT() return result; } - // === Old Tier 3 (SlowParseCommand): sequential SequenceEqual scan === + // === Tier 1: Hash table lookup (formerly in SlowParseCommand) === [Benchmark] public RespCommand ParseSUBSCRIBE() From 2c2661c54b06839dae9377f2b78c4265c70a51df Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Wed, 1 Apr 2026 13:49:50 -0700 Subject: [PATCH 15/19] updates --- .github/copilot-instructions.md | 2 +- .github/skills/add-garnet-command/SKILL.md | 19 +- libs/server/Resp/Parser/RespCommand.cs | 88 +--- .../Resp/Parser/RespCommandHashLookup.cs | 450 +----------------- .../Resp/Parser/RespCommandHashLookupData.cs | 437 +++++++++++++++++ .../Resp/Parser/RespCommandSimdPatterns.cs | 85 ++++ 6 files changed, 542 insertions(+), 539 deletions(-) create mode 100644 libs/server/Resp/Parser/RespCommandHashLookupData.cs create mode 100644 libs/server/Resp/Parser/RespCommandSimdPatterns.cs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 0f11b86116c..34b353a22f4 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -92,7 +92,7 @@ Full guide: https://microsoft.github.io/garnet/docs/dev/garnet-api ### Steps for a new built-in command: 1. **Define the command**: Add enum value to `RespCommand` in `libs/server/Resp/Parser/RespCommand.cs`. For object commands (List, SortedSet, Hash, Set), also add a value to the `[ObjectName]Operation` enum in `libs/server/Objects/[ObjectName]/[ObjectName]Object.cs`. -2. **Add parsing logic**: In `libs/server/Resp/Parser/RespCommandHashLookup.cs`, add an entry to `PopulatePrimaryTable()` (e.g., `Add("MYNEWCMD", RespCommand.MYNEWCMD)`). For commands with subcommands, set `hasSub: true` and add a subcommand table. The hash table provides O(1) lookup for all command name lengths. +2. **Add parsing logic**: In `libs/server/Resp/Parser/RespCommandHashLookupData.cs`, add an entry to `PopulatePrimaryTable()` (e.g., `Add("MYNEWCMD", RespCommand.MYNEWCMD)`). For commands with subcommands, set `hasSub: true` and add a subcommand table. The hash table provides O(1) lookup for all command name lengths. 3. **Declare the API method**: Add method signature to `IGarnetReadApi` (read-only) or `IGarnetApi` (read-write) in `libs/server/API/IGarnetApi.cs`. 4. **Implement the network handler**: Add a method to `RespServerSession` (the class is split across ~22 partial `.cs` files — object commands go in `libs/server/Resp/Objects/[ObjectName]Commands.cs`, others in `libs/server/Resp/BasicCommands.cs`, `ArrayCommands.cs`, `AdminCommands.cs`, `KeyAdminCommands.cs`, etc.). The handler parses arguments from the network buffer via `parseState.GetArgSliceByRef(i)` (returns `ref PinnedSpanByte`), calls the storage API, and writes the RESP response using `RespWriteUtils` helper methods, then calls `SendAndReset()` to flush the response buffer. 5. **Add dispatch route**: In `libs/server/Resp/RespServerSession.cs`, add a case to `ProcessBasicCommands` or `ProcessArrayCommands` calling the handler from step 4. diff --git a/.github/skills/add-garnet-command/SKILL.md b/.github/skills/add-garnet-command/SKILL.md index f1c16ced757..7a9491513e9 100644 --- a/.github/skills/add-garnet-command/SKILL.md +++ b/.github/skills/add-garnet-command/SKILL.md @@ -85,19 +85,19 @@ EVALSHA, // Note: Update LastDataCommand if adding new data commands after this Two parsing tiers exist: ### Hash table path: `RespCommandHashLookup` (primary path for most commands) -The hash table in `libs/server/Resp/Parser/RespCommandHashLookup.cs` provides O(1) lookup for all built-in commands. This is the **recommended path for all new commands**. +The hash table in `libs/server/Resp/Parser/RespCommandHashLookupData.cs` provides O(1) lookup for all built-in commands. This is the **recommended path for all new commands**. -**To add a new primary command**, add one line to `PopulatePrimaryTable()`: +**To add a new primary command**, add one line to `PopulatePrimaryTable()` in `RespCommandHashLookupData.cs`: ```csharp Add("DELIFGREATER", RespCommand.DELIFGREATER); ``` **To add a command with subcommands** (e.g., `MYPARENT SUBCMD`): -1. Add the parent with `hasSub: true`: +1. Add the parent with `hasSub: true` in `PopulatePrimaryTable()`: ```csharp Add("MYPARENT", RespCommand.MYPARENT, hasSub: true); ``` -2. Define the subcommand array: +2. Define the subcommand array in `RespCommandHashLookupData.cs`: ```csharp private static readonly (string Name, RespCommand Command)[] MyparentSubcommands = [ @@ -105,11 +105,11 @@ Add("DELIFGREATER", RespCommand.DELIFGREATER); ("SUBCMD2", RespCommand.MYPARENT_SUBCMD2), ]; ``` -3. Build the table in the static constructor: +3. Build the table in the static constructor in `RespCommandHashLookup.cs`: ```csharp myparentSubTable = BuildSubTable(MyparentSubcommands, out myparentSubTableMask); ``` -4. Wire it into `LookupSubcommand()`: +4. Wire it into `LookupSubcommand()` in `RespCommandHashLookup.cs`: ```csharp RespCommand.MYPARENT => (myparentSubTable, myparentSubTableMask), ``` @@ -124,10 +124,9 @@ public static ReadOnlySpan DELIFGREATER => "DELIFGREATER"u8; ``` ### SIMD fast path: `FastParseCommand()` (optional, for hottest commands only) -Static `Vector128` patterns that match the full RESP encoding (`*N\r\n$L\r\nCMD\r\n`) in a single 16-byte comparison. Only needed for the most performance-critical commands with: -- Fixed argument count -- Command names of 3-6 characters -- No dots or special characters +Static `Vector128` patterns defined in `libs/server/Resp/Parser/RespCommandSimdPatterns.cs` that match the full RESP encoding (`*N\r\n$L\r\nCMD\r\n`) in a single 16-byte comparison. Use the `RespPattern(argCount, "CMD")` helper to create new patterns. Only needed for the most performance-critical commands with: +- Fixed argument count (single digit) +- Command names of 3-6 characters (total encoded length must fit in 16 bytes) Most new commands should **NOT** be added here — the hash table + MRU cache provide excellent performance for all commands. Only add SIMD patterns if benchmarking shows the command is a bottleneck. diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index f8824f5c568..04126b5e729 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -655,51 +655,6 @@ internal sealed unsafe partial class RespServerSession : ServerSessionBase private Vector128 _cachedPattern1, _cachedMask1; private RespCommand _cachedCmd1; private byte _cachedLen1, _cachedCount1; - // SIMD Vector128 patterns for FastParseCommand. - // Each encodes the full RESP header + command: *N\r\n$L\r\nCMD\r\n - // Masks zero out trailing bytes for patterns shorter than 16 bytes. - private static readonly Vector128 s_mask13 = Vector128.Create( - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00).AsByte(); - private static readonly Vector128 s_mask14 = Vector128.Create( - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00).AsByte(); - private static readonly Vector128 s_mask15 = Vector128.Create( - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00).AsByte(); - - // 13-byte: *N\r\n$3\r\nXXX\r\n - private static readonly Vector128 s_GET = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'3', (byte)'\r', (byte)'\n', (byte)'G', (byte)'E', (byte)'T', (byte)'\r', (byte)'\n', 0, 0, 0); - private static readonly Vector128 s_SET = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'3', (byte)'\r', (byte)'\n', (byte)'S', (byte)'E', (byte)'T', (byte)'\r', (byte)'\n', 0, 0, 0); - private static readonly Vector128 s_DEL = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'3', (byte)'\r', (byte)'\n', (byte)'D', (byte)'E', (byte)'L', (byte)'\r', (byte)'\n', 0, 0, 0); - private static readonly Vector128 s_TTL = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'3', (byte)'\r', (byte)'\n', (byte)'T', (byte)'T', (byte)'L', (byte)'\r', (byte)'\n', 0, 0, 0); - - // 14-byte: *N\r\n$4\r\nXXXX\r\n - private static readonly Vector128 s_PING = Vector128.Create((byte)'*', (byte)'1', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'P', (byte)'I', (byte)'N', (byte)'G', (byte)'\r', (byte)'\n', 0, 0); - private static readonly Vector128 s_INCR = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'I', (byte)'N', (byte)'C', (byte)'R', (byte)'\r', (byte)'\n', 0, 0); - private static readonly Vector128 s_DECR = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'D', (byte)'E', (byte)'C', (byte)'R', (byte)'\r', (byte)'\n', 0, 0); - private static readonly Vector128 s_EXEC = Vector128.Create((byte)'*', (byte)'1', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'E', (byte)'X', (byte)'E', (byte)'C', (byte)'\r', (byte)'\n', 0, 0); - private static readonly Vector128 s_PTTL = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'P', (byte)'T', (byte)'T', (byte)'L', (byte)'\r', (byte)'\n', 0, 0); - private static readonly Vector128 s_DUMP = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'4', (byte)'\r', (byte)'\n', (byte)'D', (byte)'U', (byte)'M', (byte)'P', (byte)'\r', (byte)'\n', 0, 0); - - // 15-byte: *N\r\n$5\r\nXXXXX\r\n - private static readonly Vector128 s_MULTI = Vector128.Create((byte)'*', (byte)'1', (byte)'\r', (byte)'\n', (byte)'$', (byte)'5', (byte)'\r', (byte)'\n', (byte)'M', (byte)'U', (byte)'L', (byte)'T', (byte)'I', (byte)'\r', (byte)'\n', 0); - private static readonly Vector128 s_PFADD = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'5', (byte)'\r', (byte)'\n', (byte)'P', (byte)'F', (byte)'A', (byte)'D', (byte)'D', (byte)'\r', (byte)'\n', 0); - private static readonly Vector128 s_SETNX = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'5', (byte)'\r', (byte)'\n', (byte)'S', (byte)'E', (byte)'T', (byte)'N', (byte)'X', (byte)'\r', (byte)'\n', 0); - private static readonly Vector128 s_SETEX = Vector128.Create((byte)'*', (byte)'4', (byte)'\r', (byte)'\n', (byte)'$', (byte)'5', (byte)'\r', (byte)'\n', (byte)'S', (byte)'E', (byte)'T', (byte)'E', (byte)'X', (byte)'\r', (byte)'\n', 0); - - // 16-byte: *N\r\n$6\r\nXXXXXX\r\n (no mask needed) - private static readonly Vector128 s_EXISTS = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'E', (byte)'X', (byte)'I', (byte)'S', (byte)'T', (byte)'S', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_GETDEL = Vector128.Create((byte)'*', (byte)'2', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'G', (byte)'E', (byte)'T', (byte)'D', (byte)'E', (byte)'L', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_APPEND = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'A', (byte)'P', (byte)'P', (byte)'E', (byte)'N', (byte)'D', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_INCRBY = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'I', (byte)'N', (byte)'C', (byte)'R', (byte)'B', (byte)'Y', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_DECRBY = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'D', (byte)'E', (byte)'C', (byte)'R', (byte)'B', (byte)'Y', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_GETBIT = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'G', (byte)'E', (byte)'T', (byte)'B', (byte)'I', (byte)'T', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_SETBIT = Vector128.Create((byte)'*', (byte)'4', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'S', (byte)'E', (byte)'T', (byte)'B', (byte)'I', (byte)'T', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_GETSET = Vector128.Create((byte)'*', (byte)'3', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'G', (byte)'E', (byte)'T', (byte)'S', (byte)'E', (byte)'T', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_ASKING = Vector128.Create((byte)'*', (byte)'1', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'A', (byte)'S', (byte)'K', (byte)'I', (byte)'N', (byte)'G', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_PSETEX = Vector128.Create((byte)'*', (byte)'4', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'P', (byte)'S', (byte)'E', (byte)'T', (byte)'E', (byte)'X', (byte)'\r', (byte)'\n'); - private static readonly Vector128 s_SUBSTR = Vector128.Create((byte)'*', (byte)'4', (byte)'\r', (byte)'\n', (byte)'$', (byte)'6', (byte)'\r', (byte)'\n', (byte)'S', (byte)'U', (byte)'B', (byte)'S', (byte)'T', (byte)'R', (byte)'\r', (byte)'\n'); /// /// Fast-parses command type for inline RESP commands, starting at the current read head in the receive buffer /// and advances read head. @@ -763,19 +718,17 @@ private RespCommand FastParseCommand(out int count) if (Vector128.EqualsAll(m13, s_DEL)) { readHead += 13; count = 1; return RespCommand.DEL; } if (Vector128.EqualsAll(m13, s_TTL)) { readHead += 13; count = 1; return RespCommand.TTL; } - // 14-byte patterns: 4-char commands (PING, INCR, DECR, EXEC, PTTL, DUMP) + // 14-byte patterns: 4-char commands (PING, INCR, DECR, EXEC, PTTL) var m14 = Vector128.BitwiseAnd(input, s_mask14); if (Vector128.EqualsAll(m14, s_PING)) { readHead += 14; count = 0; return RespCommand.PING; } if (Vector128.EqualsAll(m14, s_INCR)) { readHead += 14; count = 1; return RespCommand.INCR; } if (Vector128.EqualsAll(m14, s_DECR)) { readHead += 14; count = 1; return RespCommand.DECR; } if (Vector128.EqualsAll(m14, s_EXEC)) { readHead += 14; count = 0; return RespCommand.EXEC; } if (Vector128.EqualsAll(m14, s_PTTL)) { readHead += 14; count = 1; return RespCommand.PTTL; } - if (Vector128.EqualsAll(m14, s_DUMP)) { readHead += 14; count = 1; return RespCommand.DUMP; } - // 15-byte patterns: 5-char commands (MULTI, PFADD, SETNX, SETEX) + // 15-byte patterns: 5-char commands (MULTI, SETNX, SETEX) var m15 = Vector128.BitwiseAnd(input, s_mask15); if (Vector128.EqualsAll(m15, s_MULTI)) { readHead += 15; count = 0; return RespCommand.MULTI; } - if (Vector128.EqualsAll(m15, s_PFADD)) { readHead += 15; count = 2; return RespCommand.PFADD; } if (Vector128.EqualsAll(m15, s_SETNX)) { readHead += 15; count = 2; return RespCommand.SETNX; } if (Vector128.EqualsAll(m15, s_SETEX)) { readHead += 15; count = 3; return RespCommand.SETEX; } @@ -785,12 +738,7 @@ private RespCommand FastParseCommand(out int count) if (Vector128.EqualsAll(input, s_APPEND)) { readHead += 16; count = 2; return RespCommand.APPEND; } if (Vector128.EqualsAll(input, s_INCRBY)) { readHead += 16; count = 2; return RespCommand.INCRBY; } if (Vector128.EqualsAll(input, s_DECRBY)) { readHead += 16; count = 2; return RespCommand.DECRBY; } - if (Vector128.EqualsAll(input, s_GETBIT)) { readHead += 16; count = 2; return RespCommand.GETBIT; } - if (Vector128.EqualsAll(input, s_SETBIT)) { readHead += 16; count = 3; return RespCommand.SETBIT; } - if (Vector128.EqualsAll(input, s_GETSET)) { readHead += 16; count = 2; return RespCommand.GETSET; } - if (Vector128.EqualsAll(input, s_ASKING)) { readHead += 16; count = 0; return RespCommand.ASKING; } if (Vector128.EqualsAll(input, s_PSETEX)) { readHead += 16; count = 3; return RespCommand.PSETEX; } - if (Vector128.EqualsAll(input, s_SUBSTR)) { readHead += 16; count = 3; return RespCommand.SUBSTR; } // MRU cache check: catches repeated commands that aren't in the SIMD pattern table // (e.g., HSET, LPUSH, ZADD, ZRANGEBYSCORE). Same 3-op cost as one SIMD pattern check. @@ -856,59 +804,39 @@ private RespCommand FastParseCommand(out int count) return ((count << 4) | length) switch { - // Commands without arguments + // (1) Same fixed-arg hot commands as SIMD path — fallback when remainingBytes < 16 4 when lastWord == MemoryMarshal.Read("\r\nPING\r\n"u8) => RespCommand.PING, 4 when lastWord == MemoryMarshal.Read("\r\nEXEC\r\n"u8) => RespCommand.EXEC, 5 when lastWord == MemoryMarshal.Read("\nMULTI\r\n"u8) => RespCommand.MULTI, - 6 when lastWord == MemoryMarshal.Read("ASKING\r\n"u8) => RespCommand.ASKING, - 7 when lastWord == MemoryMarshal.Read("ISCARD\r\n"u8) && ptr[8] == 'D' => RespCommand.DISCARD, - 7 when lastWord == MemoryMarshal.Read("NWATCH\r\n"u8) && ptr[8] == 'U' => RespCommand.UNWATCH, - 8 when lastWord == MemoryMarshal.Read("ADONLY\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("RE"u8) => RespCommand.READONLY, - 9 when lastWord == MemoryMarshal.Read("DWRITE\r\n"u8) && *(uint*)(ptr + 8) == MemoryMarshal.Read("READ"u8) => RespCommand.READWRITE, - - // Commands with fixed number of arguments (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nGET\r\n"u8) => RespCommand.GET, (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nDEL\r\n"u8) => RespCommand.DEL, (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nTTL\r\n"u8) => RespCommand.TTL, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDUMP\r\n"u8) => RespCommand.DUMP, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nINCR\r\n"u8) => RespCommand.INCR, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nPTTL\r\n"u8) => RespCommand.PTTL, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDECR\r\n"u8) => RespCommand.DECR, (1 << 4) | 4 when lastWord == MemoryMarshal.Read("EXISTS\r\n"u8) => RespCommand.EXISTS, (1 << 4) | 6 when lastWord == MemoryMarshal.Read("GETDEL\r\n"u8) => RespCommand.GETDEL, - (1 << 4) | 7 when lastWord == MemoryMarshal.Read("ERSIST\r\n"u8) && ptr[8] == 'P' => RespCommand.PERSIST, - (1 << 4) | 7 when lastWord == MemoryMarshal.Read("PFCOUNT\r\n"u8) && ptr[8] == 'P' => RespCommand.PFCOUNT, (2 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SET, - (2 << 4) | 5 when lastWord == MemoryMarshal.Read("\nPFADD\r\n"u8) => RespCommand.PFADD, (2 << 4) | 6 when lastWord == MemoryMarshal.Read("INCRBY\r\n"u8) => RespCommand.INCRBY, (2 << 4) | 6 when lastWord == MemoryMarshal.Read("DECRBY\r\n"u8) => RespCommand.DECRBY, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("GETBIT\r\n"u8) => RespCommand.GETBIT, (2 << 4) | 6 when lastWord == MemoryMarshal.Read("APPEND\r\n"u8) => RespCommand.APPEND, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("GETSET\r\n"u8) => RespCommand.GETSET, - (2 << 4) | 7 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && ptr[8] == 'P' => RespCommand.PUBLISH, - (2 << 4) | 7 when lastWord == MemoryMarshal.Read("FMERGE\r\n"u8) && ptr[8] == 'P' => RespCommand.PFMERGE, - (2 << 4) | 8 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SP"u8) => RespCommand.SPUBLISH, (2 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETNX\r\n"u8) => RespCommand.SETNX, (3 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETEX\r\n"u8) => RespCommand.SETEX, (3 << 4) | 6 when lastWord == MemoryMarshal.Read("PSETEX\r\n"u8) => RespCommand.PSETEX, - (3 << 4) | 6 when lastWord == MemoryMarshal.Read("SETBIT\r\n"u8) => RespCommand.SETBIT, - (3 << 4) | 6 when lastWord == MemoryMarshal.Read("SUBSTR\r\n"u8) => RespCommand.SUBSTR, - (3 << 4) | 7 when lastWord == MemoryMarshal.Read("ESTORE\r\n"u8) && ptr[8] == 'R' => RespCommand.RESTORE, + + // (2) Hot commands too long for SIMD (name > 6 chars, exceeds 16-byte Vector128) + (2 << 4) | 7 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && ptr[8] == 'P' => RespCommand.PUBLISH, + (2 << 4) | 8 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SP"u8) => RespCommand.SPUBLISH, (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SE"u8) => RespCommand.SETRANGE, (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("GE"u8) => RespCommand.GETRANGE, + // (3) Hot variable-arg commands (arg count varies, cannot be SIMD or MRU cached) _ => ((length << 4) | count) switch { - // Commands with dynamic number of arguments - >= ((6 << 4) | 2) and <= ((6 << 4) | 3) when lastWord == MemoryMarshal.Read("RENAME\r\n"u8) => RespCommand.RENAME, - >= ((8 << 4) | 2) and <= ((8 << 4) | 3) when lastWord == MemoryMarshal.Read("NAMENX\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("RE"u8) => RespCommand.RENAMENX, >= ((3 << 4) | 3) and <= ((3 << 4) | 7) when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SETEXNX, >= ((5 << 4) | 1) and <= ((5 << 4) | 3) when lastWord == MemoryMarshal.Read("\nGETEX\r\n"u8) => RespCommand.GETEX, - >= ((6 << 4) | 0) and <= ((6 << 4) | 9) when lastWord == MemoryMarshal.Read("RUNTXP\r\n"u8) => RespCommand.RUNTXP, >= ((6 << 4) | 2) and <= ((6 << 4) | 3) when lastWord == MemoryMarshal.Read("EXPIRE\r\n"u8) => RespCommand.EXPIRE, - >= ((6 << 4) | 2) and <= ((6 << 4) | 5) when lastWord == MemoryMarshal.Read("BITPOS\r\n"u8) => RespCommand.BITPOS, >= ((7 << 4) | 2) and <= ((7 << 4) | 3) when lastWord == MemoryMarshal.Read("EXPIRE\r\n"u8) && ptr[8] == 'P' => RespCommand.PEXPIRE, - >= ((8 << 4) | 1) and <= ((8 << 4) | 4) when lastWord == MemoryMarshal.Read("TCOUNT\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("BI"u8) => RespCommand.BITCOUNT, _ => MatchedNone(this, oldReadHead) } }; diff --git a/libs/server/Resp/Parser/RespCommandHashLookup.cs b/libs/server/Resp/Parser/RespCommandHashLookup.cs index 0f819e99bda..a324eea648c 100644 --- a/libs/server/Resp/Parser/RespCommandHashLookup.cs +++ b/libs/server/Resp/Parser/RespCommandHashLookup.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -11,16 +11,14 @@ namespace Garnet.server { /// /// Cache-friendly O(1) hash table for RESP command name lookup. - /// Replaces the nested switch/if-else chains in FastParseArrayCommand and SlowParseCommand. /// /// Design: /// - 32-byte entries (half a cache line) with open addressing and linear probing /// - CRC32 hardware hash (single instruction) with multiply-shift software fallback - /// - Vector128 validation for command names > 8 bytes /// - Table fits in L1 cache (~8-16KB) /// - Single-threaded read-only access after static init /// - internal static unsafe class RespCommandHashLookup + internal static unsafe partial class RespCommandHashLookup { /// /// Entry in the command hash table. Exactly 32 bytes = half a cache line. @@ -254,26 +252,6 @@ public static RespCommand Lookup(byte* name, int length, out bool hasSubcommands return RespCommand.NONE; } - /// - /// Check if the given command has subcommands. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool HasSubcommands(byte* name, int length) - { - uint hash = ComputeHash(name, length); - int idx = (int)(hash & (uint)PrimaryTableMask); - - for (int probe = 0; probe < MaxProbes; probe++) - { - ref CommandEntry entry = ref primaryTable[idx]; - if (entry.NameLength == 0) return false; - if (entry.NameLength == (byte)length && MatchName(ref entry, name, length)) - return (entry.Flags & FlagHasSubcommands) != 0; - idx = (idx + 1) & PrimaryTableMask; - } - return false; - } - /// /// Look up a subcommand for a given parent command. /// @@ -524,429 +502,5 @@ private static CommandEntry[] BuildSubTable(ReadOnlySpan<(string Name, RespComma #endregion - #region Command Definitions - - private static void PopulatePrimaryTable() - { - // Helper to insert with optional subcommand flag - void Add(string name, RespCommand cmd, bool hasSub = false) - { - InsertIntoTable(primaryTable, PrimaryTableMask, - System.Text.Encoding.ASCII.GetBytes(name), cmd, - hasSub ? FlagHasSubcommands : (byte)0); - } - - // ===== Data commands (read + write) ===== - - // String commands - Add("GET", RespCommand.GET); - Add("SET", RespCommand.SET); - Add("DEL", RespCommand.DEL); - Add("INCR", RespCommand.INCR); - Add("DECR", RespCommand.DECR); - Add("INCRBY", RespCommand.INCRBY); - Add("DECRBY", RespCommand.DECRBY); - Add("INCRBYFLOAT", RespCommand.INCRBYFLOAT); - Add("APPEND", RespCommand.APPEND); - Add("GETSET", RespCommand.GETSET); - Add("GETDEL", RespCommand.GETDEL); - Add("GETEX", RespCommand.GETEX); - Add("GETRANGE", RespCommand.GETRANGE); - Add("SETRANGE", RespCommand.SETRANGE); - Add("STRLEN", RespCommand.STRLEN); - Add("SUBSTR", RespCommand.SUBSTR); - Add("SETNX", RespCommand.SETNX); - Add("SETEX", RespCommand.SETEX); - Add("PSETEX", RespCommand.PSETEX); - Add("MGET", RespCommand.MGET); - Add("MSET", RespCommand.MSET); - Add("MSETNX", RespCommand.MSETNX); - Add("DUMP", RespCommand.DUMP); - Add("RESTORE", RespCommand.RESTORE); - Add("GETBIT", RespCommand.GETBIT); - Add("SETBIT", RespCommand.SETBIT); - Add("GETWITHETAG", RespCommand.GETWITHETAG); - Add("GETIFNOTMATCH", RespCommand.GETIFNOTMATCH); - Add("SETIFMATCH", RespCommand.SETIFMATCH); - Add("SETIFGREATER", RespCommand.SETIFGREATER); - Add("DELIFGREATER", RespCommand.DELIFGREATER); - Add("LCS", RespCommand.LCS); - - // Key commands - Add("EXISTS", RespCommand.EXISTS); - Add("TTL", RespCommand.TTL); - Add("PTTL", RespCommand.PTTL); - Add("EXPIRE", RespCommand.EXPIRE); - Add("PEXPIRE", RespCommand.PEXPIRE); - Add("EXPIREAT", RespCommand.EXPIREAT); - Add("PEXPIREAT", RespCommand.PEXPIREAT); - Add("EXPIRETIME", RespCommand.EXPIRETIME); - Add("PEXPIRETIME", RespCommand.PEXPIRETIME); - Add("PERSIST", RespCommand.PERSIST); - Add("TYPE", RespCommand.TYPE); - Add("RENAME", RespCommand.RENAME); - Add("RENAMENX", RespCommand.RENAMENX); - Add("UNLINK", RespCommand.UNLINK); - Add("KEYS", RespCommand.KEYS); - Add("SCAN", RespCommand.SCAN); - Add("DBSIZE", RespCommand.DBSIZE); - Add("SELECT", RespCommand.SELECT); - Add("SWAPDB", RespCommand.SWAPDB); - Add("MIGRATE", RespCommand.MIGRATE); - - // Bitmap commands - Add("BITCOUNT", RespCommand.BITCOUNT); - Add("BITPOS", RespCommand.BITPOS); - Add("BITFIELD", RespCommand.BITFIELD); - Add("BITFIELD_RO", RespCommand.BITFIELD_RO); - Add("BITOP", RespCommand.BITOP, hasSub: true); - - // HyperLogLog commands - Add("PFADD", RespCommand.PFADD); - Add("PFCOUNT", RespCommand.PFCOUNT); - Add("PFMERGE", RespCommand.PFMERGE); - - // Hash commands - Add("HSET", RespCommand.HSET); - Add("HGET", RespCommand.HGET); - Add("HDEL", RespCommand.HDEL); - Add("HLEN", RespCommand.HLEN); - Add("HEXISTS", RespCommand.HEXISTS); - Add("HGETALL", RespCommand.HGETALL); - Add("HKEYS", RespCommand.HKEYS); - Add("HVALS", RespCommand.HVALS); - Add("HMSET", RespCommand.HMSET); - Add("HMGET", RespCommand.HMGET); - Add("HSETNX", RespCommand.HSETNX); - Add("HINCRBY", RespCommand.HINCRBY); - Add("HINCRBYFLOAT", RespCommand.HINCRBYFLOAT); - Add("HRANDFIELD", RespCommand.HRANDFIELD); - Add("HSCAN", RespCommand.HSCAN); - Add("HSTRLEN", RespCommand.HSTRLEN); - Add("HTTL", RespCommand.HTTL); - Add("HPTTL", RespCommand.HPTTL); - Add("HEXPIRE", RespCommand.HEXPIRE); - Add("HPEXPIRE", RespCommand.HPEXPIRE); - Add("HEXPIREAT", RespCommand.HEXPIREAT); - Add("HPEXPIREAT", RespCommand.HPEXPIREAT); - Add("HEXPIRETIME", RespCommand.HEXPIRETIME); - Add("HPEXPIRETIME", RespCommand.HPEXPIRETIME); - Add("HPERSIST", RespCommand.HPERSIST); - Add("HCOLLECT", RespCommand.HCOLLECT); - - // List commands - Add("LPUSH", RespCommand.LPUSH); - Add("RPUSH", RespCommand.RPUSH); - Add("LPUSHX", RespCommand.LPUSHX); - Add("RPUSHX", RespCommand.RPUSHX); - Add("LPOP", RespCommand.LPOP); - Add("RPOP", RespCommand.RPOP); - Add("LLEN", RespCommand.LLEN); - Add("LINDEX", RespCommand.LINDEX); - Add("LINSERT", RespCommand.LINSERT); - Add("LRANGE", RespCommand.LRANGE); - Add("LREM", RespCommand.LREM); - Add("LSET", RespCommand.LSET); - Add("LTRIM", RespCommand.LTRIM); - Add("LPOS", RespCommand.LPOS); - Add("LMOVE", RespCommand.LMOVE); - Add("LMPOP", RespCommand.LMPOP); - Add("RPOPLPUSH", RespCommand.RPOPLPUSH); - Add("BLPOP", RespCommand.BLPOP); - Add("BRPOP", RespCommand.BRPOP); - Add("BLMOVE", RespCommand.BLMOVE); - Add("BRPOPLPUSH", RespCommand.BRPOPLPUSH); - Add("BLMPOP", RespCommand.BLMPOP); - - // Set commands - Add("SADD", RespCommand.SADD); - Add("SREM", RespCommand.SREM); - Add("SPOP", RespCommand.SPOP); - Add("SCARD", RespCommand.SCARD); - Add("SMEMBERS", RespCommand.SMEMBERS); - Add("SISMEMBER", RespCommand.SISMEMBER); - Add("SMISMEMBER", RespCommand.SMISMEMBER); - Add("SRANDMEMBER", RespCommand.SRANDMEMBER); - Add("SMOVE", RespCommand.SMOVE); - Add("SSCAN", RespCommand.SSCAN); - Add("SDIFF", RespCommand.SDIFF); - Add("SDIFFSTORE", RespCommand.SDIFFSTORE); - Add("SINTER", RespCommand.SINTER); - Add("SINTERCARD", RespCommand.SINTERCARD); - Add("SINTERSTORE", RespCommand.SINTERSTORE); - Add("SUNION", RespCommand.SUNION); - Add("SUNIONSTORE", RespCommand.SUNIONSTORE); - - // Sorted set commands - Add("ZADD", RespCommand.ZADD); - Add("ZREM", RespCommand.ZREM); - Add("ZCARD", RespCommand.ZCARD); - Add("ZSCORE", RespCommand.ZSCORE); - Add("ZMSCORE", RespCommand.ZMSCORE); - Add("ZRANK", RespCommand.ZRANK); - Add("ZREVRANK", RespCommand.ZREVRANK); - Add("ZCOUNT", RespCommand.ZCOUNT); - Add("ZLEXCOUNT", RespCommand.ZLEXCOUNT); - Add("ZRANGE", RespCommand.ZRANGE); - Add("ZRANGEBYLEX", RespCommand.ZRANGEBYLEX); - Add("ZRANGEBYSCORE", RespCommand.ZRANGEBYSCORE); - Add("ZRANGESTORE", RespCommand.ZRANGESTORE); - Add("ZREVRANGE", RespCommand.ZREVRANGE); - Add("ZREVRANGEBYLEX", RespCommand.ZREVRANGEBYLEX); - Add("ZREVRANGEBYSCORE", RespCommand.ZREVRANGEBYSCORE); - Add("ZPOPMIN", RespCommand.ZPOPMIN); - Add("ZPOPMAX", RespCommand.ZPOPMAX); - Add("ZRANDMEMBER", RespCommand.ZRANDMEMBER); - Add("ZSCAN", RespCommand.ZSCAN); - Add("ZINCRBY", RespCommand.ZINCRBY); - Add("ZDIFF", RespCommand.ZDIFF); - Add("ZDIFFSTORE", RespCommand.ZDIFFSTORE); - Add("ZINTER", RespCommand.ZINTER); - Add("ZINTERCARD", RespCommand.ZINTERCARD); - Add("ZINTERSTORE", RespCommand.ZINTERSTORE); - Add("ZUNION", RespCommand.ZUNION); - Add("ZUNIONSTORE", RespCommand.ZUNIONSTORE); - Add("ZMPOP", RespCommand.ZMPOP); - Add("BZMPOP", RespCommand.BZMPOP); - Add("BZPOPMAX", RespCommand.BZPOPMAX); - Add("BZPOPMIN", RespCommand.BZPOPMIN); - Add("ZREMRANGEBYLEX", RespCommand.ZREMRANGEBYLEX); - Add("ZREMRANGEBYRANK", RespCommand.ZREMRANGEBYRANK); - Add("ZREMRANGEBYSCORE", RespCommand.ZREMRANGEBYSCORE); - Add("ZTTL", RespCommand.ZTTL); - Add("ZPTTL", RespCommand.ZPTTL); - Add("ZEXPIRE", RespCommand.ZEXPIRE); - Add("ZPEXPIRE", RespCommand.ZPEXPIRE); - Add("ZEXPIREAT", RespCommand.ZEXPIREAT); - Add("ZPEXPIREAT", RespCommand.ZPEXPIREAT); - Add("ZEXPIRETIME", RespCommand.ZEXPIRETIME); - Add("ZPEXPIRETIME", RespCommand.ZPEXPIRETIME); - Add("ZPERSIST", RespCommand.ZPERSIST); - Add("ZCOLLECT", RespCommand.ZCOLLECT); - - // Geo commands - Add("GEOADD", RespCommand.GEOADD); - Add("GEOPOS", RespCommand.GEOPOS); - Add("GEOHASH", RespCommand.GEOHASH); - Add("GEODIST", RespCommand.GEODIST); - Add("GEOSEARCH", RespCommand.GEOSEARCH); - Add("GEOSEARCHSTORE", RespCommand.GEOSEARCHSTORE); - Add("GEORADIUS", RespCommand.GEORADIUS); - Add("GEORADIUS_RO", RespCommand.GEORADIUS_RO); - Add("GEORADIUSBYMEMBER", RespCommand.GEORADIUSBYMEMBER); - Add("GEORADIUSBYMEMBER_RO", RespCommand.GEORADIUSBYMEMBER_RO); - - // Scripting - Add("EVAL", RespCommand.EVAL); - Add("EVALSHA", RespCommand.EVALSHA); - - // Pub/Sub - Add("PUBLISH", RespCommand.PUBLISH); - Add("SUBSCRIBE", RespCommand.SUBSCRIBE); - Add("PSUBSCRIBE", RespCommand.PSUBSCRIBE); - Add("UNSUBSCRIBE", RespCommand.UNSUBSCRIBE); - Add("PUNSUBSCRIBE", RespCommand.PUNSUBSCRIBE); - Add("SPUBLISH", RespCommand.SPUBLISH); - Add("SSUBSCRIBE", RespCommand.SSUBSCRIBE); - - // Custom object scan - Add("CUSTOMOBJECTSCAN", RespCommand.COSCAN); - - // ===== Control / admin commands ===== - Add("PING", RespCommand.PING); - Add("ECHO", RespCommand.ECHO); - Add("QUIT", RespCommand.QUIT); - Add("AUTH", RespCommand.AUTH); - Add("HELLO", RespCommand.HELLO); - Add("INFO", RespCommand.INFO); - Add("TIME", RespCommand.TIME); - Add("ROLE", RespCommand.ROLE); - Add("SAVE", RespCommand.SAVE); - Add("LASTSAVE", RespCommand.LASTSAVE); - Add("BGSAVE", RespCommand.BGSAVE); - Add("COMMITAOF", RespCommand.COMMITAOF); - Add("FLUSHALL", RespCommand.FLUSHALL); - Add("FLUSHDB", RespCommand.FLUSHDB); - Add("FORCEGC", RespCommand.FORCEGC); - Add("PURGEBP", RespCommand.PURGEBP); - Add("FAILOVER", RespCommand.FAILOVER); - Add("MONITOR", RespCommand.MONITOR); - Add("REGISTERCS", RespCommand.REGISTERCS); - Add("ASYNC", RespCommand.ASYNC); - Add("DEBUG", RespCommand.DEBUG); - Add("EXPDELSCAN", RespCommand.EXPDELSCAN); - Add("WATCH", RespCommand.WATCH); - Add("WATCHMS", RespCommand.WATCHMS); - Add("WATCHOS", RespCommand.WATCHOS); - Add("MULTI", RespCommand.MULTI); - Add("EXEC", RespCommand.EXEC); - Add("DISCARD", RespCommand.DISCARD); - Add("UNWATCH", RespCommand.UNWATCH); - Add("RUNTXP", RespCommand.RUNTXP); - Add("ASKING", RespCommand.ASKING); - Add("READONLY", RespCommand.READONLY); - Add("READWRITE", RespCommand.READWRITE); - Add("REPLICAOF", RespCommand.REPLICAOF); - Add("SECONDARYOF", RespCommand.SECONDARYOF); - Add("SLAVEOF", RespCommand.SECONDARYOF); - - // Parent commands with subcommands - Add("SCRIPT", RespCommand.SCRIPT, hasSub: true); - Add("CONFIG", RespCommand.CONFIG, hasSub: true); - Add("CLIENT", RespCommand.CLIENT, hasSub: true); - Add("CLUSTER", RespCommand.CLUSTER, hasSub: true); - Add("ACL", RespCommand.ACL, hasSub: true); - Add("COMMAND", RespCommand.COMMAND, hasSub: true); - Add("LATENCY", RespCommand.LATENCY, hasSub: true); - Add("SLOWLOG", RespCommand.SLOWLOG, hasSub: true); - Add("MODULE", RespCommand.MODULE, hasSub: true); - Add("PUBSUB", RespCommand.PUBSUB, hasSub: true); - Add("MEMORY", RespCommand.MEMORY, hasSub: true); - } - - #endregion - - #region Subcommand Definitions - - private static readonly (string Name, RespCommand Command)[] ClusterSubcommands = - [ - ("ADDSLOTS", RespCommand.CLUSTER_ADDSLOTS), - ("ADDSLOTSRANGE", RespCommand.CLUSTER_ADDSLOTSRANGE), - ("AOFSYNC", RespCommand.CLUSTER_AOFSYNC), - ("APPENDLOG", RespCommand.CLUSTER_APPENDLOG), - ("ATTACH_SYNC", RespCommand.CLUSTER_ATTACH_SYNC), - ("BANLIST", RespCommand.CLUSTER_BANLIST), - ("BEGIN_REPLICA_RECOVER", RespCommand.CLUSTER_BEGIN_REPLICA_RECOVER), - ("BUMPEPOCH", RespCommand.CLUSTER_BUMPEPOCH), - ("COUNTKEYSINSLOT", RespCommand.CLUSTER_COUNTKEYSINSLOT), - ("DELKEYSINSLOT", RespCommand.CLUSTER_DELKEYSINSLOT), - ("DELKEYSINSLOTRANGE", RespCommand.CLUSTER_DELKEYSINSLOTRANGE), - ("DELSLOTS", RespCommand.CLUSTER_DELSLOTS), - ("DELSLOTSRANGE", RespCommand.CLUSTER_DELSLOTSRANGE), - ("ENDPOINT", RespCommand.CLUSTER_ENDPOINT), - ("FAILOVER", RespCommand.CLUSTER_FAILOVER), - ("FAILREPLICATIONOFFSET", RespCommand.CLUSTER_FAILREPLICATIONOFFSET), - ("FAILSTOPWRITES", RespCommand.CLUSTER_FAILSTOPWRITES), - ("FLUSHALL", RespCommand.CLUSTER_FLUSHALL), - ("FORGET", RespCommand.CLUSTER_FORGET), - ("GETKEYSINSLOT", RespCommand.CLUSTER_GETKEYSINSLOT), - ("GOSSIP", RespCommand.CLUSTER_GOSSIP), - ("HELP", RespCommand.CLUSTER_HELP), - ("INFO", RespCommand.CLUSTER_INFO), - ("INITIATE_REPLICA_SYNC", RespCommand.CLUSTER_INITIATE_REPLICA_SYNC), - ("KEYSLOT", RespCommand.CLUSTER_KEYSLOT), - ("MEET", RespCommand.CLUSTER_MEET), - ("MIGRATE", RespCommand.CLUSTER_MIGRATE), - ("MTASKS", RespCommand.CLUSTER_MTASKS), - ("MYID", RespCommand.CLUSTER_MYID), - ("MYPARENTID", RespCommand.CLUSTER_MYPARENTID), - ("NODES", RespCommand.CLUSTER_NODES), - ("PUBLISH", RespCommand.CLUSTER_PUBLISH), - ("SPUBLISH", RespCommand.CLUSTER_SPUBLISH), - ("REPLICAS", RespCommand.CLUSTER_REPLICAS), - ("REPLICATE", RespCommand.CLUSTER_REPLICATE), - ("RESET", RespCommand.CLUSTER_RESET), - ("SEND_CKPT_FILE_SEGMENT", RespCommand.CLUSTER_SEND_CKPT_FILE_SEGMENT), - ("SEND_CKPT_METADATA", RespCommand.CLUSTER_SEND_CKPT_METADATA), - ("SET-CONFIG-EPOCH", RespCommand.CLUSTER_SETCONFIGEPOCH), - ("SETSLOT", RespCommand.CLUSTER_SETSLOT), - ("SETSLOTSRANGE", RespCommand.CLUSTER_SETSLOTSRANGE), - ("SHARDS", RespCommand.CLUSTER_SHARDS), - ("SLOTS", RespCommand.CLUSTER_SLOTS), - ("SLOTSTATE", RespCommand.CLUSTER_SLOTSTATE), - ("SYNC", RespCommand.CLUSTER_SYNC), - ]; - - private static readonly (string Name, RespCommand Command)[] ClientSubcommands = - [ - ("ID", RespCommand.CLIENT_ID), - ("INFO", RespCommand.CLIENT_INFO), - ("LIST", RespCommand.CLIENT_LIST), - ("KILL", RespCommand.CLIENT_KILL), - ("GETNAME", RespCommand.CLIENT_GETNAME), - ("SETNAME", RespCommand.CLIENT_SETNAME), - ("SETINFO", RespCommand.CLIENT_SETINFO), - ("UNBLOCK", RespCommand.CLIENT_UNBLOCK), - ]; - - private static readonly (string Name, RespCommand Command)[] AclSubcommands = - [ - ("CAT", RespCommand.ACL_CAT), - ("DELUSER", RespCommand.ACL_DELUSER), - ("GENPASS", RespCommand.ACL_GENPASS), - ("GETUSER", RespCommand.ACL_GETUSER), - ("LIST", RespCommand.ACL_LIST), - ("LOAD", RespCommand.ACL_LOAD), - ("SAVE", RespCommand.ACL_SAVE), - ("SETUSER", RespCommand.ACL_SETUSER), - ("USERS", RespCommand.ACL_USERS), - ("WHOAMI", RespCommand.ACL_WHOAMI), - ]; - - private static readonly (string Name, RespCommand Command)[] CommandSubcommands = - [ - ("COUNT", RespCommand.COMMAND_COUNT), - ("DOCS", RespCommand.COMMAND_DOCS), - ("INFO", RespCommand.COMMAND_INFO), - ("GETKEYS", RespCommand.COMMAND_GETKEYS), - ("GETKEYSANDFLAGS", RespCommand.COMMAND_GETKEYSANDFLAGS), - ]; - - private static readonly (string Name, RespCommand Command)[] ConfigSubcommands = - [ - ("GET", RespCommand.CONFIG_GET), - ("REWRITE", RespCommand.CONFIG_REWRITE), - ("SET", RespCommand.CONFIG_SET), - ]; - - private static readonly (string Name, RespCommand Command)[] ScriptSubcommands = - [ - ("LOAD", RespCommand.SCRIPT_LOAD), - ("FLUSH", RespCommand.SCRIPT_FLUSH), - ("EXISTS", RespCommand.SCRIPT_EXISTS), - ]; - - private static readonly (string Name, RespCommand Command)[] LatencySubcommands = - [ - ("HELP", RespCommand.LATENCY_HELP), - ("HISTOGRAM", RespCommand.LATENCY_HISTOGRAM), - ("RESET", RespCommand.LATENCY_RESET), - ]; - - private static readonly (string Name, RespCommand Command)[] SlowlogSubcommands = - [ - ("HELP", RespCommand.SLOWLOG_HELP), - ("GET", RespCommand.SLOWLOG_GET), - ("LEN", RespCommand.SLOWLOG_LEN), - ("RESET", RespCommand.SLOWLOG_RESET), - ]; - - private static readonly (string Name, RespCommand Command)[] ModuleSubcommands = - [ - ("LOADCS", RespCommand.MODULE_LOADCS), - ]; - - private static readonly (string Name, RespCommand Command)[] PubsubSubcommands = - [ - ("CHANNELS", RespCommand.PUBSUB_CHANNELS), - ("NUMSUB", RespCommand.PUBSUB_NUMSUB), - ("NUMPAT", RespCommand.PUBSUB_NUMPAT), - ]; - - private static readonly (string Name, RespCommand Command)[] MemorySubcommands = - [ - ("USAGE", RespCommand.MEMORY_USAGE), - ]; - - private static readonly (string Name, RespCommand Command)[] BitopSubcommands = - [ - ("AND", RespCommand.BITOP_AND), - ("OR", RespCommand.BITOP_OR), - ("XOR", RespCommand.BITOP_XOR), - ("NOT", RespCommand.BITOP_NOT), - ("DIFF", RespCommand.BITOP_DIFF), - ]; - - #endregion } } \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommandHashLookupData.cs b/libs/server/Resp/Parser/RespCommandHashLookupData.cs new file mode 100644 index 00000000000..02829e22ee4 --- /dev/null +++ b/libs/server/Resp/Parser/RespCommandHashLookupData.cs @@ -0,0 +1,437 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Garnet.server +{ + /// + /// Command registration data for . + /// Primary command table and subcommand definitions. + /// + internal static unsafe partial class RespCommandHashLookup + { + #region Command Definitions + + private static void PopulatePrimaryTable() + { + // Helper to insert with optional subcommand flag + void Add(string name, RespCommand cmd, bool hasSub = false) + { + InsertIntoTable(primaryTable, PrimaryTableMask, + System.Text.Encoding.ASCII.GetBytes(name), cmd, + hasSub ? FlagHasSubcommands : (byte)0); + } + + // ===== Data commands (read + write) ===== + + // String commands + Add("GET", RespCommand.GET); + Add("SET", RespCommand.SET); + Add("DEL", RespCommand.DEL); + Add("INCR", RespCommand.INCR); + Add("DECR", RespCommand.DECR); + Add("INCRBY", RespCommand.INCRBY); + Add("DECRBY", RespCommand.DECRBY); + Add("INCRBYFLOAT", RespCommand.INCRBYFLOAT); + Add("APPEND", RespCommand.APPEND); + Add("GETSET", RespCommand.GETSET); + Add("GETDEL", RespCommand.GETDEL); + Add("GETEX", RespCommand.GETEX); + Add("GETRANGE", RespCommand.GETRANGE); + Add("SETRANGE", RespCommand.SETRANGE); + Add("STRLEN", RespCommand.STRLEN); + Add("SUBSTR", RespCommand.SUBSTR); + Add("SETNX", RespCommand.SETNX); + Add("SETEX", RespCommand.SETEX); + Add("PSETEX", RespCommand.PSETEX); + Add("MGET", RespCommand.MGET); + Add("MSET", RespCommand.MSET); + Add("MSETNX", RespCommand.MSETNX); + Add("DUMP", RespCommand.DUMP); + Add("RESTORE", RespCommand.RESTORE); + Add("GETBIT", RespCommand.GETBIT); + Add("SETBIT", RespCommand.SETBIT); + Add("GETWITHETAG", RespCommand.GETWITHETAG); + Add("GETIFNOTMATCH", RespCommand.GETIFNOTMATCH); + Add("SETIFMATCH", RespCommand.SETIFMATCH); + Add("SETIFGREATER", RespCommand.SETIFGREATER); + Add("DELIFGREATER", RespCommand.DELIFGREATER); + Add("LCS", RespCommand.LCS); + + // Key commands + Add("EXISTS", RespCommand.EXISTS); + Add("TTL", RespCommand.TTL); + Add("PTTL", RespCommand.PTTL); + Add("EXPIRE", RespCommand.EXPIRE); + Add("PEXPIRE", RespCommand.PEXPIRE); + Add("EXPIREAT", RespCommand.EXPIREAT); + Add("PEXPIREAT", RespCommand.PEXPIREAT); + Add("EXPIRETIME", RespCommand.EXPIRETIME); + Add("PEXPIRETIME", RespCommand.PEXPIRETIME); + Add("PERSIST", RespCommand.PERSIST); + Add("TYPE", RespCommand.TYPE); + Add("RENAME", RespCommand.RENAME); + Add("RENAMENX", RespCommand.RENAMENX); + Add("UNLINK", RespCommand.UNLINK); + Add("KEYS", RespCommand.KEYS); + Add("SCAN", RespCommand.SCAN); + Add("DBSIZE", RespCommand.DBSIZE); + Add("SELECT", RespCommand.SELECT); + Add("SWAPDB", RespCommand.SWAPDB); + Add("MIGRATE", RespCommand.MIGRATE); + + // Bitmap commands + Add("BITCOUNT", RespCommand.BITCOUNT); + Add("BITPOS", RespCommand.BITPOS); + Add("BITFIELD", RespCommand.BITFIELD); + Add("BITFIELD_RO", RespCommand.BITFIELD_RO); + Add("BITOP", RespCommand.BITOP, hasSub: true); + + // HyperLogLog commands + Add("PFADD", RespCommand.PFADD); + Add("PFCOUNT", RespCommand.PFCOUNT); + Add("PFMERGE", RespCommand.PFMERGE); + + // Hash commands + Add("HSET", RespCommand.HSET); + Add("HGET", RespCommand.HGET); + Add("HDEL", RespCommand.HDEL); + Add("HLEN", RespCommand.HLEN); + Add("HEXISTS", RespCommand.HEXISTS); + Add("HGETALL", RespCommand.HGETALL); + Add("HKEYS", RespCommand.HKEYS); + Add("HVALS", RespCommand.HVALS); + Add("HMSET", RespCommand.HMSET); + Add("HMGET", RespCommand.HMGET); + Add("HSETNX", RespCommand.HSETNX); + Add("HINCRBY", RespCommand.HINCRBY); + Add("HINCRBYFLOAT", RespCommand.HINCRBYFLOAT); + Add("HRANDFIELD", RespCommand.HRANDFIELD); + Add("HSCAN", RespCommand.HSCAN); + Add("HSTRLEN", RespCommand.HSTRLEN); + Add("HTTL", RespCommand.HTTL); + Add("HPTTL", RespCommand.HPTTL); + Add("HEXPIRE", RespCommand.HEXPIRE); + Add("HPEXPIRE", RespCommand.HPEXPIRE); + Add("HEXPIREAT", RespCommand.HEXPIREAT); + Add("HPEXPIREAT", RespCommand.HPEXPIREAT); + Add("HEXPIRETIME", RespCommand.HEXPIRETIME); + Add("HPEXPIRETIME", RespCommand.HPEXPIRETIME); + Add("HPERSIST", RespCommand.HPERSIST); + Add("HCOLLECT", RespCommand.HCOLLECT); + + // List commands + Add("LPUSH", RespCommand.LPUSH); + Add("RPUSH", RespCommand.RPUSH); + Add("LPUSHX", RespCommand.LPUSHX); + Add("RPUSHX", RespCommand.RPUSHX); + Add("LPOP", RespCommand.LPOP); + Add("RPOP", RespCommand.RPOP); + Add("LLEN", RespCommand.LLEN); + Add("LINDEX", RespCommand.LINDEX); + Add("LINSERT", RespCommand.LINSERT); + Add("LRANGE", RespCommand.LRANGE); + Add("LREM", RespCommand.LREM); + Add("LSET", RespCommand.LSET); + Add("LTRIM", RespCommand.LTRIM); + Add("LPOS", RespCommand.LPOS); + Add("LMOVE", RespCommand.LMOVE); + Add("LMPOP", RespCommand.LMPOP); + Add("RPOPLPUSH", RespCommand.RPOPLPUSH); + Add("BLPOP", RespCommand.BLPOP); + Add("BRPOP", RespCommand.BRPOP); + Add("BLMOVE", RespCommand.BLMOVE); + Add("BRPOPLPUSH", RespCommand.BRPOPLPUSH); + Add("BLMPOP", RespCommand.BLMPOP); + + // Set commands + Add("SADD", RespCommand.SADD); + Add("SREM", RespCommand.SREM); + Add("SPOP", RespCommand.SPOP); + Add("SCARD", RespCommand.SCARD); + Add("SMEMBERS", RespCommand.SMEMBERS); + Add("SISMEMBER", RespCommand.SISMEMBER); + Add("SMISMEMBER", RespCommand.SMISMEMBER); + Add("SRANDMEMBER", RespCommand.SRANDMEMBER); + Add("SMOVE", RespCommand.SMOVE); + Add("SSCAN", RespCommand.SSCAN); + Add("SDIFF", RespCommand.SDIFF); + Add("SDIFFSTORE", RespCommand.SDIFFSTORE); + Add("SINTER", RespCommand.SINTER); + Add("SINTERCARD", RespCommand.SINTERCARD); + Add("SINTERSTORE", RespCommand.SINTERSTORE); + Add("SUNION", RespCommand.SUNION); + Add("SUNIONSTORE", RespCommand.SUNIONSTORE); + + // Sorted set commands + Add("ZADD", RespCommand.ZADD); + Add("ZREM", RespCommand.ZREM); + Add("ZCARD", RespCommand.ZCARD); + Add("ZSCORE", RespCommand.ZSCORE); + Add("ZMSCORE", RespCommand.ZMSCORE); + Add("ZRANK", RespCommand.ZRANK); + Add("ZREVRANK", RespCommand.ZREVRANK); + Add("ZCOUNT", RespCommand.ZCOUNT); + Add("ZLEXCOUNT", RespCommand.ZLEXCOUNT); + Add("ZRANGE", RespCommand.ZRANGE); + Add("ZRANGEBYLEX", RespCommand.ZRANGEBYLEX); + Add("ZRANGEBYSCORE", RespCommand.ZRANGEBYSCORE); + Add("ZRANGESTORE", RespCommand.ZRANGESTORE); + Add("ZREVRANGE", RespCommand.ZREVRANGE); + Add("ZREVRANGEBYLEX", RespCommand.ZREVRANGEBYLEX); + Add("ZREVRANGEBYSCORE", RespCommand.ZREVRANGEBYSCORE); + Add("ZPOPMIN", RespCommand.ZPOPMIN); + Add("ZPOPMAX", RespCommand.ZPOPMAX); + Add("ZRANDMEMBER", RespCommand.ZRANDMEMBER); + Add("ZSCAN", RespCommand.ZSCAN); + Add("ZINCRBY", RespCommand.ZINCRBY); + Add("ZDIFF", RespCommand.ZDIFF); + Add("ZDIFFSTORE", RespCommand.ZDIFFSTORE); + Add("ZINTER", RespCommand.ZINTER); + Add("ZINTERCARD", RespCommand.ZINTERCARD); + Add("ZINTERSTORE", RespCommand.ZINTERSTORE); + Add("ZUNION", RespCommand.ZUNION); + Add("ZUNIONSTORE", RespCommand.ZUNIONSTORE); + Add("ZMPOP", RespCommand.ZMPOP); + Add("BZMPOP", RespCommand.BZMPOP); + Add("BZPOPMAX", RespCommand.BZPOPMAX); + Add("BZPOPMIN", RespCommand.BZPOPMIN); + Add("ZREMRANGEBYLEX", RespCommand.ZREMRANGEBYLEX); + Add("ZREMRANGEBYRANK", RespCommand.ZREMRANGEBYRANK); + Add("ZREMRANGEBYSCORE", RespCommand.ZREMRANGEBYSCORE); + Add("ZTTL", RespCommand.ZTTL); + Add("ZPTTL", RespCommand.ZPTTL); + Add("ZEXPIRE", RespCommand.ZEXPIRE); + Add("ZPEXPIRE", RespCommand.ZPEXPIRE); + Add("ZEXPIREAT", RespCommand.ZEXPIREAT); + Add("ZPEXPIREAT", RespCommand.ZPEXPIREAT); + Add("ZEXPIRETIME", RespCommand.ZEXPIRETIME); + Add("ZPEXPIRETIME", RespCommand.ZPEXPIRETIME); + Add("ZPERSIST", RespCommand.ZPERSIST); + Add("ZCOLLECT", RespCommand.ZCOLLECT); + + // Geo commands + Add("GEOADD", RespCommand.GEOADD); + Add("GEOPOS", RespCommand.GEOPOS); + Add("GEOHASH", RespCommand.GEOHASH); + Add("GEODIST", RespCommand.GEODIST); + Add("GEOSEARCH", RespCommand.GEOSEARCH); + Add("GEOSEARCHSTORE", RespCommand.GEOSEARCHSTORE); + Add("GEORADIUS", RespCommand.GEORADIUS); + Add("GEORADIUS_RO", RespCommand.GEORADIUS_RO); + Add("GEORADIUSBYMEMBER", RespCommand.GEORADIUSBYMEMBER); + Add("GEORADIUSBYMEMBER_RO", RespCommand.GEORADIUSBYMEMBER_RO); + + // Scripting + Add("EVAL", RespCommand.EVAL); + Add("EVALSHA", RespCommand.EVALSHA); + + // Pub/Sub + Add("PUBLISH", RespCommand.PUBLISH); + Add("SUBSCRIBE", RespCommand.SUBSCRIBE); + Add("PSUBSCRIBE", RespCommand.PSUBSCRIBE); + Add("UNSUBSCRIBE", RespCommand.UNSUBSCRIBE); + Add("PUNSUBSCRIBE", RespCommand.PUNSUBSCRIBE); + Add("SPUBLISH", RespCommand.SPUBLISH); + Add("SSUBSCRIBE", RespCommand.SSUBSCRIBE); + + // Custom object scan + Add("CUSTOMOBJECTSCAN", RespCommand.COSCAN); + + // ===== Control / admin commands ===== + Add("PING", RespCommand.PING); + Add("ECHO", RespCommand.ECHO); + Add("QUIT", RespCommand.QUIT); + Add("AUTH", RespCommand.AUTH); + Add("HELLO", RespCommand.HELLO); + Add("INFO", RespCommand.INFO); + Add("TIME", RespCommand.TIME); + Add("ROLE", RespCommand.ROLE); + Add("SAVE", RespCommand.SAVE); + Add("LASTSAVE", RespCommand.LASTSAVE); + Add("BGSAVE", RespCommand.BGSAVE); + Add("COMMITAOF", RespCommand.COMMITAOF); + Add("FLUSHALL", RespCommand.FLUSHALL); + Add("FLUSHDB", RespCommand.FLUSHDB); + Add("FORCEGC", RespCommand.FORCEGC); + Add("PURGEBP", RespCommand.PURGEBP); + Add("FAILOVER", RespCommand.FAILOVER); + Add("MONITOR", RespCommand.MONITOR); + Add("REGISTERCS", RespCommand.REGISTERCS); + Add("ASYNC", RespCommand.ASYNC); + Add("DEBUG", RespCommand.DEBUG); + Add("EXPDELSCAN", RespCommand.EXPDELSCAN); + Add("WATCH", RespCommand.WATCH); + Add("WATCHMS", RespCommand.WATCHMS); + Add("WATCHOS", RespCommand.WATCHOS); + Add("MULTI", RespCommand.MULTI); + Add("EXEC", RespCommand.EXEC); + Add("DISCARD", RespCommand.DISCARD); + Add("UNWATCH", RespCommand.UNWATCH); + Add("RUNTXP", RespCommand.RUNTXP); + Add("ASKING", RespCommand.ASKING); + Add("READONLY", RespCommand.READONLY); + Add("READWRITE", RespCommand.READWRITE); + Add("REPLICAOF", RespCommand.REPLICAOF); + Add("SECONDARYOF", RespCommand.SECONDARYOF); + Add("SLAVEOF", RespCommand.SECONDARYOF); + + // Parent commands with subcommands + Add("SCRIPT", RespCommand.SCRIPT, hasSub: true); + Add("CONFIG", RespCommand.CONFIG, hasSub: true); + Add("CLIENT", RespCommand.CLIENT, hasSub: true); + Add("CLUSTER", RespCommand.CLUSTER, hasSub: true); + Add("ACL", RespCommand.ACL, hasSub: true); + Add("COMMAND", RespCommand.COMMAND, hasSub: true); + Add("LATENCY", RespCommand.LATENCY, hasSub: true); + Add("SLOWLOG", RespCommand.SLOWLOG, hasSub: true); + Add("MODULE", RespCommand.MODULE, hasSub: true); + Add("PUBSUB", RespCommand.PUBSUB, hasSub: true); + Add("MEMORY", RespCommand.MEMORY, hasSub: true); + } + + #endregion + + #region Subcommand Definitions + + private static readonly (string Name, RespCommand Command)[] ClusterSubcommands = + [ + ("ADDSLOTS", RespCommand.CLUSTER_ADDSLOTS), + ("ADDSLOTSRANGE", RespCommand.CLUSTER_ADDSLOTSRANGE), + ("AOFSYNC", RespCommand.CLUSTER_AOFSYNC), + ("APPENDLOG", RespCommand.CLUSTER_APPENDLOG), + ("ATTACH_SYNC", RespCommand.CLUSTER_ATTACH_SYNC), + ("BANLIST", RespCommand.CLUSTER_BANLIST), + ("BEGIN_REPLICA_RECOVER", RespCommand.CLUSTER_BEGIN_REPLICA_RECOVER), + ("BUMPEPOCH", RespCommand.CLUSTER_BUMPEPOCH), + ("COUNTKEYSINSLOT", RespCommand.CLUSTER_COUNTKEYSINSLOT), + ("DELKEYSINSLOT", RespCommand.CLUSTER_DELKEYSINSLOT), + ("DELKEYSINSLOTRANGE", RespCommand.CLUSTER_DELKEYSINSLOTRANGE), + ("DELSLOTS", RespCommand.CLUSTER_DELSLOTS), + ("DELSLOTSRANGE", RespCommand.CLUSTER_DELSLOTSRANGE), + ("ENDPOINT", RespCommand.CLUSTER_ENDPOINT), + ("FAILOVER", RespCommand.CLUSTER_FAILOVER), + ("FAILREPLICATIONOFFSET", RespCommand.CLUSTER_FAILREPLICATIONOFFSET), + ("FAILSTOPWRITES", RespCommand.CLUSTER_FAILSTOPWRITES), + ("FLUSHALL", RespCommand.CLUSTER_FLUSHALL), + ("FORGET", RespCommand.CLUSTER_FORGET), + ("GETKEYSINSLOT", RespCommand.CLUSTER_GETKEYSINSLOT), + ("GOSSIP", RespCommand.CLUSTER_GOSSIP), + ("HELP", RespCommand.CLUSTER_HELP), + ("INFO", RespCommand.CLUSTER_INFO), + ("INITIATE_REPLICA_SYNC", RespCommand.CLUSTER_INITIATE_REPLICA_SYNC), + ("KEYSLOT", RespCommand.CLUSTER_KEYSLOT), + ("MEET", RespCommand.CLUSTER_MEET), + ("MIGRATE", RespCommand.CLUSTER_MIGRATE), + ("MTASKS", RespCommand.CLUSTER_MTASKS), + ("MYID", RespCommand.CLUSTER_MYID), + ("MYPARENTID", RespCommand.CLUSTER_MYPARENTID), + ("NODES", RespCommand.CLUSTER_NODES), + ("PUBLISH", RespCommand.CLUSTER_PUBLISH), + ("SPUBLISH", RespCommand.CLUSTER_SPUBLISH), + ("REPLICAS", RespCommand.CLUSTER_REPLICAS), + ("REPLICATE", RespCommand.CLUSTER_REPLICATE), + ("RESET", RespCommand.CLUSTER_RESET), + ("SEND_CKPT_FILE_SEGMENT", RespCommand.CLUSTER_SEND_CKPT_FILE_SEGMENT), + ("SEND_CKPT_METADATA", RespCommand.CLUSTER_SEND_CKPT_METADATA), + ("SET-CONFIG-EPOCH", RespCommand.CLUSTER_SETCONFIGEPOCH), + ("SETSLOT", RespCommand.CLUSTER_SETSLOT), + ("SETSLOTSRANGE", RespCommand.CLUSTER_SETSLOTSRANGE), + ("SHARDS", RespCommand.CLUSTER_SHARDS), + ("SLOTS", RespCommand.CLUSTER_SLOTS), + ("SLOTSTATE", RespCommand.CLUSTER_SLOTSTATE), + ("SYNC", RespCommand.CLUSTER_SYNC), + ]; + + private static readonly (string Name, RespCommand Command)[] ClientSubcommands = + [ + ("ID", RespCommand.CLIENT_ID), + ("INFO", RespCommand.CLIENT_INFO), + ("LIST", RespCommand.CLIENT_LIST), + ("KILL", RespCommand.CLIENT_KILL), + ("GETNAME", RespCommand.CLIENT_GETNAME), + ("SETNAME", RespCommand.CLIENT_SETNAME), + ("SETINFO", RespCommand.CLIENT_SETINFO), + ("UNBLOCK", RespCommand.CLIENT_UNBLOCK), + ]; + + private static readonly (string Name, RespCommand Command)[] AclSubcommands = + [ + ("CAT", RespCommand.ACL_CAT), + ("DELUSER", RespCommand.ACL_DELUSER), + ("GENPASS", RespCommand.ACL_GENPASS), + ("GETUSER", RespCommand.ACL_GETUSER), + ("LIST", RespCommand.ACL_LIST), + ("LOAD", RespCommand.ACL_LOAD), + ("SAVE", RespCommand.ACL_SAVE), + ("SETUSER", RespCommand.ACL_SETUSER), + ("USERS", RespCommand.ACL_USERS), + ("WHOAMI", RespCommand.ACL_WHOAMI), + ]; + + private static readonly (string Name, RespCommand Command)[] CommandSubcommands = + [ + ("COUNT", RespCommand.COMMAND_COUNT), + ("DOCS", RespCommand.COMMAND_DOCS), + ("INFO", RespCommand.COMMAND_INFO), + ("GETKEYS", RespCommand.COMMAND_GETKEYS), + ("GETKEYSANDFLAGS", RespCommand.COMMAND_GETKEYSANDFLAGS), + ]; + + private static readonly (string Name, RespCommand Command)[] ConfigSubcommands = + [ + ("GET", RespCommand.CONFIG_GET), + ("REWRITE", RespCommand.CONFIG_REWRITE), + ("SET", RespCommand.CONFIG_SET), + ]; + + private static readonly (string Name, RespCommand Command)[] ScriptSubcommands = + [ + ("LOAD", RespCommand.SCRIPT_LOAD), + ("FLUSH", RespCommand.SCRIPT_FLUSH), + ("EXISTS", RespCommand.SCRIPT_EXISTS), + ]; + + private static readonly (string Name, RespCommand Command)[] LatencySubcommands = + [ + ("HELP", RespCommand.LATENCY_HELP), + ("HISTOGRAM", RespCommand.LATENCY_HISTOGRAM), + ("RESET", RespCommand.LATENCY_RESET), + ]; + + private static readonly (string Name, RespCommand Command)[] SlowlogSubcommands = + [ + ("HELP", RespCommand.SLOWLOG_HELP), + ("GET", RespCommand.SLOWLOG_GET), + ("LEN", RespCommand.SLOWLOG_LEN), + ("RESET", RespCommand.SLOWLOG_RESET), + ]; + + private static readonly (string Name, RespCommand Command)[] ModuleSubcommands = + [ + ("LOADCS", RespCommand.MODULE_LOADCS), + ]; + + private static readonly (string Name, RespCommand Command)[] PubsubSubcommands = + [ + ("CHANNELS", RespCommand.PUBSUB_CHANNELS), + ("NUMSUB", RespCommand.PUBSUB_NUMSUB), + ("NUMPAT", RespCommand.PUBSUB_NUMPAT), + ]; + + private static readonly (string Name, RespCommand Command)[] MemorySubcommands = + [ + ("USAGE", RespCommand.MEMORY_USAGE), + ]; + + private static readonly (string Name, RespCommand Command)[] BitopSubcommands = + [ + ("AND", RespCommand.BITOP_AND), + ("OR", RespCommand.BITOP_OR), + ("XOR", RespCommand.BITOP_XOR), + ("NOT", RespCommand.BITOP_NOT), + ("DIFF", RespCommand.BITOP_DIFF), + ]; + + #endregion + } +} \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommandSimdPatterns.cs b/libs/server/Resp/Parser/RespCommandSimdPatterns.cs new file mode 100644 index 00000000000..71203fec8e4 --- /dev/null +++ b/libs/server/Resp/Parser/RespCommandSimdPatterns.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.Intrinsics; + +namespace Garnet.server +{ + /// + /// SIMD Vector128 patterns and masks for . + /// + internal sealed unsafe partial class RespServerSession + { + // SIMD Vector128 patterns for FastParseCommand. + // Each encodes the full RESP header + command: *N\r\n$L\r\nCMD\r\n + // Masks zero out trailing bytes for patterns shorter than 16 bytes. + private static readonly Vector128 s_mask13 = Vector128.Create( + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00).AsByte(); + private static readonly Vector128 s_mask14 = Vector128.Create( + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00).AsByte(); + private static readonly Vector128 s_mask15 = Vector128.Create( + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00).AsByte(); + + /// + /// Builds a Vector128 RESP pattern for SIMD matching: *{argCount}\r\n${cmdLen}\r\n{cmd}\r\n + /// Zero-padded to 16 bytes. Only called during static init. + /// + private static Vector128 RespPattern(int argCount, string cmd) + { + // Total encoded length: 4 (*N\r\n) + 4 ($L\r\n) + cmd.Length + 2 (\r\n) = cmd.Length + 10 + var totalLen = cmd.Length + 10; + if (totalLen > 16) + throw new ArgumentException($"SIMD pattern overflow: command '{cmd}' with {argCount} args requires {totalLen} bytes, max 16"); + if (argCount < 1 || argCount > 9) + throw new ArgumentException($"SIMD pattern requires single-digit arg count (1-9), got {argCount} for '{cmd}'"); + if (cmd.Length < 1 || cmd.Length > 9) + throw new ArgumentException($"SIMD pattern requires single-digit command length (1-9), got {cmd.Length} for '{cmd}'"); + + Span buf = stackalloc byte[16]; + buf.Clear(); + buf[0] = (byte)'*'; + buf[1] = (byte)('0' + argCount); + buf[2] = (byte)'\r'; + buf[3] = (byte)'\n'; + buf[4] = (byte)'$'; + buf[5] = (byte)('0' + cmd.Length); + buf[6] = (byte)'\r'; + buf[7] = (byte)'\n'; + for (int i = 0; i < cmd.Length; i++) + buf[8 + i] = (byte)cmd[i]; + buf[8 + cmd.Length] = (byte)'\r'; + buf[9 + cmd.Length] = (byte)'\n'; + return Vector128.Create(buf); + } + + // 13-byte: *N\r\n$3\r\nXXX\r\n (3-char commands) + private static readonly Vector128 s_GET = RespPattern(2, "GET"); + private static readonly Vector128 s_SET = RespPattern(3, "SET"); + private static readonly Vector128 s_DEL = RespPattern(2, "DEL"); + private static readonly Vector128 s_TTL = RespPattern(2, "TTL"); + + // 14-byte: *N\r\n$4\r\nXXXX\r\n (4-char commands) + private static readonly Vector128 s_PING = RespPattern(1, "PING"); + private static readonly Vector128 s_INCR = RespPattern(2, "INCR"); + private static readonly Vector128 s_DECR = RespPattern(2, "DECR"); + private static readonly Vector128 s_EXEC = RespPattern(1, "EXEC"); + private static readonly Vector128 s_PTTL = RespPattern(2, "PTTL"); + + // 15-byte: *N\r\n$5\r\nXXXXX\r\n (5-char commands) + private static readonly Vector128 s_MULTI = RespPattern(1, "MULTI"); + private static readonly Vector128 s_SETNX = RespPattern(3, "SETNX"); + private static readonly Vector128 s_SETEX = RespPattern(4, "SETEX"); + + // 16-byte: *N\r\n$6\r\nXXXXXX\r\n (6-char commands, no mask needed) + private static readonly Vector128 s_EXISTS = RespPattern(2, "EXISTS"); + private static readonly Vector128 s_GETDEL = RespPattern(2, "GETDEL"); + private static readonly Vector128 s_APPEND = RespPattern(3, "APPEND"); + private static readonly Vector128 s_INCRBY = RespPattern(3, "INCRBY"); + private static readonly Vector128 s_DECRBY = RespPattern(3, "DECRBY"); + private static readonly Vector128 s_PSETEX = RespPattern(4, "PSETEX"); + } +} \ No newline at end of file From b868080bc40e98956368ebf0fb22a6ce9f181d0e Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Wed, 1 Apr 2026 14:34:01 -0700 Subject: [PATCH 16/19] nits --- .../Operations/CommandParsingBenchmark.cs | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs index 2130ec67831..288c61835b6 100644 --- a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs +++ b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs @@ -22,7 +22,10 @@ public unsafe class CommandParsingBenchmark : OperationsBase static ReadOnlySpan CMD_EXISTS => "*2\r\n$6\r\nEXISTS\r\n$1\r\na\r\n"u8; static ReadOnlySpan CMD_SETEX => "*4\r\n$5\r\nSETEX\r\n$1\r\na\r\n$2\r\n60\r\n$1\r\nb\r\n"u8; - // Tier 0b: Scalar ulong switch (variable-arg commands, or when remainingBytes < 16) + // Tier 0b: Scalar path — hot commands too long for SIMD (name > 6 chars, exceeds 16-byte Vector128) + static ReadOnlySpan CMD_PUBLISH => "*3\r\n$7\r\nPUBLISH\r\n$2\r\nch\r\n$5\r\nhello\r\n"u8; + + // Tier 0c: Scalar path — variable-arg hot commands (arg count varies, cannot be SIMD or MRU cached) static ReadOnlySpan CMD_EXPIRE => "*3\r\n$6\r\nEXPIRE\r\n$1\r\na\r\n$2\r\n60\r\n"u8; // Tier 1: Hash table lookup via ArrayParseCommand → HashLookupCommand (+ MRU cache on 2nd+ call) @@ -41,7 +44,7 @@ public unsafe class CommandParsingBenchmark : OperationsBase static ReadOnlySpan CMD_SETIFMATCH => "*4\r\n$10\r\nSETIFMATCH\r\n$1\r\na\r\n$1\r\nb\r\n$1\r\n0\r\n"u8; // Pre-allocated buffers (pinned for pointer stability) - byte[] bufPing, bufGet, bufSet, bufIncr, bufExists, bufSetex, bufExpire, bufHset, bufLpush, bufZadd, bufSubscribe; + byte[] bufPing, bufGet, bufSet, bufIncr, bufExists, bufSetex, bufPublish, bufExpire, bufHset, bufLpush, bufZadd, bufSubscribe; byte[] bufZrangebyscore, bufZremrangebyscore, bufHincrbyfloat, bufGeoradius, bufSetifmatch; public override void GlobalSetup() @@ -63,6 +66,8 @@ public override void GlobalSetup() CMD_EXISTS.CopyTo(bufExists); bufSetex = GC.AllocateArray(CMD_SETEX.Length, pinned: true); CMD_SETEX.CopyTo(bufSetex); + bufPublish = GC.AllocateArray(CMD_PUBLISH.Length, pinned: true); + CMD_PUBLISH.CopyTo(bufPublish); bufExpire = GC.AllocateArray(CMD_EXPIRE.Length, pinned: true); CMD_EXPIRE.CopyTo(bufExpire); bufHset = GC.AllocateArray(CMD_HSET.Length, pinned: true); @@ -133,7 +138,6 @@ public RespCommand ParseEXISTS() } // === Tier 0a: SIMD fast path (SETEX is a 15-byte SIMD pattern) === - // === Tier 0b: Scalar ulong switch === [Benchmark] public RespCommand ParseSETEX() @@ -144,6 +148,19 @@ public RespCommand ParseSETEX() return result; } + // === Tier 0b: Scalar path — hot commands too long for SIMD === + + [Benchmark] + public RespCommand ParsePUBLISH() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufPublish); + return result; + } + + // === Tier 0c: Scalar path — variable-arg hot commands === + [Benchmark] public RespCommand ParseEXPIRE() { From b95ac6a99da3d9f693b7b97efd2b3627be83e0d1 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Wed, 1 Apr 2026 15:02:55 -0700 Subject: [PATCH 17/19] fixes --- libs/server/Resp/Parser/RespCommand.cs | 7 +++---- libs/server/Resp/Parser/RespCommandHashLookup.cs | 9 +++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 04126b5e729..26890df48b3 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -781,14 +781,13 @@ private RespCommand FastParseCommand(out int count) // Extract length of the first string header var length = ptr[5] - '0'; - Debug.Assert(length is > 0 and <= 9); var oldReadHead = readHead; - // Ensure that the complete command string is contained in the package. Otherwise exit early. - // Include 10 bytes to account for array and command string headers, and terminator + // Ensure valid command name length (1-9) and that the complete command string + // is contained in the package. Otherwise fall through to return NONE. // 10 bytes = "*_\r\n$_\r\n" (8 bytes) + "\r\n" (2 bytes) at end of command name - if (remainingBytes >= length + 10) + if (length is > 0 and <= 9 && remainingBytes >= length + 10) { // Optimistically advance read head to the end of the command name readHead += length + 10; diff --git a/libs/server/Resp/Parser/RespCommandHashLookup.cs b/libs/server/Resp/Parser/RespCommandHashLookup.cs index a324eea648c..7b5d317d599 100644 --- a/libs/server/Resp/Parser/RespCommandHashLookup.cs +++ b/libs/server/Resp/Parser/RespCommandHashLookup.cs @@ -232,7 +232,7 @@ public static RespCommand Lookup(byte* name, int length) public static RespCommand Lookup(byte* name, int length, out bool hasSubcommands) { hasSubcommands = false; - if ((uint)length > 24) + if ((uint)length - 1 > 23) return RespCommand.NONE; uint hash = ComputeHash(name, length); @@ -390,11 +390,8 @@ private static bool MatchName(ref CommandEntry entry, byte* name, int length) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static RespCommand LookupInTable(CommandEntry[] table, int tableMask, byte* name, int length) { - Debug.Assert(length > 0, "Command name length must be positive"); - Debug.Assert(table != null, "Hash table must not be null"); - - // CommandEntry stores at most 24 bytes of name; longer names can never match. - if ((uint)length > 24) return RespCommand.NONE; + // CommandEntry stores at most 24 bytes of name; empty or oversized names can never match. + if ((uint)length - 1 > 23) return RespCommand.NONE; uint hash = ComputeHash(name, length); int idx = (int)(hash & (uint)tableMask); From f6845a4767dbdf79ca149017b31a26c4feae8fcf Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Wed, 1 Apr 2026 15:54:14 -0700 Subject: [PATCH 18/19] fix flaky test --- test/Garnet.test/RespTests.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs index 47f506404c5..cb4683290b2 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/Garnet.test/RespTests.cs @@ -2672,21 +2672,22 @@ public void KeyExpireOptionsTest([Values("EXPIRE", "PEXPIRE")] string command, [ resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsFalse(resp);// NX return false existing expiry - args[1] = 50; + var ttlSmall = command.Equals("EXPIRE") ? 50 : 50000; // 50 seconds or 50000 ms + args[1] = ttlSmall; args[2] = testCaseSensitivity ? "xx" : "XX";// XX -- Set expiry only when the key has an existing expiry resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsTrue(resp);// XX return true existing expiry var time = db.KeyTimeToLive(key); ClassicAssert.Greater(time.Value.TotalSeconds, 0); - ClassicAssert.LessOrEqual(time.Value.TotalSeconds, (int)args[1]); + ClassicAssert.LessOrEqual(time.Value.TotalSeconds, ttlSmall); args[1] = 1; args[2] = testCaseSensitivity ? "Gt" : "GT";// GT -- Set expiry only when the new expiry is greater than current one resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsFalse(resp); // GT return false new expiry < current expiry - args[1] = 1000; + args[1] = command.Equals("EXPIRE") ? 1000 : 1000000; // 1000 seconds or 1000000 ms args[2] = testCaseSensitivity ? "gT" : "GT";// GT -- Set expiry only when the new expiry is greater than current one resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsTrue(resp); // GT return true new expiry > current expiry @@ -2695,12 +2696,12 @@ public void KeyExpireOptionsTest([Values("EXPIRE", "PEXPIRE")] string command, [ ClassicAssert.Greater(command.Equals("EXPIRE") ? time.Value.TotalSeconds : time.Value.TotalMilliseconds, 500); - args[1] = 2000; + args[1] = command.Equals("EXPIRE") ? 2000 : 2000000; // must be > GT value above args[2] = testCaseSensitivity ? "lt" : "LT";// LT -- Set expiry only when the new expiry is less than current one resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsFalse(resp); // LT return false new expiry > current expiry - args[1] = 500; + args[1] = command.Equals("EXPIRE") ? 500 : 500000; // must be < GT value above args[2] = testCaseSensitivity ? "lT" : "LT";// LT -- Set expiry only when the new expiry is less than current one resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsTrue(resp); // LT return true new expiry < current expiry From 5db102b5033ddd5648856dcdb1735219f1798694 Mon Sep 17 00:00:00 2001 From: Badrish Chandramouli Date: Wed, 1 Apr 2026 16:13:58 -0700 Subject: [PATCH 19/19] small cleanup --- libs/server/Resp/Parser/RespCommand.cs | 57 ++++++++++++++++---------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 26890df48b3..7c2827606cb 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -801,35 +801,48 @@ private RespCommand FastParseCommand(out int count) // Only check against commands with the correct count and length. + // (1) Same fixed-arg hot commands as SIMD path. + // On SIMD hardware, SIMD already rejected these — skip to sections (2)/(3). + // The !Vector128.IsHardwareAccelerated check is a JIT constant that eliminates + // this block entirely on SIMD hardware. + if (!Vector128.IsHardwareAccelerated) + { + var simdResult = ((count << 4) | length) switch + { + 4 when lastWord == MemoryMarshal.Read("\r\nPING\r\n"u8) => RespCommand.PING, + 4 when lastWord == MemoryMarshal.Read("\r\nEXEC\r\n"u8) => RespCommand.EXEC, + 5 when lastWord == MemoryMarshal.Read("\nMULTI\r\n"u8) => RespCommand.MULTI, + (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nGET\r\n"u8) => RespCommand.GET, + (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nDEL\r\n"u8) => RespCommand.DEL, + (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nTTL\r\n"u8) => RespCommand.TTL, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nINCR\r\n"u8) => RespCommand.INCR, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nPTTL\r\n"u8) => RespCommand.PTTL, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDECR\r\n"u8) => RespCommand.DECR, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("EXISTS\r\n"u8) => RespCommand.EXISTS, + (1 << 4) | 6 when lastWord == MemoryMarshal.Read("GETDEL\r\n"u8) => RespCommand.GETDEL, + (2 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SET, + (2 << 4) | 6 when lastWord == MemoryMarshal.Read("INCRBY\r\n"u8) => RespCommand.INCRBY, + (2 << 4) | 6 when lastWord == MemoryMarshal.Read("DECRBY\r\n"u8) => RespCommand.DECRBY, + (2 << 4) | 6 when lastWord == MemoryMarshal.Read("APPEND\r\n"u8) => RespCommand.APPEND, + (2 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETNX\r\n"u8) => RespCommand.SETNX, + (3 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETEX\r\n"u8) => RespCommand.SETEX, + (3 << 4) | 6 when lastWord == MemoryMarshal.Read("PSETEX\r\n"u8) => RespCommand.PSETEX, + _ => RespCommand.NONE + }; + if (simdResult != RespCommand.NONE) + return simdResult; + } + + // (2) Hot commands too long for SIMD (name > 6 chars, exceeds 16-byte Vector128) + // (3) Hot variable-arg commands (arg count varies, cannot be SIMD or MRU cached) + // These run on all hardware — SIMD cannot handle them. return ((count << 4) | length) switch { - // (1) Same fixed-arg hot commands as SIMD path — fallback when remainingBytes < 16 - 4 when lastWord == MemoryMarshal.Read("\r\nPING\r\n"u8) => RespCommand.PING, - 4 when lastWord == MemoryMarshal.Read("\r\nEXEC\r\n"u8) => RespCommand.EXEC, - 5 when lastWord == MemoryMarshal.Read("\nMULTI\r\n"u8) => RespCommand.MULTI, - (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nGET\r\n"u8) => RespCommand.GET, - (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nDEL\r\n"u8) => RespCommand.DEL, - (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nTTL\r\n"u8) => RespCommand.TTL, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nINCR\r\n"u8) => RespCommand.INCR, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nPTTL\r\n"u8) => RespCommand.PTTL, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDECR\r\n"u8) => RespCommand.DECR, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("EXISTS\r\n"u8) => RespCommand.EXISTS, - (1 << 4) | 6 when lastWord == MemoryMarshal.Read("GETDEL\r\n"u8) => RespCommand.GETDEL, - (2 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SET, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("INCRBY\r\n"u8) => RespCommand.INCRBY, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("DECRBY\r\n"u8) => RespCommand.DECRBY, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("APPEND\r\n"u8) => RespCommand.APPEND, - (2 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETNX\r\n"u8) => RespCommand.SETNX, - (3 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETEX\r\n"u8) => RespCommand.SETEX, - (3 << 4) | 6 when lastWord == MemoryMarshal.Read("PSETEX\r\n"u8) => RespCommand.PSETEX, - - // (2) Hot commands too long for SIMD (name > 6 chars, exceeds 16-byte Vector128) (2 << 4) | 7 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && ptr[8] == 'P' => RespCommand.PUBLISH, (2 << 4) | 8 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SP"u8) => RespCommand.SPUBLISH, (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SE"u8) => RespCommand.SETRANGE, (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("GE"u8) => RespCommand.GETRANGE, - // (3) Hot variable-arg commands (arg count varies, cannot be SIMD or MRU cached) _ => ((length << 4) | count) switch { >= ((3 << 4) | 3) and <= ((3 << 4) | 7) when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SETEXNX,