diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index ddd3947c55c..34b353a22f4 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -92,7 +92,7 @@ Full guide: https://microsoft.github.io/garnet/docs/dev/garnet-api ### Steps for a new built-in command: 1. **Define the command**: Add enum value to `RespCommand` in `libs/server/Resp/Parser/RespCommand.cs`. For object commands (List, SortedSet, Hash, Set), also add a value to the `[ObjectName]Operation` enum in `libs/server/Objects/[ObjectName]/[ObjectName]Object.cs`. -2. **Add parsing logic**: In `libs/server/Resp/Parser/RespCommand.cs`, add to `FastParseCommand` (fixed arg count) or `FastParseArrayCommand` (variable args). +2. **Add parsing logic**: In `libs/server/Resp/Parser/RespCommandHashLookupData.cs`, add an entry to `PopulatePrimaryTable()` (e.g., `Add("MYNEWCMD", RespCommand.MYNEWCMD)`). For commands with subcommands, set `hasSub: true` and add a subcommand table. The hash table provides O(1) lookup for all command name lengths. 3. **Declare the API method**: Add method signature to `IGarnetReadApi` (read-only) or `IGarnetApi` (read-write) in `libs/server/API/IGarnetApi.cs`. 4. **Implement the network handler**: Add a method to `RespServerSession` (the class is split across ~22 partial `.cs` files — object commands go in `libs/server/Resp/Objects/[ObjectName]Commands.cs`, others in `libs/server/Resp/BasicCommands.cs`, `ArrayCommands.cs`, `AdminCommands.cs`, `KeyAdminCommands.cs`, etc.). The handler parses arguments from the network buffer via `parseState.GetArgSliceByRef(i)` (returns `ref PinnedSpanByte`), calls the storage API, and writes the RESP response using `RespWriteUtils` helper methods, then calls `SendAndReset()` to flush the response buffer. 5. **Add dispatch route**: In `libs/server/Resp/RespServerSession.cs`, add a case to `ProcessBasicCommands` or `ProcessArrayCommands` calling the handler from step 4. diff --git a/.github/skills/add-garnet-command/SKILL.md b/.github/skills/add-garnet-command/SKILL.md index faeec7c6192..7a9491513e9 100644 --- a/.github/skills/add-garnet-command/SKILL.md +++ b/.github/skills/add-garnet-command/SKILL.md @@ -82,17 +82,39 @@ EVALSHA, // Note: Update LastDataCommand if adding new data commands after this **File:** `libs/server/Resp/Parser/RespCommand.cs` -Two parsing paths exist: +Two parsing tiers exist: -### Fast path: `FastParseCommand()` / `FastParseArrayCommand()` -Two fast-path methods exist with different constraints: -- **`FastParseCommand()`**: For commands with a fixed number of arguments and command names up to **9 characters**. Uses `ulong` pointer comparisons on `(count << 4) | length` patterns. -- **`FastParseArrayCommand()`**: For commands with a variable number of arguments and command names up to **16 characters**. Uses similar `ulong` comparison patterns but accommodates longer names. +### Hash table path: `RespCommandHashLookup` (primary path for most commands) +The hash table in `libs/server/Resp/Parser/RespCommandHashLookupData.cs` provides O(1) lookup for all built-in commands. This is the **recommended path for all new commands**. -Only add here if the command name is a simple word (no dots or special characters). +**To add a new primary command**, add one line to `PopulatePrimaryTable()` in `RespCommandHashLookupData.cs`: +```csharp +Add("DELIFGREATER", RespCommand.DELIFGREATER); +``` -### Slow path: `SlowParseCommand()` -For longer names, dot-prefixed names (like `RI.CREATE`), or names that don't fit the fast-path pattern. +**To add a command with subcommands** (e.g., `MYPARENT SUBCMD`): +1. Add the parent with `hasSub: true` in `PopulatePrimaryTable()`: + ```csharp + Add("MYPARENT", RespCommand.MYPARENT, hasSub: true); + ``` +2. Define the subcommand array in `RespCommandHashLookupData.cs`: + ```csharp + private static readonly (string Name, RespCommand Command)[] MyparentSubcommands = + [ + ("SUBCMD1", RespCommand.MYPARENT_SUBCMD1), + ("SUBCMD2", RespCommand.MYPARENT_SUBCMD2), + ]; + ``` +3. Build the table in the static constructor in `RespCommandHashLookup.cs`: + ```csharp + myparentSubTable = BuildSubTable(MyparentSubcommands, out myparentSubTableMask); + ``` +4. Wire it into `LookupSubcommand()` in `RespCommandHashLookup.cs`: + ```csharp + RespCommand.MYPARENT => (myparentSubTable, myparentSubTableMask), + ``` + +**⚠️ Important:** Use the exact wire-protocol spelling for the hash table name string. Some commands use hyphens (e.g., `"SET-CONFIG-EPOCH"` not `"SETCONFIGEPOCH"`). Check `CmdStrings.cs` for the canonical spelling. **⚠️ Convention:** Define the command name string in **`libs/server/Resp/CmdStrings.cs`** and reference it from the parser, rather than using inline `"..."u8` literals. This keeps command name strings centralized and reusable (e.g., for error messages). @@ -101,26 +123,12 @@ For longer names, dot-prefixed names (like `RI.CREATE`), or names that don't fit public static ReadOnlySpan DELIFGREATER => "DELIFGREATER"u8; ``` -**Pattern for slow-path commands:** -```csharp -else if (command.SequenceEqual(CmdStrings.DELIFGREATER)) -{ - return RespCommand.DELIFGREATER; -} -``` - -**Pattern for dot-prefixed commands (e.g., `RI.CREATE`):** -```csharp -else if (command.SequenceEqual(CmdStrings.RICREATE)) -{ - return RespCommand.RICREATE; -} -``` - -Add this before the final `return RespCommand.INVALID;` at the end of `SlowParseCommand`. +### SIMD fast path: `FastParseCommand()` (optional, for hottest commands only) +Static `Vector128` patterns defined in `libs/server/Resp/Parser/RespCommandSimdPatterns.cs` that match the full RESP encoding (`*N\r\n$L\r\nCMD\r\n`) in a single 16-byte comparison. Use the `RespPattern(argCount, "CMD")` helper to create new patterns. Only needed for the most performance-critical commands with: +- Fixed argument count (single digit) +- Command names of 3-6 characters (total encoded length must fit in 16 bytes) -**⚠️ Caveat: Dot-prefixed commands and ACL** -If your command uses a dot (e.g., `RI.CREATE`), you must also update **`libs/server/ACL/ACLParser.cs`** so that the ACL system can map the dotted wire name to the enum name. Search for how existing dot-handling works (look for `Replace(".", "")` or similar normalization). +Most new commands should **NOT** be added here — the hash table + MRU cache provide excellent performance for all commands. Only add SIMD patterns if benchmarking shows the command is a bottleneck. --- diff --git a/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs new file mode 100644 index 00000000000..288c61835b6 --- /dev/null +++ b/benchmark/BDN.benchmark/Operations/CommandParsingBenchmark.cs @@ -0,0 +1,260 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using BenchmarkDotNet.Attributes; +using Garnet.server; + +namespace BDN.benchmark.Operations +{ + /// + /// Benchmark for RESP command parsing only (no storage operations). + /// Calls ParseRespCommandBuffer directly to measure pure parsing throughput + /// across all optimization tiers. + /// + [MemoryDiagnoser] + public unsafe class CommandParsingBenchmark : OperationsBase + { + // Tier 0a: SIMD Vector128 fast path (3-6 char commands with fixed arg counts) + static ReadOnlySpan CMD_PING => "*1\r\n$4\r\nPING\r\n"u8; + static ReadOnlySpan CMD_GET => "*2\r\n$3\r\nGET\r\n$1\r\na\r\n"u8; + static ReadOnlySpan CMD_SET => "*3\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\nb\r\n"u8; + static ReadOnlySpan CMD_INCR => "*2\r\n$4\r\nINCR\r\n$1\r\ni\r\n"u8; + static ReadOnlySpan CMD_EXISTS => "*2\r\n$6\r\nEXISTS\r\n$1\r\na\r\n"u8; + static ReadOnlySpan CMD_SETEX => "*4\r\n$5\r\nSETEX\r\n$1\r\na\r\n$2\r\n60\r\n$1\r\nb\r\n"u8; + + // Tier 0b: Scalar path — hot commands too long for SIMD (name > 6 chars, exceeds 16-byte Vector128) + static ReadOnlySpan CMD_PUBLISH => "*3\r\n$7\r\nPUBLISH\r\n$2\r\nch\r\n$5\r\nhello\r\n"u8; + + // Tier 0c: Scalar path — variable-arg hot commands (arg count varies, cannot be SIMD or MRU cached) + static ReadOnlySpan CMD_EXPIRE => "*3\r\n$6\r\nEXPIRE\r\n$1\r\na\r\n$2\r\n60\r\n"u8; + + // Tier 1: Hash table lookup via ArrayParseCommand → HashLookupCommand (+ MRU cache on 2nd+ call) + static ReadOnlySpan CMD_HSET => "*4\r\n$4\r\nHSET\r\n$1\r\nh\r\n$1\r\nf\r\n$1\r\nv\r\n"u8; + static ReadOnlySpan CMD_LPUSH => "*3\r\n$5\r\nLPUSH\r\n$1\r\nl\r\n$1\r\nv\r\n"u8; + static ReadOnlySpan CMD_ZADD => "*4\r\n$4\r\nZADD\r\n$1\r\nz\r\n$1\r\n1\r\n$1\r\nm\r\n"u8; + + // Tier 1: Hash table lookup (long command names, double-digit $ header) + static ReadOnlySpan CMD_ZRANGEBYSCORE => "*4\r\n$13\r\nZRANGEBYSCORE\r\n$1\r\nz\r\n$1\r\n0\r\n$2\r\n10\r\n"u8; + static ReadOnlySpan CMD_ZREMRANGEBYSCORE => "*4\r\n$16\r\nZREMRANGEBYSCORE\r\n$1\r\nz\r\n$1\r\n0\r\n$2\r\n10\r\n"u8; + static ReadOnlySpan CMD_HINCRBYFLOAT => "*4\r\n$12\r\nHINCRBYFLOAT\r\n$1\r\nh\r\n$1\r\nf\r\n$3\r\n1.5\r\n"u8; + + // Tier 1: Hash table lookup (commands formerly in SlowParseCommand) + static ReadOnlySpan CMD_SUBSCRIBE => "*2\r\n$9\r\nSUBSCRIBE\r\n$2\r\nch\r\n"u8; + static ReadOnlySpan CMD_GEORADIUS => "*6\r\n$9\r\nGEORADIUS\r\n$1\r\ng\r\n$1\r\n0\r\n$1\r\n0\r\n$3\r\n100\r\n$2\r\nkm\r\n"u8; + static ReadOnlySpan CMD_SETIFMATCH => "*4\r\n$10\r\nSETIFMATCH\r\n$1\r\na\r\n$1\r\nb\r\n$1\r\n0\r\n"u8; + + // Pre-allocated buffers (pinned for pointer stability) + byte[] bufPing, bufGet, bufSet, bufIncr, bufExists, bufSetex, bufPublish, bufExpire, bufHset, bufLpush, bufZadd, bufSubscribe; + byte[] bufZrangebyscore, bufZremrangebyscore, bufHincrbyfloat, bufGeoradius, bufSetifmatch; + + public override void GlobalSetup() + { + base.GlobalSetup(); + + // Pre-seed a key so GET/EXISTS don't return NOT_FOUND + SlowConsumeMessage("*3\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\nb\r\n"u8); + + bufPing = GC.AllocateArray(CMD_PING.Length, pinned: true); + CMD_PING.CopyTo(bufPing); + bufGet = GC.AllocateArray(CMD_GET.Length, pinned: true); + CMD_GET.CopyTo(bufGet); + bufSet = GC.AllocateArray(CMD_SET.Length, pinned: true); + CMD_SET.CopyTo(bufSet); + bufIncr = GC.AllocateArray(CMD_INCR.Length, pinned: true); + CMD_INCR.CopyTo(bufIncr); + bufExists = GC.AllocateArray(CMD_EXISTS.Length, pinned: true); + CMD_EXISTS.CopyTo(bufExists); + bufSetex = GC.AllocateArray(CMD_SETEX.Length, pinned: true); + CMD_SETEX.CopyTo(bufSetex); + bufPublish = GC.AllocateArray(CMD_PUBLISH.Length, pinned: true); + CMD_PUBLISH.CopyTo(bufPublish); + bufExpire = GC.AllocateArray(CMD_EXPIRE.Length, pinned: true); + CMD_EXPIRE.CopyTo(bufExpire); + bufHset = GC.AllocateArray(CMD_HSET.Length, pinned: true); + CMD_HSET.CopyTo(bufHset); + bufLpush = GC.AllocateArray(CMD_LPUSH.Length, pinned: true); + CMD_LPUSH.CopyTo(bufLpush); + bufZadd = GC.AllocateArray(CMD_ZADD.Length, pinned: true); + CMD_ZADD.CopyTo(bufZadd); + bufSubscribe = GC.AllocateArray(CMD_SUBSCRIBE.Length, pinned: true); + CMD_SUBSCRIBE.CopyTo(bufSubscribe); + bufZrangebyscore = GC.AllocateArray(CMD_ZRANGEBYSCORE.Length, pinned: true); + CMD_ZRANGEBYSCORE.CopyTo(bufZrangebyscore); + bufZremrangebyscore = GC.AllocateArray(CMD_ZREMRANGEBYSCORE.Length, pinned: true); + CMD_ZREMRANGEBYSCORE.CopyTo(bufZremrangebyscore); + bufHincrbyfloat = GC.AllocateArray(CMD_HINCRBYFLOAT.Length, pinned: true); + CMD_HINCRBYFLOAT.CopyTo(bufHincrbyfloat); + bufGeoradius = GC.AllocateArray(CMD_GEORADIUS.Length, pinned: true); + CMD_GEORADIUS.CopyTo(bufGeoradius); + bufSetifmatch = GC.AllocateArray(CMD_SETIFMATCH.Length, pinned: true); + CMD_SETIFMATCH.CopyTo(bufSetifmatch); + } + + // === Tier 0a: SIMD Vector128 fast path === + + [Benchmark] + public RespCommand ParsePING() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufPing); + return result; + } + + [Benchmark] + public RespCommand ParseGET() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufGet); + return result; + } + + [Benchmark] + public RespCommand ParseSET() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufSet); + return result; + } + + [Benchmark] + public RespCommand ParseINCR() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufIncr); + return result; + } + + [Benchmark] + public RespCommand ParseEXISTS() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufExists); + return result; + } + + // === Tier 0a: SIMD fast path (SETEX is a 15-byte SIMD pattern) === + + [Benchmark] + public RespCommand ParseSETEX() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufSetex); + return result; + } + + // === Tier 0b: Scalar path — hot commands too long for SIMD === + + [Benchmark] + public RespCommand ParsePUBLISH() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufPublish); + return result; + } + + // === Tier 0c: Scalar path — variable-arg hot commands === + + [Benchmark] + public RespCommand ParseEXPIRE() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufExpire); + return result; + } + + // === Tier 1: Hash table lookup (short names, MRU cache on repeat) === + + [Benchmark] + public RespCommand ParseHSET() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufHset); + return result; + } + + [Benchmark] + public RespCommand ParseLPUSH() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufLpush); + return result; + } + + [Benchmark] + public RespCommand ParseZADD() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufZadd); + return result; + } + + // === Tier 1: Hash table lookup (long names) === + + [Benchmark] + public RespCommand ParseZRANGEBYSCORE() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufZrangebyscore); + return result; + } + + [Benchmark] + public RespCommand ParseZREMRANGEBYSCORE() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufZremrangebyscore); + return result; + } + + [Benchmark] + public RespCommand ParseHINCRBYFLOAT() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufHincrbyfloat); + return result; + } + + // === Tier 1: Hash table lookup (formerly in SlowParseCommand) === + + [Benchmark] + public RespCommand ParseSUBSCRIBE() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufSubscribe); + return result; + } + + [Benchmark] + public RespCommand ParseGEORADIUS() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufGeoradius); + return result; + } + + [Benchmark] + public RespCommand ParseSETIFMATCH() + { + RespCommand result = default; + for (int i = 0; i < batchSize; i++) + result = session.ParseRespCommandBuffer(bufSetifmatch); + return result; + } + } +} \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index cc81121b1df..7c2827606cb 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; using System.Text; using Garnet.common; using Microsoft.Extensions.Logging; @@ -642,6 +643,18 @@ enum RespCommandOption : byte /// internal sealed unsafe partial class RespServerSession : ServerSessionBase { + // Per-session MRU command cache: 2 entries caching the last matched command patterns. + // Sits after SIMD patterns but before scalar switch — catches repeated Tier 1b/2 commands + // (HSET, LPUSH, ZADD etc.) in 3 ops instead of falling through to the hash table. + // NOTE: All fields default to zero (RespCommand.NONE = 0x00, Vector128 = all zeros), + // so the cache starts empty and the _cachedCmd0 != RespCommand.NONE check is safe. + private Vector128 _cachedPattern0, _cachedMask0; + private RespCommand _cachedCmd0; + private byte _cachedLen0, _cachedCount0; + + private Vector128 _cachedPattern1, _cachedMask1; + private RespCommand _cachedCmd1; + private byte _cachedLen1, _cachedCount1; /// /// Fast-parses command type for inline RESP commands, starting at the current read head in the receive buffer /// and advances read head. @@ -680,8 +693,8 @@ private RespCommand FastParseInlineCommand(out int count) } /// - /// Fast-parses for command type, starting at the current read head in the receive buffer - /// and advances the read head to the position after the parsed command. + /// Fast-parses for command type using SIMD Vector128 matching for the most common commands, + /// falling back to scalar ulong matching for the rest. /// /// Outputs the number of arguments stored with the command /// RespCommand that was parsed or RespCommand.NONE, if no command was matched in this pass. @@ -691,1918 +704,201 @@ private RespCommand FastParseCommand(out int count) var ptr = recvBufferPtr + readHead; var remainingBytes = bytesRead - readHead; - // Check if the package starts with "*_\r\n$_\r\n" (_ = masked out), - // i.e. an array with a single-digit length and single-digit first string length. - if ((remainingBytes >= 8) && (*(ulong*)ptr & 0xFFFF00FFFFFF00FF) == MemoryMarshal.Read("*\0\r\n$\0\r\n"u8)) - { - // Extract total element count from the array header. - // NOTE: Subtracting one to account for first token being parsed. - count = ptr[1] - '1'; - - // Extract length of the first string header - var length = ptr[5] - '0'; - Debug.Assert(length is > 0 and <= 9); - - var oldReadHead = readHead; - - // Ensure that the complete command string is contained in the package. Otherwise exit early. - // Include 10 bytes to account for array and command string headers, and terminator - // 10 bytes = "*_\r\n$_\r\n" (8 bytes) + "\r\n" (2 bytes) at end of command name - if (remainingBytes >= length + 10) - { - // Optimistically advance read head to the end of the command name - readHead += length + 10; - - // Last 8 byte word of the command name, for quick comparison - var lastWord = *(ulong*)(ptr + length + 2); - - // - // Fast path for common commands with fixed numbers of arguments - // - - // Only check against commands with the correct count and length. - - return ((count << 4) | length) switch - { - // Commands without arguments - 4 when lastWord == MemoryMarshal.Read("\r\nPING\r\n"u8) => RespCommand.PING, - 4 when lastWord == MemoryMarshal.Read("\r\nEXEC\r\n"u8) => RespCommand.EXEC, - 5 when lastWord == MemoryMarshal.Read("\nMULTI\r\n"u8) => RespCommand.MULTI, - 6 when lastWord == MemoryMarshal.Read("ASKING\r\n"u8) => RespCommand.ASKING, - 7 when lastWord == MemoryMarshal.Read("ISCARD\r\n"u8) && ptr[8] == 'D' => RespCommand.DISCARD, - 7 when lastWord == MemoryMarshal.Read("NWATCH\r\n"u8) && ptr[8] == 'U' => RespCommand.UNWATCH, - 8 when lastWord == MemoryMarshal.Read("ADONLY\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("RE"u8) => RespCommand.READONLY, - 9 when lastWord == MemoryMarshal.Read("DWRITE\r\n"u8) && *(uint*)(ptr + 8) == MemoryMarshal.Read("READ"u8) => RespCommand.READWRITE, - - // Commands with fixed number of arguments - (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nGET\r\n"u8) => RespCommand.GET, - (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nDEL\r\n"u8) => RespCommand.DEL, - (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nTTL\r\n"u8) => RespCommand.TTL, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDUMP\r\n"u8) => RespCommand.DUMP, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nINCR\r\n"u8) => RespCommand.INCR, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nPTTL\r\n"u8) => RespCommand.PTTL, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDECR\r\n"u8) => RespCommand.DECR, - (1 << 4) | 4 when lastWord == MemoryMarshal.Read("EXISTS\r\n"u8) => RespCommand.EXISTS, - (1 << 4) | 6 when lastWord == MemoryMarshal.Read("GETDEL\r\n"u8) => RespCommand.GETDEL, - (1 << 4) | 7 when lastWord == MemoryMarshal.Read("ERSIST\r\n"u8) && ptr[8] == 'P' => RespCommand.PERSIST, - (1 << 4) | 7 when lastWord == MemoryMarshal.Read("PFCOUNT\r\n"u8) && ptr[8] == 'P' => RespCommand.PFCOUNT, - (2 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SET, - (2 << 4) | 5 when lastWord == MemoryMarshal.Read("\nPFADD\r\n"u8) => RespCommand.PFADD, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("INCRBY\r\n"u8) => RespCommand.INCRBY, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("DECRBY\r\n"u8) => RespCommand.DECRBY, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("GETBIT\r\n"u8) => RespCommand.GETBIT, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("APPEND\r\n"u8) => RespCommand.APPEND, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("GETSET\r\n"u8) => RespCommand.GETSET, - (2 << 4) | 7 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && ptr[8] == 'P' => RespCommand.PUBLISH, - (2 << 4) | 7 when lastWord == MemoryMarshal.Read("FMERGE\r\n"u8) && ptr[8] == 'P' => RespCommand.PFMERGE, - (2 << 4) | 8 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SP"u8) => RespCommand.SPUBLISH, - (2 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETNX\r\n"u8) => RespCommand.SETNX, - (3 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETEX\r\n"u8) => RespCommand.SETEX, - (3 << 4) | 6 when lastWord == MemoryMarshal.Read("PSETEX\r\n"u8) => RespCommand.PSETEX, - (3 << 4) | 6 when lastWord == MemoryMarshal.Read("SETBIT\r\n"u8) => RespCommand.SETBIT, - (3 << 4) | 6 when lastWord == MemoryMarshal.Read("SUBSTR\r\n"u8) => RespCommand.SUBSTR, - (3 << 4) | 7 when lastWord == MemoryMarshal.Read("ESTORE\r\n"u8) && ptr[8] == 'R' => RespCommand.RESTORE, - (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SE"u8) => RespCommand.SETRANGE, - (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("GE"u8) => RespCommand.GETRANGE, - - _ => ((length << 4) | count) switch - { - // Commands with dynamic number of arguments - >= ((6 << 4) | 2) and <= ((6 << 4) | 3) when lastWord == MemoryMarshal.Read("RENAME\r\n"u8) => RespCommand.RENAME, - >= ((8 << 4) | 2) and <= ((8 << 4) | 3) when lastWord == MemoryMarshal.Read("NAMENX\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("RE"u8) => RespCommand.RENAMENX, - >= ((3 << 4) | 3) and <= ((3 << 4) | 7) when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SETEXNX, - >= ((5 << 4) | 1) and <= ((5 << 4) | 3) when lastWord == MemoryMarshal.Read("\nGETEX\r\n"u8) => RespCommand.GETEX, - >= ((6 << 4) | 0) and <= ((6 << 4) | 9) when lastWord == MemoryMarshal.Read("RUNTXP\r\n"u8) => RespCommand.RUNTXP, - >= ((6 << 4) | 2) and <= ((6 << 4) | 3) when lastWord == MemoryMarshal.Read("EXPIRE\r\n"u8) => RespCommand.EXPIRE, - >= ((6 << 4) | 2) and <= ((6 << 4) | 5) when lastWord == MemoryMarshal.Read("BITPOS\r\n"u8) => RespCommand.BITPOS, - >= ((7 << 4) | 2) and <= ((7 << 4) | 3) when lastWord == MemoryMarshal.Read("EXPIRE\r\n"u8) && ptr[8] == 'P' => RespCommand.PEXPIRE, - >= ((8 << 4) | 1) and <= ((8 << 4) | 4) when lastWord == MemoryMarshal.Read("TCOUNT\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("BI"u8) => RespCommand.BITCOUNT, - _ => MatchedNone(this, oldReadHead) - } - }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static RespCommand MatchedNone(RespServerSession session, int oldReadHead) - { - // Backup the read head, if we didn't find a command and need to continue in the more expensive parsing loop - session.readHead = oldReadHead; - - return RespCommand.NONE; - } - } - } - else - { - return FastParseInlineCommand(out count); - } - - // Couldn't find a matching command in this pass - count = -1; - return RespCommand.NONE; - } - - /// - /// Fast parsing function for common command names. - /// Parses the receive buffer starting from the current read head and advances it to the end of - /// the parsed command/subcommand name. - /// - /// Reference to the number of remaining tokens in the packet. Will be reduced to number of command arguments. - /// The parsed command name. - private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan specificErrorMessage) - { - // Bytes remaining in the read buffer - int remainingBytes = bytesRead - readHead; - - // The current read head to continue reading from - byte* ptr = recvBufferPtr + readHead; - - // - // Fast-path parsing by (1) command string length, (2) First character of command name (optional) and (3) priority (manual order) - // - - // NOTE: A valid RESP string is at a minimum 7 characters long "$_\r\n_\r\n" - if (remainingBytes >= 7) - { - var oldReadHead = readHead; - - // Check if this is a string with a single-digit length ("$_\r\n" -> _ omitted) - if ((*(uint*)ptr & 0xFFFF00FF) == MemoryMarshal.Read("$\0\r\n"u8)) - { - // Extract length from string header - var length = ptr[1] - '0'; - - // Ensure that the complete command string is contained in the package. Otherwise exit early. - // Include 6 bytes to account for command string header and name terminator. - // 6 bytes = "$_\r\n" (4 bytes) + "\r\n" (2 bytes) at end of command name - if (remainingBytes >= length + 6) - { - // Optimistically increase read head and decrease the number of remaining elements - readHead += length + 6; - count -= 1; - - switch (length) - { - case 3: - if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("3\r\nDEL\r\n"u8)) - { - return RespCommand.DEL; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("3\r\nLCS\r\n"u8)) - { - return RespCommand.LCS; - } - - break; - - case 4: - switch ((ushort)ptr[4]) - { - case 'E': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nEVAL\r\n"u8)) - { - return RespCommand.EVAL; - } - break; - - case 'H': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHSET\r\n"u8)) - { - return RespCommand.HSET; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHGET\r\n"u8)) - { - return RespCommand.HGET; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHDEL\r\n"u8)) - { - return RespCommand.HDEL; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHLEN\r\n"u8)) - { - return RespCommand.HLEN; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nHTTL\r\n"u8)) - { - return RespCommand.HTTL; - } - break; - - case 'K': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nKEYS\r\n"u8)) - { - return RespCommand.KEYS; - } - break; - - case 'L': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLPOP\r\n"u8)) - { - return RespCommand.LPOP; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLLEN\r\n"u8)) - { - return RespCommand.LLEN; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLREM\r\n"u8)) - { - return RespCommand.LREM; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLSET\r\n"u8)) - { - return RespCommand.LSET; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nLPOS\r\n"u8)) - { - return RespCommand.LPOS; - } - break; - - case 'M': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nMGET\r\n"u8)) - { - return RespCommand.MGET; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nMSET\r\n"u8)) - { - return RespCommand.MSET; - } - break; - - case 'R': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nRPOP\r\n"u8)) - { - return RespCommand.RPOP; - } - break; - - case 'S': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nSCAN\r\n"u8)) - { - return RespCommand.SCAN; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nSADD\r\n"u8)) - { - return RespCommand.SADD; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nSREM\r\n"u8)) - { - return RespCommand.SREM; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nSPOP\r\n"u8)) - { - return RespCommand.SPOP; - } - break; - - case 'T': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nTYPE\r\n"u8)) - { - return RespCommand.TYPE; - } - break; - - case 'Z': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nZADD\r\n"u8)) - { - return RespCommand.ZADD; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nZREM\r\n"u8)) - { - return RespCommand.ZREM; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nZTTL\r\n"u8)) - { - return RespCommand.ZTTL; - } - break; - } - break; - - case 5: - switch ((ushort)ptr[4]) - { - case 'B': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nBITOP\r\n"u8)) - { - // Check for matching bit-operation - if (remainingBytes > length + 6 + 8) - { - // TODO: AND|OR|XOR|NOT|DIFF may not correctly handle mixed cases? - - var tag64 = *(ulong*)(ptr + 11); - var tag32 = (uint)tag64; - - if (tag32 == MemoryMarshal.Read("$2\r\n"u8)) - { - if (tag64 == MemoryMarshal.Read("$2\r\nOR\r\n"u8) || tag64 == MemoryMarshal.Read("$2\r\nor\r\n"u8)) - { - readHead += 8; // "$2\r\n" + "OR" + "\r\n" - count -= 1; - return RespCommand.BITOP_OR; - } - } - else if (tag32 == MemoryMarshal.Read("$3\r\n"u8) && remainingBytes > length + 6 + 9) - { - // Optimistically adjust - readHead += 9; // "$3\r\n" + AND|XOR|NOT + "\r\n" - count -= 1; - - tag64 = *(ulong*)(ptr + 12); - - if (tag64 == MemoryMarshal.Read("3\r\nAND\r\n"u8) || tag64 == MemoryMarshal.Read("3\r\nand\r\n"u8)) - { - return RespCommand.BITOP_AND; - } - else if (tag64 == MemoryMarshal.Read("3\r\nXOR\r\n"u8) || tag64 == MemoryMarshal.Read("3\r\nxor\r\n"u8)) - { - return RespCommand.BITOP_XOR; - } - else if (tag64 == MemoryMarshal.Read("3\r\nNOT\r\n"u8) || tag64 == MemoryMarshal.Read("3\r\nnot\r\n"u8)) - { - return RespCommand.BITOP_NOT; - } - - // Reset if no match - readHead -= 9; - count += 1; - } - else if (tag32 == MemoryMarshal.Read("$4\r\n"u8) && remainingBytes > length + 6 + 10) - { - // Optimistically adjust - readHead += 10; // "$4\r\nDIFF\r\n" - count -= 1; - - tag64 = *(ulong*)(ptr + 12); - - // Compare first 8 bytes then the trailing '\n' for "4\r\nDIFF\r\n" - if ((*(ulong*)(ptr + 12) == MemoryMarshal.Read("4\r\nDIFF\r"u8) || - *(ulong*)(ptr + 12) == MemoryMarshal.Read("4\r\ndiff\r"u8)) && - *(ptr + 20) == (byte)'\n') - { - return RespCommand.BITOP_DIFF; - } - - // Reset if no match - readHead -= 10; - count += 1; - } - - // Although we recognize BITOP, the pseudo-subcommand isn't recognized so fail early - specificErrorMessage = CmdStrings.RESP_SYNTAX_ERROR; - return RespCommand.NONE; - } - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nBRPOP\r\n"u8)) - { - return RespCommand.BRPOP; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nBLPOP\r\n"u8)) - { - return RespCommand.BLPOP; - } - break; - - case 'H': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHMSET\r\n"u8)) - { - return RespCommand.HMSET; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHMGET\r\n"u8)) - { - return RespCommand.HMGET; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHKEYS\r\n"u8)) - { - return RespCommand.HKEYS; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHVALS\r\n"u8)) - { - return RespCommand.HVALS; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHSCAN\r\n"u8)) - { - return RespCommand.HSCAN; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nHPTTL\r\n"u8)) - { - return RespCommand.HPTTL; - } - break; - - case 'L': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nLPUSH\r\n"u8)) - { - return RespCommand.LPUSH; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nLTRIM\r\n"u8)) - { - return RespCommand.LTRIM; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nLMOVE\r\n"u8)) - { - return RespCommand.LMOVE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nLMPOP\r\n"u8)) - { - return RespCommand.LMPOP; - } - break; - - case 'P': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nPFADD\r\n"u8)) - { - return RespCommand.PFADD; - } - break; - - case 'R': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nRPUSH\r\n"u8)) - { - return RespCommand.RPUSH; - } - break; - - case 'S': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nSCARD\r\n"u8)) - { - return RespCommand.SCARD; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nSSCAN\r\n"u8)) - { - return RespCommand.SSCAN; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nSMOVE\r\n"u8)) - { - return RespCommand.SMOVE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nSDIFF\r\n"u8)) - { - return RespCommand.SDIFF; - } - break; - - case 'W': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nWATCH\r\n"u8)) - { - return RespCommand.WATCH; - } - break; - - case 'Z': - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZCARD\r\n"u8)) - { - return RespCommand.ZCARD; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZRANK\r\n"u8)) - { - return RespCommand.ZRANK; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZDIFF\r\n"u8)) - { - return RespCommand.ZDIFF; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZSCAN\r\n"u8)) - { - return RespCommand.ZSCAN; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZMPOP\r\n"u8)) - { - return RespCommand.ZMPOP; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\nZPTTL\r\n"u8)) - { - return RespCommand.ZPTTL; - } - break; - } - break; - - case 6: - switch ((ushort)ptr[4]) - { - case 'B': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BLMOVE\r\n"u8)) - { - return RespCommand.BLMOVE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BLMPOP\r\n"u8)) - { - return RespCommand.BLMPOP; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BZMPOP\r\n"u8)) - { - return RespCommand.BZMPOP; - } - break; - case 'D': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("DBSIZE\r\n"u8)) - { - return RespCommand.DBSIZE; - } - break; - - case 'E': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("EXISTS\r\n"u8)) - { - return RespCommand.EXISTS; - } - break; - - case 'G': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEOADD\r\n"u8)) - { - return RespCommand.GEOADD; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEOPOS\r\n"u8)) - { - return RespCommand.GEOPOS; - } - break; - - case 'H': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HSETNX\r\n"u8)) - { - return RespCommand.HSETNX; - } - break; - - case 'L': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("LPUSHX\r\n"u8)) - { - return RespCommand.LPUSHX; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("LRANGE\r\n"u8)) - { - return RespCommand.LRANGE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("LINDEX\r\n"u8)) - { - return RespCommand.LINDEX; - } - break; - - case 'M': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("MSETNX\r\n"u8)) - { - return RespCommand.MSETNX; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("MEMORY\r\n"u8)) - { - // MEMORY USAGE - // 11 = "$5\r\nUSAGE\r\n".Length - if (remainingBytes >= length + 11) - { - if (*(ulong*)(ptr + 12) == MemoryMarshal.Read("$5\r\nUSAG"u8) && *(ulong*)(ptr + 15) == MemoryMarshal.Read("\nUSAGE\r\n"u8)) - { - count--; - readHead += 11; - return RespCommand.MEMORY_USAGE; - } - } - } - break; - - case 'R': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("RPUSHX\r\n"u8)) - { - return RespCommand.RPUSHX; - } - break; - - case 'S': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SELECT\r\n"u8)) - { - return RespCommand.SELECT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("STRLEN\r\n"u8)) - { - return RespCommand.STRLEN; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SUNION\r\n"u8)) - { - return RespCommand.SUNION; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SINTER\r\n"u8)) - { - return RespCommand.SINTER; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SCRIPT\r\n"u8)) - { - // SCRIPT EXISTS => "$6\r\nEXISTS\r\n".Length == 12 - // SCRIPT FLUSH => "$5\r\nFLUSH\r\n".Length == 11 - // SCRIPT LOAD => "$4\r\nLOAD\r\n".Length == 10 - - if (remainingBytes >= length + 10) - { - if (*(ulong*)(ptr + 4 + 8) == MemoryMarshal.Read("$4\r\nLOAD"u8) && *(ulong*)(ptr + 4 + 8 + 2) == MemoryMarshal.Read("\r\nLOAD\r\n"u8)) - { - count--; - readHead += 10; - return RespCommand.SCRIPT_LOAD; - } - - if (remainingBytes >= length + 11) - { - if (*(ulong*)(ptr + 4 + 8) == MemoryMarshal.Read("$5\r\nFLUS"u8) && *(ulong*)(ptr + 4 + 8 + 3) == MemoryMarshal.Read("\nFLUSH\r\n"u8)) - { - count--; - readHead += 11; - return RespCommand.SCRIPT_FLUSH; - } - - if (remainingBytes >= length + 12) - { - if (*(ulong*)(ptr + 4 + 8) == MemoryMarshal.Read("$6\r\nEXIS"u8) && *(ulong*)(ptr + 4 + 8 + 4) == MemoryMarshal.Read("EXISTS\r\n"u8)) - { - count--; - readHead += 12; - return RespCommand.SCRIPT_EXISTS; - } - } - } - } - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SWAPDB\r\n"u8)) - { - return RespCommand.SWAPDB; - } - break; - - case 'U': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("UNLINK\r\n"u8)) - { - return RespCommand.UNLINK; - } - break; - - case 'Z': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZCOUNT\r\n"u8)) - { - return RespCommand.ZCOUNT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZRANGE\r\n"u8)) - { - return RespCommand.ZRANGE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZUNION\r\n"u8)) - { - return RespCommand.ZUNION; - } - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZSCORE\r\n"u8)) - { - return RespCommand.ZSCORE; - } - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZINTER\r\n"u8)) - { - return RespCommand.ZINTER; - } - break; - } - - break; - case 7: - switch ((ushort)ptr[4]) - { - case 'E': - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("\r\nEVALSHA\r\n"u8)) - { - return RespCommand.EVALSHA; - } - break; - - case 'G': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEOHASH\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.GEOHASH; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEODIST\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.GEODIST; - } - break; - - case 'H': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HGETALL\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HGETALL; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HEXISTS\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HEXISTS; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HEXPIRE\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HEXPIRE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HINCRBY\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HINCRBY; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HSTRLEN\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.HSTRLEN; - } - break; - - case 'L': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("LINSERT\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.LINSERT; - } - break; - - case 'M': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("MONITOR\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.MONITOR; - } - break; - - case 'P': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("PFCOUNT\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.PFCOUNT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("PFMERGE\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.PFMERGE; - } - break; - case 'W': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("WATCHMS\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.WATCHMS; - } - - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("WATCHOS\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.WATCHOS; - } - - break; - - case 'Z': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPOPMIN\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZPOPMIN; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZEXPIRE\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZEXPIRE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPOPMAX\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZPOPMAX; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZINCRBY\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZINCRBY; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZMSCORE\r"u8) && *(byte*)(ptr + 12) == '\n') - { - return RespCommand.ZMSCORE; - } - break; - } - break; - case 8: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZREVRANK"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZREVRANK; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SMEMBERS"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.SMEMBERS; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BITFIELD"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.BITFIELD; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("EXPIREAT"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.EXPIREAT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HPEXPIRE"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.HPEXPIRE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HPERSIST"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.HPERSIST; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPEXPIRE"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZPEXPIRE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZPERSIST"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZPERSIST; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BZPOPMAX"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.BZPOPMAX; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("BZPOPMIN"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.BZPOPMIN; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SPUBLISH"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.SPUBLISH; - } - break; - case 9: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SUBSCRIB"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("BE\r\n"u8)) - { - return RespCommand.SUBSCRIBE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SISMEMBE"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("ER\r\n"u8)) - { - return RespCommand.SISMEMBER; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZLEXCOUN"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("NT\r\n"u8)) - { - return RespCommand.ZLEXCOUNT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("GEOSEARC"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("CH\r\n"u8)) - { - return RespCommand.GEOSEARCH; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZREVRANG"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("GE\r\n"u8)) - { - return RespCommand.ZREVRANGE; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("RPOPLPUS"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("SH\r\n"u8)) - { - return RespCommand.RPOPLPUSH; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("PEXPIREA"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("AT\r\n"u8)) - { - return RespCommand.PEXPIREAT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("HEXPIREA"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("AT\r\n"u8)) - { - return RespCommand.HEXPIREAT; - } - else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("ZEXPIREA"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("AT\r\n"u8)) - { - return RespCommand.ZEXPIREAT; - } - break; - case 10: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SSUBSCRI"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("BE\r\n"u8)) - { - return RespCommand.SSUBSCRIBE; - } - break; - } - - // Reset optimistically changed state, if no matching command was found - count += 1; - readHead = oldReadHead; - } - } - // Check if this is a string with a double-digit length ("$__\r" -> _ omitted) - else if ((*(uint*)ptr & 0xFF0000FF) == MemoryMarshal.Read("$\0\0\r"u8)) - { - // Extract length from string header - var length = ptr[2] - '0' + 10; - - // Ensure that the complete command string is contained in the package. Otherwise exit early. - // Include 7 bytes to account for command string header and name terminator. - // 7 bytes = "$__\r\n" (5 bytes) + "\r\n" (2 bytes) at end of command name - if (remainingBytes >= length + 7) - { - // Optimistically increase read head and decrease the number of remaining elements - readHead += length + 7; - count -= 1; - - // Match remaining character by length - // NOTE: Check should include the remaining array length terminator '\n' - switch (length) - { - case 10: - if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nPSUB"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("SCRIBE\r\n"u8)) - { - return RespCommand.PSUBSCRIBE; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nHRAN"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("DFIELD\r\n"u8)) - { - return RespCommand.HRANDFIELD; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nSDIF"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("FSTORE\r\n"u8)) - { - return RespCommand.SDIFFSTORE; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nEXPI"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.EXPIRETIME; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nSMIS"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("MEMBER\r\n"u8)) - { - return RespCommand.SMISMEMBER; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nSINT"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("ERCARD\r\n"u8)) - { - return RespCommand.SINTERCARD; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nZDIF"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("FSTORE\r\n"u8)) - { - return RespCommand.ZDIFFSTORE; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nBRPO"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("PLPUSH\r\n"u8)) - { - return RespCommand.BRPOPLPUSH; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nZINT"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("ERCARD\r\n"u8)) - { - return RespCommand.ZINTERCARD; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nHPEX"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("PIREAT\r\n"u8)) - { - return RespCommand.HPEXPIREAT; - } - else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nZPEX"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("PIREAT\r\n"u8)) - { - return RespCommand.ZPEXPIREAT; - } - break; - case 11: - if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nUNSUB"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("SCRIBE\r\n"u8)) - { - return RespCommand.UNSUBSCRIBE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZRAND"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("MEMBER\r\n"u8)) - { - return RespCommand.ZRANDMEMBER; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nBITFI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("ELD_RO\r\n"u8)) - { - return RespCommand.BITFIELD_RO; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nSRAND"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("MEMBER\r\n"u8)) - { - return RespCommand.SRANDMEMBER; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nSUNIO"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("NSTORE\r\n"u8)) - { - return RespCommand.SUNIONSTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nSINTE"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RSTORE\r\n"u8)) - { - return RespCommand.SINTERSTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nPEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.PEXPIRETIME; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nHEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.HEXPIRETIME; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nINCRB"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("YFLOAT\r\n"u8)) - { - return RespCommand.INCRBYFLOAT; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZRANG"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("ESTORE\r\n"u8)) - { - return RespCommand.ZRANGESTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZRANG"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("EBYLEX\r\n"u8)) - { - return RespCommand.ZRANGEBYLEX; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZINTE"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RSTORE\r\n"u8)) - { - return RespCommand.ZINTERSTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZUNIO"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("NSTORE\r\n"u8)) - { - return RespCommand.ZUNIONSTORE; - } - else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nZEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.ZEXPIRETIME; - } - break; - - case 12: - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nPUNSUB"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("SCRIBE\r\n"u8)) - { - return RespCommand.PUNSUBSCRIBE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nHINCRB"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("YFLOAT\r\n"u8)) - { - return RespCommand.HINCRBYFLOAT; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nHPEXPI"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.HPEXPIRETIME; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZPEXPI"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("RETIME\r\n"u8)) - { - return RespCommand.ZPEXPIRETIME; - } - break; - - case 13: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("\nZRANGEB"u8) && *(ulong*)(ptr + 12) == MemoryMarshal.Read("YSCORE\r\n"u8)) - { - return RespCommand.ZRANGEBYSCORE; - } - break; - - case 14: - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZREMRA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("NGEBYLEX"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZREMRANGEBYLEX; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nGEOSEA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("RCHSTORE"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.GEOSEARCHSTORE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZREVRA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("NGEBYLEX"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZREVRANGEBYLEX; - } - break; - - case 15: - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("\nZREMRAN"u8) && *(ulong*)(ptr + 12) == MemoryMarshal.Read("GEBYRANK"u8) && *(ushort*)(ptr + 20) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.ZREMRANGEBYRANK; - } - break; - - case 16: - if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nCUSTOM"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("OBJECTSC"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("AN\r\n"u8)) - { - return RespCommand.COSCAN; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZREMRA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("NGEBYSCO"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("RE\r\n"u8)) - { - return RespCommand.ZREMRANGEBYSCORE; - } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nZREVRA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("NGEBYSCO"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("RE\r\n"u8)) - { - return RespCommand.ZREVRANGEBYSCORE; - } - break; - } - - // Reset optimistically changed state, if no matching command was found - count += 1; - readHead = oldReadHead; - } - } - } - - // No matching command name found in this pass - return RespCommand.NONE; - } - - private bool TryParseCustomCommand(ReadOnlySpan command, out RespCommand cmd) - { - if (customCommandManagerSession.Match(command, out currentCustomTransaction)) - { - cmd = RespCommand.CustomTxn; - return true; - } - else if (customCommandManagerSession.Match(command, out currentCustomProcedure)) - { - cmd = RespCommand.CustomProcedure; - return true; - } - else if (customCommandManagerSession.Match(command, out currentCustomRawStringCommand)) - { - cmd = RespCommand.CustomRawStringCmd; - return true; - } - else if (customCommandManagerSession.Match(command, out currentCustomObjectCommand)) - { - cmd = RespCommand.CustomObjCmd; - return true; - } - cmd = RespCommand.NONE; - return false; - } - - /// - /// Parses the receive buffer, starting from the current read head, for all command names that are - /// not covered by FastParseArrayCommand() and advances the read head to the end of the command name. - /// - /// NOTE: Assumes the input command names have already been converted to upper-case. - /// - /// Reference to the number of remaining tokens in the packet. Will be reduced to number of command arguments. - /// If the command could not be parsed, will be non-empty if a specific error message should be returned. - /// True if the input RESP string was completely included in the buffer, false if we couldn't read the full command name. - /// The parsed command name. - private RespCommand SlowParseCommand(ref int count, ref ReadOnlySpan specificErrorMsg, out bool success) - { - // Try to extract the current string from the front of the read head - var command = GetCommand(out success); - - if (!success) - { - return RespCommand.INVALID; - } - - // Account for the command name being taken off the read head - count -= 1; - - if (TryParseCustomCommand(command, out var cmd)) - { - return cmd; - } - else - { - return SlowParseCommand(command, ref count, ref specificErrorMsg, out success); - } - } - - private RespCommand SlowParseCommand(ReadOnlySpan command, ref int count, ref ReadOnlySpan specificErrorMsg, out bool success) - { - success = true; - if (command.SequenceEqual(CmdStrings.SUBSCRIBE)) - { - return RespCommand.SUBSCRIBE; - } - else if (command.SequenceEqual(CmdStrings.SSUBSCRIBE)) - { - return RespCommand.SSUBSCRIBE; - } - else if (command.SequenceEqual(CmdStrings.RUNTXP)) - { - return RespCommand.RUNTXP; - } - else if (command.SequenceEqual(CmdStrings.SCRIPT)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.SCRIPT))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.LOAD)) - { - return RespCommand.SCRIPT_LOAD; - } - - if (subCommand.SequenceEqual(CmdStrings.FLUSH)) - { - return RespCommand.SCRIPT_FLUSH; - } - - if (subCommand.SequenceEqual(CmdStrings.EXISTS)) - { - return RespCommand.SCRIPT_EXISTS; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.SCRIPT)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.ECHO)) - { - return RespCommand.ECHO; - } - else if (command.SequenceEqual(CmdStrings.GEORADIUS)) - { - return RespCommand.GEORADIUS; - } - else if (command.SequenceEqual(CmdStrings.GEORADIUS_RO)) - { - return RespCommand.GEORADIUS_RO; - } - else if (command.SequenceEqual(CmdStrings.GEORADIUSBYMEMBER)) - { - return RespCommand.GEORADIUSBYMEMBER; - } - else if (command.SequenceEqual(CmdStrings.GEORADIUSBYMEMBER_RO)) - { - return RespCommand.GEORADIUSBYMEMBER_RO; - } - else if (command.SequenceEqual(CmdStrings.REPLICAOF)) - { - return RespCommand.REPLICAOF; - } - else if (command.SequenceEqual(CmdStrings.SECONDARYOF) || command.SequenceEqual(CmdStrings.SLAVEOF)) - { - return RespCommand.SECONDARYOF; - } - else if (command.SequenceEqual(CmdStrings.CONFIG)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.CONFIG))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.GET)) - { - return RespCommand.CONFIG_GET; - } - else if (subCommand.SequenceEqual(CmdStrings.REWRITE)) - { - return RespCommand.CONFIG_REWRITE; - } - else if (subCommand.SequenceEqual(CmdStrings.SET)) - { - return RespCommand.CONFIG_SET; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.CONFIG)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.CLIENT)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.CLIENT))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.ID)) - { - return RespCommand.CLIENT_ID; - } - else if (subCommand.SequenceEqual(CmdStrings.INFO)) - { - return RespCommand.CLIENT_INFO; - } - else if (subCommand.SequenceEqual(CmdStrings.LIST)) - { - return RespCommand.CLIENT_LIST; - } - else if (subCommand.SequenceEqual(CmdStrings.KILL)) - { - return RespCommand.CLIENT_KILL; - } - else if (subCommand.SequenceEqual(CmdStrings.GETNAME)) - { - return RespCommand.CLIENT_GETNAME; - } - else if (subCommand.SequenceEqual(CmdStrings.SETNAME)) - { - return RespCommand.CLIENT_SETNAME; - } - else if (subCommand.SequenceEqual(CmdStrings.SETINFO)) - { - return RespCommand.CLIENT_SETINFO; - } - else if (subCommand.SequenceEqual(CmdStrings.UNBLOCK)) - { - return RespCommand.CLIENT_UNBLOCK; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.CLIENT)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.AUTH)) - { - return RespCommand.AUTH; - } - else if (command.SequenceEqual(CmdStrings.INFO)) - { - return RespCommand.INFO; - } - else if (command.SequenceEqual(CmdStrings.ROLE)) - { - return RespCommand.ROLE; - } - else if (command.SequenceEqual(CmdStrings.COMMAND)) - { - if (count == 0) - { - return RespCommand.COMMAND; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.COUNT)) - { - return RespCommand.COMMAND_COUNT; - } - - if (subCommand.SequenceEqual(CmdStrings.INFO)) - { - return RespCommand.COMMAND_INFO; - } - - if (subCommand.SequenceEqual(CmdStrings.DOCS)) - { - return RespCommand.COMMAND_DOCS; - } - - if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.GETKEYS)) - { - return RespCommand.COMMAND_GETKEYS; - } - - if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.GETKEYSANDFLAGS)) - { - return RespCommand.COMMAND_GETKEYSANDFLAGS; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.COMMAND)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.PING)) - { - return RespCommand.PING; - } - else if (command.SequenceEqual(CmdStrings.HELLO)) - { - return RespCommand.HELLO; - } - else if (command.SequenceEqual(CmdStrings.CLUSTER)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.CLUSTER))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.BUMPEPOCH)) - { - return RespCommand.CLUSTER_BUMPEPOCH; - } - else if (subCommand.SequenceEqual(CmdStrings.FORGET)) - { - return RespCommand.CLUSTER_FORGET; - } - else if (subCommand.SequenceEqual(CmdStrings.gossip)) - { - return RespCommand.CLUSTER_GOSSIP; - } - else if (subCommand.SequenceEqual(CmdStrings.INFO)) - { - return RespCommand.CLUSTER_INFO; - } - else if (subCommand.SequenceEqual(CmdStrings.MEET)) - { - return RespCommand.CLUSTER_MEET; - } - else if (subCommand.SequenceEqual(CmdStrings.MYID)) - { - return RespCommand.CLUSTER_MYID; - } - else if (subCommand.SequenceEqual(CmdStrings.myparentid)) - { - return RespCommand.CLUSTER_MYPARENTID; - } - else if (subCommand.SequenceEqual(CmdStrings.NODES)) - { - return RespCommand.CLUSTER_NODES; - } - else if (subCommand.SequenceEqual(CmdStrings.SHARDS)) - { - return RespCommand.CLUSTER_SHARDS; - } - else if (subCommand.SequenceEqual(CmdStrings.RESET)) - { - return RespCommand.CLUSTER_RESET; - } - else if (subCommand.SequenceEqual(CmdStrings.FAILOVER)) - { - return RespCommand.CLUSTER_FAILOVER; - } - else if (subCommand.SequenceEqual(CmdStrings.ADDSLOTS)) - { - return RespCommand.CLUSTER_ADDSLOTS; - } - else if (subCommand.SequenceEqual(CmdStrings.ADDSLOTSRANGE)) - { - return RespCommand.CLUSTER_ADDSLOTSRANGE; - } - else if (subCommand.SequenceEqual(CmdStrings.COUNTKEYSINSLOT)) - { - return RespCommand.CLUSTER_COUNTKEYSINSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.DELSLOTS)) - { - return RespCommand.CLUSTER_DELSLOTS; - } - else if (subCommand.SequenceEqual(CmdStrings.DELSLOTSRANGE)) - { - return RespCommand.CLUSTER_DELSLOTSRANGE; - } - else if (subCommand.SequenceEqual(CmdStrings.GETKEYSINSLOT)) - { - return RespCommand.CLUSTER_GETKEYSINSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.HELP)) - { - return RespCommand.CLUSTER_HELP; - } - else if (subCommand.SequenceEqual(CmdStrings.KEYSLOT)) - { - return RespCommand.CLUSTER_KEYSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.SETSLOT)) - { - return RespCommand.CLUSTER_SETSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.SLOTS)) - { - return RespCommand.CLUSTER_SLOTS; - } - else if (subCommand.SequenceEqual(CmdStrings.REPLICAS)) - { - return RespCommand.CLUSTER_REPLICAS; - } - else if (subCommand.SequenceEqual(CmdStrings.REPLICATE)) - { - return RespCommand.CLUSTER_REPLICATE; - } - else if (subCommand.SequenceEqual(CmdStrings.delkeysinslot)) - { - return RespCommand.CLUSTER_DELKEYSINSLOT; - } - else if (subCommand.SequenceEqual(CmdStrings.delkeysinslotrange)) - { - return RespCommand.CLUSTER_DELKEYSINSLOTRANGE; - } - else if (subCommand.SequenceEqual(CmdStrings.setslotsrange)) - { - return RespCommand.CLUSTER_SETSLOTSRANGE; - } - else if (subCommand.SequenceEqual(CmdStrings.slotstate)) - { - return RespCommand.CLUSTER_SLOTSTATE; - } - else if (subCommand.SequenceEqual(CmdStrings.publish)) - { - return RespCommand.CLUSTER_PUBLISH; - } - else if (subCommand.SequenceEqual(CmdStrings.spublish)) - { - return RespCommand.CLUSTER_SPUBLISH; - } - else if (subCommand.SequenceEqual(CmdStrings.MIGRATE)) - { - return RespCommand.CLUSTER_MIGRATE; - } - else if (subCommand.SequenceEqual(CmdStrings.mtasks)) - { - return RespCommand.CLUSTER_MTASKS; - } - else if (subCommand.SequenceEqual(CmdStrings.aofsync)) - { - return RespCommand.CLUSTER_AOFSYNC; - } - else if (subCommand.SequenceEqual(CmdStrings.appendlog)) - { - return RespCommand.CLUSTER_APPENDLOG; - } - else if (subCommand.SequenceEqual(CmdStrings.attach_sync)) - { - return RespCommand.CLUSTER_ATTACH_SYNC; - } - else if (subCommand.SequenceEqual(CmdStrings.banlist)) - { - return RespCommand.CLUSTER_BANLIST; - } - else if (subCommand.SequenceEqual(CmdStrings.begin_replica_recover)) - { - return RespCommand.CLUSTER_BEGIN_REPLICA_RECOVER; - } - else if (subCommand.SequenceEqual(CmdStrings.endpoint)) - { - return RespCommand.CLUSTER_ENDPOINT; - } - else if (subCommand.SequenceEqual(CmdStrings.failreplicationoffset)) - { - return RespCommand.CLUSTER_FAILREPLICATIONOFFSET; - } - else if (subCommand.SequenceEqual(CmdStrings.failstopwrites)) - { - return RespCommand.CLUSTER_FAILSTOPWRITES; - } - else if (subCommand.SequenceEqual(CmdStrings.FLUSHALL)) - { - return RespCommand.CLUSTER_FLUSHALL; - } - else if (subCommand.SequenceEqual(CmdStrings.SETCONFIGEPOCH)) - { - return RespCommand.CLUSTER_SETCONFIGEPOCH; - } - else if (subCommand.SequenceEqual(CmdStrings.initiate_replica_sync)) - { - return RespCommand.CLUSTER_INITIATE_REPLICA_SYNC; - } - else if (subCommand.SequenceEqual(CmdStrings.send_ckpt_file_segment)) - { - return RespCommand.CLUSTER_SEND_CKPT_FILE_SEGMENT; - } - else if (subCommand.SequenceEqual(CmdStrings.send_ckpt_metadata)) - { - return RespCommand.CLUSTER_SEND_CKPT_METADATA; - } - else if (subCommand.SequenceEqual(CmdStrings.cluster_sync)) - { - return RespCommand.CLUSTER_SYNC; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommand, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.CLUSTER)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.LATENCY)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.LATENCY))); - return RespCommand.INVALID; - } - - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.HELP)) - { - return RespCommand.LATENCY_HELP; - } - else if (subCommand.SequenceEqual(CmdStrings.HISTOGRAM)) - { - return RespCommand.LATENCY_HISTOGRAM; - } - else if (subCommand.SequenceEqual(CmdStrings.RESET)) - { - return RespCommand.LATENCY_RESET; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommand, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.LATENCY)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.SLOWLOG)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.SLOWLOG))); - } - else if (count >= 1) - { - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; - - if (subCommand.SequenceEqual(CmdStrings.HELP)) - { - return RespCommand.SLOWLOG_HELP; - } - else if (subCommand.SequenceEqual(CmdStrings.GET)) - { - return RespCommand.SLOWLOG_GET; - } - else if (subCommand.SequenceEqual(CmdStrings.LEN)) + // SIMD fast path: match the full RESP-encoded pattern (*N\r\n$L\r\nCMD\r\n) in a + // single Vector128 comparison. Each check validates the header AND command name + // simultaneously: 1 load + 1 AND (mask) + 1 EqualsAll = 3 ops per candidate. + if (Vector128.IsHardwareAccelerated && remainingBytes >= 16) + { + var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); + + // 13-byte patterns: 3-char commands (GET, SET, DEL, TTL) + var m13 = Vector128.BitwiseAnd(input, s_mask13); + if (Vector128.EqualsAll(m13, s_GET)) { readHead += 13; count = 1; return RespCommand.GET; } + if (Vector128.EqualsAll(m13, s_SET)) { readHead += 13; count = 2; return RespCommand.SET; } + if (Vector128.EqualsAll(m13, s_DEL)) { readHead += 13; count = 1; return RespCommand.DEL; } + if (Vector128.EqualsAll(m13, s_TTL)) { readHead += 13; count = 1; return RespCommand.TTL; } + + // 14-byte patterns: 4-char commands (PING, INCR, DECR, EXEC, PTTL) + var m14 = Vector128.BitwiseAnd(input, s_mask14); + if (Vector128.EqualsAll(m14, s_PING)) { readHead += 14; count = 0; return RespCommand.PING; } + if (Vector128.EqualsAll(m14, s_INCR)) { readHead += 14; count = 1; return RespCommand.INCR; } + if (Vector128.EqualsAll(m14, s_DECR)) { readHead += 14; count = 1; return RespCommand.DECR; } + if (Vector128.EqualsAll(m14, s_EXEC)) { readHead += 14; count = 0; return RespCommand.EXEC; } + if (Vector128.EqualsAll(m14, s_PTTL)) { readHead += 14; count = 1; return RespCommand.PTTL; } + + // 15-byte patterns: 5-char commands (MULTI, SETNX, SETEX) + var m15 = Vector128.BitwiseAnd(input, s_mask15); + if (Vector128.EqualsAll(m15, s_MULTI)) { readHead += 15; count = 0; return RespCommand.MULTI; } + if (Vector128.EqualsAll(m15, s_SETNX)) { readHead += 15; count = 2; return RespCommand.SETNX; } + if (Vector128.EqualsAll(m15, s_SETEX)) { readHead += 15; count = 3; return RespCommand.SETEX; } + + // 16-byte patterns: 6-char commands (no mask — exact 16-byte match) + if (Vector128.EqualsAll(input, s_EXISTS)) { readHead += 16; count = 1; return RespCommand.EXISTS; } + if (Vector128.EqualsAll(input, s_GETDEL)) { readHead += 16; count = 1; return RespCommand.GETDEL; } + if (Vector128.EqualsAll(input, s_APPEND)) { readHead += 16; count = 2; return RespCommand.APPEND; } + if (Vector128.EqualsAll(input, s_INCRBY)) { readHead += 16; count = 2; return RespCommand.INCRBY; } + if (Vector128.EqualsAll(input, s_DECRBY)) { readHead += 16; count = 2; return RespCommand.DECRBY; } + if (Vector128.EqualsAll(input, s_PSETEX)) { readHead += 16; count = 3; return RespCommand.PSETEX; } + + // MRU cache check: catches repeated commands that aren't in the SIMD pattern table + // (e.g., HSET, LPUSH, ZADD, ZRANGEBYSCORE). Same 3-op cost as one SIMD pattern check. + if (_cachedCmd0 != RespCommand.NONE) + { + if (Vector128.EqualsAll(Vector128.BitwiseAnd(input, _cachedMask0), _cachedPattern0)) { - return RespCommand.SLOWLOG_LEN; + readHead += _cachedLen0; + count = _cachedCount0; + return _cachedCmd0; } - else if (subCommand.SequenceEqual(CmdStrings.RESET)) + + if (_cachedCmd1 != RespCommand.NONE && + Vector128.EqualsAll(Vector128.BitwiseAnd(input, _cachedMask1), _cachedPattern1)) { - return RespCommand.SLOWLOG_RESET; + readHead += _cachedLen1; + count = _cachedCount1; + // Promote slot 1 → slot 0 (swap) + (_cachedPattern0, _cachedPattern1) = (_cachedPattern1, _cachedPattern0); + (_cachedMask0, _cachedMask1) = (_cachedMask1, _cachedMask0); + (_cachedCmd0, _cachedCmd1) = (_cachedCmd1, _cachedCmd0); + (_cachedLen0, _cachedLen1) = (_cachedLen1, _cachedLen0); + (_cachedCount0, _cachedCount1) = (_cachedCount1, _cachedCount0); + return _cachedCmd0; } } } - else if (command.SequenceEqual(CmdStrings.TIME)) - { - return RespCommand.TIME; - } - else if (command.SequenceEqual(CmdStrings.QUIT)) - { - return RespCommand.QUIT; - } - else if (command.SequenceEqual(CmdStrings.SAVE)) - { - return RespCommand.SAVE; - } - else if (command.SequenceEqual(CmdStrings.EXPDELSCAN)) - { - return RespCommand.EXPDELSCAN; - } - else if (command.SequenceEqual(CmdStrings.LASTSAVE)) - { - return RespCommand.LASTSAVE; - } - else if (command.SequenceEqual(CmdStrings.BGSAVE)) - { - return RespCommand.BGSAVE; - } - else if (command.SequenceEqual(CmdStrings.COMMITAOF)) - { - return RespCommand.COMMITAOF; - } - else if (command.SequenceEqual(CmdStrings.FLUSHALL)) - { - return RespCommand.FLUSHALL; - } - else if (command.SequenceEqual(CmdStrings.FLUSHDB)) - { - return RespCommand.FLUSHDB; - } - else if (command.SequenceEqual(CmdStrings.FORCEGC)) - { - return RespCommand.FORCEGC; - } - else if (command.SequenceEqual(CmdStrings.MIGRATE)) - { - return RespCommand.MIGRATE; - } - else if (command.SequenceEqual(CmdStrings.PURGEBP)) - { - return RespCommand.PURGEBP; - } - else if (command.SequenceEqual(CmdStrings.FAILOVER)) - { - return RespCommand.FAILOVER; - } - else if (command.SequenceEqual(CmdStrings.MEMORY)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.MEMORY))); - return RespCommand.INVALID; - } - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } - - count--; + // Scalar fast path: uses ulong comparisons to match common commands. + // On SIMD hardware, this catches commands when remainingBytes < 16 (SIMD needs 16). + // On non-SIMD hardware, this is the primary fast path. - if (subCommand.EqualsUpperCaseSpanIgnoringCase(CmdStrings.USAGE)) - { - return RespCommand.MEMORY_USAGE; - } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.MEMORY)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.MONITOR)) - { - return RespCommand.MONITOR; - } - else if (command.SequenceEqual(CmdStrings.ACL)) + // Check if the package starts with "*_\r\n$_\r\n" (_ = masked out), + // i.e. an array with a single-digit length and single-digit first string length. + if ((remainingBytes >= 8) && (*(ulong*)ptr & 0xFFFF00FFFFFF00FF) == MemoryMarshal.Read("*\0\r\n$\0\r\n"u8)) { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.ACL))); - return RespCommand.INVALID; - } + // Extract total element count from the array header. + // NOTE: Subtracting one to account for first token being parsed. + count = ptr[1] - '1'; - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } + // Extract length of the first string header + var length = ptr[5] - '0'; - count--; + var oldReadHead = readHead; - if (subCommand.SequenceEqual(CmdStrings.CAT)) - { - return RespCommand.ACL_CAT; - } - else if (subCommand.SequenceEqual(CmdStrings.DELUSER)) - { - return RespCommand.ACL_DELUSER; - } - else if (subCommand.SequenceEqual(CmdStrings.GENPASS)) - { - return RespCommand.ACL_GENPASS; - } - else if (subCommand.SequenceEqual(CmdStrings.GETUSER)) - { - return RespCommand.ACL_GETUSER; - } - else if (subCommand.SequenceEqual(CmdStrings.LIST)) - { - return RespCommand.ACL_LIST; - } - else if (subCommand.SequenceEqual(CmdStrings.LOAD)) - { - return RespCommand.ACL_LOAD; - } - else if (subCommand.SequenceEqual(CmdStrings.SAVE)) - { - return RespCommand.ACL_SAVE; - } - else if (subCommand.SequenceEqual(CmdStrings.SETUSER)) - { - return RespCommand.ACL_SETUSER; - } - else if (subCommand.SequenceEqual(CmdStrings.USERS)) - { - return RespCommand.ACL_USERS; - } - else if (subCommand.SequenceEqual(CmdStrings.WHOAMI)) + // Ensure valid command name length (1-9) and that the complete command string + // is contained in the package. Otherwise fall through to return NONE. + // 10 bytes = "*_\r\n$_\r\n" (8 bytes) + "\r\n" (2 bytes) at end of command name + if (length is > 0 and <= 9 && remainingBytes >= length + 10) { - return RespCommand.ACL_WHOAMI; - } + // Optimistically advance read head to the end of the command name + readHead += length + 10; - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.ACL)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.REGISTERCS)) - { - return RespCommand.REGISTERCS; - } - else if (command.SequenceEqual(CmdStrings.ASYNC)) - { - return RespCommand.ASYNC; - } - else if (command.SequenceEqual(CmdStrings.MODULE)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.MODULE))); - return RespCommand.INVALID; - } + // Last 8 byte word of the command name, for quick comparison + var lastWord = *(ulong*)(ptr + length + 2); - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } + // + // Fast path for common commands with fixed numbers of arguments + // - count--; + // Only check against commands with the correct count and length. - if (subCommand.SequenceEqual(CmdStrings.LOADCS)) - { - return RespCommand.MODULE_LOADCS; - } + // (1) Same fixed-arg hot commands as SIMD path. + // On SIMD hardware, SIMD already rejected these — skip to sections (2)/(3). + // The !Vector128.IsHardwareAccelerated check is a JIT constant that eliminates + // this block entirely on SIMD hardware. + if (!Vector128.IsHardwareAccelerated) + { + var simdResult = ((count << 4) | length) switch + { + 4 when lastWord == MemoryMarshal.Read("\r\nPING\r\n"u8) => RespCommand.PING, + 4 when lastWord == MemoryMarshal.Read("\r\nEXEC\r\n"u8) => RespCommand.EXEC, + 5 when lastWord == MemoryMarshal.Read("\nMULTI\r\n"u8) => RespCommand.MULTI, + (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nGET\r\n"u8) => RespCommand.GET, + (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nDEL\r\n"u8) => RespCommand.DEL, + (1 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nTTL\r\n"u8) => RespCommand.TTL, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nINCR\r\n"u8) => RespCommand.INCR, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nPTTL\r\n"u8) => RespCommand.PTTL, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("\r\nDECR\r\n"u8) => RespCommand.DECR, + (1 << 4) | 4 when lastWord == MemoryMarshal.Read("EXISTS\r\n"u8) => RespCommand.EXISTS, + (1 << 4) | 6 when lastWord == MemoryMarshal.Read("GETDEL\r\n"u8) => RespCommand.GETDEL, + (2 << 4) | 3 when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SET, + (2 << 4) | 6 when lastWord == MemoryMarshal.Read("INCRBY\r\n"u8) => RespCommand.INCRBY, + (2 << 4) | 6 when lastWord == MemoryMarshal.Read("DECRBY\r\n"u8) => RespCommand.DECRBY, + (2 << 4) | 6 when lastWord == MemoryMarshal.Read("APPEND\r\n"u8) => RespCommand.APPEND, + (2 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETNX\r\n"u8) => RespCommand.SETNX, + (3 << 4) | 5 when lastWord == MemoryMarshal.Read("\nSETEX\r\n"u8) => RespCommand.SETEX, + (3 << 4) | 6 when lastWord == MemoryMarshal.Read("PSETEX\r\n"u8) => RespCommand.PSETEX, + _ => RespCommand.NONE + }; + if (simdResult != RespCommand.NONE) + return simdResult; + } - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.MODULE)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.PUBSUB)) - { - if (count == 0) - { - specificErrorMsg = Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, - nameof(RespCommand.PUBSUB))); - return RespCommand.INVALID; - } + // (2) Hot commands too long for SIMD (name > 6 chars, exceeds 16-byte Vector128) + // (3) Hot variable-arg commands (arg count varies, cannot be SIMD or MRU cached) + // These run on all hardware — SIMD cannot handle them. + return ((count << 4) | length) switch + { + (2 << 4) | 7 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && ptr[8] == 'P' => RespCommand.PUBLISH, + (2 << 4) | 8 when lastWord == MemoryMarshal.Read("UBLISH\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SP"u8) => RespCommand.SPUBLISH, + (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("SE"u8) => RespCommand.SETRANGE, + (3 << 4) | 8 when lastWord == MemoryMarshal.Read("TRANGE\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("GE"u8) => RespCommand.GETRANGE, - var subCommand = GetUpperCaseCommand(out var gotSubCommand); - if (!gotSubCommand) - { - success = false; - return RespCommand.NONE; - } + _ => ((length << 4) | count) switch + { + >= ((3 << 4) | 3) and <= ((3 << 4) | 7) when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SETEXNX, + >= ((5 << 4) | 1) and <= ((5 << 4) | 3) when lastWord == MemoryMarshal.Read("\nGETEX\r\n"u8) => RespCommand.GETEX, + >= ((6 << 4) | 2) and <= ((6 << 4) | 3) when lastWord == MemoryMarshal.Read("EXPIRE\r\n"u8) => RespCommand.EXPIRE, + >= ((7 << 4) | 2) and <= ((7 << 4) | 3) when lastWord == MemoryMarshal.Read("EXPIRE\r\n"u8) && ptr[8] == 'P' => RespCommand.PEXPIRE, + _ => MatchedNone(this, oldReadHead) + } + }; - count--; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static RespCommand MatchedNone(RespServerSession session, int oldReadHead) + { + // Backup the read head, if we didn't find a command and need to continue in the more expensive parsing loop + session.readHead = oldReadHead; - if (subCommand.SequenceEqual(CmdStrings.CHANNELS)) - { - return RespCommand.PUBSUB_CHANNELS; - } - else if (subCommand.SequenceEqual(CmdStrings.NUMSUB)) - { - return RespCommand.PUBSUB_NUMSUB; - } - else if (subCommand.SequenceEqual(CmdStrings.NUMPAT)) - { - return RespCommand.PUBSUB_NUMPAT; + return RespCommand.NONE; + } } - - string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, - Encoding.UTF8.GetString(subCommand), - nameof(RespCommand.PUBSUB)); - specificErrorMsg = Encoding.UTF8.GetBytes(errMsg); - return RespCommand.INVALID; - } - else if (command.SequenceEqual(CmdStrings.HCOLLECT)) - { - return RespCommand.HCOLLECT; } - else if (command.SequenceEqual(CmdStrings.DEBUG)) - { - return RespCommand.DEBUG; - } - else if (command.SequenceEqual(CmdStrings.ZCOLLECT)) - { - return RespCommand.ZCOLLECT; - } - // Note: The commands below are not slow path commands, so they should probably move to earlier. - else if (command.SequenceEqual(CmdStrings.SETIFMATCH)) + else { - return RespCommand.SETIFMATCH; + return FastParseInlineCommand(out count); } - else if (command.SequenceEqual(CmdStrings.SETIFGREATER)) + + count = -1; + return RespCommand.NONE; + } + + private bool TryParseCustomCommand(ReadOnlySpan command, out RespCommand cmd) + { + if (customCommandManagerSession.Match(command, out currentCustomTransaction)) { - return RespCommand.SETIFGREATER; + cmd = RespCommand.CustomTxn; + return true; } - else if (command.SequenceEqual(CmdStrings.GETWITHETAG)) + else if (customCommandManagerSession.Match(command, out currentCustomProcedure)) { - return RespCommand.GETWITHETAG; + cmd = RespCommand.CustomProcedure; + return true; } - else if (command.SequenceEqual(CmdStrings.GETIFNOTMATCH)) + else if (customCommandManagerSession.Match(command, out currentCustomRawStringCommand)) { - return RespCommand.GETIFNOTMATCH; + cmd = RespCommand.CustomRawStringCmd; + return true; } - else if (command.SequenceEqual(CmdStrings.DELIFGREATER)) + else if (customCommandManagerSession.Match(command, out currentCustomObjectCommand)) { - return RespCommand.DELIFGREATER; + cmd = RespCommand.CustomObjCmd; + return true; } - - // If this command name was not known to the slow pass, we are out of options and the command is unknown. - return RespCommand.INVALID; + cmd = RespCommand.NONE; + return false; } + /// /// Attempts to skip to the end of the line ("\r\n") under the current read head. /// @@ -2738,10 +1034,24 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) // If we have not found a command, continue parsing on slow path if (cmd == RespCommand.NONE) { + var cmdStartOffset = readHead; // Save position before ArrayParseCommand advances it cmd = ArrayParseCommand(writeErrorOnFailure, ref count, ref success); if (!success) return cmd; + + // Update MRU cache for commands resolved by hash table. + // Exclude custom commands — they have runtime-registered names. + if (Vector128.IsHardwareAccelerated && + cmd != RespCommand.INVALID && cmd != RespCommand.NONE && + cmd != RespCommand.CustomTxn && cmd != RespCommand.CustomProcedure && + cmd != RespCommand.CustomRawStringCmd && cmd != RespCommand.CustomObjCmd) + { + UpdateCommandCache(cmdStartOffset, cmd, count); + } } + Debug.Assert(cmd != RespCommand.NONE, "ParseCommand should never return NONE - expected INVALID for unrecognized commands"); + Debug.Assert(count >= 0 || cmd == RespCommand.INVALID, "Argument count must be non-negative for valid commands"); + // Set up parse state parseState.Initialize(count); var ptr = recvBufferPtr + readHead; @@ -2761,6 +1071,56 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) return cmd; } + /// + /// Update the MRU command cache with a newly matched command from the hash table. + /// Captures the first 16 bytes of the RESP encoding and the appropriate mask so that + /// the cache check in FastParseCommand can match it via Vector128.EqualsAll. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private void UpdateCommandCache(int cmdStartOffset, RespCommand cmd, int argCount) + { + var ptr = recvBufferPtr + cmdStartOffset; + var availableBytes = bytesRead - cmdStartOffset; + + // Only cache commands where we have at least 16 bytes to load a full Vector128 + if (availableBytes < 16) return; + + // Compute how many bytes the parse actually consumed (command + subcommand if any) + var consumedBytes = readHead - cmdStartOffset; + + // Only cache if the full parse fits within 16 bytes (one Vector128) + if (consumedBytes < 13 || consumedBytes > 16) return; + + var input = Vector128.LoadUnsafe(ref Unsafe.AsRef(ptr)); + + // Select the appropriate mask — covers exactly the consumed bytes + Vector128 mask = consumedBytes switch + { + 16 => Vector128.AllBitsSet, + 15 => s_mask15, + 14 => s_mask14, + 13 => s_mask13, + _ => Vector128.Zero + }; + + if (mask == Vector128.Zero) return; + + var pattern = Vector128.BitwiseAnd(input, mask); + + // Demote slot 0 → slot 1, promote new match → slot 0 + _cachedPattern1 = _cachedPattern0; + _cachedMask1 = _cachedMask0; + _cachedCmd1 = _cachedCmd0; + _cachedLen1 = _cachedLen0; + _cachedCount1 = _cachedCount0; + + _cachedPattern0 = pattern; + _cachedMask0 = mask; + _cachedCmd0 = cmd; + _cachedLen0 = (byte)consumedBytes; + _cachedCount0 = (byte)argCount; + } + [MethodImpl(MethodImplOptions.NoInlining)] private void HandleAofCommitMode(RespCommand cmd) { @@ -2816,13 +1176,8 @@ private RespCommand ArrayParseCommand(bool writeErrorOnFailure, ref int count, r // Move readHead to start of command payload readHead = (int)(ptr - recvBufferPtr); - // Try parsing the most important variable-length commands - cmd = FastParseArrayCommand(ref count, ref specificErrorMessage); - - if (cmd == RespCommand.NONE) - { - cmd = SlowParseCommand(ref count, ref specificErrorMessage, out success); - } + // Extract command name via HashLookupCommand (reads $len\r\n header via GetCommand, relies on prior MakeUpperCase() pass, and advances readHead) + cmd = HashLookupCommand(ref count, ref specificErrorMessage, out success); // Parsing for command name was successful, but the command is unknown if (writeErrorOnFailure && success && cmd == RespCommand.INVALID) @@ -2841,5 +1196,113 @@ private RespCommand ArrayParseCommand(bool writeErrorOnFailure, ref int count, r } return cmd; } + + /// + /// Hash-based command lookup. Extracts the command name from the RESP buffer, + /// looks it up in the static hash table, and handles subcommand dispatch. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private RespCommand HashLookupCommand(ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) + { + // Extract the command name (reads $len\r\n...\r\n, advances readHead). + // MakeUpperCase has already uppercased the first command name token in the buffer, + // so we use GetCommand (no redundant ToUpperInPlace call). + // NOTE: MakeUpperCase only uppercases the first token — subcommand names are + // uppercased separately via GetUpperCaseCommand in HandleSubcommandLookup. + var command = GetCommand(out success); + if (!success) + { + return RespCommand.INVALID; + } + + // Account for the command name being taken off the read head + count -= 1; + + // Hash table lookup for the primary command (checked before custom commands + // since built-in commands are far more common). + // Single probe returns both the command and whether it has subcommands. + fixed (byte* namePtr = command) + { + var cmd = RespCommandHashLookup.Lookup(namePtr, command.Length, out var hasSubcommands); + + if (cmd == RespCommand.NONE) + { + // Not a built-in command — check custom commands + if (TryParseCustomCommand(command, out var customCmd)) + { + return customCmd; + } + + // Not a built-in or custom command + return RespCommand.INVALID; + } + + // Commands with subcommands — dispatch via subcommand hash table + if (hasSubcommands) + { + return HandleSubcommandLookup(cmd, ref count, ref specificErrorMessage, out success); + } + + return cmd; + } + } + + /// + /// Handles subcommand dispatch for parent commands (CLUSTER, CONFIG, CLIENT, etc.) + /// using per-parent hash tables. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private RespCommand HandleSubcommandLookup(RespCommand parentCmd, ref int count, ref ReadOnlySpan specificErrorMessage, out bool success) + { + success = true; + + // COMMAND with no args returns RespCommand.COMMAND (lists all commands) + if (parentCmd == RespCommand.COMMAND && count == 0) + { + return RespCommand.COMMAND; + } + + // Most parent commands require at least one subcommand argument + if (count == 0) + { + specificErrorMessage = parentCmd == RespCommand.BITOP + ? CmdStrings.RESP_SYNTAX_ERROR + : Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrWrongNumArgs, parentCmd.ToString())); + return RespCommand.INVALID; + } + + // Extract and uppercase the subcommand name + var subCommand = GetUpperCaseCommand(out var gotSubCommand); + if (!gotSubCommand) + { + success = false; + return RespCommand.NONE; + } + + count--; + + // Hash table lookup for the subcommand + fixed (byte* subNamePtr = subCommand) + { + var subCmd = RespCommandHashLookup.LookupSubcommand(parentCmd, subNamePtr, subCommand.Length); + if (subCmd != RespCommand.NONE) + { + return subCmd; + } + } + + // Generate error message for unknown subcommand + specificErrorMessage = parentCmd switch + { + RespCommand.BITOP => CmdStrings.RESP_SYNTAX_ERROR, + RespCommand.CLUSTER or RespCommand.LATENCY => + Encoding.UTF8.GetBytes(string.Format(CmdStrings.GenericErrUnknownSubCommand, + Encoding.UTF8.GetString(subCommand), parentCmd.ToString())), + _ => + Encoding.UTF8.GetBytes(string.Format(CmdStrings.GenericErrUnknownSubCommandNoHelp, + Encoding.UTF8.GetString(subCommand), parentCmd.ToString())), + }; + return RespCommand.INVALID; + } } } \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommandHashLookup.cs b/libs/server/Resp/Parser/RespCommandHashLookup.cs new file mode 100644 index 00000000000..7b5d317d599 --- /dev/null +++ b/libs/server/Resp/Parser/RespCommandHashLookup.cs @@ -0,0 +1,503 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace Garnet.server +{ + /// + /// Cache-friendly O(1) hash table for RESP command name lookup. + /// + /// Design: + /// - 32-byte entries (half a cache line) with open addressing and linear probing + /// - CRC32 hardware hash (single instruction) with multiply-shift software fallback + /// - Table fits in L1 cache (~8-16KB) + /// - Single-threaded read-only access after static init + /// + internal static unsafe partial class RespCommandHashLookup + { + /// + /// Entry in the command hash table. Exactly 32 bytes = half a cache line. + /// Two entries per cache line gives excellent spatial locality during linear probing. + /// + [StructLayout(LayoutKind.Explicit, Size = 32)] + private struct CommandEntry + { + /// The command enum value. + [FieldOffset(0)] + public RespCommand Command; + + /// Length of the command name in bytes. + [FieldOffset(2)] + public byte NameLength; + + /// Flags (e.g., HasSubcommands). + [FieldOffset(3)] + public byte Flags; + + /// First 8 bytes of the uppercase command name. + [FieldOffset(8)] + public ulong NameWord0; + + /// Bytes 8-15 of the uppercase command name (zero-padded). + [FieldOffset(16)] + public ulong NameWord1; + + /// Bytes 16-23 of the uppercase command name (zero-padded). + [FieldOffset(24)] + public ulong NameWord2; + } + + /// Flag indicating the command has subcommands that require a second lookup. + internal const byte FlagHasSubcommands = 1; + + // Primary command table: 512 entries = 16KB, fits in L1 cache + private const int PrimaryTableBits = 9; + private const int PrimaryTableSize = 1 << PrimaryTableBits; + private const int PrimaryTableMask = PrimaryTableSize - 1; + private const int MaxProbes = 16; + + private static readonly CommandEntry[] primaryTable; + + // Subcommand tables (per parent command) + private static readonly CommandEntry[] clusterSubTable; + private static readonly int clusterSubTableMask; + + private static readonly CommandEntry[] clientSubTable; + private static readonly int clientSubTableMask; + + private static readonly CommandEntry[] aclSubTable; + private static readonly int aclSubTableMask; + + private static readonly CommandEntry[] commandSubTable; + private static readonly int commandSubTableMask; + + private static readonly CommandEntry[] configSubTable; + private static readonly int configSubTableMask; + + private static readonly CommandEntry[] scriptSubTable; + private static readonly int scriptSubTableMask; + + private static readonly CommandEntry[] latencySubTable; + private static readonly int latencySubTableMask; + + private static readonly CommandEntry[] slowlogSubTable; + private static readonly int slowlogSubTableMask; + + private static readonly CommandEntry[] moduleSubTable; + private static readonly int moduleSubTableMask; + + private static readonly CommandEntry[] pubsubSubTable; + private static readonly int pubsubSubTableMask; + + private static readonly CommandEntry[] memorySubTable; + private static readonly int memorySubTableMask; + + private static readonly CommandEntry[] bitopSubTable; + private static readonly int bitopSubTableMask; + + static RespCommandHashLookup() + { + // Build primary command table + primaryTable = GC.AllocateArray(PrimaryTableSize, pinned: true); + PopulatePrimaryTable(); + + // Build subcommand tables + clusterSubTable = BuildSubTable(ClusterSubcommands, out clusterSubTableMask); + clientSubTable = BuildSubTable(ClientSubcommands, out clientSubTableMask); + aclSubTable = BuildSubTable(AclSubcommands, out aclSubTableMask); + commandSubTable = BuildSubTable(CommandSubcommands, out commandSubTableMask); + configSubTable = BuildSubTable(ConfigSubcommands, out configSubTableMask); + scriptSubTable = BuildSubTable(ScriptSubcommands, out scriptSubTableMask); + latencySubTable = BuildSubTable(LatencySubcommands, out latencySubTableMask); + slowlogSubTable = BuildSubTable(SlowlogSubcommands, out slowlogSubTableMask); + moduleSubTable = BuildSubTable(ModuleSubcommands, out moduleSubTableMask); + pubsubSubTable = BuildSubTable(PubsubSubcommands, out pubsubSubTableMask); + memorySubTable = BuildSubTable(MemorySubcommands, out memorySubTableMask); + bitopSubTable = BuildSubTable(BitopSubcommands, out bitopSubTableMask); + + // Validate all subcommand tables are round-trip correct + ValidateSubTable(RespCommand.CLUSTER, ClusterSubcommands, clusterSubTable, clusterSubTableMask); + ValidateSubTable(RespCommand.CLIENT, ClientSubcommands, clientSubTable, clientSubTableMask); + ValidateSubTable(RespCommand.ACL, AclSubcommands, aclSubTable, aclSubTableMask); + ValidateSubTable(RespCommand.COMMAND, CommandSubcommands, commandSubTable, commandSubTableMask); + ValidateSubTable(RespCommand.CONFIG, ConfigSubcommands, configSubTable, configSubTableMask); + ValidateSubTable(RespCommand.SCRIPT, ScriptSubcommands, scriptSubTable, scriptSubTableMask); + ValidateSubTable(RespCommand.LATENCY, LatencySubcommands, latencySubTable, latencySubTableMask); + ValidateSubTable(RespCommand.SLOWLOG, SlowlogSubcommands, slowlogSubTable, slowlogSubTableMask); + ValidateSubTable(RespCommand.MODULE, ModuleSubcommands, moduleSubTable, moduleSubTableMask); + ValidateSubTable(RespCommand.PUBSUB, PubsubSubcommands, pubsubSubTable, pubsubSubTableMask); + ValidateSubTable(RespCommand.MEMORY, MemorySubcommands, memorySubTable, memorySubTableMask); + ValidateSubTable(RespCommand.BITOP, BitopSubcommands, bitopSubTable, bitopSubTableMask); + + // Validate primary table: every inserted command must round-trip via Lookup + ValidatePrimaryTable(); + } + + /// + /// Validate that every command inserted into the primary table can be looked up. + /// Called once during static init; throws on any mismatch to catch registration bugs early. + /// + private static void ValidatePrimaryTable() + { + // Scan all occupied slots and verify each round-trips via Lookup + Span word0Bytes = stackalloc byte[8]; + Span word1Bytes = stackalloc byte[8]; + Span word2Bytes = stackalloc byte[8]; + + for (int i = 0; i < primaryTable.Length; i++) + { + ref CommandEntry entry = ref primaryTable[i]; + if (entry.NameLength == 0) continue; + + // Reconstruct the name from the stored words + var nameBytes = new byte[entry.NameLength]; + MemoryMarshal.Write(word0Bytes, in entry.NameWord0); + MemoryMarshal.Write(word1Bytes, in entry.NameWord1); + MemoryMarshal.Write(word2Bytes, in entry.NameWord2); + + if (entry.NameLength <= 8) + { + word0Bytes.Slice(0, entry.NameLength).CopyTo(nameBytes); + } + else if (entry.NameLength <= 16) + { + word0Bytes.CopyTo(nameBytes); + // Word1 stores last 8 bytes (overlapping for lengths 9-16) + word1Bytes.CopyTo(nameBytes.AsSpan(entry.NameLength - 8)); + } + else + { + word0Bytes.CopyTo(nameBytes); + word1Bytes.CopyTo(nameBytes.AsSpan(8)); + word2Bytes.CopyTo(nameBytes.AsSpan(entry.NameLength - 8)); + } + + fixed (byte* p = nameBytes) + { + var found = Lookup(p, nameBytes.Length); + if (found != entry.Command) + { + throw new InvalidOperationException( + $"Primary hash table validation failed: '{System.Text.Encoding.ASCII.GetString(nameBytes)}' expected {entry.Command} but Lookup returned {found}"); + } + } + } + } + + /// + /// Validate that every entry in a subcommand definition array can be looked up in the hash table. + /// Called once during static init; throws on any mismatch to catch typos early. + /// + private static void ValidateSubTable(RespCommand parent, ReadOnlySpan<(string Name, RespCommand Command)> subcommands, + CommandEntry[] table, int mask) + { + foreach (var (name, expectedCmd) in subcommands) + { + var nameBytes = System.Text.Encoding.ASCII.GetBytes(name); + fixed (byte* p = nameBytes) + { + var found = LookupInTable(table, mask, p, nameBytes.Length); + if (found != expectedCmd) + { + throw new InvalidOperationException( + $"Hash table validation failed: {parent} subcommand '{name}' expected {expectedCmd} but got {found}"); + } + } + } + } + + #region Public API + + /// + /// Look up a primary command name in the hash table. + /// + /// Pointer to the uppercase command name bytes. + /// Length of the command name. + /// The matching RespCommand, or RespCommand.NONE if not found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RespCommand Lookup(byte* name, int length) + { + return LookupInTable(primaryTable, PrimaryTableMask, name, length); + } + + /// + /// Look up a primary command name and return whether it has subcommands, in a single probe. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RespCommand Lookup(byte* name, int length, out bool hasSubcommands) + { + hasSubcommands = false; + if ((uint)length - 1 > 23) + return RespCommand.NONE; + + uint hash = ComputeHash(name, length); + int idx = (int)(hash & (uint)PrimaryTableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref primaryTable[idx]; + if (entry.NameLength == 0) return RespCommand.NONE; + if (entry.NameLength == (byte)length && MatchName(ref entry, name, length)) + { + hasSubcommands = (entry.Flags & FlagHasSubcommands) != 0; + return entry.Command; + } + idx = (idx + 1) & PrimaryTableMask; + } + return RespCommand.NONE; + } + + /// + /// Look up a subcommand for a given parent command. + /// + /// The parent command. + /// Pointer to the uppercase subcommand name bytes. + /// Length of the subcommand name. + /// The matching RespCommand, or RespCommand.NONE if not found. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RespCommand LookupSubcommand(RespCommand parent, byte* name, int length) + { + var (table, mask) = parent switch + { + RespCommand.CLUSTER => (clusterSubTable, clusterSubTableMask), + RespCommand.CLIENT => (clientSubTable, clientSubTableMask), + RespCommand.ACL => (aclSubTable, aclSubTableMask), + RespCommand.COMMAND => (commandSubTable, commandSubTableMask), + RespCommand.CONFIG => (configSubTable, configSubTableMask), + RespCommand.SCRIPT => (scriptSubTable, scriptSubTableMask), + RespCommand.LATENCY => (latencySubTable, latencySubTableMask), + RespCommand.SLOWLOG => (slowlogSubTable, slowlogSubTableMask), + RespCommand.MODULE => (moduleSubTable, moduleSubTableMask), + RespCommand.PUBSUB => (pubsubSubTable, pubsubSubTableMask), + RespCommand.MEMORY => (memorySubTable, memorySubTableMask), + RespCommand.BITOP => (bitopSubTable, bitopSubTableMask), + _ => (null, 0) + }; + + if (table == null) return RespCommand.NONE; + return LookupInTable(table, mask, name, length); + } + + #endregion + + #region Hash and Match + + /// + /// Compute hash from command name bytes using hardware CRC32 when available. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint ComputeHash(byte* name, int length) + { + ulong word0 = length >= 8 ? *(ulong*)name : ReadPartialWord(name, length); + + if (Sse42.X64.IsSupported) + { + uint crc = (uint)Sse42.X64.Crc32(0UL, word0); + return Sse42.Crc32(crc, (uint)length); + } + + if (System.Runtime.Intrinsics.Arm.Crc32.Arm64.IsSupported) + { + uint crc = (uint)System.Runtime.Intrinsics.Arm.Crc32.Arm64.ComputeCrc32C(0U, word0); + return System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C(crc, (uint)length); + } + + // Software fallback: Fibonacci multiply-shift + return (uint)((word0 * 0x9E3779B97F4A7C15UL) >> 32) ^ (uint)(length * 2654435761U); + } + + /// + /// Compute hash from a ReadOnlySpan (used during table construction). + /// + private static uint ComputeHash(ReadOnlySpan name) + { + ulong word0 = GetWordFromSpan(name, 0); + + if (Sse42.X64.IsSupported) + { + uint crc = (uint)Sse42.X64.Crc32(0UL, word0); + return Sse42.Crc32(crc, (uint)name.Length); + } + + if (System.Runtime.Intrinsics.Arm.Crc32.Arm64.IsSupported) + { + uint crc = (uint)System.Runtime.Intrinsics.Arm.Crc32.Arm64.ComputeCrc32C(0U, word0); + return System.Runtime.Intrinsics.Arm.Crc32.ComputeCrc32C(crc, (uint)name.Length); + } + + return (uint)((word0 * 0x9E3779B97F4A7C15UL) >> 32) ^ (uint)(name.Length * 2654435761U); + } + + /// + /// Read up to 8 bytes from a pointer, zero-extending short reads. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong ReadPartialWord(byte* p, int len) + { + // For len >= 8, just read the full ulong + // For len < 8, read and mask + Debug.Assert(len > 0 && len < 8); + + return len switch + { + 1 => *p, + 2 => *(ushort*)p, + 3 => *(ushort*)p | ((ulong)p[2] << 16), + 4 => *(uint*)p, + 5 => *(uint*)p | ((ulong)p[4] << 32), + 6 => *(uint*)p | ((ulong)*(ushort*)(p + 4) << 32), + 7 => *(uint*)p | ((ulong)*(ushort*)(p + 4) << 32) | ((ulong)p[6] << 48), + _ => 0 + }; + } + + /// + /// Compare entry name against input name bytes. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool MatchName(ref CommandEntry entry, byte* name, int length) + { + Debug.Assert(length > 0, "MatchName: length must be positive"); + + if (length <= 8) + { + ulong inputWord = length == 8 ? *(ulong*)name : ReadPartialWord(name, length); + return entry.NameWord0 == inputWord; + } + else if (length <= 16) + { + // Compare first 8 bytes and last 8 bytes (may overlap for 9-15 byte names) + return entry.NameWord0 == *(ulong*)name && + entry.NameWord1 == *(ulong*)(name + length - 8); + } + else + { + // Compare first 8, middle 8, and last 8 bytes + return entry.NameWord0 == *(ulong*)name && + entry.NameWord1 == *(ulong*)(name + 8) && + entry.NameWord2 == *(ulong*)(name + length - 8); + } + } + + /// + /// Core lookup in any hash table (primary or subcommand). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static RespCommand LookupInTable(CommandEntry[] table, int tableMask, byte* name, int length) + { + // CommandEntry stores at most 24 bytes of name; empty or oversized names can never match. + if ((uint)length - 1 > 23) return RespCommand.NONE; + + uint hash = ComputeHash(name, length); + int idx = (int)(hash & (uint)tableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref table[idx]; + + // Empty slot — command not found + if (entry.NameLength == 0) return RespCommand.NONE; + + // Fast rejection: length mismatch (single byte compare) + if (entry.NameLength == (byte)length && MatchName(ref entry, name, length)) + return entry.Command; + + idx = (idx + 1) & tableMask; + } + + return RespCommand.NONE; + } + + #endregion + + #region Table Construction + + /// + /// Read a ulong from a span at the given offset, zero-padding short reads. + /// + private static ulong GetWordFromSpan(ReadOnlySpan span, int offset) + { + if (offset >= span.Length) return 0; + int remaining = span.Length - offset; + if (remaining >= 8) return MemoryMarshal.Read(span.Slice(offset)); + + ulong word = 0; + for (int i = 0; i < remaining; i++) + word |= (ulong)span[offset + i] << (i * 8); + return word; + } + + /// + /// Insert a command into a hash table. + /// + private static void InsertIntoTable(CommandEntry[] table, int tableMask, ReadOnlySpan name, RespCommand command, byte flags = 0) + { + if (name.Length == 0 || name.Length > 24) + throw new ArgumentException($"Command name must be 1-24 bytes, got {name.Length}: {System.Text.Encoding.ASCII.GetString(name)}"); + + uint hash = ComputeHash(name); + int idx = (int)(hash & (uint)tableMask); + + for (int probe = 0; probe < MaxProbes; probe++) + { + ref CommandEntry entry = ref table[idx]; + if (entry.NameLength == 0) + { + entry.Command = command; + entry.NameLength = (byte)name.Length; + entry.Flags = flags; + + // Store name words to match the layout expected by MatchName: + // length <= 8: Word0 = bytes 0..len-1 (zero-padded) + // length 9-16: Word0 = bytes 0..7, Word1 = bytes len-8..len-1 (overlapping) + // length 17-24: Word0 = bytes 0..7, Word1 = bytes 8..15, Word2 = bytes len-8..len-1 + entry.NameWord0 = GetWordFromSpan(name, 0); + entry.NameWord1 = 0; + entry.NameWord2 = 0; + + if (name.Length > 16) + { + entry.NameWord1 = GetWordFromSpan(name, 8); + entry.NameWord2 = GetWordFromSpan(name, name.Length - 8); + } + else if (name.Length > 8) + { + entry.NameWord1 = GetWordFromSpan(name, name.Length - 8); + } + + return; + } + idx = (idx + 1) & tableMask; + } + + throw new InvalidOperationException( + $"Hash table overflow: could not insert command '{System.Text.Encoding.ASCII.GetString(name)}' after {MaxProbes} probes. Increase table size."); + } + + /// + /// Build a subcommand hash table from a list of (name, command) pairs. + /// + private static CommandEntry[] BuildSubTable(ReadOnlySpan<(string Name, RespCommand Command)> subcommands, out int mask) + { + // Find next power of 2 that gives at most ~70% load factor + int size = 16; + while (size * 7 / 10 < subcommands.Length) size <<= 1; + mask = size - 1; + + var table = GC.AllocateArray(size, pinned: true); + foreach (var (name, command) in subcommands) + { + InsertIntoTable(table, mask, System.Text.Encoding.ASCII.GetBytes(name), command); + } + return table; + } + + #endregion + + } +} \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommandHashLookupData.cs b/libs/server/Resp/Parser/RespCommandHashLookupData.cs new file mode 100644 index 00000000000..02829e22ee4 --- /dev/null +++ b/libs/server/Resp/Parser/RespCommandHashLookupData.cs @@ -0,0 +1,437 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Garnet.server +{ + /// + /// Command registration data for . + /// Primary command table and subcommand definitions. + /// + internal static unsafe partial class RespCommandHashLookup + { + #region Command Definitions + + private static void PopulatePrimaryTable() + { + // Helper to insert with optional subcommand flag + void Add(string name, RespCommand cmd, bool hasSub = false) + { + InsertIntoTable(primaryTable, PrimaryTableMask, + System.Text.Encoding.ASCII.GetBytes(name), cmd, + hasSub ? FlagHasSubcommands : (byte)0); + } + + // ===== Data commands (read + write) ===== + + // String commands + Add("GET", RespCommand.GET); + Add("SET", RespCommand.SET); + Add("DEL", RespCommand.DEL); + Add("INCR", RespCommand.INCR); + Add("DECR", RespCommand.DECR); + Add("INCRBY", RespCommand.INCRBY); + Add("DECRBY", RespCommand.DECRBY); + Add("INCRBYFLOAT", RespCommand.INCRBYFLOAT); + Add("APPEND", RespCommand.APPEND); + Add("GETSET", RespCommand.GETSET); + Add("GETDEL", RespCommand.GETDEL); + Add("GETEX", RespCommand.GETEX); + Add("GETRANGE", RespCommand.GETRANGE); + Add("SETRANGE", RespCommand.SETRANGE); + Add("STRLEN", RespCommand.STRLEN); + Add("SUBSTR", RespCommand.SUBSTR); + Add("SETNX", RespCommand.SETNX); + Add("SETEX", RespCommand.SETEX); + Add("PSETEX", RespCommand.PSETEX); + Add("MGET", RespCommand.MGET); + Add("MSET", RespCommand.MSET); + Add("MSETNX", RespCommand.MSETNX); + Add("DUMP", RespCommand.DUMP); + Add("RESTORE", RespCommand.RESTORE); + Add("GETBIT", RespCommand.GETBIT); + Add("SETBIT", RespCommand.SETBIT); + Add("GETWITHETAG", RespCommand.GETWITHETAG); + Add("GETIFNOTMATCH", RespCommand.GETIFNOTMATCH); + Add("SETIFMATCH", RespCommand.SETIFMATCH); + Add("SETIFGREATER", RespCommand.SETIFGREATER); + Add("DELIFGREATER", RespCommand.DELIFGREATER); + Add("LCS", RespCommand.LCS); + + // Key commands + Add("EXISTS", RespCommand.EXISTS); + Add("TTL", RespCommand.TTL); + Add("PTTL", RespCommand.PTTL); + Add("EXPIRE", RespCommand.EXPIRE); + Add("PEXPIRE", RespCommand.PEXPIRE); + Add("EXPIREAT", RespCommand.EXPIREAT); + Add("PEXPIREAT", RespCommand.PEXPIREAT); + Add("EXPIRETIME", RespCommand.EXPIRETIME); + Add("PEXPIRETIME", RespCommand.PEXPIRETIME); + Add("PERSIST", RespCommand.PERSIST); + Add("TYPE", RespCommand.TYPE); + Add("RENAME", RespCommand.RENAME); + Add("RENAMENX", RespCommand.RENAMENX); + Add("UNLINK", RespCommand.UNLINK); + Add("KEYS", RespCommand.KEYS); + Add("SCAN", RespCommand.SCAN); + Add("DBSIZE", RespCommand.DBSIZE); + Add("SELECT", RespCommand.SELECT); + Add("SWAPDB", RespCommand.SWAPDB); + Add("MIGRATE", RespCommand.MIGRATE); + + // Bitmap commands + Add("BITCOUNT", RespCommand.BITCOUNT); + Add("BITPOS", RespCommand.BITPOS); + Add("BITFIELD", RespCommand.BITFIELD); + Add("BITFIELD_RO", RespCommand.BITFIELD_RO); + Add("BITOP", RespCommand.BITOP, hasSub: true); + + // HyperLogLog commands + Add("PFADD", RespCommand.PFADD); + Add("PFCOUNT", RespCommand.PFCOUNT); + Add("PFMERGE", RespCommand.PFMERGE); + + // Hash commands + Add("HSET", RespCommand.HSET); + Add("HGET", RespCommand.HGET); + Add("HDEL", RespCommand.HDEL); + Add("HLEN", RespCommand.HLEN); + Add("HEXISTS", RespCommand.HEXISTS); + Add("HGETALL", RespCommand.HGETALL); + Add("HKEYS", RespCommand.HKEYS); + Add("HVALS", RespCommand.HVALS); + Add("HMSET", RespCommand.HMSET); + Add("HMGET", RespCommand.HMGET); + Add("HSETNX", RespCommand.HSETNX); + Add("HINCRBY", RespCommand.HINCRBY); + Add("HINCRBYFLOAT", RespCommand.HINCRBYFLOAT); + Add("HRANDFIELD", RespCommand.HRANDFIELD); + Add("HSCAN", RespCommand.HSCAN); + Add("HSTRLEN", RespCommand.HSTRLEN); + Add("HTTL", RespCommand.HTTL); + Add("HPTTL", RespCommand.HPTTL); + Add("HEXPIRE", RespCommand.HEXPIRE); + Add("HPEXPIRE", RespCommand.HPEXPIRE); + Add("HEXPIREAT", RespCommand.HEXPIREAT); + Add("HPEXPIREAT", RespCommand.HPEXPIREAT); + Add("HEXPIRETIME", RespCommand.HEXPIRETIME); + Add("HPEXPIRETIME", RespCommand.HPEXPIRETIME); + Add("HPERSIST", RespCommand.HPERSIST); + Add("HCOLLECT", RespCommand.HCOLLECT); + + // List commands + Add("LPUSH", RespCommand.LPUSH); + Add("RPUSH", RespCommand.RPUSH); + Add("LPUSHX", RespCommand.LPUSHX); + Add("RPUSHX", RespCommand.RPUSHX); + Add("LPOP", RespCommand.LPOP); + Add("RPOP", RespCommand.RPOP); + Add("LLEN", RespCommand.LLEN); + Add("LINDEX", RespCommand.LINDEX); + Add("LINSERT", RespCommand.LINSERT); + Add("LRANGE", RespCommand.LRANGE); + Add("LREM", RespCommand.LREM); + Add("LSET", RespCommand.LSET); + Add("LTRIM", RespCommand.LTRIM); + Add("LPOS", RespCommand.LPOS); + Add("LMOVE", RespCommand.LMOVE); + Add("LMPOP", RespCommand.LMPOP); + Add("RPOPLPUSH", RespCommand.RPOPLPUSH); + Add("BLPOP", RespCommand.BLPOP); + Add("BRPOP", RespCommand.BRPOP); + Add("BLMOVE", RespCommand.BLMOVE); + Add("BRPOPLPUSH", RespCommand.BRPOPLPUSH); + Add("BLMPOP", RespCommand.BLMPOP); + + // Set commands + Add("SADD", RespCommand.SADD); + Add("SREM", RespCommand.SREM); + Add("SPOP", RespCommand.SPOP); + Add("SCARD", RespCommand.SCARD); + Add("SMEMBERS", RespCommand.SMEMBERS); + Add("SISMEMBER", RespCommand.SISMEMBER); + Add("SMISMEMBER", RespCommand.SMISMEMBER); + Add("SRANDMEMBER", RespCommand.SRANDMEMBER); + Add("SMOVE", RespCommand.SMOVE); + Add("SSCAN", RespCommand.SSCAN); + Add("SDIFF", RespCommand.SDIFF); + Add("SDIFFSTORE", RespCommand.SDIFFSTORE); + Add("SINTER", RespCommand.SINTER); + Add("SINTERCARD", RespCommand.SINTERCARD); + Add("SINTERSTORE", RespCommand.SINTERSTORE); + Add("SUNION", RespCommand.SUNION); + Add("SUNIONSTORE", RespCommand.SUNIONSTORE); + + // Sorted set commands + Add("ZADD", RespCommand.ZADD); + Add("ZREM", RespCommand.ZREM); + Add("ZCARD", RespCommand.ZCARD); + Add("ZSCORE", RespCommand.ZSCORE); + Add("ZMSCORE", RespCommand.ZMSCORE); + Add("ZRANK", RespCommand.ZRANK); + Add("ZREVRANK", RespCommand.ZREVRANK); + Add("ZCOUNT", RespCommand.ZCOUNT); + Add("ZLEXCOUNT", RespCommand.ZLEXCOUNT); + Add("ZRANGE", RespCommand.ZRANGE); + Add("ZRANGEBYLEX", RespCommand.ZRANGEBYLEX); + Add("ZRANGEBYSCORE", RespCommand.ZRANGEBYSCORE); + Add("ZRANGESTORE", RespCommand.ZRANGESTORE); + Add("ZREVRANGE", RespCommand.ZREVRANGE); + Add("ZREVRANGEBYLEX", RespCommand.ZREVRANGEBYLEX); + Add("ZREVRANGEBYSCORE", RespCommand.ZREVRANGEBYSCORE); + Add("ZPOPMIN", RespCommand.ZPOPMIN); + Add("ZPOPMAX", RespCommand.ZPOPMAX); + Add("ZRANDMEMBER", RespCommand.ZRANDMEMBER); + Add("ZSCAN", RespCommand.ZSCAN); + Add("ZINCRBY", RespCommand.ZINCRBY); + Add("ZDIFF", RespCommand.ZDIFF); + Add("ZDIFFSTORE", RespCommand.ZDIFFSTORE); + Add("ZINTER", RespCommand.ZINTER); + Add("ZINTERCARD", RespCommand.ZINTERCARD); + Add("ZINTERSTORE", RespCommand.ZINTERSTORE); + Add("ZUNION", RespCommand.ZUNION); + Add("ZUNIONSTORE", RespCommand.ZUNIONSTORE); + Add("ZMPOP", RespCommand.ZMPOP); + Add("BZMPOP", RespCommand.BZMPOP); + Add("BZPOPMAX", RespCommand.BZPOPMAX); + Add("BZPOPMIN", RespCommand.BZPOPMIN); + Add("ZREMRANGEBYLEX", RespCommand.ZREMRANGEBYLEX); + Add("ZREMRANGEBYRANK", RespCommand.ZREMRANGEBYRANK); + Add("ZREMRANGEBYSCORE", RespCommand.ZREMRANGEBYSCORE); + Add("ZTTL", RespCommand.ZTTL); + Add("ZPTTL", RespCommand.ZPTTL); + Add("ZEXPIRE", RespCommand.ZEXPIRE); + Add("ZPEXPIRE", RespCommand.ZPEXPIRE); + Add("ZEXPIREAT", RespCommand.ZEXPIREAT); + Add("ZPEXPIREAT", RespCommand.ZPEXPIREAT); + Add("ZEXPIRETIME", RespCommand.ZEXPIRETIME); + Add("ZPEXPIRETIME", RespCommand.ZPEXPIRETIME); + Add("ZPERSIST", RespCommand.ZPERSIST); + Add("ZCOLLECT", RespCommand.ZCOLLECT); + + // Geo commands + Add("GEOADD", RespCommand.GEOADD); + Add("GEOPOS", RespCommand.GEOPOS); + Add("GEOHASH", RespCommand.GEOHASH); + Add("GEODIST", RespCommand.GEODIST); + Add("GEOSEARCH", RespCommand.GEOSEARCH); + Add("GEOSEARCHSTORE", RespCommand.GEOSEARCHSTORE); + Add("GEORADIUS", RespCommand.GEORADIUS); + Add("GEORADIUS_RO", RespCommand.GEORADIUS_RO); + Add("GEORADIUSBYMEMBER", RespCommand.GEORADIUSBYMEMBER); + Add("GEORADIUSBYMEMBER_RO", RespCommand.GEORADIUSBYMEMBER_RO); + + // Scripting + Add("EVAL", RespCommand.EVAL); + Add("EVALSHA", RespCommand.EVALSHA); + + // Pub/Sub + Add("PUBLISH", RespCommand.PUBLISH); + Add("SUBSCRIBE", RespCommand.SUBSCRIBE); + Add("PSUBSCRIBE", RespCommand.PSUBSCRIBE); + Add("UNSUBSCRIBE", RespCommand.UNSUBSCRIBE); + Add("PUNSUBSCRIBE", RespCommand.PUNSUBSCRIBE); + Add("SPUBLISH", RespCommand.SPUBLISH); + Add("SSUBSCRIBE", RespCommand.SSUBSCRIBE); + + // Custom object scan + Add("CUSTOMOBJECTSCAN", RespCommand.COSCAN); + + // ===== Control / admin commands ===== + Add("PING", RespCommand.PING); + Add("ECHO", RespCommand.ECHO); + Add("QUIT", RespCommand.QUIT); + Add("AUTH", RespCommand.AUTH); + Add("HELLO", RespCommand.HELLO); + Add("INFO", RespCommand.INFO); + Add("TIME", RespCommand.TIME); + Add("ROLE", RespCommand.ROLE); + Add("SAVE", RespCommand.SAVE); + Add("LASTSAVE", RespCommand.LASTSAVE); + Add("BGSAVE", RespCommand.BGSAVE); + Add("COMMITAOF", RespCommand.COMMITAOF); + Add("FLUSHALL", RespCommand.FLUSHALL); + Add("FLUSHDB", RespCommand.FLUSHDB); + Add("FORCEGC", RespCommand.FORCEGC); + Add("PURGEBP", RespCommand.PURGEBP); + Add("FAILOVER", RespCommand.FAILOVER); + Add("MONITOR", RespCommand.MONITOR); + Add("REGISTERCS", RespCommand.REGISTERCS); + Add("ASYNC", RespCommand.ASYNC); + Add("DEBUG", RespCommand.DEBUG); + Add("EXPDELSCAN", RespCommand.EXPDELSCAN); + Add("WATCH", RespCommand.WATCH); + Add("WATCHMS", RespCommand.WATCHMS); + Add("WATCHOS", RespCommand.WATCHOS); + Add("MULTI", RespCommand.MULTI); + Add("EXEC", RespCommand.EXEC); + Add("DISCARD", RespCommand.DISCARD); + Add("UNWATCH", RespCommand.UNWATCH); + Add("RUNTXP", RespCommand.RUNTXP); + Add("ASKING", RespCommand.ASKING); + Add("READONLY", RespCommand.READONLY); + Add("READWRITE", RespCommand.READWRITE); + Add("REPLICAOF", RespCommand.REPLICAOF); + Add("SECONDARYOF", RespCommand.SECONDARYOF); + Add("SLAVEOF", RespCommand.SECONDARYOF); + + // Parent commands with subcommands + Add("SCRIPT", RespCommand.SCRIPT, hasSub: true); + Add("CONFIG", RespCommand.CONFIG, hasSub: true); + Add("CLIENT", RespCommand.CLIENT, hasSub: true); + Add("CLUSTER", RespCommand.CLUSTER, hasSub: true); + Add("ACL", RespCommand.ACL, hasSub: true); + Add("COMMAND", RespCommand.COMMAND, hasSub: true); + Add("LATENCY", RespCommand.LATENCY, hasSub: true); + Add("SLOWLOG", RespCommand.SLOWLOG, hasSub: true); + Add("MODULE", RespCommand.MODULE, hasSub: true); + Add("PUBSUB", RespCommand.PUBSUB, hasSub: true); + Add("MEMORY", RespCommand.MEMORY, hasSub: true); + } + + #endregion + + #region Subcommand Definitions + + private static readonly (string Name, RespCommand Command)[] ClusterSubcommands = + [ + ("ADDSLOTS", RespCommand.CLUSTER_ADDSLOTS), + ("ADDSLOTSRANGE", RespCommand.CLUSTER_ADDSLOTSRANGE), + ("AOFSYNC", RespCommand.CLUSTER_AOFSYNC), + ("APPENDLOG", RespCommand.CLUSTER_APPENDLOG), + ("ATTACH_SYNC", RespCommand.CLUSTER_ATTACH_SYNC), + ("BANLIST", RespCommand.CLUSTER_BANLIST), + ("BEGIN_REPLICA_RECOVER", RespCommand.CLUSTER_BEGIN_REPLICA_RECOVER), + ("BUMPEPOCH", RespCommand.CLUSTER_BUMPEPOCH), + ("COUNTKEYSINSLOT", RespCommand.CLUSTER_COUNTKEYSINSLOT), + ("DELKEYSINSLOT", RespCommand.CLUSTER_DELKEYSINSLOT), + ("DELKEYSINSLOTRANGE", RespCommand.CLUSTER_DELKEYSINSLOTRANGE), + ("DELSLOTS", RespCommand.CLUSTER_DELSLOTS), + ("DELSLOTSRANGE", RespCommand.CLUSTER_DELSLOTSRANGE), + ("ENDPOINT", RespCommand.CLUSTER_ENDPOINT), + ("FAILOVER", RespCommand.CLUSTER_FAILOVER), + ("FAILREPLICATIONOFFSET", RespCommand.CLUSTER_FAILREPLICATIONOFFSET), + ("FAILSTOPWRITES", RespCommand.CLUSTER_FAILSTOPWRITES), + ("FLUSHALL", RespCommand.CLUSTER_FLUSHALL), + ("FORGET", RespCommand.CLUSTER_FORGET), + ("GETKEYSINSLOT", RespCommand.CLUSTER_GETKEYSINSLOT), + ("GOSSIP", RespCommand.CLUSTER_GOSSIP), + ("HELP", RespCommand.CLUSTER_HELP), + ("INFO", RespCommand.CLUSTER_INFO), + ("INITIATE_REPLICA_SYNC", RespCommand.CLUSTER_INITIATE_REPLICA_SYNC), + ("KEYSLOT", RespCommand.CLUSTER_KEYSLOT), + ("MEET", RespCommand.CLUSTER_MEET), + ("MIGRATE", RespCommand.CLUSTER_MIGRATE), + ("MTASKS", RespCommand.CLUSTER_MTASKS), + ("MYID", RespCommand.CLUSTER_MYID), + ("MYPARENTID", RespCommand.CLUSTER_MYPARENTID), + ("NODES", RespCommand.CLUSTER_NODES), + ("PUBLISH", RespCommand.CLUSTER_PUBLISH), + ("SPUBLISH", RespCommand.CLUSTER_SPUBLISH), + ("REPLICAS", RespCommand.CLUSTER_REPLICAS), + ("REPLICATE", RespCommand.CLUSTER_REPLICATE), + ("RESET", RespCommand.CLUSTER_RESET), + ("SEND_CKPT_FILE_SEGMENT", RespCommand.CLUSTER_SEND_CKPT_FILE_SEGMENT), + ("SEND_CKPT_METADATA", RespCommand.CLUSTER_SEND_CKPT_METADATA), + ("SET-CONFIG-EPOCH", RespCommand.CLUSTER_SETCONFIGEPOCH), + ("SETSLOT", RespCommand.CLUSTER_SETSLOT), + ("SETSLOTSRANGE", RespCommand.CLUSTER_SETSLOTSRANGE), + ("SHARDS", RespCommand.CLUSTER_SHARDS), + ("SLOTS", RespCommand.CLUSTER_SLOTS), + ("SLOTSTATE", RespCommand.CLUSTER_SLOTSTATE), + ("SYNC", RespCommand.CLUSTER_SYNC), + ]; + + private static readonly (string Name, RespCommand Command)[] ClientSubcommands = + [ + ("ID", RespCommand.CLIENT_ID), + ("INFO", RespCommand.CLIENT_INFO), + ("LIST", RespCommand.CLIENT_LIST), + ("KILL", RespCommand.CLIENT_KILL), + ("GETNAME", RespCommand.CLIENT_GETNAME), + ("SETNAME", RespCommand.CLIENT_SETNAME), + ("SETINFO", RespCommand.CLIENT_SETINFO), + ("UNBLOCK", RespCommand.CLIENT_UNBLOCK), + ]; + + private static readonly (string Name, RespCommand Command)[] AclSubcommands = + [ + ("CAT", RespCommand.ACL_CAT), + ("DELUSER", RespCommand.ACL_DELUSER), + ("GENPASS", RespCommand.ACL_GENPASS), + ("GETUSER", RespCommand.ACL_GETUSER), + ("LIST", RespCommand.ACL_LIST), + ("LOAD", RespCommand.ACL_LOAD), + ("SAVE", RespCommand.ACL_SAVE), + ("SETUSER", RespCommand.ACL_SETUSER), + ("USERS", RespCommand.ACL_USERS), + ("WHOAMI", RespCommand.ACL_WHOAMI), + ]; + + private static readonly (string Name, RespCommand Command)[] CommandSubcommands = + [ + ("COUNT", RespCommand.COMMAND_COUNT), + ("DOCS", RespCommand.COMMAND_DOCS), + ("INFO", RespCommand.COMMAND_INFO), + ("GETKEYS", RespCommand.COMMAND_GETKEYS), + ("GETKEYSANDFLAGS", RespCommand.COMMAND_GETKEYSANDFLAGS), + ]; + + private static readonly (string Name, RespCommand Command)[] ConfigSubcommands = + [ + ("GET", RespCommand.CONFIG_GET), + ("REWRITE", RespCommand.CONFIG_REWRITE), + ("SET", RespCommand.CONFIG_SET), + ]; + + private static readonly (string Name, RespCommand Command)[] ScriptSubcommands = + [ + ("LOAD", RespCommand.SCRIPT_LOAD), + ("FLUSH", RespCommand.SCRIPT_FLUSH), + ("EXISTS", RespCommand.SCRIPT_EXISTS), + ]; + + private static readonly (string Name, RespCommand Command)[] LatencySubcommands = + [ + ("HELP", RespCommand.LATENCY_HELP), + ("HISTOGRAM", RespCommand.LATENCY_HISTOGRAM), + ("RESET", RespCommand.LATENCY_RESET), + ]; + + private static readonly (string Name, RespCommand Command)[] SlowlogSubcommands = + [ + ("HELP", RespCommand.SLOWLOG_HELP), + ("GET", RespCommand.SLOWLOG_GET), + ("LEN", RespCommand.SLOWLOG_LEN), + ("RESET", RespCommand.SLOWLOG_RESET), + ]; + + private static readonly (string Name, RespCommand Command)[] ModuleSubcommands = + [ + ("LOADCS", RespCommand.MODULE_LOADCS), + ]; + + private static readonly (string Name, RespCommand Command)[] PubsubSubcommands = + [ + ("CHANNELS", RespCommand.PUBSUB_CHANNELS), + ("NUMSUB", RespCommand.PUBSUB_NUMSUB), + ("NUMPAT", RespCommand.PUBSUB_NUMPAT), + ]; + + private static readonly (string Name, RespCommand Command)[] MemorySubcommands = + [ + ("USAGE", RespCommand.MEMORY_USAGE), + ]; + + private static readonly (string Name, RespCommand Command)[] BitopSubcommands = + [ + ("AND", RespCommand.BITOP_AND), + ("OR", RespCommand.BITOP_OR), + ("XOR", RespCommand.BITOP_XOR), + ("NOT", RespCommand.BITOP_NOT), + ("DIFF", RespCommand.BITOP_DIFF), + ]; + + #endregion + } +} \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommandSimdPatterns.cs b/libs/server/Resp/Parser/RespCommandSimdPatterns.cs new file mode 100644 index 00000000000..71203fec8e4 --- /dev/null +++ b/libs/server/Resp/Parser/RespCommandSimdPatterns.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.Intrinsics; + +namespace Garnet.server +{ + /// + /// SIMD Vector128 patterns and masks for . + /// + internal sealed unsafe partial class RespServerSession + { + // SIMD Vector128 patterns for FastParseCommand. + // Each encodes the full RESP header + command: *N\r\n$L\r\nCMD\r\n + // Masks zero out trailing bytes for patterns shorter than 16 bytes. + private static readonly Vector128 s_mask13 = Vector128.Create( + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00).AsByte(); + private static readonly Vector128 s_mask14 = Vector128.Create( + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00).AsByte(); + private static readonly Vector128 s_mask15 = Vector128.Create( + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00).AsByte(); + + /// + /// Builds a Vector128 RESP pattern for SIMD matching: *{argCount}\r\n${cmdLen}\r\n{cmd}\r\n + /// Zero-padded to 16 bytes. Only called during static init. + /// + private static Vector128 RespPattern(int argCount, string cmd) + { + // Total encoded length: 4 (*N\r\n) + 4 ($L\r\n) + cmd.Length + 2 (\r\n) = cmd.Length + 10 + var totalLen = cmd.Length + 10; + if (totalLen > 16) + throw new ArgumentException($"SIMD pattern overflow: command '{cmd}' with {argCount} args requires {totalLen} bytes, max 16"); + if (argCount < 1 || argCount > 9) + throw new ArgumentException($"SIMD pattern requires single-digit arg count (1-9), got {argCount} for '{cmd}'"); + if (cmd.Length < 1 || cmd.Length > 9) + throw new ArgumentException($"SIMD pattern requires single-digit command length (1-9), got {cmd.Length} for '{cmd}'"); + + Span buf = stackalloc byte[16]; + buf.Clear(); + buf[0] = (byte)'*'; + buf[1] = (byte)('0' + argCount); + buf[2] = (byte)'\r'; + buf[3] = (byte)'\n'; + buf[4] = (byte)'$'; + buf[5] = (byte)('0' + cmd.Length); + buf[6] = (byte)'\r'; + buf[7] = (byte)'\n'; + for (int i = 0; i < cmd.Length; i++) + buf[8 + i] = (byte)cmd[i]; + buf[8 + cmd.Length] = (byte)'\r'; + buf[9 + cmd.Length] = (byte)'\n'; + return Vector128.Create(buf); + } + + // 13-byte: *N\r\n$3\r\nXXX\r\n (3-char commands) + private static readonly Vector128 s_GET = RespPattern(2, "GET"); + private static readonly Vector128 s_SET = RespPattern(3, "SET"); + private static readonly Vector128 s_DEL = RespPattern(2, "DEL"); + private static readonly Vector128 s_TTL = RespPattern(2, "TTL"); + + // 14-byte: *N\r\n$4\r\nXXXX\r\n (4-char commands) + private static readonly Vector128 s_PING = RespPattern(1, "PING"); + private static readonly Vector128 s_INCR = RespPattern(2, "INCR"); + private static readonly Vector128 s_DECR = RespPattern(2, "DECR"); + private static readonly Vector128 s_EXEC = RespPattern(1, "EXEC"); + private static readonly Vector128 s_PTTL = RespPattern(2, "PTTL"); + + // 15-byte: *N\r\n$5\r\nXXXXX\r\n (5-char commands) + private static readonly Vector128 s_MULTI = RespPattern(1, "MULTI"); + private static readonly Vector128 s_SETNX = RespPattern(3, "SETNX"); + private static readonly Vector128 s_SETEX = RespPattern(4, "SETEX"); + + // 16-byte: *N\r\n$6\r\nXXXXXX\r\n (6-char commands, no mask needed) + private static readonly Vector128 s_EXISTS = RespPattern(2, "EXISTS"); + private static readonly Vector128 s_GETDEL = RespPattern(2, "GETDEL"); + private static readonly Vector128 s_APPEND = RespPattern(3, "APPEND"); + private static readonly Vector128 s_INCRBY = RespPattern(3, "INCRBY"); + private static readonly Vector128 s_DECRBY = RespPattern(3, "DECRBY"); + private static readonly Vector128 s_PSETEX = RespPattern(4, "PSETEX"); + } +} \ No newline at end of file diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs index 47f506404c5..cb4683290b2 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/Garnet.test/RespTests.cs @@ -2672,21 +2672,22 @@ public void KeyExpireOptionsTest([Values("EXPIRE", "PEXPIRE")] string command, [ resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsFalse(resp);// NX return false existing expiry - args[1] = 50; + var ttlSmall = command.Equals("EXPIRE") ? 50 : 50000; // 50 seconds or 50000 ms + args[1] = ttlSmall; args[2] = testCaseSensitivity ? "xx" : "XX";// XX -- Set expiry only when the key has an existing expiry resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsTrue(resp);// XX return true existing expiry var time = db.KeyTimeToLive(key); ClassicAssert.Greater(time.Value.TotalSeconds, 0); - ClassicAssert.LessOrEqual(time.Value.TotalSeconds, (int)args[1]); + ClassicAssert.LessOrEqual(time.Value.TotalSeconds, ttlSmall); args[1] = 1; args[2] = testCaseSensitivity ? "Gt" : "GT";// GT -- Set expiry only when the new expiry is greater than current one resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsFalse(resp); // GT return false new expiry < current expiry - args[1] = 1000; + args[1] = command.Equals("EXPIRE") ? 1000 : 1000000; // 1000 seconds or 1000000 ms args[2] = testCaseSensitivity ? "gT" : "GT";// GT -- Set expiry only when the new expiry is greater than current one resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsTrue(resp); // GT return true new expiry > current expiry @@ -2695,12 +2696,12 @@ public void KeyExpireOptionsTest([Values("EXPIRE", "PEXPIRE")] string command, [ ClassicAssert.Greater(command.Equals("EXPIRE") ? time.Value.TotalSeconds : time.Value.TotalMilliseconds, 500); - args[1] = 2000; + args[1] = command.Equals("EXPIRE") ? 2000 : 2000000; // must be > GT value above args[2] = testCaseSensitivity ? "lt" : "LT";// LT -- Set expiry only when the new expiry is less than current one resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsFalse(resp); // LT return false new expiry > current expiry - args[1] = 500; + args[1] = command.Equals("EXPIRE") ? 500 : 500000; // must be < GT value above args[2] = testCaseSensitivity ? "lT" : "LT";// LT -- Set expiry only when the new expiry is less than current one resp = (bool)db.Execute($"{command}", args); ClassicAssert.IsTrue(resp); // LT return true new expiry < current expiry