From 27a72734e552c21ada7e6c84d04b8ce639f60c25 Mon Sep 17 00:00:00 2001 From: John Uhlmann Date: Wed, 1 Apr 2026 15:16:34 +0800 Subject: [PATCH 1/2] replace name->property map with hinted linear scan --- O365.Security.Native.ETW.Debug.nuspec | 7 ++-- O365.Security.Native.ETW.nuspec | 7 ++-- krabs/krabs/parser.hpp | 49 ++++++++++++++------------- krabs/krabs/schema.hpp | 4 --- krabs/krabs/schema_locator.hpp | 49 ++------------------------- krabsetw.nuspec | 7 ++-- 6 files changed, 37 insertions(+), 86 deletions(-) diff --git a/O365.Security.Native.ETW.Debug.nuspec b/O365.Security.Native.ETW.Debug.nuspec index 4961ca74..2c7d7826 100644 --- a/O365.Security.Native.ETW.Debug.nuspec +++ b/O365.Security.Native.ETW.Debug.nuspec @@ -2,7 +2,7 @@ Microsoft.O365.Security.Native.ETW.Debug - 4.4.8 + 4.4.9 Microsoft.O365.Security.Native.ETW Debug - managed wrappers for krabsetw Microsoft Microsoft @@ -12,9 +12,8 @@ Microsoft.O365.Security.Native.ETW Debug is a managed wrapper around the krabsetw ETW library. This is the Debug build. Microsoft.O365.Security.Native.ETW Debug is a managed wrapper around the krabsetw ETW library. This is the Debug build. - Version 4.4.8: - - Optimize parser property lookup with persistent name-to-index map - - Use std::wstring_view for parser property name parameters + Version 4.4.9: + - Optimize parser property lookup with hinted linear scan © Microsoft Corporation. All rights reserved. diff --git a/O365.Security.Native.ETW.nuspec b/O365.Security.Native.ETW.nuspec index 7d0945ad..6cff0182 100644 --- a/O365.Security.Native.ETW.nuspec +++ b/O365.Security.Native.ETW.nuspec @@ -2,7 +2,7 @@ Microsoft.O365.Security.Native.ETW - 4.4.8 + 4.4.9 Microsoft.O365.Security.Native.ETW - managed wrappers for krabsetw Microsoft Microsoft @@ -12,9 +12,8 @@ Microsoft.O365.Security.Native.ETW is a managed wrapper around the krabsetw ETW library. Microsoft.O365.Security.Native.ETW is a managed wrapper around the krabsetw ETW library. - Version 4.4.8: - - Optimize parser property lookup with persistent name-to-index map - - Use std::wstring_view for parser property name parameters + Version 4.4.9: + - Optimize parser property lookup with hinted linear scan © Microsoft Corporation. All rights reserved. diff --git a/krabs/krabs/parser.hpp b/krabs/krabs/parser.hpp index 2736857f..d322d7e4 100644 --- a/krabs/krabs/parser.hpp +++ b/krabs/krabs/parser.hpp @@ -99,8 +99,6 @@ namespace krabs { const BYTE *pEndBuffer_; BYTE *pBufferIndex_; ULONG lastPropertyIndex_; - // Persistent name to index map shared across all events of the same type. - const property_name_map *pPropertyNames_; // Maintain a mapping from property index to blob data location. std::vector propertyCache_; }; @@ -113,7 +111,6 @@ namespace krabs { , pEndBuffer_((BYTE*)s.record_.UserData + s.record_.UserDataLength) , pBufferIndex_((BYTE*)s.record_.UserData) , lastPropertyIndex_(0) - , pPropertyNames_(s.pPropertyNames_) , propertyCache_(s.pSchema_->PropertyCount) {} @@ -127,25 +124,33 @@ namespace krabs { // A schema contains a collection of properties that are keyed by name. // These properties are stored in a blob of bytes that needs to be // interpreted according to information that is packaged up in the - // schema and that can be retrieved using the Tdh* APIs. This format - // requires a linear traversal over the blob, incrementing according to - // the contents within it. This is janky, so our strategy is to - // minimize this as much as possible via caching. + // schema and that can be retrieved using the Tdh* APIs. const ULONG totalPropCount = schema_.pSchema_->PropertyCount; - // Resolve property name to index. + // Resolve property name to index via hinted linear scan. + // Optimisitically start at lastPropertyIndex_, then wrap around if not found. + // ** This assumes that callers typically access properties in event order. ** + // An alternative is to maintain a static name->index map, but this was slower + // in practice for events with < ~12 properties. + ULONG index = totalPropCount; // sentinel = not found - if (pPropertyNames_) { - // Fast path: use the persistent name to index map shared across - // all events of the same type. - auto it = pPropertyNames_->find(name); - if (it != pPropertyNames_->end()) { - index = it->second; + + // Scan [lastPropertyIndex_..totalPropCount) first + for (ULONG i = lastPropertyIndex_; i < totalPropCount; ++i) { + auto &propInfo = schema_.pSchema_->EventPropertyInfoArray[i]; + const wchar_t *pName = reinterpret_cast( + reinterpret_cast(schema_.pSchema_) + + propInfo.NameOffset); + if (name == pName) { + index = i; + break; } - } else { - // Fallback: linear scan of property names in the schema. - for (ULONG i = 0; i < totalPropCount; ++i) { + } + + if (index == totalPropCount) { // not found + // Scan the remainder [0..lastPropertyIndex_) + for (ULONG i = 0; i < lastPropertyIndex_; ++i) { auto &propInfo = schema_.pSchema_->EventPropertyInfoArray[i]; const wchar_t *pName = reinterpret_cast( reinterpret_cast(schema_.pSchema_) + @@ -157,7 +162,7 @@ namespace krabs { } } - if (index >= totalPropCount) { + if (index >= totalPropCount) { // not found return property_info(); } @@ -180,11 +185,9 @@ namespace krabs { // We've not looked up this property before, so we have to do the work // to find it. While we're going through the blob to find it, we'll // remember what we've seen to save time later. - // - // Note: The name-to-index map is built once per schema type (cheap - // metadata scan). But the blob walk below is lazy per-event -- we - // only walk forward to the requested index, avoiding overhead when - // only a subset of properties are needed. + // The blob walk is lazy per-event -- we only walk forward to the + // requested index, avoiding overhead when only a subset of properties + // are needed. while (lastPropertyIndex_ <= index) { auto ¤tPropInfo = schema_.pSchema_->EventPropertyInfoArray[lastPropertyIndex_]; diff --git a/krabs/krabs/schema.hpp b/krabs/krabs/schema.hpp index 5212e8c9..9016f7b5 100644 --- a/krabs/krabs/schema.hpp +++ b/krabs/krabs/schema.hpp @@ -308,8 +308,6 @@ namespace krabs { private: const EVENT_RECORD &record_; const TRACE_EVENT_INFO *pSchema_; - // Persistent name to index map, owned by schema_locator. May be nullptr. - const property_name_map *pPropertyNames_; private: friend std::wstring event_name(const schema &); @@ -339,13 +337,11 @@ namespace krabs { inline schema::schema(const EVENT_RECORD &record, const krabs::schema_locator &schema_locator) : record_(record) , pSchema_(schema_locator.get_event_schema(record)) - , pPropertyNames_(schema_locator.get_property_names(pSchema_)) { } inline schema::schema(const EVENT_RECORD &record, const PTRACE_EVENT_INFO pSchema) : record_(record) , pSchema_(pSchema) - , pPropertyNames_(nullptr) { } inline bool schema::operator==(const schema &other) const diff --git a/krabs/krabs/schema_locator.hpp b/krabs/krabs/schema_locator.hpp index db00d569..4dffb7d2 100644 --- a/krabs/krabs/schema_locator.hpp +++ b/krabs/krabs/schema_locator.hpp @@ -173,14 +173,6 @@ namespace krabs { */ std::string_view get_trace_logger_event_name(const EVENT_RECORD &); - /** - * - * Maps property names to their index in the schema. - * Keys are wstring_views pointing into stable TRACE_EVENT_INFO memory. - * - */ - using property_name_map = std::unordered_map; - /** * * Fetches and caches schemas from TDH. @@ -215,21 +207,8 @@ namespace krabs { */ bool has_event_schema(const EVENT_RECORD& record) const; - /** - * - * Returns the persistent property name to index map for a schema. - * The map is built when the schema is first cached. - * Returns nullptr if pSchema is null or not in the cache. - * - */ - const property_name_map* get_property_names(const TRACE_EVENT_INFO* pSchema) const; - private: - void build_property_names(const TRACE_EVENT_INFO* pSchema) const; - mutable std::unordered_map, TDHSTATUS>> cache_; - // Persistent property name to index maps, keyed by schema pointer. - mutable std::unordered_map property_name_cache_; }; // Implementation @@ -332,10 +311,9 @@ namespace krabs { // Add the new instance to the cache. // NB: key's 'internalize_name' gets called by the cctor here. - if (status == ERROR_SUCCESS) { + if (status == ERROR_SUCCESS) cache_.emplace(key, std::move(buffer)); - build_property_names(returnVal); - } else + else cache_.emplace(key, status); return returnVal; @@ -348,29 +326,6 @@ namespace krabs { return status == ERROR_SUCCESS; } - inline void schema_locator::build_property_names(const TRACE_EVENT_INFO* pSchema) const - { - property_name_map names; - for (ULONG i = 0; i < pSchema->PropertyCount; ++i) { - const wchar_t* pName = reinterpret_cast( - reinterpret_cast(pSchema) + - pSchema->EventPropertyInfoArray[i].NameOffset); - names.emplace(std::wstring_view(pName), i); - } - property_name_cache_.emplace(pSchema, std::move(names)); - } - - inline const property_name_map* schema_locator::get_property_names(const TRACE_EVENT_INFO* pSchema) const - { - if (!pSchema) return nullptr; - - auto it = property_name_cache_.find(pSchema); - if (it != property_name_cache_.end()) { - return &it->second; - } - return nullptr; - } - inline std::unique_ptr get_event_schema_from_tdh(const EVENT_RECORD &record) { TDHSTATUS status = ERROR_SUCCESS; diff --git a/krabsetw.nuspec b/krabsetw.nuspec index f1ce3b58..7ba3adf4 100644 --- a/krabsetw.nuspec +++ b/krabsetw.nuspec @@ -2,7 +2,7 @@ Microsoft.O365.Security.Krabsetw - 4.4.8 + 4.4.9 Krabs ETW Wrappers Microsoft Microsoft @@ -12,9 +12,8 @@ Krabs ETW provides a modern C++ wrapper around the low-level ETW trace consumption functions Krabs ETW provides a modern C++ wrapper around the low-level ETW trace consumption functions - Version 4.4.8: - - Optimize parser property lookup with persistent name-to-index map - - Use std::wstring_view for parser property name parameters + Version 4.4.9: + - Optimize parser property lookup with hinted linear scan © Microsoft Corporation. All rights reserved. From 99869be3e8995ca1762bddfe52b37627de3895a7 Mon Sep 17 00:00:00 2001 From: John Uhlmann Date: Fri, 10 Apr 2026 10:21:26 +0800 Subject: [PATCH 2/2] Add parse hints (native-layer only) --- O365.Security.Native.ETW.Debug.nuspec | 3 +- O365.Security.Native.ETW.nuspec | 3 +- krabs/krabs/parser.hpp | 81 ++++++++++++++++++++------- krabsetw.nuspec | 3 +- 4 files changed, 68 insertions(+), 22 deletions(-) diff --git a/O365.Security.Native.ETW.Debug.nuspec b/O365.Security.Native.ETW.Debug.nuspec index 2c7d7826..d05938d7 100644 --- a/O365.Security.Native.ETW.Debug.nuspec +++ b/O365.Security.Native.ETW.Debug.nuspec @@ -13,7 +13,8 @@ Microsoft.O365.Security.Native.ETW Debug is a managed wrapper around the krabsetw ETW library. This is the Debug build. Version 4.4.9: - - Optimize parser property lookup with hinted linear scan + - Optimize parser property lookup with last property hinted linear scan + - Add hinted parse/try_parse/view_of overloads accepting a ULONG property index hint © Microsoft Corporation. All rights reserved. diff --git a/O365.Security.Native.ETW.nuspec b/O365.Security.Native.ETW.nuspec index 6cff0182..ca20d974 100644 --- a/O365.Security.Native.ETW.nuspec +++ b/O365.Security.Native.ETW.nuspec @@ -13,7 +13,8 @@ Microsoft.O365.Security.Native.ETW is a managed wrapper around the krabsetw ETW library. Version 4.4.9: - - Optimize parser property lookup with hinted linear scan + - Optimize parser property lookup with last property hinted linear scan + - Add hinted parse/try_parse/view_of overloads accepting a ULONG property index hint © Microsoft Corporation. All rights reserved. diff --git a/krabs/krabs/parser.hpp b/krabs/krabs/parser.hpp index d322d7e4..67b6dc68 100644 --- a/krabs/krabs/parser.hpp +++ b/krabs/krabs/parser.hpp @@ -78,6 +78,15 @@ namespace krabs { template bool try_parse(std::wstring_view name, T &out); + /** + * + * Attempts to retrieve the given property by name and type, + * starting the name scan at the given hint index. + * + */ + template + bool try_parse(std::wstring_view name, T &out, ULONG hint); + /** * * Attempts to parse the given property by name and type. If the @@ -87,11 +96,24 @@ namespace krabs { template T parse(std::wstring_view name); + /** + * + * Attempts to parse the given property by name and type, + * starting the name scan at the given hint index. + * + */ + template + T parse(std::wstring_view name, ULONG hint); + template auto view_of(std::wstring_view name, Adapter &adapter) -> collection_view; + template + auto view_of(std::wstring_view name, ULONG hint, Adapter &adapter) -> collection_view; + private: property_info find_property(std::wstring_view name); + property_info find_property(std::wstring_view name, ULONG hint); void cache_property(ULONG index, property_info info); private: @@ -99,6 +121,7 @@ namespace krabs { const BYTE *pEndBuffer_; BYTE *pBufferIndex_; ULONG lastPropertyIndex_; + ULONG nextHint_; // Maintain a mapping from property index to blob data location. std::vector propertyCache_; }; @@ -111,6 +134,7 @@ namespace krabs { , pEndBuffer_((BYTE*)s.record_.UserData + s.record_.UserDataLength) , pBufferIndex_((BYTE*)s.record_.UserData) , lastPropertyIndex_(0) + , nextHint_(0) , propertyCache_(s.pSchema_->PropertyCount) {} @@ -120,6 +144,11 @@ namespace krabs { } inline property_info parser::find_property(std::wstring_view name) + { + return find_property(name, nextHint_); + } + + inline property_info parser::find_property(std::wstring_view name, ULONG hint) { // A schema contains a collection of properties that are keyed by name. // These properties are stored in a blob of bytes that needs to be @@ -127,18 +156,22 @@ namespace krabs { // schema and that can be retrieved using the Tdh* APIs. const ULONG totalPropCount = schema_.pSchema_->PropertyCount; + if (totalPropCount == 0) { + return property_info(); + } // Resolve property name to index via hinted linear scan. - // Optimisitically start at lastPropertyIndex_, then wrap around if not found. + // Optimistically start at hint, then wrap around if not found. // ** This assumes that callers typically access properties in event order. ** // An alternative is to maintain a static name->index map, but this was slower // in practice for events with < ~12 properties. - ULONG index = totalPropCount; // sentinel = not found + if (hint >= totalPropCount) hint = 0; - // Scan [lastPropertyIndex_..totalPropCount) first - for (ULONG i = lastPropertyIndex_; i < totalPropCount; ++i) { - auto &propInfo = schema_.pSchema_->EventPropertyInfoArray[i]; + ULONG index = totalPropCount; // sentinel = not found + for (ULONG n = 0; n < totalPropCount; ++n) { + const ULONG i = (hint + n) % totalPropCount; + const auto &propInfo = schema_.pSchema_->EventPropertyInfoArray[i]; const wchar_t *pName = reinterpret_cast( reinterpret_cast(schema_.pSchema_) + propInfo.NameOffset); @@ -148,24 +181,12 @@ namespace krabs { } } - if (index == totalPropCount) { // not found - // Scan the remainder [0..lastPropertyIndex_) - for (ULONG i = 0; i < lastPropertyIndex_; ++i) { - auto &propInfo = schema_.pSchema_->EventPropertyInfoArray[i]; - const wchar_t *pName = reinterpret_cast( - reinterpret_cast(schema_.pSchema_) + - propInfo.NameOffset); - if (name == pName) { - index = i; - break; - } - } - } - if (index >= totalPropCount) { // not found return property_info(); } + nextHint_ = (index + 1) % totalPropCount; + // The first step is to use our cache for the property to see if we've // discovered it already. if (index < lastPropertyIndex_) { @@ -247,6 +268,13 @@ namespace krabs { // try_parse // ------------------------------------------------------------------------ + template + bool parser::try_parse(std::wstring_view name, T &out, ULONG hint) + { + nextHint_ = hint; + return try_parse(name, out); + } + template bool parser::try_parse(std::wstring_view name, T &out) { @@ -272,6 +300,13 @@ namespace krabs { // parse // ------------------------------------------------------------------------ + template + T parser::parse(std::wstring_view name, ULONG hint) + { + nextHint_ = hint; + return parse(name); + } + template T parser::parse(std::wstring_view name) { @@ -438,6 +473,14 @@ namespace krabs { // view_of // ------------------------------------------------------------------------ + template + auto parser::view_of(std::wstring_view name, ULONG hint, Adapter &adapter) + -> collection_view + { + nextHint_ = hint; + return view_of(name, adapter); + } + template auto parser::view_of(std::wstring_view name, Adapter &adapter) -> collection_view diff --git a/krabsetw.nuspec b/krabsetw.nuspec index 7ba3adf4..9202b86c 100644 --- a/krabsetw.nuspec +++ b/krabsetw.nuspec @@ -13,7 +13,8 @@ Krabs ETW provides a modern C++ wrapper around the low-level ETW trace consumption functions Version 4.4.9: - - Optimize parser property lookup with hinted linear scan + - Optimize parser property lookup with last property hinted linear scan + - Add hinted parse/try_parse/view_of overloads accepting a ULONG property index hint © Microsoft Corporation. All rights reserved.