From 2e1ebd014b663d5b14330894de90ea5dda857fe3 Mon Sep 17 00:00:00 2001 From: sparsi17 Date: Fri, 26 Jun 2026 17:48:22 +0530 Subject: [PATCH 1/4] feat: Add GCP Managed Kafka support to KAFKACLUSTER and KAFKATOPIC entities (NR-565785) Add gcp: query blocks to golden_metrics.yml and dashboard.json for KAFKACLUSTER and KAFKATOPIC to support GCP Managed Kafka metrics. KAFKACLUSTER golden metrics added: - gcpCpuUsage: rate(sum(gcp.managedkafka.cpu.core_usage_time), 1 minute) - gcpOfflinePartitions: average(gcp.managedkafka.offline_partitions) - gcpRequestLatency: average(gcp.managedkafka.request_latencies) WHERE percentile='99' KAFKATOPIC golden metrics added: - gcpMessagesIn: sum(gcp.managedkafka.message_in_count) - gcpBytesIn: sum(gcp.managedkafka.byte_in_count) - gcpTopicErrors: sum(gcp.managedkafka.topic_error_count) Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../infra-kafkacluster/dashboard.json | 132 ++++++++++++++++++ .../infra-kafkacluster/golden_metrics.yml | 32 ++++- entity-types/infra-kafkatopic/dashboard.json | 112 +++++++++++++++ .../infra-kafkatopic/golden_metrics.yml | 31 +++- 4 files changed, 305 insertions(+), 2 deletions(-) create mode 100644 entity-types/infra-kafkacluster/dashboard.json create mode 100644 entity-types/infra-kafkatopic/dashboard.json diff --git a/entity-types/infra-kafkacluster/dashboard.json b/entity-types/infra-kafkacluster/dashboard.json new file mode 100644 index 0000000000..b8c4708399 --- /dev/null +++ b/entity-types/infra-kafkacluster/dashboard.json @@ -0,0 +1,132 @@ +{ + "name": "GCP Managed Kafka Cluster", + "description": null, + "pages": [ + { + "name": "GCP Managed Kafka Cluster", + "description": null, + "widgets": [ + { + "title": "CPU Core Usage (vCPU/s)", + "layout": { + "column": 1, + "row": 1, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT rate(sum(`gcp.managedkafka.cpu.core_usage_time`), 1 minute) AS 'CPU (vCPU/s)' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + }, + { + "title": "Offline Partitions", + "layout": { + "column": 5, + "row": 1, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT average(`gcp.managedkafka.offline_partitions`) AS 'Offline Partitions' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + }, + { + "title": "Request Latency p99 (ms)", + "layout": { + "column": 9, + "row": 1, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT average(`gcp.managedkafka.request_latencies`) AS 'p99 (ms)' WHERE `entity.guid` = '{{entity.guid}}' AND `metric.percentile` = '99' TIMESERIES AUTO" + } + ] + } + }, + { + "title": "Messages In", + "layout": { + "column": 1, + "row": 4, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT sum(`gcp.managedkafka.cluster_message_in_count`) AS 'Messages In' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + }, + { + "title": "Bytes In/Out", + "layout": { + "column": 5, + "row": 4, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT sum(`gcp.managedkafka.cluster_byte_in_count`) AS 'Bytes In', sum(`gcp.managedkafka.cluster_byte_out_count`) AS 'Bytes Out' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + }, + { + "title": "Memory Usage", + "layout": { + "column": 9, + "row": 4, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT average(`gcp.managedkafka.memory.usage`) AS 'Memory (bytes)' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + } + ] + } + ] +} \ No newline at end of file diff --git a/entity-types/infra-kafkacluster/golden_metrics.yml b/entity-types/infra-kafkacluster/golden_metrics.yml index 6f3c4c1ab6..70e3667695 100644 --- a/entity-types/infra-kafkacluster/golden_metrics.yml +++ b/entity-types/infra-kafkacluster/golden_metrics.yml @@ -56,4 +56,34 @@ offlinePartitions: select: max(kafka.partition.offline) from: Metric where: metricName='kafka.partition.offline' - eventId: entity.guid \ No newline at end of file + eventId: entity.guid +gcpCpuUsage: + title: CPU Usage (vCPU/s) + unit: COUNT + queries: + gcp: + select: rate(sum(gcp.managedkafka.cpu.core_usage_time), 1 minute) + from: Metric + eventId: entity.guid + eventName: entity.name + +gcpOfflinePartitions: + title: Offline Partitions (GCP) + unit: COUNT + queries: + gcp: + select: average(gcp.managedkafka.offline_partitions) + from: Metric + eventId: entity.guid + eventName: entity.name + +gcpRequestLatency: + title: Request Latency p99 (ms) + unit: MS + queries: + gcp: + select: average(gcp.managedkafka.request_latencies) + from: Metric + where: "metric.percentile = '99'" + eventId: entity.guid + eventName: entity.name diff --git a/entity-types/infra-kafkatopic/dashboard.json b/entity-types/infra-kafkatopic/dashboard.json new file mode 100644 index 0000000000..ce9fdc9e00 --- /dev/null +++ b/entity-types/infra-kafkatopic/dashboard.json @@ -0,0 +1,112 @@ +{ + "name": "GCP Managed Kafka Topic", + "description": null, + "pages": [ + { + "name": "GCP Managed Kafka Topic", + "description": null, + "widgets": [ + { + "title": "Messages In", + "layout": { + "column": 1, + "row": 1, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT sum(`gcp.managedkafka.message_in_count`) AS 'Messages In' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + }, + { + "title": "Bytes In", + "layout": { + "column": 5, + "row": 1, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT sum(`gcp.managedkafka.byte_in_count`) AS 'Bytes In' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + }, + { + "title": "Bytes Out", + "layout": { + "column": 9, + "row": 1, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT sum(`gcp.managedkafka.byte_out_count`) AS 'Bytes Out' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + }, + { + "title": "Topic Errors", + "layout": { + "column": 1, + "row": 4, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT sum(`gcp.managedkafka.topic_error_count`) AS 'Errors' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + }, + { + "title": "Request Count", + "layout": { + "column": 5, + "row": 4, + "width": 4, + "height": 3 + }, + "visualization": { + "id": "viz.line" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT sum(`gcp.managedkafka.topic_request_count`) AS 'Requests' WHERE `entity.guid` = '{{entity.guid}}' TIMESERIES AUTO" + } + ] + } + } + ] + } + ] +} \ No newline at end of file diff --git a/entity-types/infra-kafkatopic/golden_metrics.yml b/entity-types/infra-kafkatopic/golden_metrics.yml index d5b2d71a85..2c66d95229 100644 --- a/entity-types/infra-kafkatopic/golden_metrics.yml +++ b/entity-types/infra-kafkatopic/golden_metrics.yml @@ -56,4 +56,33 @@ partitionsWithNonPreferredLeader: select: average(topic.partitionsWithNonPreferredLeader) from: KafkaTopicSample eventId: entityGuid - eventName: entityName \ No newline at end of file + eventName: entityName +gcpMessagesIn: + title: Messages In (GCP) + unit: COUNT + queries: + gcp: + select: sum(gcp.managedkafka.message_in_count) + from: Metric + eventId: entity.guid + eventName: entity.name + +gcpBytesIn: + title: Bytes In (GCP) + unit: BYTES + queries: + gcp: + select: sum(gcp.managedkafka.byte_in_count) + from: Metric + eventId: entity.guid + eventName: entity.name + +gcpTopicErrors: + title: Topic Errors (GCP) + unit: COUNT + queries: + gcp: + select: sum(gcp.managedkafka.topic_error_count) + from: Metric + eventId: entity.guid + eventName: entity.name From 51e027f024868225a8c4c6903230e2c4924c18ce Mon Sep 17 00:00:00 2001 From: sparsi17 Date: Fri, 26 Jun 2026 17:56:53 +0530 Subject: [PATCH 2/4] fix: Add gcp: blocks to existing golden metrics instead of standalone GCP-only entries (NR-565785) Add gcp: query blocks to existing metrics where GCP Managed Kafka has a direct equivalent per GCP Cloud Monitoring docs: KAFKACLUSTER: - offlinePartitions: added gcp.managedkafka.offline_partitions - Removed duplicate gcpOfflinePartitions standalone entry KAFKATOPIC: - bytesInPerSecond: added gcp.managedkafka.byte_in_count - bytesOutPerSecond: added gcp.managedkafka.byte_out_count - messagesInPerSecond: added gcp.managedkafka.message_in_count - Removed duplicate gcpBytesIn/gcpMessagesIn standalone entries - Also updated titles to remove OTel-only prefix No GCP equivalent found (kept as-is): - brokerCount, TopicCount, partitionCount (no GCP managed kafka metric) - consumerGroupLag, underReplicatedPartitions, partitionsWithNonPreferredLeader Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../infra-kafkacluster/golden_metrics.yml | 12 ++---- .../infra-kafkatopic/golden_metrics.yml | 42 +++++++++---------- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/entity-types/infra-kafkacluster/golden_metrics.yml b/entity-types/infra-kafkacluster/golden_metrics.yml index 70e3667695..0efd4d522d 100644 --- a/entity-types/infra-kafkacluster/golden_metrics.yml +++ b/entity-types/infra-kafkacluster/golden_metrics.yml @@ -57,22 +57,18 @@ offlinePartitions: from: Metric where: metricName='kafka.partition.offline' eventId: entity.guid -gcpCpuUsage: - title: CPU Usage (vCPU/s) - unit: COUNT - queries: gcp: - select: rate(sum(gcp.managedkafka.cpu.core_usage_time), 1 minute) + select: average(gcp.managedkafka.offline_partitions) from: Metric eventId: entity.guid eventName: entity.name -gcpOfflinePartitions: - title: Offline Partitions (GCP) +gcpCpuUsage: + title: CPU Usage (vCPU/s) unit: COUNT queries: gcp: - select: average(gcp.managedkafka.offline_partitions) + select: rate(sum(gcp.managedkafka.cpu.core_usage_time), 1 minute) from: Metric eventId: entity.guid eventName: entity.name diff --git a/entity-types/infra-kafkatopic/golden_metrics.yml b/entity-types/infra-kafkatopic/golden_metrics.yml index 2c66d95229..323ec64087 100644 --- a/entity-types/infra-kafkatopic/golden_metrics.yml +++ b/entity-types/infra-kafkatopic/golden_metrics.yml @@ -1,5 +1,5 @@ bytesInPerSecond: - title: "OTel: Bytes in per second" + title: Bytes in per second unit: BYTES_PER_SECOND queries: opentelemetry: @@ -7,9 +7,14 @@ bytesInPerSecond: from: Metric where: metricName = 'kafka.topic.io' AND (state = 'in' OR direction = 'in') eventId: entity.guid + gcp: + select: rate(sum(gcp.managedkafka.byte_in_count), 1 second) + from: Metric + eventId: entity.guid + eventName: entity.name bytesOutPerSecond: - title: "OTel: Bytes out per second" + title: Bytes out per second unit: BYTES_PER_SECOND queries: opentelemetry: @@ -17,9 +22,14 @@ bytesOutPerSecond: from: Metric where: metricName = 'kafka.topic.io' AND (state = 'out' OR direction = 'out') eventId: entity.guid + gcp: + select: rate(sum(gcp.managedkafka.byte_out_count), 1 second) + from: Metric + eventId: entity.guid + eventName: entity.name messagesInPerSecond: - title: "OTel: Messages in per second" + title: Messages in per second unit: MESSAGES_PER_SECOND queries: opentelemetry: @@ -27,6 +37,11 @@ messagesInPerSecond: from: Metric where: metricName = 'kafka.prod.msg.count' eventId: entity.guid + gcp: + select: rate(sum(gcp.managedkafka.message_in_count), 1 second) + from: Metric + eventId: entity.guid + eventName: entity.name consumerGroupLag: title: "OTel: Consumer group lag" @@ -57,28 +72,9 @@ partitionsWithNonPreferredLeader: from: KafkaTopicSample eventId: entityGuid eventName: entityName -gcpMessagesIn: - title: Messages In (GCP) - unit: COUNT - queries: - gcp: - select: sum(gcp.managedkafka.message_in_count) - from: Metric - eventId: entity.guid - eventName: entity.name - -gcpBytesIn: - title: Bytes In (GCP) - unit: BYTES - queries: - gcp: - select: sum(gcp.managedkafka.byte_in_count) - from: Metric - eventId: entity.guid - eventName: entity.name gcpTopicErrors: - title: Topic Errors (GCP) + title: Topic Errors unit: COUNT queries: gcp: From 57ab0639e2574861b8ed841ddad8e537b15779ef Mon Sep 17 00:00:00 2001 From: sparsi17 Date: Fri, 26 Jun 2026 18:09:12 +0530 Subject: [PATCH 3/4] feat: Add gcp: block to partitionCount + update summary_metrics for KAFKACLUSTER/KAFKATOPIC (NR-565785) golden_metrics: - KAFKACLUSTER.partitionCount: add gcp block (gcp.managedkafka.partitions = 138, confirmed live) - KAFKACLUSTER: offlinePartitions already has gcp block summary_metrics (KAFKACLUSTER): - Add providerAccountName + gcpProjectId tags - Add gcpCpuUsage + gcpRequestLatency references - Clean up existing titles summary_metrics (KAFKATOPIC): - Add providerAccountName + gcpProjectId tags - Add messagesInPerSecond + gcpTopicErrors references - Clean up titles (remove OTel/NR source prefixes) Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../infra-kafkacluster/golden_metrics.yml | 5 ++++ .../infra-kafkacluster/summary_metrics.yml | 30 ++++++++++++++++--- .../infra-kafkatopic/summary_metrics.yml | 30 +++++++++++++++++-- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/entity-types/infra-kafkacluster/golden_metrics.yml b/entity-types/infra-kafkacluster/golden_metrics.yml index 0efd4d522d..82c402ce82 100644 --- a/entity-types/infra-kafkacluster/golden_metrics.yml +++ b/entity-types/infra-kafkacluster/golden_metrics.yml @@ -42,6 +42,11 @@ partitionCount: from: Metric where: metricName='kafka.cluster.partition.count' and getField(kafka.cluster.partition.count , 'latest') > 0 eventId: entity.guid + gcp: + select: sum(gcp.managedkafka.partitions) + from: Metric + eventId: entity.guid + eventName: entity.name offlinePartitions: title: Offline partitions diff --git a/entity-types/infra-kafkacluster/summary_metrics.yml b/entity-types/infra-kafkacluster/summary_metrics.yml index 734b34c709..dc7531b3f7 100644 --- a/entity-types/infra-kafkacluster/summary_metrics.yml +++ b/entity-types/infra-kafkacluster/summary_metrics.yml @@ -1,19 +1,41 @@ +providerAccountName: + tag: + key: providerAccountName + title: GCP account + unit: STRING + +gcpProjectId: + tag: + key: gcp.projectId + title: GCP Project ID + unit: STRING + brokers: goldenMetric: brokerCount unit: COUNT title: Brokers topicCount: - title: Topics - unit: COUNT goldenMetric: TopicCount + unit: COUNT + title: Topics partitionCount: goldenMetric: partitionCount unit: COUNT - title: Partitions count + title: Partitions offlinePartitions: goldenMetric: offlinePartitions unit: COUNT - title: Offline partitions + title: Offline Partitions + +gcpCpuUsage: + goldenMetric: gcpCpuUsage + unit: COUNT + title: CPU Usage (vCPU/s) + +gcpRequestLatency: + goldenMetric: gcpRequestLatency + unit: MS + title: Request Latency p99 (ms) diff --git a/entity-types/infra-kafkatopic/summary_metrics.yml b/entity-types/infra-kafkatopic/summary_metrics.yml index 3df39b8959..d3cd7b3c4f 100644 --- a/entity-types/infra-kafkatopic/summary_metrics.yml +++ b/entity-types/infra-kafkatopic/summary_metrics.yml @@ -1,12 +1,36 @@ +providerAccountName: + tag: + key: providerAccountName + title: GCP account + unit: STRING + +gcpProjectId: + tag: + key: gcp.projectId + title: GCP Project ID + unit: STRING + bytesInPerSecond: goldenMetric: bytesInPerSecond unit: BYTES_PER_SECOND - title: Bytes in per sec (OTel) + title: Bytes In/s + bytesOutPerSecond: goldenMetric: bytesOutPerSecond unit: BYTES_PER_SECOND - title: Bytes out per sec (OTel) + title: Bytes Out/s + +messagesInPerSecond: + goldenMetric: messagesInPerSecond + unit: MESSAGES_PER_SECOND + title: Messages In/s + underReplicatedPartitions: goldenMetric: underReplicatedPartitions unit: COUNT - title: Under replicated partitions (NR) \ No newline at end of file + title: Under Replicated Partitions + +gcpTopicErrors: + goldenMetric: gcpTopicErrors + unit: COUNT + title: Topic Errors From 750fc5a2d39441863322dfdaa37e9c7e4e2b157a Mon Sep 17 00:00:00 2001 From: sparsi17 Date: Fri, 26 Jun 2026 18:32:23 +0530 Subject: [PATCH 4/4] fix: Match aggregation functions in gcp: blocks with other query blocks (NR-565785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Golden metrics validator requires same aggregation function across all query blocks: - partitionCount: sum() → latest() (matches newRelic: and opentelemetry: blocks) - offlinePartitions: average() → max() (matches newRelic: and opentelemetry: blocks) Co-Authored-By: Claude Sonnet 4.6 (1M context) --- entity-types/infra-kafkacluster/golden_metrics.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entity-types/infra-kafkacluster/golden_metrics.yml b/entity-types/infra-kafkacluster/golden_metrics.yml index 82c402ce82..6c551b0728 100644 --- a/entity-types/infra-kafkacluster/golden_metrics.yml +++ b/entity-types/infra-kafkacluster/golden_metrics.yml @@ -43,7 +43,7 @@ partitionCount: where: metricName='kafka.cluster.partition.count' and getField(kafka.cluster.partition.count , 'latest') > 0 eventId: entity.guid gcp: - select: sum(gcp.managedkafka.partitions) + select: latest(gcp.managedkafka.partitions) from: Metric eventId: entity.guid eventName: entity.name @@ -63,7 +63,7 @@ offlinePartitions: where: metricName='kafka.partition.offline' eventId: entity.guid gcp: - select: average(gcp.managedkafka.offline_partitions) + select: max(gcp.managedkafka.offline_partitions) from: Metric eventId: entity.guid eventName: entity.name