Skip to content

Redact OTLP header and Datadog key configs in telemetry #11583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
bm1549 wants to merge 1 commit into
base: master
Choose a base branch
from
+98 −35
Draft

Redact OTLP header and Datadog key configs in telemetry #11583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 23 additions & 19 deletions internal-api/src/test/groovy/datadog/trace/api/ConfigCollectorTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,43 +27,45 @@ class ConfigCollectorTest extends DDSpecification {
expect:
def envConfigByKey = ConfigCollector.get().collect().get(ConfigOrigin.ENV)
def config = envConfigByKey.get(configKey)
config.stringValue() == configValue
config.stringValue() == expectedValue
config.origin == ConfigOrigin.ENV

where:
configKey | configValue
// expectedValue equals configValue for every setting except those redacted from configuration
// telemetry (e.g. the application key), where the collected value is rendered as "<hidden>".
configKey | configValue | expectedValue
// ConfigProvider.getEnum
IastConfig.IAST_TELEMETRY_VERBOSITY | Verbosity.DEBUG.toString()
IastConfig.IAST_TELEMETRY_VERBOSITY | Verbosity.DEBUG.toString() | configValue
// ConfigProvider.getString
TracerConfig.TRACE_SPAN_ATTRIBUTE_SCHEMA | "v1"
TracerConfig.TRACE_SPAN_ATTRIBUTE_SCHEMA | "v1" | configValue
// ConfigProvider.getStringNotEmpty
AppSecConfig.APPSEC_AUTOMATED_USER_EVENTS_TRACKING | UserEventTrackingMode.EXTENDED.toString()
AppSecConfig.APPSEC_AUTOMATED_USER_EVENTS_TRACKING | UserEventTrackingMode.EXTENDED.toString() | configValue
// ConfigProvider.getStringExcludingSource
GeneralConfig.APPLICATION_KEY | "app-key"
GeneralConfig.APPLICATION_KEY | "app-key" | "<hidden>"
// ConfigProvider.getBoolean
TraceInstrumentationConfig.RESOLVER_USE_URL_CACHES | "true"
TraceInstrumentationConfig.RESOLVER_USE_URL_CACHES | "true" | configValue
// ConfigProvider.getInteger
JmxFetchConfig.JMX_FETCH_CHECK_PERIOD | "60"
JmxFetchConfig.JMX_FETCH_CHECK_PERIOD | "60" | configValue
// ConfigProvider.getLong
CiVisibilityConfig.CIVISIBILITY_GIT_COMMAND_TIMEOUT_MILLIS | "450273"
CiVisibilityConfig.CIVISIBILITY_GIT_COMMAND_TIMEOUT_MILLIS | "450273" | configValue
// ConfigProvider.getFloat
GeneralConfig.TELEMETRY_HEARTBEAT_INTERVAL | "1.5"
GeneralConfig.TELEMETRY_HEARTBEAT_INTERVAL | "1.5" | configValue
// ConfigProvider.getDouble
TracerConfig.TRACE_SAMPLE_RATE | "2.2"
TracerConfig.TRACE_SAMPLE_RATE | "2.2" | configValue
// ConfigProvider.getList
TraceInstrumentationConfig.JMS_PROPAGATION_DISABLED_TOPICS | "someTopic,otherTopic"
TraceInstrumentationConfig.JMS_PROPAGATION_DISABLED_TOPICS | "someTopic,otherTopic" | configValue
// ConfigProvider.getSet
IastConfig.IAST_WEAK_HASH_ALGORITHMS | "SHA1,SHA-1"
IastConfig.IAST_WEAK_HASH_ALGORITHMS | "SHA1,SHA-1" | configValue
// ConfigProvider.getSpacedList
TracerConfig.PROXY_NO_PROXY | "a b c"
TracerConfig.PROXY_NO_PROXY | "a b c" | configValue
// ConfigProvider.getMergedMap
TracerConfig.TRACE_PEER_SERVICE_MAPPING | "service1:best_service,userService:my_service"
TracerConfig.TRACE_PEER_SERVICE_MAPPING | "service1:best_service,userService:my_service" | configValue
// ConfigProvider.getOrderedMap
TracerConfig.TRACE_HTTP_SERVER_PATH_RESOURCE_NAME_MAPPING | "/asdf/*:/test"
TracerConfig.TRACE_HTTP_SERVER_PATH_RESOURCE_NAME_MAPPING | "/asdf/*:/test" | configValue
// ConfigProvider.getMergedMapWithOptionalMappings
TracerConfig.HEADER_TAGS | "e:five"
TracerConfig.HEADER_TAGS | "e:five" | configValue
// ConfigProvider.getIntegerRange
TracerConfig.TRACE_HTTP_CLIENT_ERROR_STATUSES | "400-402"
TracerConfig.TRACE_HTTP_CLIENT_ERROR_STATUSES | "400-402" | configValue
}

def "should collect merged data from multiple sources"() {
Expand Down Expand Up @@ -131,8 +133,10 @@ class ConfigCollectorTest extends DDSpecification {
cs.origin == ConfigOrigin.DEFAULT

where:
// GeneralConfig.APPLICATION_KEY is redacted from configuration telemetry, so its collected
// value is rendered as "<hidden>" rather than null; that redaction is verified in the
// "non-default config settings get collected" feature above.
configKey << [
GeneralConfig.APPLICATION_KEY,
TraceInstrumentationConfig.RESOLVER_USE_URL_CACHES,
JmxFetchConfig.JMX_FETCH_CHECK_PERIOD,
CiVisibilityConfig.CIVISIBILITY_DEBUG_PORT,
Expand Down
27 changes: 18 additions & 9 deletions metadata/supported-configurations.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,8 @@
"version": "A",
"type": "string",
"default": null,
"aliases": []
"aliases": [],
"sensitive": true
}
],
"DD_API_KEY_FILE": [
Expand Down Expand Up @@ -198,7 +199,8 @@
"version": "A",
"type": "string",
"default": null,
"aliases": ["DD_APP_KEY"]
"aliases": ["DD_APP_KEY"],
"sensitive": true
}
],
"DD_APPLICATION_KEY_FILE": [
Expand Down Expand Up @@ -2310,7 +2312,8 @@
"version": "A",
"type": "map",
"default": null,
"aliases": []
"aliases": [],
"sensitive": true
}
],
"DD_OTLP_LOGS_PROTOCOL": [
Expand Down Expand Up @@ -2454,7 +2457,8 @@
"version": "B",
"type": "map",
"default": null,
"aliases": []
"aliases": [],
"sensitive": true
}
],
"DD_OTLP_METRICS_PROTOCOL": [
Expand Down Expand Up @@ -2502,7 +2506,8 @@
"version": "B",
"type": "map",
"default": null,
"aliases": []
"aliases": [],
"sensitive": true
}
],
"DD_OTLP_TRACES_PROTOCOL": [
Expand Down Expand Up @@ -11646,7 +11651,8 @@
"version": "B",
"type": "map",
"default": null,
"aliases": []
"aliases": [],
"sensitive": true
}
],
"OTEL_EXPORTER_OTLP_PROTOCOL": [
Expand Down Expand Up @@ -11686,7 +11692,8 @@
"version": "A",
"type": "string",
"default": null,
"aliases": []
"aliases": [],
"sensitive": true
}
],
"OTEL_EXPORTER_OTLP_LOGS_PROTOCOL": [
Expand Down Expand Up @@ -11726,7 +11733,8 @@
"version": "B",
"type": "string",
"default": null,
"aliases": []
"aliases": [],
"sensitive": true
}
],
"OTEL_EXPORTER_OTLP_METRICS_PROTOCOL": [
Expand Down Expand Up @@ -11782,7 +11790,8 @@
"version": "A",
"type": "string",
"default": null,
"aliases": []
"aliases": [],
"sensitive": true
}
],
"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": [
Expand Down
21 changes: 20 additions & 1 deletion utils/config-utils/src/main/java/datadog/trace/api/ConfigSetting.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,28 @@ public final class ConfigSetting {
/** The config ID associated with this setting, or {@code null} if not applicable. */
public final String configId;

// Configuration keys whose values are excluded from configuration telemetry by replacing them
// with "<hidden>". Keys are listed in every form that may reach this constructor: the dotted
// configuration name (used by ConfigProvider) and the environment-variable name.
private static final Set<String> CONFIG_FILTER_LIST =
new HashSet<>(
Arrays.asList("DD_API_KEY", "dd.api-key", "dd.profiling.api-key", "dd.profiling.apikey"));
Arrays.asList(
"DD_API_KEY",
"dd.api-key",
"dd.profiling.api-key",
"dd.profiling.apikey",
"application-key",
"dd.application-key",
"DD_APPLICATION_KEY",
"app-key",
"dd.app-key",
"otlp.traces.headers",
"otlp.metrics.headers",
"otlp.logs.headers",
"OTEL_EXPORTER_OTLP_HEADERS",
"OTEL_EXPORTER_OTLP_TRACES_HEADERS",
"OTEL_EXPORTER_OTLP_METRICS_HEADERS",
"OTEL_EXPORTER_OTLP_LOGS_HEADERS"));

public static ConfigSetting of(String key, Object value, ConfigOrigin origin) {
return new ConfigSetting(key, value, origin, ABSENT_SEQ_ID, null);
Expand Down
43 changes: 37 additions & 6 deletions utils/config-utils/src/test/java/datadog/trace/api/ConfigSettingTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package datadog.trace.api;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;

import datadog.trace.junit.utils.tabletest.BoxedValueConverter;
Expand Down Expand Up @@ -42,17 +43,47 @@ void supportsEqualityCheck(
}

@TableTest({
"scenario | key | value | filteredValue",
"DD_API_KEY | DD_API_KEY | somevalue | <hidden> ",
"dd.api-key | dd.api-key | somevalue | <hidden> ",
"dd.profiling.api-key | dd.profiling.api-key | somevalue | <hidden> ",
"dd.profiling.apikey | dd.profiling.apikey | somevalue | <hidden> ",
"some.other.key | some.other.key | somevalue | somevalue "
"scenario | key | value | filteredValue",
"dd api key env | DD_API_KEY | somevalue | <hidden> ",
"dd api key prop | dd.api-key | somevalue | <hidden> ",
"profiling api key | dd.profiling.api-key | somevalue | <hidden> ",
"profiling apikey | dd.profiling.apikey | somevalue | <hidden> ",
"application key name | application-key | somevalue | <hidden> ",
"application key prop | dd.application-key | somevalue | <hidden> ",
"application key env | DD_APPLICATION_KEY | somevalue | <hidden> ",
"app key alias name | app-key | somevalue | <hidden> ",
"app key alias prop | dd.app-key | somevalue | <hidden> ",
"otlp traces headers | otlp.traces.headers | somevalue | <hidden> ",
"otlp metrics headers | otlp.metrics.headers | somevalue | <hidden> ",
"otlp logs headers | otlp.logs.headers | somevalue | <hidden> ",
"otel otlp headers | OTEL_EXPORTER_OTLP_HEADERS | somevalue | <hidden> ",
"otel traces headers | OTEL_EXPORTER_OTLP_TRACES_HEADERS | somevalue | <hidden> ",
"otel metrics headers | OTEL_EXPORTER_OTLP_METRICS_HEADERS | somevalue | <hidden> ",
"otel logs headers | OTEL_EXPORTER_OTLP_LOGS_HEADERS | somevalue | <hidden> ",
"other key | some.other.key | somevalue | somevalue "
})
void filtersKeyValues(String key, String value, String filteredValue) {
assertEquals(filteredValue, ConfigSetting.of(key, value, ConfigOrigin.DEFAULT).stringValue());
}

@TableTest({
"scenario | key | value ",
"otlp traces | otlp.traces.headers | dd-api-key=secret-traces ",
"otlp metrics | otlp.metrics.headers | dd-api-key=secret-metrics",
"otlp logs | otlp.logs.headers | dd-api-key=secret-logs ",
"otel base | OTEL_EXPORTER_OTLP_HEADERS | dd-api-key=secret-base ",
"otel traces | OTEL_EXPORTER_OTLP_TRACES_HEADERS | dd-api-key=secret-traces ",
"otel metrics | OTEL_EXPORTER_OTLP_METRICS_HEADERS | dd-api-key=secret-metrics",
"otel logs | OTEL_EXPORTER_OTLP_LOGS_HEADERS | dd-api-key=secret-logs ",
"dd api key | DD_API_KEY | secret-api-key "
})
void doesNotExposeSensitiveValues(String key, String value) {
String rendered = ConfigSetting.of(key, value, ConfigOrigin.ENV).stringValue();
assertEquals("<hidden>", rendered);
assertFalse(
rendered.contains(value), "rendered telemetry value must not contain the configured value");
}

@TableTest({
"scenario | value | rendered",
"null | | ",
Expand Down