diff --git a/exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java b/exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java index 5f443dba0ab..69ae44024d7 100644 --- a/exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java +++ b/exporters/prometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java @@ -75,6 +75,9 @@ final class Otel2PrometheusConverter { private static final Logger LOGGER = Logger.getLogger(Otel2PrometheusConverter.class.getName()); private static final ThrottlingLogger THROTTLING_LOGGER = new ThrottlingLogger(LOGGER); + // Prometheus limits the total UTF-8 character count across all exemplar label names and values + // to 128. See https://github.com/open-telemetry/opentelemetry-java/issues/6770 + static final int EXEMPLAR_MAX_LABEL_SET_LENGTH = 128; private static final String OTEL_SCOPE_NAME = "otel_scope_name"; private static final String OTEL_SCOPE_VERSION = "otel_scope_version"; private static final String OTEL_SCOPE_SCHEMA_URL = "otel_scope_schema_url"; @@ -418,8 +421,7 @@ private Exemplars convertDoubleExemplars(List exemplars) { private Exemplar convertExemplar(double value, ExemplarData exemplar) { SpanContext spanContext = exemplar.getSpanContext(); if (spanContext.isValid()) { - return new Exemplar( - value, + Labels labels = convertAttributes( null, // resource attributes are only copied for point's attributes null, // scope attributes are only needed for point's attributes @@ -427,17 +429,52 @@ private Exemplar convertExemplar(double value, ExemplarData exemplar) { "trace_id", spanContext.getTraceId(), "span_id", - spanContext.getSpanId()), - exemplar.getEpochNanos() / NANOS_PER_MILLISECOND); + spanContext.getSpanId()); + if (labelSetLength(labels) > EXEMPLAR_MAX_LABEL_SET_LENGTH) { + // Drop filtered attributes to stay within Prometheus 128-char exemplar label limit, + // keeping trace_id and span_id which are the most valuable for correlation. + THROTTLING_LOGGER.log( + Level.WARNING, + "Exemplar attributes exceeded Prometheus limit of " + + EXEMPLAR_MAX_LABEL_SET_LENGTH + + " UTF-8 characters; dropping filtered attributes."); + labels = + convertAttributes( + null, // resource attributes are only copied for point's attributes + null, // scope attributes are only needed for point's attributes + Attributes.empty(), + "trace_id", + spanContext.getTraceId(), + "span_id", + spanContext.getSpanId()); + } + return new Exemplar(value, labels, exemplar.getEpochNanos() / NANOS_PER_MILLISECOND); } else { - return new Exemplar( - value, + Labels labels = convertAttributes( null, // resource attributes are only copied for point's attributes null, // scope attributes are only needed for point's attributes - exemplar.getFilteredAttributes()), - exemplar.getEpochNanos() / NANOS_PER_MILLISECOND); + exemplar.getFilteredAttributes()); + if (labelSetLength(labels) > EXEMPLAR_MAX_LABEL_SET_LENGTH) { + THROTTLING_LOGGER.log( + Level.WARNING, + "Exemplar attributes exceeded Prometheus limit of " + + EXEMPLAR_MAX_LABEL_SET_LENGTH + + " UTF-8 characters; dropping filtered attributes."); + labels = Labels.EMPTY; + } + return new Exemplar(value, labels, exemplar.getEpochNanos() / NANOS_PER_MILLISECOND); + } + } + + private static int labelSetLength(Labels labels) { + int length = 0; + for (int i = 0; i < labels.size(); i++) { + length += + labels.getName(i).codePointCount(0, labels.getName(i).length()) + + labels.getValue(i).codePointCount(0, labels.getValue(i).length()); } + return length; } private InfoSnapshot makeTargetInfo(Resource resource) { diff --git a/exporters/prometheus/src/test/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverterTest.java b/exporters/prometheus/src/test/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverterTest.java index c7618769d53..ab851bd236b 100644 --- a/exporters/prometheus/src/test/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverterTest.java +++ b/exporters/prometheus/src/test/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverterTest.java @@ -20,10 +20,14 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.common.KeyValue; import io.opentelemetry.api.common.Value; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.sdk.common.InstrumentationScopeInfo; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; import io.opentelemetry.sdk.metrics.data.MetricData; import io.opentelemetry.sdk.metrics.data.MetricDataType; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableDoubleExemplarData; import io.opentelemetry.sdk.metrics.internal.data.ImmutableDoublePointData; import io.opentelemetry.sdk.metrics.internal.data.ImmutableExponentialHistogramBuckets; import io.opentelemetry.sdk.metrics.internal.data.ImmutableExponentialHistogramData; @@ -39,6 +43,7 @@ import io.opentelemetry.sdk.resources.Resource; import io.prometheus.metrics.expositionformats.ExpositionFormats; import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot; import io.prometheus.metrics.model.snapshots.Labels; import io.prometheus.metrics.model.snapshots.MetricSnapshot; import io.prometheus.metrics.model.snapshots.MetricSnapshots; @@ -555,4 +560,83 @@ void validateCacheIsBounded() { // it never saw those resources before. assertThat(predicateCalledCount.get()).isEqualTo(2); } + + @ParameterizedTest + @MethodSource("exemplarLabelLimitArgs") + void exemplarLabelLimit( + String testName, + SpanContext spanContext, + Attributes filteredAttributes, + String[] expectedPresentKeys, + String[] expectedAbsentKeys) { + ImmutableDoubleExemplarData exemplar = + (ImmutableDoubleExemplarData) + ImmutableDoubleExemplarData.create(filteredAttributes, 1000L, spanContext, 1.0); + + MetricData metricData = + ImmutableMetricData.createDoubleGauge( + Resource.getDefault(), + InstrumentationScopeInfo.create("test"), + "my.gauge", + "desc", + "unit", + ImmutableGaugeData.create( + Collections.singletonList( + ImmutableDoublePointData.create( + 0, 1000, Attributes.empty(), 1.0, Collections.singletonList(exemplar))))); + + MetricSnapshots snapshots = converter.convert(Collections.singletonList(metricData)); + assertThat(snapshots).isNotNull(); + GaugeDataPointSnapshot point = (GaugeDataPointSnapshot) snapshots.get(0).getDataPoints().get(0); + Labels exemplarLabels = point.getExemplar().getLabels(); + for (String key : expectedPresentKeys) { + assertThat(exemplarLabels.get(key)).as("expected label '%s' to be present", key).isNotNull(); + } + for (String key : expectedAbsentKeys) { + assertThat(exemplarLabels.get(key)).as("expected label '%s' to be absent", key).isNull(); + } + } + + private static Stream exemplarLabelLimitArgs() { + SpanContext validSpanContext = + SpanContext.create( + "00000000000000000000000000000001", + "0000000000000001", + TraceFlags.getSampled(), + TraceState.getDefault()); + + char[] chars = new char[100]; + Arrays.fill(chars, 'x'); + String longValue100 = new String(chars); + + chars = new char[150]; + Arrays.fill(chars, 'x'); + String longValue150 = new String(chars); + + return Stream.of( + Arguments.of( + "withSpanContext_withinLimit", + validSpanContext, + Attributes.of(stringKey("short"), "val"), + new String[] {"trace_id", "span_id", "short"}, + new String[] {}), + Arguments.of( + "withSpanContext_exceedingLimit", + validSpanContext, + Attributes.of(stringKey("long_attr"), longValue100), + new String[] {"trace_id", "span_id"}, + new String[] {"long_attr"}), + Arguments.of( + "withoutSpanContext_exceedingLimit", + SpanContext.getInvalid(), + Attributes.of(stringKey("long_attr"), longValue150), + new String[] {}, + new String[] {"long_attr"}), + Arguments.of( + "withoutSpanContext_withinLimit", + SpanContext.getInvalid(), + Attributes.of(stringKey("short"), "val"), + new String[] {"short"}, + new String[] {})); + } }