From ad6630e3f3486747c33f6c3c2fcd71201be60b91 Mon Sep 17 00:00:00 2001 From: Dan Markwat Date: Sat, 6 Jul 2024 19:08:26 -0700 Subject: [PATCH] feat: instruments com.twitter.util-stats --- .../javaagent/build.gradle.kts | 52 +++ .../LoadedStatsReceiverInstrumentation.java | 57 +++ .../v23_11/OtelStatsReceiver.java | 419 ++++++++++++++++++ .../v23_11/OtelStatsReceiverProxy.java | 19 + ...TwitterUtilStatsInstrumentationModule.java | 25 ++ .../twitterutilstats/v23_11/ClientTest.java | 211 +++++++++ .../v23_11/TestStatsReceiver.java | 246 ++++++++++ .../com.twitter.finagle.stats.StatsReceiver | 1 + settings.gradle.kts | 1 + 9 files changed, 1031 insertions(+) create mode 100644 instrumentation/twitter-util-stats-23.11/javaagent/build.gradle.kts create mode 100644 instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/LoadedStatsReceiverInstrumentation.java create mode 100644 instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiver.java create mode 100644 instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiverProxy.java create mode 100644 instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TwitterUtilStatsInstrumentationModule.java create mode 100644 instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/ClientTest.java create mode 100644 instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TestStatsReceiver.java create mode 100644 instrumentation/twitter-util-stats-23.11/javaagent/src/test/resources/META-INF/services/com.twitter.finagle.stats.StatsReceiver diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/build.gradle.kts b/instrumentation/twitter-util-stats-23.11/javaagent/build.gradle.kts new file mode 100644 index 000000000000..455d6e9567b8 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/build.gradle.kts @@ -0,0 +1,52 @@ +plugins { + id("otel.javaagent-instrumentation") + id("otel.scala-conventions") +} + +muzzle { + pass { + group.set("com.twitter") + module.set("util-stats_2.12") + versions.set("[23.11.0,]") + } + + pass { + group.set("com.twitter") + module.set("util-stats_2.13") + versions.set("[23.11.0,]") + } +} + +val twitterUtilVersion = "23.11.0" +val scalaVersion = "2.13.10" + +val scalaMinor = Regex("""^([0-9]+\.[0-9]+)\.?.*$""").find(scalaVersion)!!.run { + val (minorVersion) = this.destructured + minorVersion +} + +val scalified = fun(pack: String): String { + return "${pack}_$scalaMinor" +} + +dependencies { + bootstrap(project(":instrumentation:executors:bootstrap")) + + compileOnly("com.google.auto.value:auto-value-annotations") + annotationProcessor("com.google.auto.value:auto-value") + + library("${scalified("com.twitter:util-stats")}:$twitterUtilVersion") + + testImplementation("${scalified("com.twitter:finagle-http")}:$twitterUtilVersion") + // get all the metric services loaded + testImplementation("${scalified("com.twitter:finagle-stats")}:$twitterUtilVersion") + + // should wire netty contexts +// testInstrumentation(project(":instrumentation:netty:netty-4.1:javaagent")) +} + +tasks { + test { + jvmArgs("-Dotel.instrumentation.twitter-util-stats.stats-receiver.mode=additive") + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/LoadedStatsReceiverInstrumentation.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/LoadedStatsReceiverInstrumentation.java new file mode 100644 index 000000000000..d87db17b728f --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/LoadedStatsReceiverInstrumentation.java @@ -0,0 +1,57 @@ +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import static net.bytebuddy.matcher.ElementMatchers.isTypeInitializer; +import static net.bytebuddy.matcher.ElementMatchers.named; + +import com.twitter.finagle.stats.BroadcastStatsReceiver; +import com.twitter.finagle.stats.StatsReceiver; +import io.opentelemetry.javaagent.bootstrap.internal.AgentInstrumentationConfig; +import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation; +import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +public class LoadedStatsReceiverInstrumentation implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + // $ => we want the scala object, not the class + return named("com.twitter.finagle.stats.LoadedStatsReceiver$"); + } + + @Override + public void transform(TypeTransformer transformer) { + // perform this on the class initializer bc LoadedStatsReceiver::self is a var, not a val; + // we don't want to wrap this every time, and we need to respect behavior when someone sets it + transformer.applyAdviceToMethod( + isTypeInitializer(), LoadedStatsReceiverInstrumentation.class.getName() + "$ClinitAdvice"); + } + + @SuppressWarnings("unused") + public static class ClinitAdvice { + + @Advice.OnMethodExit(suppress = Throwable.class) + public static void methodEnter( + @Advice.FieldValue(value = "self", readOnly = false) StatsReceiver self) { + String mode = + AgentInstrumentationConfig.get() + .getString("otel.instrumentation.twitter-util-stats.metrics.mode", "additive"); + List sr; + if (Objects.equals(mode, "additive")) { + sr = Arrays.asList(new OtelStatsReceiverProxy(), self); + + } else { + sr = Collections.singletonList(new OtelStatsReceiverProxy()); + } + // emulate the original invocation to avoid downstream side effects; + // iow make no assumptions about how BroadcastStatsReceiver behaves + self = + BroadcastStatsReceiver.apply( + scala.jdk.CollectionConverters.ListHasAsScala(sr).asScala().toSeq()); + } + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiver.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiver.java new file mode 100644 index 000000000000..fbaf8195e55f --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiver.java @@ -0,0 +1,419 @@ +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import com.google.auto.value.AutoValue; +import com.google.auto.value.extension.memoized.Memoized; +import com.google.common.annotations.VisibleForTesting; +import com.twitter.finagle.stats.Bytes$; +import com.twitter.finagle.stats.Counter; +import com.twitter.finagle.stats.CustomUnit; +import com.twitter.finagle.stats.Gauge; +import com.twitter.finagle.stats.Kilobytes$; +import com.twitter.finagle.stats.Megabytes$; +import com.twitter.finagle.stats.Metadata; +import com.twitter.finagle.stats.MetricBuilder; +import com.twitter.finagle.stats.MetricUnit; +import com.twitter.finagle.stats.Microseconds$; +import com.twitter.finagle.stats.Milliseconds$; +import com.twitter.finagle.stats.Percentage$; +import com.twitter.finagle.stats.Requests$; +import com.twitter.finagle.stats.Seconds$; +import com.twitter.finagle.stats.Stat; +import com.twitter.finagle.stats.StatsReceiver; +import com.twitter.finagle.stats.Unspecified$; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleGaugeBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongCounterBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.ObservableDoubleGauge; +import io.opentelemetry.javaagent.bootstrap.internal.AgentInstrumentationConfig; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import scala.jdk.CollectionConverters; + +public class OtelStatsReceiver implements StatsReceiver { + + private static final Logger logger = Logger.getLogger(OtelStatsReceiver.class.getName()); + + private static final String DEFAULT_DELIM = "."; + + private static final Pattern NAME_REGEX = Pattern.compile("^[A-Za-z][_.-/A-Za-z0-9]{0,254}$"); + + private static final Map HISTO_MAP = new ConcurrentHashMap<>(); + private static final Map GAUGE_MAP = new ConcurrentHashMap<>(); + private static final Map COUNTER_MAP = new ConcurrentHashMap<>(); + + private static final Meter meter = GlobalOpenTelemetry.get().getMeter("twitter-util-stats"); + + private static final AttributeKey> HIERARCHICAL_NAME_LABEL_KEY = + AttributeKey.stringArrayKey("twitter-util-stats/hierarchical_name"); + + private static final String delimiter; + + static { + String delim = + AgentInstrumentationConfig.get() + .getString( + "otel.instrumentation.twitter-util-stats.metrics.name.delimiter", DEFAULT_DELIM); + + if (delim.isEmpty()) { + logger.warning("Delimiter must be non-empty"); + delim = DEFAULT_DELIM; + } else if (!delim.equals(".") && !delim.equals("/") && !delim.equals("-")) { + logger.warning("Unsupported delimiter detected: " + delim); + delim = DEFAULT_DELIM; + } + + delimiter = delim; + } + + public OtelStatsReceiver() {} + + @Override + public Object repr() { + return this; + } + + @VisibleForTesting + static String nameConversion(MetricBuilder schema) { + return schema.identity().dimensionalName().mkString(delimiter); + } + + /* + Identical semantics to the finagle-stats PrometheusExporter. + */ + @VisibleForTesting + static boolean shouldEmit(MetricBuilder schema) { + return schema + .identity() + .identityType() + .bias(com.twitter.finagle.stats.MetricBuilder$IdentityType$HierarchicalOnly$.MODULE$) + == com.twitter.finagle.stats.MetricBuilder$IdentityType$Full$.MODULE$; + } + + /* + Guard against avoidable, incorrect metric names. + Finagle produces and emits a number of metrics which contain invalid chars and entirely free-form, + making them hard to adapt to the otel spec, whitespace, in particular. This, along with names + containing other identifying attributes which are unpredictable except in specific known cases + but which are indistinguishable from other potential metrics. IOW, a metric bearing some pattern + of identifying attributes in its name is indistinguishable from others created for a different + purpose and in a different context and is therefore not reasonably translated into otel metrics. + + Suggestion: write another instrumentation that adapts those specific cases. + */ + @VisibleForTesting + static boolean canEmit(MetricBuilder schema) { + return CollectionConverters.SeqHasAsJava(schema.identity().dimensionalName()).asJava().stream() + .allMatch(NAME_REGEX.asPredicate()); + } + + @VisibleForTesting + static Attributes attributesFromLabels(MetricBuilder builder) { + return builder + .identity() + .labels() + .foldLeft( + // put the hierarchical name as a label bc hierarchical names are not cleanly mapped + // to dimensional names via twitter stats or the final usage in finagle in a way otel + // can concisely reason about; by adding the label, downstream systems can aggregate + // on their own -- or not -- achieving a finer granularity without compromising + // the general patterns applied herein, and without the complexities applied in + // the finagle PrometheusExporter (implicitly, how the MetricsView & registries work, + // etc.), for example + Attributes.builder() + .put( + HIERARCHICAL_NAME_LABEL_KEY, + scala.jdk.CollectionConverters.SeqHasAsJava( + builder.identity().hierarchicalName()) + .asJava()), + (v1, v2) -> v1.put(v2._1(), v2._2())) + .build(); + } + + /* + Always use "byte" for size units to avoid precision loss. + Why: the aggregating services treat unit scales uniformly, so this should present little issue. + */ + // unchecked casting to efficiently handle the generic number conversion for long vs float + @SuppressWarnings("unchecked") + private static UnitConversion unitConverter( + MetricUnit unit, MetricBuilder.MetricType metricType) { + if (unit instanceof CustomUnit) { + return UnitConversion.create(((CustomUnit) unit).name().toLowerCase(Locale.getDefault())); + } else if (unit instanceof Unspecified$) { + return UnitConversion.create(); + } else if (unit instanceof Bytes$) { + return UnitConversion.create("byte"); + } else if (unit instanceof Kilobytes$) { + // base-10 to base-2 translation + if (metricType == MetricBuilder.CounterType$.MODULE$) { + return (UnitConversion) UnitConversion.create("byte", (Long x) -> (x * 1000)); + } else { + return (UnitConversion) UnitConversion.create("byte", (Float x) -> (x * 1000)); + } + } else if (unit instanceof Megabytes$) { + // base-10 to base-2 translation + if (metricType == MetricBuilder.CounterType$.MODULE$) { + return (UnitConversion) UnitConversion.create("byte", (Long x) -> (x * 1000 * 1000)); + } else { + return (UnitConversion) UnitConversion.create("byte", (Float x) -> (x * 1000 * 1000)); + } + } else if (unit instanceof Seconds$) { + return UnitConversion.create("second"); + } else if (unit instanceof Milliseconds$) { + return UnitConversion.create("millisecond"); + } else if (unit instanceof Microseconds$) { + return UnitConversion.create("microsecond"); + } else if (unit instanceof Requests$) { + return UnitConversion.create("request"); + } else if (unit instanceof Percentage$) { + return UnitConversion.create("percent"); + } else { + throw new IllegalArgumentException("unsupported metric unit: " + unit.toString()); + } + } + + @Override + public Stat stat(MetricBuilder schema) { + Attributes attributes = attributesFromLabels(schema); + UnitConversion conversion = unitConverter(schema.units(), schema.metricType()); + + if (!canEmit(schema) && !shouldEmit(schema)) { + logger.fine("skipping stat: " + nameConversion(schema)); + return OtelStat.createNull(schema, attributes, conversion); + } + + logger.fine("instrumenting stat: " + nameConversion(schema)); + return OtelStat.create( + schema, + attributes, + conversion, + HISTO_MAP.computeIfAbsent( + nameConversion(schema), + name -> { + DoubleHistogramBuilder builder = + meter.histogramBuilder(name).setDescription(schema.description()); + if (conversion.getUnits().isPresent()) { + builder = builder.setUnit(conversion.getUnits().get()); + } + if (schema.percentiles().nonEmpty()) { + builder = + builder.setExplicitBucketBoundariesAdvice( + CollectionConverters.SeqHasAsJava(schema.percentiles()).asJava().stream() + .map(scala.Double::unbox) + .collect(Collectors.toList())); + } + return builder.build(); + })); + } + + @Override + public Gauge addGauge(MetricBuilder metricBuilder, scala.Function0 f) { + if (!canEmit(metricBuilder) && !shouldEmit(metricBuilder)) { + logger.fine("skipping gauge: " + nameConversion(metricBuilder)); + return OtelGauge.createNull(metricBuilder); + } + + logger.fine("instrumenting gauge: " + nameConversion(metricBuilder)); + return OtelGauge.create( + metricBuilder, + GAUGE_MAP.computeIfAbsent( + nameConversion(metricBuilder), + name -> { + Attributes attributes = attributesFromLabels(metricBuilder); + + UnitConversion conversion = + unitConverter(metricBuilder.units(), metricBuilder.metricType()); + + DoubleGaugeBuilder builder = + meter.gaugeBuilder(name).setDescription(metricBuilder.description()); + if (conversion.getUnits().isPresent()) { + builder = builder.setUnit(conversion.getUnits().get()); + } + return builder.buildWithCallback( + odm -> + odm.record(conversion.getConverter().apply((float) f.apply()), attributes)); + })); + } + + @Override + public Counter counter(MetricBuilder schema) { + Attributes attributes = attributesFromLabels(schema); + UnitConversion conversion = unitConverter(schema.units(), schema.metricType()); + + if (!canEmit(schema) && !shouldEmit(schema)) { + logger.fine("skipping counter: " + nameConversion(schema)); + return OtelCounter.createNull(schema, attributes, conversion); + } + + logger.fine("instrumenting counter: " + nameConversion(schema)); + return OtelCounter.create( + schema, + attributes, + conversion, + COUNTER_MAP.computeIfAbsent( + nameConversion(schema), + name -> { + logger.fine("creating counter: " + nameConversion(schema)); + LongCounterBuilder builder = + meter.counterBuilder(name).setDescription(schema.description()); + if (conversion.getUnits().isPresent()) { + builder = builder.setUnit(conversion.getUnits().get()); + } + return builder.build(); + })); + } + + @AutoValue + abstract static class UnitConversion { + abstract Optional getUnits(); + + abstract Function getConverter(); + + static UnitConversion create(String units, Function converter) { + return new AutoValue_OtelStatsReceiver_UnitConversion<>(Optional.of(units), converter); + } + + static UnitConversion create(String units) { + return create(units, Function.identity()); + } + + static UnitConversion create() { + return new AutoValue_OtelStatsReceiver_UnitConversion<>( + Optional.empty(), Function.identity()); + } + } + + @AutoValue + abstract static class OtelGauge implements Gauge { + abstract MetricBuilder getSchema(); + + @Nullable + abstract ObservableDoubleGauge getGauge(); + + static OtelGauge createNull(MetricBuilder schema) { + return new AutoValue_OtelStatsReceiver_OtelGauge(schema, null); + } + + static OtelGauge create(MetricBuilder schema, ObservableDoubleGauge gauge) { + return new AutoValue_OtelStatsReceiver_OtelGauge(schema, gauge); + } + + @Memoized + boolean isEmitted() { + return getGauge() != null; + } + + @Override + public void remove() { + if (getGauge() == null) { + return; + } + getGauge().close(); + } + + @Override + public Metadata metadata() { + return getSchema(); + } + } + + @AutoValue + abstract static class OtelCounter implements Counter { + abstract MetricBuilder getSchema(); + + abstract Attributes getAttributes(); + + abstract UnitConversion getConversion(); + + @Nullable + abstract LongCounter getCounter(); + + static OtelCounter createNull( + MetricBuilder schema, Attributes attributes, UnitConversion conversion) { + return new AutoValue_OtelStatsReceiver_OtelCounter(schema, attributes, conversion, null); + } + + static OtelCounter create( + MetricBuilder schema, + Attributes attributes, + UnitConversion conversion, + LongCounter counter) { + return new AutoValue_OtelStatsReceiver_OtelCounter(schema, attributes, conversion, counter); + } + + @Memoized + boolean isEmitted() { + return getCounter() != null; + } + + @Override + public void incr(long delta) { + if (getCounter() == null) { + return; + } + getCounter().add(getConversion().getConverter().apply(delta), getAttributes()); + } + + @Override + public Metadata metadata() { + return getSchema(); + } + } + + @AutoValue + abstract static class OtelStat implements Stat { + abstract MetricBuilder getSchema(); + + abstract Attributes getAttributes(); + + abstract UnitConversion getConversion(); + + @Nullable + abstract DoubleHistogram getHistogram(); + + static OtelStat createNull( + MetricBuilder schema, Attributes attributes, UnitConversion conversion) { + return new AutoValue_OtelStatsReceiver_OtelStat(schema, attributes, conversion, null); + } + + static OtelStat create( + MetricBuilder schema, + Attributes attributes, + UnitConversion conversion, + DoubleHistogram histogram) { + return new AutoValue_OtelStatsReceiver_OtelStat(schema, attributes, conversion, histogram); + } + + @Memoized + boolean isEmitted() { + return getHistogram() != null; + } + + @Override + public void add(float value) { + if (getHistogram() == null) { + return; + } + getHistogram().record(getConversion().getConverter().apply(value), getAttributes()); + } + + @Override + public Metadata metadata() { + return getSchema(); + } + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiverProxy.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiverProxy.java new file mode 100644 index 000000000000..c82f21bd755a --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiverProxy.java @@ -0,0 +1,19 @@ +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import com.google.common.annotations.VisibleForTesting; +import com.twitter.finagle.stats.StatsReceiver; +import com.twitter.finagle.stats.StatsReceiverProxy; + +public class OtelStatsReceiverProxy implements StatsReceiverProxy { + private static final OtelStatsReceiver INSTANCE = new OtelStatsReceiver(); + + @VisibleForTesting + static OtelStatsReceiver getInstance() { + return INSTANCE; + } + + @Override + public StatsReceiver self() { + return INSTANCE; + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TwitterUtilStatsInstrumentationModule.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TwitterUtilStatsInstrumentationModule.java new file mode 100644 index 000000000000..04645ae40fb7 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TwitterUtilStatsInstrumentationModule.java @@ -0,0 +1,25 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import com.google.auto.service.AutoService; +import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule; +import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation; +import java.util.Collections; +import java.util.List; + +@AutoService(InstrumentationModule.class) +public class TwitterUtilStatsInstrumentationModule extends InstrumentationModule { + + public TwitterUtilStatsInstrumentationModule() { + super("twitter-util-stats", "twitter-util-stats-23.11"); + } + + @Override + public List typeInstrumentations() { + return Collections.singletonList(new LoadedStatsReceiverInstrumentation()); + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/ClientTest.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/ClientTest.java new file mode 100644 index 000000000000..7fa72e30336d --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/ClientTest.java @@ -0,0 +1,211 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import static io.opentelemetry.instrumentation.testing.junit.http.AbstractHttpClientTest.CONNECTION_TIMEOUT; +import static io.opentelemetry.instrumentation.testing.junit.http.AbstractHttpClientTest.READ_TIMEOUT; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; + +import com.twitter.finagle.Http; +import com.twitter.finagle.ListeningServer; +import com.twitter.finagle.Service; +import com.twitter.finagle.http.Method; +import com.twitter.finagle.http.Request; +import com.twitter.finagle.http.Response; +import com.twitter.finagle.service.RetryBudget; +import com.twitter.finagle.stats.Counter; +import com.twitter.finagle.stats.CustomUnit; +import com.twitter.finagle.stats.DefaultStatsReceiver$; +import com.twitter.finagle.stats.MetricBuilder; +import com.twitter.util.Await; +import com.twitter.util.Duration; +import com.twitter.util.Future; +import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension; +import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; +import io.opentelemetry.sdk.testing.assertj.DoublePointAssert; +import io.opentelemetry.sdk.testing.assertj.LongPointAssert; +import io.opentelemetry.testing.internal.armeria.internal.shaded.guava.collect.ImmutableMap; +import java.net.URI; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Consumer; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +class ClientTest { + private static final Logger logger = Logger.getLogger(ClientTest.class.getName()); + + @RegisterExtension + static final InstrumentationExtension testing = AgentInstrumentationExtension.create(); + + static Request buildRequest(String method, URI uri, Map headers) { + Request request = + Request.apply( + Method.apply(method.toUpperCase(Locale.ENGLISH)), + uri.getPath() + (uri.getQuery() == null ? "" : "?" + uri.getRawQuery())); + request.host(uri.getHost() + ":" + safePort(uri)); + headers.forEach((key, value) -> request.headerMap().put(key, value)); + return request; + } + + static int safePort(URI uri) { + int port = uri.getPort(); + if (port == -1) { + port = uri.getScheme().equals("https") ? 443 : 80; + } + return port; + } + + @Test + void sometest() throws Exception { + Service service = + new Service() { + final Counter counter = + DefaultStatsReceiver$.MODULE$ + .scope("mine") + .counter( + MetricBuilder.forCounter() + .withName("requests2") + .withUnits(new CustomUnit("widgets"))); + + @Override + public Future apply(Request request) { + counter.incr(); + Response response = Response.apply(); + response.setContentString("Hello, World!"); + return Future.value(response); + } + }; + + ListeningServer serve = + Http.server() + .withLabel("http_server") + .withStatsReceiver(DefaultStatsReceiver$.MODULE$) + .serve(":8080", service); + + Service client = + Http.client() + .withNoHttp2() + .withTransport() + .readTimeout(Duration.fromMilliseconds(READ_TIMEOUT.toMillis())) + .withTransport() + .connectTimeout(Duration.fromMilliseconds(CONNECTION_TIMEOUT.toMillis())) + // disable automatic retries for sanity and result predictability/uniformity + .withRetryBudget(RetryBudget.Empty()) + .newService("127.0.0.1:8080"); + + IntStream.range(0, 10) + .forEach( + ignored -> { + Future response = + client.apply( + buildRequest("GET", URI.create("http://localhost:8080/"), ImmutableMap.of())); + + try { + Await.result(response); + Thread.sleep(250L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + Await.result(client.close(Duration.fromMilliseconds(1000L))); + Await.result(serve.close(Duration.fromMilliseconds(1000L))); + + TestStatsReceiver.getInstance() + .getCounters() + .asMap() + .forEach( + (name, counters) -> { + List> collect = + counters.stream() + .filter( + counter -> { + if (!counter.isInitialized()) { + logger.info("uninitialized: " + name); + // skip uninitialized bc they won't have + // default values assigned in otel + return false; + } + if (!counter.getCounterpart().isEmitted()) { + logger.info("skipped: " + name); + return false; + } + logger.info(name + ": " + counter.getCounter()); + return true; + }) + .map( + counter -> + (Consumer) + points -> points.hasValue(counter.getCounter().apply())) + .collect(Collectors.toList()); + + if (collect.isEmpty()) { + logger.info("unset; skipping all assertions: " + name); + return; + } + + testing.waitAndAssertMetrics( + "twitter-util-stats", + name, + metrics -> + metrics.anySatisfy( + metric -> + assertThat(metric) + .hasLongSumSatisfying(sum -> sum.hasPointsSatisfying(collect)))); + }); + + TestStatsReceiver.getInstance() + .getGauges() + .asMap() + .forEach( + (name, gauges) -> { + List> collect = + gauges.stream() + .filter( + gauge -> { + if (!gauge.isInitialized()) { + logger.info("uninitialized: " + name); + // skip uninitialized bc they won't have + // default values assigned in otel + return false; + } + if (!gauge.getCounterpart().isEmitted()) { + logger.info("skipped: " + name); + return false; + } + logger.info(name + ": " + gauge.getLast()); + return true; + }) + .map( + gauge -> + (Consumer) + points -> points.hasValue(gauge.getLast())) + .collect(Collectors.toList()); + + if (collect.isEmpty()) { + logger.info("unset; skipping all assertions: " + name); + return; + } + + testing.waitAndAssertMetrics( + "twitter-util-stats", + name, + metrics -> + metrics.anySatisfy( + metric -> + assertThat(metric) + .hasDoubleGaugeSatisfying( + sum -> sum.hasPointsSatisfying(collect)))); + }); + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TestStatsReceiver.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TestStatsReceiver.java new file mode 100644 index 000000000000..6843c6da6166 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TestStatsReceiver.java @@ -0,0 +1,246 @@ +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Multimap; +import com.twitter.finagle.stats.Counter; +import com.twitter.finagle.stats.Gauge; +import com.twitter.finagle.stats.InMemoryStatsReceiver; +import com.twitter.finagle.stats.Metadata; +import com.twitter.finagle.stats.MetricBuilder; +import com.twitter.finagle.stats.ReadableCounter; +import com.twitter.finagle.stats.ReadableStat; +import com.twitter.finagle.stats.Stat; +import com.twitter.finagle.stats.StatsReceiver; +import com.twitter.finagle.stats.StatsReceiverProxy; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import scala.Function0; + +public class TestStatsReceiver implements StatsReceiverProxy { + @Override + public StatsReceiver self() { + return IMPL; + } + + static Impl getInstance() { + return IMPL; + } + + private static final Impl IMPL = + new Impl() { + // used to generate finagle-computed values to establish baseline for comparisons + final InMemoryStatsReceiver self = new InMemoryStatsReceiver(); + + private final Map counters = new ConcurrentHashMap<>(); + private final Map stats = new ConcurrentHashMap<>(); + private final Map gauges = new ConcurrentHashMap<>(); + + @Override + public Multimap getCounters() { + return counters.entrySet().stream() + .collect( + ImmutableSetMultimap.toImmutableSetMultimap( + k -> OtelStatsReceiver.nameConversion(k.getKey()), Map.Entry::getValue)); + } + + @Override + public Multimap getStats() { + return stats.entrySet().stream() + .collect( + ImmutableSetMultimap.toImmutableSetMultimap( + k -> OtelStatsReceiver.nameConversion(k.getKey()), Map.Entry::getValue)); + } + + @Override + public Multimap getGauges() { + return gauges.entrySet().stream() + .collect( + ImmutableSetMultimap.toImmutableSetMultimap( + k -> OtelStatsReceiver.nameConversion(k.getKey()), Map.Entry::getValue)); + } + + @Override + public Object repr() { + return this; + } + + @Override + public RichCounter counter(MetricBuilder schema) { + return counters.computeIfAbsent( + schema, + key -> { + ReadableCounter counter = self.counter(schema); + AtomicBoolean isInitialized = new AtomicBoolean(false); + return new RichCounter() { + final AtomicReference counterpart = + new AtomicReference<>(); + + @Override + public void incr(long delta) { + isInitialized.set(true); + counter.incr(delta); + } + + @Override + public Metadata metadata() { + return counter.metadata(); + } + + @Override + public boolean isInitialized() { + return isInitialized.get(); + } + + @Override + public ReadableCounter getCounter() { + return counter; + } + + @Override + public OtelStatsReceiver.OtelCounter getCounterpart() { + // safe to cast + return counterpart.updateAndGet( + value -> + value == null + ? (OtelStatsReceiver.OtelCounter) + OtelStatsReceiverProxy.getInstance().counter(schema) + : value); + } + }; + }); + } + + @Override + public RichStat stat(MetricBuilder schema) { + return stats.computeIfAbsent( + schema, + key -> { + ReadableStat stat = self.stat(schema); + AtomicBoolean isInitialized = new AtomicBoolean(false); + return new RichStat() { + final AtomicReference counterpart = + new AtomicReference<>(); + + @Override + public void add(float value) { + isInitialized.set(true); + stat.add(value); + } + + @Override + public Metadata metadata() { + return stat.metadata(); + } + + @Override + public boolean isInitialized() { + return isInitialized.get(); + } + + @Override + public ReadableStat getStat() { + return stat; + } + + @Override + public OtelStatsReceiver.OtelStat getCounterpart() { + // safe to cast + return counterpart.updateAndGet( + value -> + value == null + ? (OtelStatsReceiver.OtelStat) + OtelStatsReceiverProxy.getInstance().stat(schema) + : value); + } + }; + }); + } + + @Override + public GaugeWithMemory addGauge(MetricBuilder metricBuilder, Function0 f) { + return gauges.computeIfAbsent( + metricBuilder, + key -> { + AtomicReference ref = new AtomicReference<>(); + Gauge gauge = + self.addGauge( + metricBuilder, + () -> { + float value = (float) f.apply(); + ref.getAndSet(value); + return value; + }); + return new GaugeWithMemory() { + final AtomicReference counterpart = + new AtomicReference<>(); + + @Override + public boolean isInitialized() { + return ref.get() != null; + } + + @Override + public Float getLast() { + return ref.get(); + } + + @Override + public OtelStatsReceiver.OtelGauge getCounterpart() { + // safe to cast + return counterpart.updateAndGet( + value -> + value == null + ? (OtelStatsReceiver.OtelGauge) + OtelStatsReceiverProxy.getInstance().addGauge(metricBuilder, f) + : value); + } + + @Override + public void remove() { + gauge.remove(); + } + + @Override + public Metadata metadata() { + return gauge.metadata(); + } + }; + }); + } + }; + + interface Impl extends StatsReceiver { + + Multimap getCounters(); + + Multimap getStats(); + + Multimap getGauges(); + } + + interface GaugeWithMemory extends Gauge { + boolean isInitialized(); + + Float getLast(); + + OtelStatsReceiver.OtelGauge getCounterpart(); + } + + interface RichCounter extends Counter { + boolean isInitialized(); + + ReadableCounter getCounter(); + + OtelStatsReceiver.OtelCounter getCounterpart(); + } + + interface RichStat extends Stat { + boolean isInitialized(); + + ReadableStat getStat(); + + OtelStatsReceiver.OtelStat getCounterpart(); + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/test/resources/META-INF/services/com.twitter.finagle.stats.StatsReceiver b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/resources/META-INF/services/com.twitter.finagle.stats.StatsReceiver new file mode 100644 index 000000000000..fc0e2f4cad41 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/resources/META-INF/services/com.twitter.finagle.stats.StatsReceiver @@ -0,0 +1 @@ +io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11.TestStatsReceiver diff --git a/settings.gradle.kts b/settings.gradle.kts index 828170472690..439a4f0b0caf 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -597,6 +597,7 @@ include(":instrumentation:tomcat:tomcat-10.0:javaagent") include(":instrumentation:tomcat:tomcat-common:javaagent") include(":instrumentation:tomcat:tomcat-jdbc") include(":instrumentation:twilio-6.6:javaagent") +include(":instrumentation:twitter-util-stats-23.11:javaagent") include(":instrumentation:undertow-1.4:bootstrap") include(":instrumentation:undertow-1.4:javaagent") include(":instrumentation:vaadin-14.2:javaagent")