diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/build.gradle.kts b/instrumentation/twitter-util-stats-23.11/javaagent/build.gradle.kts new file mode 100644 index 000000000000..da96a9d18265 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/build.gradle.kts @@ -0,0 +1,50 @@ +plugins { + id("otel.javaagent-instrumentation") + id("otel.scala-conventions") +} + +muzzle { + pass { + group.set("com.twitter") + module.set("util-stats_2.12") + versions.set("[23.11.0,]") + } + + pass { + group.set("com.twitter") + module.set("util-stats_2.13") + versions.set("[23.11.0,]") + } +} + +val twitterUtilVersion = "23.11.0" +val scalaVersion = "2.13.10" + +val scalaMinor = Regex("""^([0-9]+\.[0-9]+)\.?.*$""").find(scalaVersion)!!.run { + val (minorVersion) = this.destructured + minorVersion +} + +val scalified = fun(pack: String): String { + return "${pack}_$scalaMinor" +} + +dependencies { + compileOnly("com.google.auto.value:auto-value-annotations") + annotationProcessor("com.google.auto.value:auto-value") + + library("${scalified("com.twitter:util-stats")}:$twitterUtilVersion") + + testImplementation("${scalified("com.twitter:finagle-http")}:$twitterUtilVersion") + // get all the metric services loaded + testImplementation("${scalified("com.twitter:finagle-stats")}:$twitterUtilVersion") + + // should wire netty contexts +// testInstrumentation(project(":instrumentation:netty:netty-4.1:javaagent")) +} + +tasks { + test { + jvmArgs("-Dotel.instrumentation.twitter-util-stats.stats-receiver.mode=additive") + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/Helpers.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/Helpers.java new file mode 100644 index 000000000000..6665c5e08d6f --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/Helpers.java @@ -0,0 +1,182 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; +import com.twitter.finagle.stats.Bytes$; +import com.twitter.finagle.stats.CustomUnit; +import com.twitter.finagle.stats.Kilobytes$; +import com.twitter.finagle.stats.Megabytes$; +import com.twitter.finagle.stats.MetricBuilder; +import com.twitter.finagle.stats.MetricUnit; +import com.twitter.finagle.stats.Microseconds$; +import com.twitter.finagle.stats.Milliseconds$; +import com.twitter.finagle.stats.Percentage$; +import com.twitter.finagle.stats.Requests$; +import com.twitter.finagle.stats.Seconds$; +import com.twitter.finagle.stats.Unspecified$; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.javaagent.bootstrap.internal.AgentInstrumentationConfig; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.regex.Pattern; +import scala.jdk.CollectionConverters; + +public class Helpers { + + private static final Logger logger = Logger.getLogger(Helpers.class.getName()); + + private static final AttributeKey> HIERARCHICAL_NAME_LABEL_KEY = + AttributeKey.stringArrayKey("twitter-util-stats/hierarchical_name"); + + private static final Pattern NAME_REGEX = Pattern.compile("^[A-Za-z][_.-/A-Za-z0-9]{0,254}$"); + + private static final String DEFAULT_DELIM = "."; + + public static final String delimiter; + + static { + String delim = + AgentInstrumentationConfig.get() + .getString( + "otel.instrumentation.twitter-util-stats.metrics.name.delimiter", DEFAULT_DELIM); + + if (delim.isEmpty()) { + logger.warning("Delimiter must be non-empty"); + delim = DEFAULT_DELIM; + } else if (!delim.equals(".") && !delim.equals("/") && !delim.equals("-")) { + logger.warning("Unsupported delimiter detected: " + delim); + delim = DEFAULT_DELIM; + } + + delimiter = delim; + } + + private Helpers() {} + + public static Attributes attributesFromLabels(MetricBuilder builder) { + return builder + .identity() + .labels() + .foldLeft( + // put the hierarchical name as a label bc hierarchical names are not cleanly mapped + // to dimensional names via twitter stats or the final usage in finagle in a way otel + // can concisely reason about; by adding the label, downstream systems can aggregate + // on their own -- or not -- achieving a finer granularity without compromising + // the general patterns applied herein, and without the complexities applied in + // the finagle PrometheusExporter (implicitly, how the MetricsView & registries work, + // etc.), for example + Attributes.builder() + .put( + HIERARCHICAL_NAME_LABEL_KEY, + scala.jdk.CollectionConverters.SeqHasAsJava( + builder.identity().hierarchicalName()) + .asJava()), + (v1, v2) -> v1.put(v2._1(), v2._2())) + .build(); + } + + @VisibleForTesting + public static String nameConversion(MetricBuilder schema) { + return schema.identity().dimensionalName().mkString(delimiter); + } + + /* + Always use "byte" for size units to avoid precision loss. + Why: the aggregating services treat unit scales uniformly, so this should present little issue. + */ + // unchecked casting to efficiently handle the generic number conversion for long vs float + @SuppressWarnings("unchecked") + public static UnitConversion unitConverter( + MetricUnit unit, MetricBuilder.MetricType metricType) { + if (unit instanceof CustomUnit) { + return UnitConversion.create(((CustomUnit) unit).name().toLowerCase(Locale.getDefault())); + } else if (unit instanceof Unspecified$) { + return UnitConversion.create(); + } else if (unit instanceof Bytes$) { + return UnitConversion.create("byte"); + } else if (unit instanceof Kilobytes$) { + // base-10 to base-2 translation + if (metricType == MetricBuilder.CounterType$.MODULE$) { + return (UnitConversion) UnitConversion.create("byte", (Long x) -> (x * 1000)); + } else { + return (UnitConversion) UnitConversion.create("byte", (Float x) -> (x * 1000)); + } + } else if (unit instanceof Megabytes$) { + // base-10 to base-2 translation + if (metricType == MetricBuilder.CounterType$.MODULE$) { + return (UnitConversion) UnitConversion.create("byte", (Long x) -> (x * 1000 * 1000)); + } else { + return (UnitConversion) UnitConversion.create("byte", (Float x) -> (x * 1000 * 1000)); + } + } else if (unit instanceof Seconds$) { + return UnitConversion.create("second"); + } else if (unit instanceof Milliseconds$) { + return UnitConversion.create("millisecond"); + } else if (unit instanceof Microseconds$) { + return UnitConversion.create("microsecond"); + } else if (unit instanceof Requests$) { + return UnitConversion.create("request"); + } else if (unit instanceof Percentage$) { + return UnitConversion.create("percent"); + } else { + throw new IllegalArgumentException("unsupported metric unit: " + unit.toString()); + } + } + + /* + Identical semantics to the finagle-stats PrometheusExporter. + */ + @VisibleForTesting + public static boolean shouldEmit(MetricBuilder schema) { + return schema + .identity() + .identityType() + .bias(com.twitter.finagle.stats.MetricBuilder$IdentityType$HierarchicalOnly$.MODULE$) + == com.twitter.finagle.stats.MetricBuilder$IdentityType$Full$.MODULE$; + } + + /* + Guard against avoidable, incorrect metric names. + Finagle produces and emits a number of metrics which contain invalid chars and entirely free-form, + making them hard to adapt to the otel spec, whitespace, in particular. This, along with names + containing other identifying attributes which are unpredictable except in specific known cases + but which are indistinguishable from other potential metrics. IOW, a metric bearing some pattern + of identifying attributes in its name is indistinguishable from others created for a different + purpose and in a different context and is therefore not reasonably translated into otel metrics. + + Suggestion: write another instrumentation that adapts those specific cases. + */ + @VisibleForTesting + public static boolean canEmit(MetricBuilder schema) { + return CollectionConverters.SeqHasAsJava(schema.identity().dimensionalName()).asJava().stream() + .allMatch(NAME_REGEX.asPredicate()); + } + + @AutoValue + public abstract static class UnitConversion { + public abstract Optional getUnits(); + + public abstract Function getConverter(); + + static UnitConversion create(String units, Function converter) { + return new AutoValue_Helpers_UnitConversion<>(Optional.of(units), converter); + } + + static UnitConversion create(String units) { + return create(units, Function.identity()); + } + + static UnitConversion create() { + return new AutoValue_Helpers_UnitConversion<>(Optional.empty(), Function.identity()); + } + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/LoadedStatsReceiverInstrumentation.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/LoadedStatsReceiverInstrumentation.java new file mode 100644 index 000000000000..e2ec6c1899b3 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/LoadedStatsReceiverInstrumentation.java @@ -0,0 +1,62 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import static net.bytebuddy.matcher.ElementMatchers.isTypeInitializer; +import static net.bytebuddy.matcher.ElementMatchers.named; + +import com.twitter.finagle.stats.BroadcastStatsReceiver; +import com.twitter.finagle.stats.StatsReceiver; +import io.opentelemetry.javaagent.bootstrap.internal.AgentInstrumentationConfig; +import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation; +import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +public class LoadedStatsReceiverInstrumentation implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + // $ => we want the scala object, not the class + return named("com.twitter.finagle.stats.LoadedStatsReceiver$"); + } + + @Override + public void transform(TypeTransformer transformer) { + // perform this on the class initializer bc LoadedStatsReceiver::self is a var, not a val; + // we don't want to wrap this every time, and we need to respect behavior when someone sets it + transformer.applyAdviceToMethod( + isTypeInitializer(), LoadedStatsReceiverInstrumentation.class.getName() + "$ClinitAdvice"); + } + + @SuppressWarnings("unused") + public static class ClinitAdvice { + + @Advice.OnMethodExit(suppress = Throwable.class) + public static void methodEnter( + @Advice.FieldValue(value = "self", readOnly = false) StatsReceiver self) { + String mode = + AgentInstrumentationConfig.get() + .getString("otel.instrumentation.twitter-util-stats.metrics.mode", "additive"); + List sr; + if (Objects.equals(mode, "additive")) { + sr = Arrays.asList(new OtelStatsReceiverProxy(), self); + + } else { + sr = Collections.singletonList(new OtelStatsReceiverProxy()); + } + // emulate the original invocation to avoid downstream side effects; + // iow make no assumptions about how BroadcastStatsReceiver behaves + self = + BroadcastStatsReceiver.apply( + scala.jdk.CollectionConverters.ListHasAsScala(sr).asScala().toSeq()); + } + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiver.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiver.java new file mode 100644 index 000000000000..9737417b037b --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiver.java @@ -0,0 +1,259 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import com.google.auto.value.AutoValue; +import com.twitter.finagle.stats.Counter; +import com.twitter.finagle.stats.Gauge; +import com.twitter.finagle.stats.Metadata; +import com.twitter.finagle.stats.MetricBuilder; +import com.twitter.finagle.stats.Stat; +import com.twitter.finagle.stats.StatsReceiver; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleGaugeBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongCounterBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.ObservableDoubleGauge; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +public class OtelStatsReceiver implements StatsReceiver { + + private static final Logger logger = Logger.getLogger(OtelStatsReceiver.class.getName()); + + private static final Map HISTO_MAP = new ConcurrentHashMap<>(); + private static final Map GAUGE_MAP = new ConcurrentHashMap<>(); + private static final Map COUNTER_MAP = new ConcurrentHashMap<>(); + + private static final Meter meter = GlobalOpenTelemetry.get().getMeter("twitter-util-stats"); + + public OtelStatsReceiver() {} + + @Override + public Object repr() { + return this; + } + + @Override + public Stat stat(MetricBuilder schema) { + Attributes attributes = Helpers.attributesFromLabels(schema); + Helpers.UnitConversion conversion = + Helpers.unitConverter(schema.units(), schema.metricType()); + + if (!Helpers.canEmit(schema) && !Helpers.shouldEmit(schema)) { + logger.fine("skipping stat: " + Helpers.nameConversion(schema)); + return OtelStat.createNull(schema, attributes, conversion); + } + + logger.fine("instrumenting stat: " + Helpers.nameConversion(schema)); + return OtelStat.create( + schema, + attributes, + conversion, + HISTO_MAP.computeIfAbsent( + // NOTE: histograms aren't native to twitter-util-stats but we're using the stat name + Helpers.nameConversion(schema), + name -> { + DoubleHistogramBuilder builder = + meter.histogramBuilder(name).setDescription(schema.description()); + if (conversion.getUnits().isPresent()) { + builder = builder.setUnit(conversion.getUnits().get()); + } + return builder.build(); + })); + } + + @Override + public Gauge addGauge(MetricBuilder metricBuilder, scala.Function0 f) { + Attributes attributes = Helpers.attributesFromLabels(metricBuilder); + Helpers.UnitConversion conversion = + Helpers.unitConverter(metricBuilder.units(), metricBuilder.metricType()); + + if (!Helpers.canEmit(metricBuilder) && !Helpers.shouldEmit(metricBuilder)) { + logger.fine("skipping gauge: " + Helpers.nameConversion(metricBuilder)); + return OtelGauge.createNull(metricBuilder, attributes, conversion); + } + + logger.fine("instrumenting gauge: " + Helpers.nameConversion(metricBuilder)); + return OtelGauge.create( + metricBuilder, + attributes, + conversion, + GAUGE_MAP.computeIfAbsent( + Helpers.nameConversion(metricBuilder), + name -> { + DoubleGaugeBuilder builder = + meter.gaugeBuilder(name).setDescription(metricBuilder.description()); + if (conversion.getUnits().isPresent()) { + builder = builder.setUnit(conversion.getUnits().get()); + } + return builder.buildWithCallback( + odm -> + odm.record(conversion.getConverter().apply((float) f.apply()), attributes)); + })); + } + + @Override + public Counter counter(MetricBuilder schema) { + Attributes attributes = Helpers.attributesFromLabels(schema); + Helpers.UnitConversion conversion = + Helpers.unitConverter(schema.units(), schema.metricType()); + + if (!Helpers.canEmit(schema) && !Helpers.shouldEmit(schema)) { + logger.fine("skipping counter: " + Helpers.nameConversion(schema)); + return OtelCounter.createNull(schema, attributes, conversion); + } + + logger.fine("instrumenting counter: " + Helpers.nameConversion(schema)); + return OtelCounter.create( + schema, + attributes, + conversion, + COUNTER_MAP.computeIfAbsent( + Helpers.nameConversion(schema), + name -> { + logger.fine("creating counter: " + Helpers.nameConversion(schema)); + LongCounterBuilder builder = + meter.counterBuilder(name).setDescription(schema.description()); + if (conversion.getUnits().isPresent()) { + builder = builder.setUnit(conversion.getUnits().get()); + } + return builder.build(); + })); + } + + @AutoValue + abstract static class OtelGauge implements Gauge { + abstract MetricBuilder getSchema(); + + abstract Attributes getAttributes(); + + abstract Helpers.UnitConversion getConversion(); + + @Nullable + abstract ObservableDoubleGauge getGauge(); + + static OtelGauge createNull( + MetricBuilder schema, Attributes attributes, Helpers.UnitConversion conversion) { + return new AutoValue_OtelStatsReceiver_OtelGauge(schema, attributes, conversion, null); + } + + static OtelGauge create( + MetricBuilder schema, + Attributes attributes, + Helpers.UnitConversion conversion, + ObservableDoubleGauge gauge) { + return new AutoValue_OtelStatsReceiver_OtelGauge(schema, attributes, conversion, gauge); + } + + boolean isEmitted() { + return getGauge() != null; + } + + @Override + public void remove() { + if (getGauge() == null) { + return; + } + getGauge().close(); + } + + @Override + public Metadata metadata() { + return getSchema(); + } + } + + @AutoValue + abstract static class OtelCounter implements Counter { + abstract MetricBuilder getSchema(); + + abstract Attributes getAttributes(); + + abstract Helpers.UnitConversion getConversion(); + + @Nullable + abstract LongCounter getCounter(); + + static OtelCounter createNull( + MetricBuilder schema, Attributes attributes, Helpers.UnitConversion conversion) { + return new AutoValue_OtelStatsReceiver_OtelCounter(schema, attributes, conversion, null); + } + + static OtelCounter create( + MetricBuilder schema, + Attributes attributes, + Helpers.UnitConversion conversion, + LongCounter counter) { + return new AutoValue_OtelStatsReceiver_OtelCounter(schema, attributes, conversion, counter); + } + + boolean isEmitted() { + return getCounter() != null; + } + + @Override + public void incr(long delta) { + if (getCounter() == null) { + return; + } + getCounter().add(getConversion().getConverter().apply(delta), getAttributes()); + } + + @Override + public Metadata metadata() { + return getSchema(); + } + } + + @AutoValue + abstract static class OtelStat implements Stat { + abstract MetricBuilder getSchema(); + + abstract Attributes getAttributes(); + + abstract Helpers.UnitConversion getConversion(); + + @Nullable + abstract DoubleHistogram getHistogram(); + + static OtelStat createNull( + MetricBuilder schema, Attributes attributes, Helpers.UnitConversion conversion) { + return new AutoValue_OtelStatsReceiver_OtelStat(schema, attributes, conversion, null); + } + + static OtelStat create( + MetricBuilder schema, + Attributes attributes, + Helpers.UnitConversion conversion, + DoubleHistogram histogram) { + return new AutoValue_OtelStatsReceiver_OtelStat(schema, attributes, conversion, histogram); + } + + boolean isEmitted() { + return getHistogram() != null; + } + + @Override + public void add(float value) { + if (getHistogram() == null) { + return; + } + getHistogram().record(getConversion().getConverter().apply(value), getAttributes()); + } + + @Override + public Metadata metadata() { + return getSchema(); + } + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiverProxy.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiverProxy.java new file mode 100644 index 000000000000..8b8788c5f630 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/OtelStatsReceiverProxy.java @@ -0,0 +1,24 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import com.google.common.annotations.VisibleForTesting; +import com.twitter.finagle.stats.StatsReceiver; +import com.twitter.finagle.stats.StatsReceiverProxy; + +public class OtelStatsReceiverProxy implements StatsReceiverProxy { + private static final OtelStatsReceiver INSTANCE = new OtelStatsReceiver(); + + @VisibleForTesting + static OtelStatsReceiver getInstance() { + return INSTANCE; + } + + @Override + public StatsReceiver self() { + return INSTANCE; + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TwitterUtilStatsInstrumentationModule.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TwitterUtilStatsInstrumentationModule.java new file mode 100644 index 000000000000..04645ae40fb7 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TwitterUtilStatsInstrumentationModule.java @@ -0,0 +1,25 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import com.google.auto.service.AutoService; +import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule; +import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation; +import java.util.Collections; +import java.util.List; + +@AutoService(InstrumentationModule.class) +public class TwitterUtilStatsInstrumentationModule extends InstrumentationModule { + + public TwitterUtilStatsInstrumentationModule() { + super("twitter-util-stats", "twitter-util-stats-23.11"); + } + + @Override + public List typeInstrumentations() { + return Collections.singletonList(new LoadedStatsReceiverInstrumentation()); + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/ClientTest.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/ClientTest.java new file mode 100644 index 000000000000..c1ad295edd60 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/ClientTest.java @@ -0,0 +1,295 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import static io.opentelemetry.instrumentation.testing.junit.http.AbstractHttpClientTest.CONNECTION_TIMEOUT; +import static io.opentelemetry.instrumentation.testing.junit.http.AbstractHttpClientTest.READ_TIMEOUT; +import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat; + +import com.google.common.collect.ImmutableMap; +import com.twitter.finagle.Http; +import com.twitter.finagle.ListeningServer; +import com.twitter.finagle.Service; +import com.twitter.finagle.http.Method; +import com.twitter.finagle.http.Request; +import com.twitter.finagle.http.Response; +import com.twitter.finagle.netty4.HashedWheelTimer$; +import com.twitter.finagle.service.RetryBudget; +import com.twitter.finagle.stats.Counter; +import com.twitter.finagle.stats.CustomUnit; +import com.twitter.finagle.stats.DefaultStatsReceiver$; +import com.twitter.finagle.stats.MetricBuilder; +import com.twitter.util.Await; +import com.twitter.util.Duration; +import com.twitter.util.Future; +import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension; +import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension; +import io.opentelemetry.sdk.testing.assertj.DoublePointAssert; +import io.opentelemetry.sdk.testing.assertj.HistogramPointAssert; +import io.opentelemetry.sdk.testing.assertj.LongPointAssert; +import java.net.URI; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Consumer; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import scala.jdk.CollectionConverters; + +class ClientTest { + private static final Logger logger = Logger.getLogger(ClientTest.class.getName()); + + @RegisterExtension + static final InstrumentationExtension testing = AgentInstrumentationExtension.create(); + + static Request buildRequest(String method, URI uri, Map headers) { + Request request = + Request.apply( + Method.apply(method.toUpperCase(Locale.ENGLISH)), + uri.getPath() + (uri.getQuery() == null ? "" : "?" + uri.getRawQuery())); + request.host(uri.getHost() + ":" + safePort(uri)); + headers.forEach((key, value) -> request.headerMap().put(key, value)); + return request; + } + + static int safePort(URI uri) { + int port = uri.getPort(); + if (port == -1) { + port = uri.getScheme().equals("https") ? 443 : 80; + } + return port; + } + + @Test + void sometest() throws Exception { + Service service = + new Service() { + final Counter counter = + DefaultStatsReceiver$.MODULE$ + .scope("mine") + .counter( + MetricBuilder.forCounter() + .withName("requests2") + .withUnits(new CustomUnit("widgets"))); + + @Override + public Future apply(Request request) { + counter.incr(); + Response response = Response.apply(); + response.setContentString("Hello, World!"); + return Future.value(response); + } + }; + + ListeningServer serve = + Http.server() + .withLabel("http_server") + .withStatsReceiver(DefaultStatsReceiver$.MODULE$) + .serve(":8080", service); + + Service client = + Http.client() + .withNoHttp2() + .withTransport() + .readTimeout(Duration.fromMilliseconds(READ_TIMEOUT.toMillis())) + .withTransport() + .connectTimeout(Duration.fromMilliseconds(CONNECTION_TIMEOUT.toMillis())) + // disable automatic retries for sanity and result predictability/uniformity + .withRetryBudget(RetryBudget.Empty()) + .newService("127.0.0.1:8080"); + + IntStream.range(0, 10) + .forEach( + ignored -> { + Future response = + client.apply( + buildRequest("GET", URI.create("http://localhost:8080/"), ImmutableMap.of())); + + try { + Await.result(response); + Thread.sleep(250L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + Await.result(client.close(Duration.fromMilliseconds(1000L))); + Await.result(serve.close(Duration.fromMilliseconds(1000L))); + + TestStatsReceiver.getInstance() + .getCounters() + .asMap() + .forEach( + (name, counters) -> { + List> collect = + counters.stream() + .filter( + counter -> { + if (!counter.isInitialized()) { + logger.info("uninitialized: " + name); + // skip uninitialized bc they won't have + // default values assigned in otel + return false; + } + if (!counter.getCounterpart().isEmitted()) { + logger.info("skipped: " + name); + return false; + } + logger.info(name + ": " + counter.getCounter()); + return true; + }) + .map( + counter -> + (Consumer) + points -> points.hasValue(counter.getCounter().apply())) + .collect(Collectors.toList()); + + if (collect.isEmpty()) { + logger.info("unset; skipping all assertions: " + name); + return; + } + + testing.waitAndAssertMetrics( + "twitter-util-stats", + name, + metrics -> + metrics.anySatisfy( + metric -> + assertThat(metric) + .hasLongSumSatisfying(sum -> sum.hasPointsSatisfying(collect)))); + }); + + TestStatsReceiver.getInstance() + .getGauges() + .asMap() + .forEach( + (name, gauges) -> { + List> collect = + gauges.stream() + .filter( + gauge -> { + if (!gauge.isInitialized()) { + logger.info("uninitialized: " + name); + // skip uninitialized bc they won't have + // default values assigned in otel + return false; + } + if (!gauge.getCounterpart().isEmitted()) { + logger.info("skipped: " + name); + return false; + } + logger.info(name + ": " + gauge.getLast()); + return true; + }) + .map( + gauge -> + (Consumer) + points -> points.hasValue(gauge.getLast())) + .collect(Collectors.toList()); + + if (collect.isEmpty()) { + logger.info("unset; skipping all assertions: " + name); + return; + } + + testing.waitAndAssertMetrics( + "twitter-util-stats", + name, + metrics -> + metrics.anySatisfy( + metric -> + assertThat(metric) + .hasDoubleGaugeSatisfying( + sum -> sum.hasPointsSatisfying(collect)))); + }); + + // stop this otherwise some metrics will keep emitting + HashedWheelTimer$.MODULE$.stop(); + + TestStatsReceiver.getInstance() + .getStats() + .asMap() + .forEach( + (name, stats) -> { + List> collect = + stats.stream() + .filter( + stat -> { + if (!stat.isInitialized()) { + logger.info("uninitialized: " + name); + // skip uninitialized bc they won't have + // default values assigned in otel + return false; + } + if (!stat.getCounterpart().isEmitted()) { + logger.info("skipped: " + name); + return false; + } + logger.info(name + ": " + stat.getStat()); + return true; + }) + .map( + stat -> { + List collected = + CollectionConverters.SeqHasAsJava(stat.getStat().apply()) + .asJava() + .stream() + .mapToDouble(x -> (float) x) + .boxed() + .collect(Collectors.toList()); + System.out.println( + "stat: " + + stat.getStat() + + " -> " + + stat.getStat().apply() + + " { count=" + + collected.size() + + ", sum=" + + collected.stream().reduce(0d, Double::sum) + + ", min=" + + collected.stream().min(Comparator.naturalOrder()).orElse(0d) + + ", max=" + + collected.stream().max(Comparator.naturalOrder()).orElse(0d) + + " }"); + return (Consumer) + points -> + points + .hasMin( + collected.stream() + .min(Comparator.naturalOrder()) + .orElse(0d)) + .hasMax( + collected.stream() + .max(Comparator.naturalOrder()) + .orElse(0d)); + }) + .collect(Collectors.toList()); + + if (collect.isEmpty()) { + logger.info("unset; skipping all assertions: " + name); + return; + } + + // NOTE: choosing to be very lax with the histogram assertions as twitter-util-stats + // only supports Summary types + testing.waitAndAssertMetrics( + "twitter-util-stats", + name, + metrics -> + metrics.anySatisfy( + metric -> + assertThat(metric) + .hasHistogramSatisfying( + histo -> histo.hasPointsSatisfying(collect)))); + }); + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TestStatsReceiver.java b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TestStatsReceiver.java new file mode 100644 index 000000000000..e80f18e21b84 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/twitterutilstats/v23_11/TestStatsReceiver.java @@ -0,0 +1,251 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11; + +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.Multimap; +import com.twitter.finagle.stats.Counter; +import com.twitter.finagle.stats.Gauge; +import com.twitter.finagle.stats.InMemoryStatsReceiver; +import com.twitter.finagle.stats.Metadata; +import com.twitter.finagle.stats.MetricBuilder; +import com.twitter.finagle.stats.ReadableCounter; +import com.twitter.finagle.stats.ReadableStat; +import com.twitter.finagle.stats.Stat; +import com.twitter.finagle.stats.StatsReceiver; +import com.twitter.finagle.stats.StatsReceiverProxy; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import scala.Function0; + +public class TestStatsReceiver implements StatsReceiverProxy { + @Override + public StatsReceiver self() { + return IMPL; + } + + static Impl getInstance() { + return IMPL; + } + + private static final Impl IMPL = + new Impl() { + // used to generate finagle-computed values to establish baseline for comparisons + final InMemoryStatsReceiver self = new InMemoryStatsReceiver(); + + private final Map counters = new ConcurrentHashMap<>(); + private final Map stats = new ConcurrentHashMap<>(); + private final Map gauges = new ConcurrentHashMap<>(); + + @Override + public Multimap getCounters() { + return counters.entrySet().stream() + .collect( + ImmutableSetMultimap.toImmutableSetMultimap( + k -> Helpers.nameConversion(k.getKey()), Map.Entry::getValue)); + } + + @Override + public Multimap getStats() { + return stats.entrySet().stream() + .collect( + ImmutableSetMultimap.toImmutableSetMultimap( + k -> Helpers.nameConversion(k.getKey()), Map.Entry::getValue)); + } + + @Override + public Multimap getGauges() { + return gauges.entrySet().stream() + .collect( + ImmutableSetMultimap.toImmutableSetMultimap( + k -> Helpers.nameConversion(k.getKey()), Map.Entry::getValue)); + } + + @Override + public Object repr() { + return this; + } + + @Override + public RichCounter counter(MetricBuilder schema) { + return counters.computeIfAbsent( + schema, + key -> { + ReadableCounter counter = self.counter(schema); + AtomicBoolean isInitialized = new AtomicBoolean(false); + return new RichCounter() { + final AtomicReference counterpart = + new AtomicReference<>(); + + @Override + public void incr(long delta) { + isInitialized.set(true); + counter.incr(delta); + } + + @Override + public Metadata metadata() { + return counter.metadata(); + } + + @Override + public boolean isInitialized() { + return isInitialized.get(); + } + + @Override + public ReadableCounter getCounter() { + return counter; + } + + @Override + public OtelStatsReceiver.OtelCounter getCounterpart() { + // safe to cast + return counterpart.updateAndGet( + value -> + value == null + ? (OtelStatsReceiver.OtelCounter) + OtelStatsReceiverProxy.getInstance().counter(schema) + : value); + } + }; + }); + } + + @Override + public RichStat stat(MetricBuilder schema) { + return stats.computeIfAbsent( + schema, + key -> { + ReadableStat stat = self.stat(schema); + AtomicBoolean isInitialized = new AtomicBoolean(false); + return new RichStat() { + final AtomicReference counterpart = + new AtomicReference<>(); + + @Override + public void add(float value) { + isInitialized.set(true); + stat.add(value); + } + + @Override + public Metadata metadata() { + return stat.metadata(); + } + + @Override + public boolean isInitialized() { + return isInitialized.get(); + } + + @Override + public ReadableStat getStat() { + return stat; + } + + @Override + public OtelStatsReceiver.OtelStat getCounterpart() { + // safe to cast + return counterpart.updateAndGet( + value -> + value == null + ? (OtelStatsReceiver.OtelStat) + OtelStatsReceiverProxy.getInstance().stat(schema) + : value); + } + }; + }); + } + + @Override + public GaugeWithMemory addGauge(MetricBuilder metricBuilder, Function0 f) { + return gauges.computeIfAbsent( + metricBuilder, + key -> { + AtomicReference ref = new AtomicReference<>(); + Gauge gauge = + self.addGauge( + metricBuilder, + () -> { + float value = (float) f.apply(); + ref.getAndSet(value); + return value; + }); + return new GaugeWithMemory() { + final AtomicReference counterpart = + new AtomicReference<>(); + + @Override + public boolean isInitialized() { + return ref.get() != null; + } + + @Override + public Float getLast() { + return ref.get(); + } + + @Override + public OtelStatsReceiver.OtelGauge getCounterpart() { + // safe to cast + return counterpart.updateAndGet( + value -> + value == null + ? (OtelStatsReceiver.OtelGauge) + OtelStatsReceiverProxy.getInstance().addGauge(metricBuilder, f) + : value); + } + + @Override + public void remove() { + gauge.remove(); + } + + @Override + public Metadata metadata() { + return gauge.metadata(); + } + }; + }); + } + }; + + interface Impl extends StatsReceiver { + + Multimap getCounters(); + + Multimap getStats(); + + Multimap getGauges(); + } + + interface GaugeWithMemory extends Gauge { + boolean isInitialized(); + + Float getLast(); + + OtelStatsReceiver.OtelGauge getCounterpart(); + } + + interface RichCounter extends Counter { + boolean isInitialized(); + + ReadableCounter getCounter(); + + OtelStatsReceiver.OtelCounter getCounterpart(); + } + + interface RichStat extends Stat { + boolean isInitialized(); + + ReadableStat getStat(); + + OtelStatsReceiver.OtelStat getCounterpart(); + } +} diff --git a/instrumentation/twitter-util-stats-23.11/javaagent/src/test/resources/META-INF/services/com.twitter.finagle.stats.StatsReceiver b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/resources/META-INF/services/com.twitter.finagle.stats.StatsReceiver new file mode 100644 index 000000000000..fc0e2f4cad41 --- /dev/null +++ b/instrumentation/twitter-util-stats-23.11/javaagent/src/test/resources/META-INF/services/com.twitter.finagle.stats.StatsReceiver @@ -0,0 +1 @@ +io.opentelemetry.javaagent.instrumentation.twitterutilstats.v23_11.TestStatsReceiver diff --git a/settings.gradle.kts b/settings.gradle.kts index 06ee5fe6f725..1a4bab6356d1 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -601,6 +601,7 @@ include(":instrumentation:tomcat:tomcat-10.0:javaagent") include(":instrumentation:tomcat:tomcat-common:javaagent") include(":instrumentation:tomcat:tomcat-jdbc") include(":instrumentation:twilio-6.6:javaagent") +include(":instrumentation:twitter-util-stats-23.11:javaagent") include(":instrumentation:undertow-1.4:bootstrap") include(":instrumentation:undertow-1.4:javaagent") include(":instrumentation:vaadin-14.2:javaagent")