Add Open Telemetry tracing on Server and Client

* Bump Go version to 1.22

* Fix producer and load-test commands to run in docker

* Upgrade ginkgo library

* Update ginkgo on Makefile

* Upgrade multiple dependencies

* Remove Open Telemetry Sarama dependency

* Bump Sarama to IBM package latest version

* Add docker volume to load-test and producer commands

* Fix makefile

* Trying to fix action not handling error code

* Force Server tests to fail for testing github action error handling

* Fixing CI

* Revert force fail server tests

* Added healthcheck to eventsgateway api on docker compose

* Testing tests on docker

* Fix docker run client tests

* Debugging CI

* Debugging CI

* Debugging CI

* Debugging CI

* Add custom test methods for CI

* Debugging CI

* Decouple CI and local test commands

* Enable Tracing on local server tests

* Using GRPC OpenTelemetry on Client and Server

* Configure OTLP tracing for server calls

* Add OTEL support and configuration for server

* Add support for OTEL tracing on Client

* Add debug mode to server container

* Added prometheus and grafana into main docker-compose. Added default prometheus conf to ser ver local.yaml

* Set default compose to run only the EG API direct dependencies

* Remove unecessary OTLP config

* Fix Tracer Sampling settings

* Add client tracing config and properly set tracer sampling

* Added Jaeger propagator on OTEL eventsgateway server

* Add OpenTracing setting on LoadTest tool

* Remove custom tracing from clinet

* Support Open Tracing and Open Telemetry on EG client for backward compatibility

* Add examples to using OpenTracing and OpenTelemetry from EG Client

* Remove unecessary traces from Server

* Add example on how to keep tracing with decoupled context

client/async.go

@@ -14,8 +14,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
-	"github.com/opentracing/opentracing-go"
 	uuid "github.com/satori/go.uuid"
 	"github.com/spf13/viper"
 	"github.com/topfreegames/eventsgateway/v4/logger"
@@ -114,12 +112,10 @@ func (a *gRPCClientAsync) configureGRPCForwarderClient(
 	a.logger.WithFields(map[string]interface{}{
 		"operation": "configureGRPCForwarderClient",
 	}).Info("connecting to grpc server")
-	tracer := opentracing.GlobalTracer()
 	dialOpts := append(
 		[]grpc.DialOption{
 			grpc.WithInsecure(),
 			grpc.WithChainUnaryInterceptor(
-				otgrpc.OpenTracingClientInterceptor(tracer),
 				a.metricsReporterInterceptor,
 			),
 		},

client/client.go

@@ -10,8 +10,12 @@ package client
 import (
 	"context"
 	"fmt"
+	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
+	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/topfreegames/eventsgateway/v4/metrics"
+	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
+	"go.opentelemetry.io/otel"
 	"strings"
 	"sync"
 	"time"
@@ -73,7 +77,21 @@ func New(
 		"topic":  c.topic,
 	})
 	var err error
-	if c.client, err = c.newGRPCClient(configPrefix, client, opts...); err != nil {
+
+	dialOpts := append(
+		[]grpc.DialOption{
+			grpc.WithChainUnaryInterceptor(
+				otelgrpc.UnaryClientInterceptor(
+					otelgrpc.WithPropagators(otel.GetTextMapPropagator()),
+					otelgrpc.WithTracerProvider(otel.GetTracerProvider()),
+				),
+				otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer()),
+			),
+		},
+		opts...,
+	)
+
+	if c.client, err = c.newGRPCClient(configPrefix, client, dialOpts...); err != nil {
 		return nil, err
 	}
 	return c, nil
@@ -127,7 +145,7 @@ func (c *Client) registerMetrics(configPrefix string) {
 		metrics.ClientRequestsDroppedCounter,
 	}
 	err := metrics.RegisterMetrics(collectors)
-        if err != nil {
+	if err != nil {
 		c.logger.WithError(err).Error("failed to register metric")
 	}
 }

client/sync.go

@@ -10,15 +10,12 @@ package client
 import (
 	"context"
 	"fmt"
-	"time"
-
-	"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc"
-	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/spf13/viper"
 	"github.com/topfreegames/eventsgateway/v4/logger"
 	"github.com/topfreegames/eventsgateway/v4/metrics"
 	pb "github.com/topfreegames/protos/eventsgateway/grpc/generated"
 	"google.golang.org/grpc"
+	"time"
 )
 
 type gRPCClientSync struct {
@@ -69,18 +66,19 @@ func (s *gRPCClientSync) configureGRPCForwarderClient(
 	s.logger.WithFields(map[string]interface{}{
 		"operation": "configureGRPCForwarderClient",
 	}).Info("connecting to grpc server")
-	tracer := opentracing.GlobalTracer()
+
 	dialOpts := append(
 		[]grpc.DialOption{
 			grpc.WithInsecure(),
 			grpc.WithChainUnaryInterceptor(
-				otgrpc.OpenTracingClientInterceptor(tracer),
 				s.metricsReporterInterceptor,
 			),
 		},
 		opts...,
 	)
+
 	conn, err := grpc.Dial(serverAddress, dialOpts...)
+
 	if err != nil {
 		return err
 	}

config/local.yaml

@@ -1,3 +1,4 @@
+# Those properties can also be loaded as ENV Vars with EVENTSGATEWAY_ prefix.
 kafka:
   logger:
     enabled: true
@@ -15,10 +16,24 @@ client:
   numRoutines: 2
   retryInterval: 2s
   grpc:
-    serverAddress: eventsgateway-api:5000
+    serverAddress: localhost:5000 #eventsgateway-api:5000
     timeout: 500ms
 loadtestclient:
-  duration: 10s
-  threads: 2
+  duration: 20s
+  threads: 20
   randSleepCeilingMs: 500
   randPropsSize: small # small, medium, large, jumbo
+  opentelemetry:
+    enabled: false
+    serviceName: "eventsgateway-client-loadtest-opentelemetry"
+    traceSamplingRatio: 1.0
+    jaegerHost: localhost
+    jaegerPort: 4317
+  opentracing:
+    enabled: true
+    serviceName: "eventsgateway-client-loadtest-opentracing"
+    jaegerHost: localhost
+    jaegerPort: 6831
+    samplerType: const
+    samplerParam: 1.0
+    logSpans: false

docker-compose-ci.yaml

@@ -30,30 +30,13 @@ services:
     networks:
       - eventsgateway
 
-  jaeger:
-    image: jaegertracing/all-in-one:latest
-    environment:
-      - COLLECTOR_OTLP_ENABLED=true
-    ports:
-      - "5775:5775/udp"
-      - "6831:6831/udp"
-      - "6832:6832/udp"
-      - "5778:5778" 
-      - "16686:16686"
-      - "4317:4317"
-    networks:
-        - eventsgateway
-
   eventsgateway-api:
     image: eventsgateway-server
     ports:
       - "5000:5000"
-      - "6060:6060"
     depends_on:
       kafka:
         condition: service_healthy
-      jaeger:
-        condition: service_started
     healthcheck:
       test: [ "CMD", "nc", "-z", "localhost", "5000" ]
       interval: 5s

docker-compose.yaml

@@ -1,5 +1,3 @@
-version: '2'
-
 services:
   zookeeper:
     image: confluentinc/cp-zookeeper:6.2.15
@@ -40,6 +38,7 @@ services:
       - ./spark-notebook/work:/home/jovyan/work
     networks:
       - eventsgateway
+
   jaeger:
     image: jaegertracing/all-in-one:latest
     environment:
@@ -56,16 +55,12 @@ services:
 
   eventsgateway-api:
     image: eventsgateway-server-dev
-#    environment:
-#      - "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=jaeger:4317"
     ports:
       - "5000:5000"
-      - "6060:6060"
+      - "9091:9091"
     depends_on:
       kafka:
         condition: service_healthy
-      jaeger:
-        condition: service_started
     healthcheck:
       test: [ "CMD", "nc", "-z", "localhost", "5000" ]
       interval: 5s
@@ -90,5 +85,42 @@ services:
     networks:
       - eventsgateway
 
+  prometheus:
+    image: prom/prometheus
+    restart: always
+    volumes:
+      - ./testing/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+    ports:
+      - "9090:9090"
+    networks:
+      - eventsgateway
+
+  grafana:
+    image: grafana/grafana
+    restart: always
+    environment:
+      GF_INSTALL_PLUGINS: 'grafana-clock-panel,grafana-simple-json-datasource'
+      GF_SECURITY_ADMIN_USER: 'admin'
+      GF_SECURITY_ADMIN_PASSWORD: 'foobar'
+      GF_USERS_ALLOW_SIGN_UP: 'false'
+    volumes:
+      - grafana_data:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    depends_on:
+      - prometheus
+    networks:
+      - eventsgateway
+
 networks:
   eventsgateway:
+
+volumes:
+  prometheus_data: {}
+  grafana_data: {}

go.mod

@@ -9,13 +9,18 @@ require (
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645
 	github.com/onsi/ginkgo/v2 v2.1.4
 	github.com/onsi/gomega v1.20.0
-	github.com/opentracing/opentracing-go v1.1.0
+	github.com/opentracing/opentracing-go v1.2.0
 	github.com/prometheus/client_golang v1.12.2
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v0.0.1
 	github.com/spf13/viper v1.3.2
 	github.com/topfreegames/protos v1.6.1
+	github.com/uber/jaeger-client-go v2.30.0+incompatible
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.34.0
+	go.opentelemetry.io/otel v1.10.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0
+	go.opentelemetry.io/otel/sdk v1.10.0
 	google.golang.org/grpc v1.48.0
 )
 
@@ -64,4 +69,18 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-require github.com/inconshreveable/mousetrap v1.0.0 // indirect
+require (
+	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect
+	go.opentelemetry.io/otel/trace v1.10.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+)