feat(kuma-dp): rework on the virtual probes to support probing tcp and grpc ports

api/mesh/v1alpha1/dataplane.proto

@@ -310,33 +310,5 @@ message Dataplane {
   // defined at a Mesh level.
   MetricsBackend metrics = 2;
 
-  message Probes {
-    // Port on which the probe endpoints will be exposed. This cannot overlap
-    // with any other ports.
-    uint32 port = 1;
-
-    message Endpoint {
-      // Inbound port is a port of the application from which we expose the
-      // endpoint.
-      uint32 inbound_port = 1;
-      // Inbound path is a path of the application from which we expose the
-      // endpoint. It is recommended to be as specific as possible.
-      string inbound_path = 2;
-      // Path is a path on which we expose inbound path on the probes port.
-      string path = 3;
-    }
-
-    // List of endpoints to expose without mTLS.
-    repeated Endpoint endpoints = 2;
-  }
-
-  // Probes describe a list of endpoints that will be exposed without mTLS.
-  // This is useful to expose the health endpoints of the application so the
-  // orchestration system (e.g. Kubernetes) can still health check the
-  // application.
-  //
-  // See
-  // https://kuma.io/docs/latest/policies/service-health-probes/#virtual-probes
-  // for more information.
-  Probes probes = 3;
+  reserved 3;
 }

app/cni/pkg/cni/annotations_linux.go

@@ -17,6 +17,8 @@ const (
 	defaultIPFamilyMode        = "unspecified"
 	defaultBuiltinDNSPort      = "15053"
 	defaultNoRedirectUID       = "5678"
+	defaultVirtualProbeEnabled = "true"
+	defaultVirtualProbePorts   = "9000"
 	defaultRedirectExcludePort = defaultProxyStatusPort
 )
 
@@ -34,6 +36,8 @@ var annotationRegistry = map[string]*annotationParam{
 	"builtinDNSPort":              {"kuma.io/builtin-dns-port", defaultBuiltinDNSPort, validatePortList},
 	"excludeOutboundPortsForUIDs": {"traffic.kuma.io/exclude-outbound-ports-for-uids", "", alwaysValidFunc},
 	"noRedirectUID":               {"kuma.io/sidecar-uid", defaultNoRedirectUID, alwaysValidFunc},
+	"virtualProbesEnabled":        {"kuma.io/virtual-probes", defaultVirtualProbeEnabled, alwaysValidFunc},
+	"virtualProbesPort":           {"kuma.io/virtual-probes-port", defaultVirtualProbePorts, validateSinglePort},
 }
 
 type IntermediateConfig struct {
@@ -99,6 +103,13 @@ func validatePortList(ports string) error {
 	return nil
 }
 
+func validateSinglePort(portString string) error {
+	if _, err := parsePort(portString); err != nil {
+		return err
+	}
+	return nil
+}
+
 func validateIpFamilyMode(val string) error {
 	if val == "" {
 		return errors.New("value is empty")
@@ -132,6 +143,8 @@ func getAnnotationOrDefault(name string, annotations map[string]string) (string,
 // NewIntermediateConfig returns a new IntermediateConfig Object constructed from a list of ports and annotations
 func NewIntermediateConfig(annotations map[string]string) (*IntermediateConfig, error) {
 	intermediateConfig := &IntermediateConfig{}
+	valTrue := "true"
+	valDefaultVirtualPort := defaultVirtualProbePorts
 
 	allFields := map[string]*string{
 		"outboundPort":                &intermediateConfig.targetPort,
@@ -145,6 +158,8 @@ func NewIntermediateConfig(annotations map[string]string) (*IntermediateConfig,
 		"builtinDNSPort":              &intermediateConfig.builtinDNSPort,
 		"excludeOutboundPortsForUIDs": &intermediateConfig.excludeOutboundPortsForUIDs,
 		"noRedirectUID":               &intermediateConfig.noRedirectUID,
+		"virtualProbesEnabled":        &valTrue,
+		"virtualProbesPort":           &valDefaultVirtualPort,
 	}
 
 	for fieldName, fieldPointer := range allFields {
@@ -155,6 +170,7 @@ func NewIntermediateConfig(annotations map[string]string) (*IntermediateConfig,
 
 	// defaults to the ipv4 port if ipv6 port is not set
 	assignIPv6InboundRedirectPort(allFields)
+	excludeVirtualProbePort(allFields)
 	return intermediateConfig, nil
 }
 
@@ -186,3 +202,24 @@ func assignIPv6InboundRedirectPort(allFields map[string]*string) {
 		*v6PortFieldPointer = *allFields["inboundPort"]
 	}
 }
+
+func excludeVirtualProbePort(allFields map[string]*string) {
+	inboundPortsToExclude := allFields["excludeInboundPorts"]
+	enabledPtr := allFields["virtualProbesEnabled"]
+	enabled, err := GetEnabled(*enabledPtr)
+	if err != nil {
+		enabled = true
+	}
+
+	if !enabled {
+		return
+	}
+
+	virtualProbesPort := *allFields["virtualProbesPort"]
+	existingExcludes := *inboundPortsToExclude
+	if existingExcludes == "" {
+		*inboundPortsToExclude = virtualProbesPort
+	} else {
+		*inboundPortsToExclude = fmt.Sprintf("%s,%s", existingExcludes, virtualProbesPort)
+	}
+}

app/cni/pkg/cni/annotations_linux_test.go

@@ -48,4 +48,15 @@ var _ = Describe("NewIntermediateConfig", func() {
 		Expect(cfg.ipFamilyMode).To(Equal("ipv4"))
 		Expect(cfg.inboundPortV6).To(Equal("0"))
 	})
+
+	It("should exclude virtual probe ports", func() {
+		a := map[string]string{
+			"kuma.io/virtual-probes":                "true",
+			"kuma.io/virtual-probes-port":           "19988",
+			"traffic.kuma.io/exclude-inbound-ports": "3355",
+		}
+		cfg, err := NewIntermediateConfig(a)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(cfg.excludeInboundPorts).To(Equal("3355,19988"))
+	})
 })

app/cni/pkg/cni/injector_linux.go

@@ -160,11 +160,11 @@ func mapToConfig(intermediateConfig *IntermediateConfig, logWriter *bufio.Writer
 
 func GetEnabled(value string) (bool, error) {
 	switch strings.ToLower(value) {
-	case "enabled", "true":
+	case "enabled", "true", "yes":
 		return true, nil
-	case "disabled", "false":
+	case "disabled", "false", "no":
 		return false, nil
 	default:
-		return false, errors.Errorf(`wrong value "%s", available values are: "enabled", "disabled", "true", "false"`, value)
+		return false, errors.Errorf(`wrong value "%s", available values are: "enabled", "disabled", "true", "false", "yes", "no"`, value)
 	}
 }

app/cni/pkg/cni/main_linux.go

@@ -124,7 +124,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 	ctx := context.Background()
 	conf, err := parseConfig(args.StdinData)
 	if err != nil {
-		return errors.Wrap(err, "error parsing kuma-cni cmdAdd config")
+		return errorLogged(log, err, "error parsing kuma-cni cmdAdd config")
 	}
 
 	mainProcessStderr, err := hijackMainProcessStderr(conf.LogLevel)
@@ -139,7 +139,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 	// Determine if running under k8s by checking the CNI args
 	k8sArgs := K8sArgs{}
 	if err := types.LoadArgs(args.Args, &k8sArgs); err != nil {
-		return errors.Wrap(err, "error loading kuma-cni cmdAdd args")
+		return errorLogged(log, err, "error loading kuma-cni cmdAdd args")
 	}
 	logger := log.WithValues(
 		"pod", string(k8sArgs.K8S_POD_NAME),
@@ -162,7 +162,7 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 	containerCount, initContainersMap, annotations, err := getPodInfoWithRetries(ctx, conf, k8sArgs)
 	if err != nil {
-		return errors.Wrap(err, "pod excluded - error getting pod info")
+		return errorLogged(logger, err, "pod excluded - error getting pod info")
 	}
 
 	if isInitContainerPresent(initContainersMap) {
@@ -184,10 +184,10 @@ func cmdAdd(args *skel.CmdArgs) error {
 	}
 
 	if intermediateConfig, configErr := NewIntermediateConfig(annotations); configErr != nil {
-		return errors.Wrap(configErr, "pod excluded - pod intermediateConfig failed due to bad params")
+		return errorLogged(logger, configErr, "pod excluded - pod intermediateConfig failed due to bad params")
 	} else {
 		if err := Inject(args.Netns, logger, intermediateConfig); err != nil {
-			return errors.Wrap(err, "pod excluded - could not inject rules into namespace")
+			return errorLogged(logger, err, "pod excluded - could not inject rules into namespace")
 		}
 	}
 	logger.Info("successfully injected iptables rules")
@@ -208,6 +208,11 @@ func prepareResult(conf *PluginConf, logger logr.Logger) error {
 	return types.PrintResult(result, conf.CNIVersion)
 }
 
+func errorLogged(logger logr.Logger, err error, message string) error {
+	logger.Info(fmt.Sprintf("[WARNING] %s", message), "err", err)
+	return errors.Wrap(err, message)
+}
+
 func excludeByMissingSidecarInjectedAnnotation(annotations map[string]string) bool {
 	excludePod := false
 	val, ok := annotations[metadata.KumaSidecarInjectedAnnotation]

app/kuma-dp/cmd/run.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/envoy"
 	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/meshmetrics"
 	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/metrics"
+	"github.com/kumahq/kuma/app/kuma-dp/pkg/dataplane/probes"
 	kuma_cmd "github.com/kumahq/kuma/pkg/cmd"
 	"github.com/kumahq/kuma/pkg/config"
 	kumadp "github.com/kumahq/kuma/pkg/config/app/kuma-dp"
@@ -240,6 +241,13 @@ func newRunCmd(opts kuma_cmd.RunCmdOpts, rootCtx *RootContext) *cobra.Command {
 				return err
 			}
 
+			if opts.Config.VirtualProbesServer.Enabled {
+				prober := probes.NewProber(kumaSidecarConfiguration.Networking.Address, opts.Config.VirtualProbesServer.Port)
+				if err := rootCtx.ComponentManager.Add(prober); err != nil {
+					return err
+				}
+			}
+
 			stopComponents := make(chan struct{})
 			go func() {
 				var draining bool