From 10ebd59c3edb6a8b726347093cc96b48394aa360 Mon Sep 17 00:00:00 2001 From: Ketian Xu Date: Fri, 6 Dec 2024 15:27:50 +0900 Subject: [PATCH] Add nodeName template var and update doc Signed-off-by: Ketian Xu --- .../configuring_logging_links_in_the_ui.md | 4 ++- flyteplugins/go/tasks/logs/logging_utils.go | 1 + .../go/tasks/logs/logging_utils_test.go | 33 +++++++++++++++---- .../tasks/pluginmachinery/tasklog/plugin.go | 1 + .../tasks/pluginmachinery/tasklog/template.go | 3 ++ .../pluginmachinery/tasklog/template_test.go | 8 +++-- 6 files changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/user_guide/productionizing/configuring_logging_links_in_the_ui.md b/docs/user_guide/productionizing/configuring_logging_links_in_the_ui.md index 84c1c9d0cb..818ed476a0 100644 --- a/docs/user_guide/productionizing/configuring_logging_links_in_the_ui.md +++ b/docs/user_guide/productionizing/configuring_logging_links_in_the_ui.md @@ -40,7 +40,9 @@ The parameters can be used to generate a unique URL to the logs using a template * - ``{{ .logName }}`` - A deployment specific name where to expect the logs to be * - ``{{ .hostname }}`` - - The hostname where the pod is running and logs reside + - The value used to override the hostname the pod uses internally within its own network namespace (i.e., the pod's ``.spec.hostname``) + * - ``{{ .nodeName }}`` + - The hostname of the node where the pod is running and logs reside (i.e., the pod's ``.spec.nodeName``) * - ``{{ .podRFC3339StartTime }}`` - The pod creation time (in RFC3339 format, e.g. "2021-01-01T02:07:14Z", also conforming to ISO 8601) * - ``{{ .podRFC3339FinishTime }}`` diff --git a/flyteplugins/go/tasks/logs/logging_utils.go b/flyteplugins/go/tasks/logs/logging_utils.go index c237fc15c8..be1ab6438b 100644 --- a/flyteplugins/go/tasks/logs/logging_utils.go +++ b/flyteplugins/go/tasks/logs/logging_utils.go @@ -64,6 +64,7 @@ func GetLogsForContainerInPod(ctx context.Context, logPlugin tasklog.Plugin, tas ExtraTemplateVars: extraLogTemplateVars, TaskTemplate: taskTemplate, HostName: pod.Spec.Hostname, + NodeName: pod.Spec.NodeName, }, ) diff --git a/flyteplugins/go/tasks/logs/logging_utils_test.go b/flyteplugins/go/tasks/logs/logging_utils_test.go index e8c716e341..3b80ae8b6d 100644 --- a/flyteplugins/go/tasks/logs/logging_utils_test.go +++ b/flyteplugins/go/tasks/logs/logging_utils_test.go @@ -302,7 +302,7 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "Stackdriver Logs my-Suffix", }, - }, "") + }, "", "") }) t.Run("StackDriver", func(t *testing.T) { @@ -315,11 +315,11 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "Stackdriver Logs my-Suffix", }, - }, "") + }, "", "") }) } -func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog, hostname string) { +func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog, hostname string, nodeName string) { logPlugin, err := InitializeLogPlugins(config) assert.NoError(tb, err) @@ -335,6 +335,7 @@ func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.Ta }, }, Hostname: hostname, + NodeName: nodeName, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -382,7 +383,7 @@ func TestGetLogsForContainerInPod_Templates(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "Internal my-Suffix", }, - }, "") + }, "", "") } func TestGetLogsForContainerInPodTemplates_Hostname(t *testing.T) { @@ -402,7 +403,27 @@ func TestGetLogsForContainerInPodTemplates_Hostname(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "StackDriver my-Suffix", }, - }, "my-hostname") + }, "my-hostname", "") +} + +func TestGetLogsForContainerInPodTemplates_NodeName(t *testing.T) { + assertTestSucceeded(t, &LogConfig{ + Templates: []tasklog.TemplateLogPlugin{ + { + DisplayName: "MyLog", + TemplateURIs: []string{ + "{{ .nodeName }}/{{ .namespace }}/{{ .podName }}/{{ .containerName }}/{{ .containerId }}", + }, + MessageFormat: core.TaskLog_JSON, + }, + }, + }, nil, []*core.TaskLog{ + { + Uri: "ip-1-2-3-4/my-namespace/my-pod/ContainerName/ContainerID", + MessageFormat: core.TaskLog_JSON, + Name: "MyLog my-Suffix", + }, + }, "my-hostname", "ip-1-2-3-4") } func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) { @@ -562,7 +583,7 @@ func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs, "") + assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs, "", "") }) } } diff --git a/flyteplugins/go/tasks/pluginmachinery/tasklog/plugin.go b/flyteplugins/go/tasks/pluginmachinery/tasklog/plugin.go index 143cf02e43..85a5961f0f 100644 --- a/flyteplugins/go/tasks/pluginmachinery/tasklog/plugin.go +++ b/flyteplugins/go/tasks/pluginmachinery/tasklog/plugin.go @@ -28,6 +28,7 @@ type TemplateVar struct { // log links. type Input struct { HostName string + NodeName string PodName string Namespace string ContainerName string diff --git a/flyteplugins/go/tasks/pluginmachinery/tasklog/template.go b/flyteplugins/go/tasks/pluginmachinery/tasklog/template.go index e8b7a4abed..c3f60268ed 100644 --- a/flyteplugins/go/tasks/pluginmachinery/tasklog/template.go +++ b/flyteplugins/go/tasks/pluginmachinery/tasklog/template.go @@ -27,6 +27,7 @@ type templateRegexes struct { ContainerName *regexp.Regexp ContainerID *regexp.Regexp Hostname *regexp.Regexp + NodeName *regexp.Regexp PodRFC3339StartTime *regexp.Regexp PodRFC3339FinishTime *regexp.Regexp PodUnixStartTime *regexp.Regexp @@ -52,6 +53,7 @@ func initDefaultRegexes() templateRegexes { MustCreateRegex("containerName"), MustCreateRegex("containerID"), MustCreateRegex("hostname"), + MustCreateRegex("nodeName"), MustCreateRegex("podRFC3339StartTime"), MustCreateRegex("podRFC3339FinishTime"), MustCreateRegex("podUnixStartTime"), @@ -105,6 +107,7 @@ func (input Input) templateVars() []TemplateVar { TemplateVar{defaultRegexes.ContainerName, input.ContainerName}, TemplateVar{defaultRegexes.ContainerID, containerID}, TemplateVar{defaultRegexes.Hostname, input.HostName}, + TemplateVar{defaultRegexes.NodeName, input.NodeName}, ) if input.TaskExecutionID != nil { taskExecutionIdentifier := input.TaskExecutionID.GetID() diff --git a/flyteplugins/go/tasks/pluginmachinery/tasklog/template_test.go b/flyteplugins/go/tasks/pluginmachinery/tasklog/template_test.go index 2e885732f8..da282b084a 100644 --- a/flyteplugins/go/tasks/pluginmachinery/tasklog/template_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/tasklog/template_test.go @@ -63,6 +63,7 @@ func Test_Input_templateVars(t *testing.T) { } podBase := Input{ HostName: "my-host", + NodeName: "my-node-name", PodName: "my-pod", PodUID: "my-pod-uid", Namespace: "my-namespace", @@ -103,6 +104,7 @@ func Test_Input_templateVars(t *testing.T) { {defaultRegexes.ContainerName, "my-container"}, {defaultRegexes.ContainerID, "containerID"}, {defaultRegexes.Hostname, "my-host"}, + {defaultRegexes.NodeName, "my-node-name"}, {defaultRegexes.PodRFC3339StartTime, "1970-01-01T01:02:03+01:00"}, {defaultRegexes.PodRFC3339FinishTime, "1970-01-01T04:25:45+01:00"}, {defaultRegexes.PodUnixStartTime, "123"}, @@ -139,6 +141,7 @@ func Test_Input_templateVars(t *testing.T) { {defaultRegexes.ContainerName, ""}, {defaultRegexes.ContainerID, ""}, {defaultRegexes.Hostname, ""}, + {defaultRegexes.NodeName, ""}, {defaultRegexes.NodeID, "n0-0-n0"}, {defaultRegexes.GeneratedName, "generated-name"}, {defaultRegexes.TaskRetryAttempt, "1"}, @@ -327,12 +330,13 @@ func TestTemplateLogPlugin(t *testing.T) { { "ddog", TemplateLogPlugin{ - TemplateURIs: []TemplateURI{"https://app.datadoghq.com/logs?event&from_ts={{ .podUnixStartTime }}&live=true&query=pod_name%3A{{ .podName }}&to_ts={{ .podUnixFinishTime }}"}, + TemplateURIs: []TemplateURI{"https://app.datadoghq.com/logs?event&from_ts={{ .podUnixStartTime }}&live=true&query=pod_name%3A{{ .podName }}&to_ts={{ .podUnixFinishTime }}&host={{ .nodeName }}"}, MessageFormat: core.TaskLog_JSON, }, args{ input: Input{ HostName: "my-host", + NodeName: "ip-1-2-3-4", PodName: "my-pod", Namespace: "my-namespace", ContainerName: "my-container", @@ -347,7 +351,7 @@ func TestTemplateLogPlugin(t *testing.T) { Output{ TaskLogs: []*core.TaskLog{ { - Uri: "https://app.datadoghq.com/logs?event&from_ts=123&live=true&query=pod_name%3Amy-pod&to_ts=12345", + Uri: "https://app.datadoghq.com/logs?event&from_ts=123&live=true&query=pod_name%3Amy-pod&to_ts=12345&host=ip-1-2-3-4", MessageFormat: core.TaskLog_JSON, Name: "main_logs", },