Dev/headroom rnb

go.mod

@@ -161,6 +161,7 @@ require (
 )
 
 replace (
+	github.com/kubewharf/katalyst-api => github.com/luomingmeng/katalyst-api v0.0.0-20241008091724-de1c08556aab // FIXME
 	k8s.io/api => k8s.io/api v0.24.6
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.24.6
 	k8s.io/apimachinery => k8s.io/apimachinery v0.24.6

go.sum

@@ -568,8 +568,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kubewharf/katalyst-api v0.5.1-0.20240929080014-ae613a8935d8 h1:rIwZVD4iia7pTiB6h1xR8muc4jx4GgHWzhVCPKlEhXw=
-github.com/kubewharf/katalyst-api v0.5.1-0.20240929080014-ae613a8935d8/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
 github.com/kubewharf/kubelet v1.24.6-kubewharf.9 h1:jOTYZt7h/J7I8xQMKMUcJjKf5UFBv37jHWvNp5VRFGc=
 github.com/kubewharf/kubelet v1.24.6-kubewharf.9/go.mod h1:MxbSZUx3wXztFneeelwWWlX7NAAStJ6expqq7gY2J3c=
 github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8=
@@ -581,6 +579,8 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA=
+github.com/luomingmeng/katalyst-api v0.0.0-20241008091724-de1c08556aab h1:2m1zZnUXbwpWpHA9fGnnPonEc5rJRyT0O41NUpXztk4=
+github.com/luomingmeng/katalyst-api v0.0.0-20241008091724-de1c08556aab/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=

pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpuadvisor/const.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cpuadvisor
+
+type CPUControlKnobName string
+
+const (
+	ControlKnobKeyCPUNUMAHeadroom CPUControlKnobName = "cpu_numa_headroom"
+)

pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpuadvisor/cpu.proto

@@ -34,6 +34,7 @@ option go_package = "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cp
 message ListAndWatchResponse {
     map<string,CalculationEntries> entries = 1; // keyed by pool name or podUID
     bool allow_shared_cores_overlap_reclaimed_cores= 2; // if set to true, cpuset of shared_cores may overlap with reclaimed_cores
+    repeated advisorsvc.CalculationInfo extra_entries = 3; // for non-container level adjustment (eg. /kubepods/besteffort)
 }
 
 message CalculationEntries {

pkg/agent/qrm-plugins/memory/dynamicpolicy/memoryadvisor/const.go → pkg/agent/qrm-plugins/memory/dynamicpolicy/memoryadvisor/types.go

@@ -26,4 +26,7 @@ const (
 	ControlKnobKeyBalanceNumaMemory  MemoryControlKnobName = "balance_numa_memory"
 	ControlKnobKeySwapMax            MemoryControlKnobName = "swap_max"
 	ControlKnowKeyMemoryOffloading   MemoryControlKnobName = "memory_offloading"
+	ControlKnobKeyMemoryNUMAHeadroom MemoryControlKnobName = "memory_numa_headroom"
 )
+
+type MemoryNUMAHeadroom map[int]int64

pkg/agent/sysadvisor/plugin/qosaware/qos_aware.go

@@ -18,6 +18,7 @@ package qosaware
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
@@ -69,11 +70,8 @@ func NewQoSAwarePlugin(pluginName string, conf *config.Configuration, extraConf
 		return nil, err
 	}
 
-	qrmServer, err := server.NewQRMServer(resourceAdvisor, conf, metaCache, metaServer, emitter)
-	if err != nil {
-		return nil, err
-	}
-
+	var resourceGetter reporter.HeadroomResourceGetter
+	headroomReporterExists := false
 	reporters := make([]reporter.Reporter, 0)
 	for _, reporterName := range conf.Reporters {
 		switch reporterName {
@@ -83,6 +81,8 @@ func NewQoSAwarePlugin(pluginName string, conf *config.Configuration, extraConf
 				return nil, err
 			}
 			reporters = append(reporters, headroomReporter)
+			resourceGetter = headroomReporter
+			headroomReporterExists = true
 		case types.NodeMetricReporter:
 			nodeMetricsReporter, err := reporter.NewNodeMetricsReporter(emitter, metaServer, metaCache, conf)
 			if err != nil {
@@ -92,6 +92,15 @@ func NewQoSAwarePlugin(pluginName string, conf *config.Configuration, extraConf
 		}
 	}
 
+	if !headroomReporterExists {
+		return nil, fmt.Errorf("headroom reporter must be specified")
+	}
+
+	qrmServer, err := server.NewQRMServer(resourceAdvisor, resourceGetter, conf, metaCache, metaServer, emitter)
+	if err != nil {
+		return nil, err
+	}
+
 	// add AdminQos dynamic config watcher
 	err = metaServer.ConfigurationManager.AddConfigWatcher(crd.AdminQoSConfigurationGVR)
 	if err != nil {