support reclaimed numa binding test

Makefile

@@ -122,7 +122,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: ## Run go test against code.
-	go test -v -json -coverprofile=coverage.txt -parallel=16 -p=16 -covermode=atomic -race -coverpkg=./... \
+	go test -v -coverprofile=coverage.txt -parallel=16 -p=16 -covermode=atomic -race -coverpkg=./... \
 		`go list ./pkg/... | grep -E -v "pkg/scheduler|pkg/controller/resource-recommend|pkg/util/resource-recommend"`
 
 .PHONY: license

cmd/katalyst-agent/app/options/global/base.go

@@ -21,7 +21,6 @@ import (
 	cliflag "k8s.io/component-base/cli/flag"
 
 	"github.com/kubewharf/katalyst-core/pkg/config/agent/global"
-	"github.com/kubewharf/katalyst-core/pkg/util/cgroup/common"
 )
 
 const (
@@ -163,6 +162,8 @@ func (o *BaseOptions) ApplyTo(c *global.BaseConfiguration) error {
 	c.ReclaimRelativeRootCgroupPath = o.ReclaimRelativeRootCgroupPath
 	c.GeneralRelativeCgroupPaths = o.GeneralRelativeCgroupPaths
 	c.OptionalRelativeCgroupPaths = o.OptionalRelativeCgroupPaths
+	c.CgroupType = o.CgroupType
+	c.AdditionalK8sCgroupPaths = o.AdditionalCgroupPaths
 
 	c.NetMultipleNS = o.MachineNetMultipleNS
 	c.NetNSDirAbsPath = o.MachineNetNSDirAbsPath
@@ -179,7 +180,5 @@ func (o *BaseOptions) ApplyTo(c *global.BaseConfiguration) error {
 	c.APIAuthTokenFile = o.APIAuthTokenFile
 
 	c.RuntimeEndpoint = o.RuntimeEndpoint
-
-	common.InitKubernetesCGroupPath(common.CgroupType(o.CgroupType), o.AdditionalCgroupPaths)
 	return nil
 }

cmd/katalyst-agent/app/options/qrm/cpu_plugin.go

@@ -34,13 +34,15 @@ type CPUOptions struct {
 }
 
 type CPUDynamicPolicyOptions struct {
-	EnableCPUAdvisor              bool
-	EnableCPUPressureEviction     bool
-	LoadPressureEvictionSkipPools []string
-	EnableSyncingCPUIdle          bool
-	EnableCPUIdle                 bool
-	CPUNUMAHintPreferPolicy       string
-	CPUNUMAHintPreferLowThreshold float64
+	EnableCPUAdvisor                       bool
+	EnableCPUPressureEviction              bool
+	LoadPressureEvictionSkipPools          []string
+	EnableSyncingCPUIdle                   bool
+	EnableCPUIdle                          bool
+	CPUNUMAHintPreferPolicy                string
+	CPUNUMAHintPreferLowThreshold          float64
+	CPUNUMAReclaimedHintPreferPolicy       string
+	CPUNUMAReclaimedHintPreferLowThreshold float64
 }
 
 type CPUNativePolicyOptions struct {
@@ -54,11 +56,12 @@ func NewCPUOptions() *CPUOptions {
 		ReservedCPUCores:       0,
 		SkipCPUStateCorruption: false,
 		CPUDynamicPolicyOptions: CPUDynamicPolicyOptions{
-			EnableCPUAdvisor:          false,
-			EnableCPUPressureEviction: false,
-			EnableSyncingCPUIdle:      false,
-			EnableCPUIdle:             false,
-			CPUNUMAHintPreferPolicy:   cpuconsts.CPUNUMAHintPreferPolicySpreading,
+			EnableCPUAdvisor:                 false,
+			EnableCPUPressureEviction:        false,
+			EnableSyncingCPUIdle:             false,
+			EnableCPUIdle:                    false,
+			CPUNUMAHintPreferPolicy:          cpuconsts.CPUNUMAHintPreferPolicySpreading,
+			CPUNUMAReclaimedHintPreferPolicy: cpuconsts.CPUNUMAHintPreferPolicyNone,
 			LoadPressureEvictionSkipPools: []string{
 				commonstate.PoolNameReclaim,
 				commonstate.PoolNameDedicated,
@@ -97,6 +100,10 @@ func (o *CPUOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 		"it decides hint preference calculation strategy")
 	fs.Float64Var(&o.CPUNUMAHintPreferLowThreshold, "cpu-numa-hint-prefer-low-threshold", o.CPUNUMAHintPreferLowThreshold,
 		"it indicates threshold to apply CPUNUMAHintPreferPolicy dynamically, and it's working when CPUNUMAHintPreferPolicy is set to dynamic_packing")
+	fs.StringVar(&o.CPUNUMAReclaimedHintPreferPolicy, "cpu-numa-reclaimed-hint-prefer-policy", o.CPUNUMAReclaimedHintPreferPolicy,
+		"it decides hint preference calculation strategy for reclaimed cpu")
+	fs.Float64Var(&o.CPUNUMAReclaimedHintPreferLowThreshold, "cpu-numa-reclaimed-hint-prefer-low-threshold", o.CPUNUMAReclaimedHintPreferLowThreshold,
+		"it indicates threshold to apply CPUNUMAReclaimedHintPreferPolicy dynamically, and it's working when CPUNUMAReclaimedHintPreferPolicy is set to dynamic_packing")
 	fs.StringVar(&o.CPUAllocationOption, "cpu-allocation-option",
 		o.CPUAllocationOption, "The allocation option of cpu (packed/distributed). The default value is packed."+
 			"in cases where more than one NUMA node is required to satisfy the allocation.")
@@ -118,5 +125,7 @@ func (o *CPUOptions) ApplyTo(conf *qrmconfig.CPUQRMPluginConfig) error {
 	conf.CPUAllocationOption = o.CPUAllocationOption
 	conf.CPUNUMAHintPreferPolicy = o.CPUNUMAHintPreferPolicy
 	conf.CPUNUMAHintPreferLowThreshold = o.CPUNUMAHintPreferLowThreshold
+	conf.CPUNUMAReclaimedHintPreferPolicy = o.CPUNUMAHintPreferPolicy
+	conf.CPUNUMAReclaimedHintPreferLowThreshold = o.CPUNUMAReclaimedHintPreferLowThreshold
 	return nil
 }

cmd/katalyst-agent/app/options/qrm/memory_plugin.go

@@ -22,6 +22,7 @@ import (
 	cliflag "k8s.io/component-base/cli/flag"
 
 	qrmconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/qrm"
+	"github.com/kubewharf/katalyst-core/pkg/consts"
 )
 
 type MemoryOptions struct {
@@ -34,6 +35,8 @@ type MemoryOptions struct {
 	EnableOOMPriority                             bool
 	OOMPriorityPinnedMapAbsPath                   string
 	EnableNonBindingShareCoresMemoryResourceCheck bool
+	EnableNUMAAllocationReactor                   bool
+	NUMABindResultResourceAllocationAnnotationKey string
 
 	SockMemOptions
 	LogCacheOptions
@@ -81,6 +84,7 @@ func NewMemoryOptions() *MemoryOptions {
 		EnableMemoryAdvisor:        false,
 		EnableOOMPriority:          false,
 		EnableNonBindingShareCoresMemoryResourceCheck: true,
+		NUMABindResultResourceAllocationAnnotationKey: consts.QRMResourceAnnotationKeyNUMABindResult,
 		SockMemOptions: SockMemOptions{
 			EnableSettingSockMem: false,
 			SetGlobalTCPMemRatio: 20,  // default: 20% * {host total memory}
@@ -121,6 +125,10 @@ func (o *MemoryOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 		o.EnableOOMPriority, "if set true, we will enable oom priority enhancement")
 	fs.BoolVar(&o.EnableNonBindingShareCoresMemoryResourceCheck, "enable-non-binding-share-cores-memory-resource-check",
 		o.EnableNonBindingShareCoresMemoryResourceCheck, "enable the topology check for non-binding shares cores pods")
+	fs.BoolVar(&o.EnableNUMAAllocationReactor, "enable-numa-allocation-reactor",
+		o.EnableNUMAAllocationReactor, "enable numa allocation reactor for numa binding pods to patch pod numa binding result annotation")
+	fs.StringVar(&o.NUMABindResultResourceAllocationAnnotationKey, "numa-bind-result-resource-allocation-annotation-key",
+		o.NUMABindResultResourceAllocationAnnotationKey, "the key of numa bind result resource allocation annotation")
 	fs.StringVar(&o.OOMPriorityPinnedMapAbsPath, "oom-priority-pinned-bpf-map-path",
 		o.OOMPriorityPinnedMapAbsPath, "the absolute path of oom priority pinned bpf map")
 	fs.BoolVar(&o.EnableSettingSockMem, "enable-setting-sockmem",
@@ -158,6 +166,8 @@ func (o *MemoryOptions) ApplyTo(conf *qrmconfig.MemoryQRMPluginConfig) error {
 	conf.ExtraControlKnobConfigFile = o.ExtraControlKnobConfigFile
 	conf.EnableOOMPriority = o.EnableOOMPriority
 	conf.EnableNonBindingShareCoresMemoryResourceCheck = o.EnableNonBindingShareCoresMemoryResourceCheck
+	conf.EnableNUMAAllocationReactor = o.EnableNUMAAllocationReactor
+	conf.NUMABindResultResourceAllocationAnnotationKey = o.NUMABindResultResourceAllocationAnnotationKey
 	conf.OOMPriorityPinnedMapAbsPath = o.OOMPriorityPinnedMapAbsPath
 	conf.EnableSettingSockMem = o.EnableSettingSockMem
 	conf.SetGlobalTCPMemRatio = o.SetGlobalTCPMemRatio

cmd/katalyst-agent/app/options/qrm/qrm_base.go

@@ -30,6 +30,7 @@ type GenericQRMPluginOptions struct {
 	UseKubeletReservedConfig bool
 	PodAnnotationKeptKeys    []string
 	PodLabelKeptKeys         []string
+	EnableReclaimNUMABinding bool
 }
 
 func NewGenericQRMPluginOptions() *GenericQRMPluginOptions {
@@ -57,6 +58,8 @@ func (o *GenericQRMPluginOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 		o.PodAnnotationKeptKeys, "pod annotation keys will be kept in qrm state")
 	fs.StringSliceVar(&o.PodLabelKeptKeys, "pod-label-kept-keys",
 		o.PodLabelKeptKeys, "pod label keys will be kept in qrm state")
+	fs.BoolVar(&o.EnableReclaimNUMABinding, "enable-reclaim-numa-binding",
+		o.EnableReclaimNUMABinding, "if set true, reclaim pod will be allocated on a specific NUMA node best-effort, otherwise, reclaim pod will be allocated on multi NUMA nodes")
 }
 
 func (o *GenericQRMPluginOptions) ApplyTo(conf *qrmconfig.GenericQRMPluginConfiguration) error {
@@ -67,6 +70,7 @@ func (o *GenericQRMPluginOptions) ApplyTo(conf *qrmconfig.GenericQRMPluginConfig
 	conf.UseKubeletReservedConfig = o.UseKubeletReservedConfig
 	conf.PodAnnotationKeptKeys = append(conf.PodAnnotationKeptKeys, o.PodAnnotationKeptKeys...)
 	conf.PodLabelKeptKeys = append(conf.PodLabelKeptKeys, o.PodLabelKeptKeys...)
+	conf.EnableReclaimNUMABinding = o.EnableReclaimNUMABinding
 	return nil
 }
 

go.mod

@@ -166,6 +166,7 @@ require (
 )
 
 replace (
+	github.com/kubewharf/katalyst-api => github.com/luomingmeng/katalyst-api v0.0.0-20241008091724-de1c08556aab
 	k8s.io/api => k8s.io/api v0.24.6
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.24.6
 	k8s.io/apimachinery => k8s.io/apimachinery v0.24.6

go.sum

@@ -570,8 +570,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kubewharf/katalyst-api v0.5.1-0.20240929080014-ae613a8935d8 h1:rIwZVD4iia7pTiB6h1xR8muc4jx4GgHWzhVCPKlEhXw=
-github.com/kubewharf/katalyst-api v0.5.1-0.20240929080014-ae613a8935d8/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
 github.com/kubewharf/kubelet v1.24.6-kubewharf.9 h1:jOTYZt7h/J7I8xQMKMUcJjKf5UFBv37jHWvNp5VRFGc=
 github.com/kubewharf/kubelet v1.24.6-kubewharf.9/go.mod h1:MxbSZUx3wXztFneeelwWWlX7NAAStJ6expqq7gY2J3c=
 github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8=
@@ -583,6 +581,8 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA=
+github.com/luomingmeng/katalyst-api v0.0.0-20241008091724-de1c08556aab h1:2m1zZnUXbwpWpHA9fGnnPonEc5rJRyT0O41NUpXztk4=
+github.com/luomingmeng/katalyst-api v0.0.0-20241008091724-de1c08556aab/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=

pkg/agent/qrm-plugins/advisorsvc/advisor_svc.pb.go

@@ -2033,7 +2033,7 @@ func (m *ContainerMetadata) Unmarshal(dAtA []byte) error {
 					if err != nil {
 						return err
 					}
-					if skippy < 0 {
+					if (skippy < 0) || (iNdEx+skippy) < 0 {
 						return ErrInvalidLengthAdvisorSvc
 					}
 					if (iNdEx + skippy) > postIndex {
@@ -2160,7 +2160,7 @@ func (m *ContainerMetadata) Unmarshal(dAtA []byte) error {
 					if err != nil {
 						return err
 					}
-					if skippy < 0 {
+					if (skippy < 0) || (iNdEx+skippy) < 0 {
 						return ErrInvalidLengthAdvisorSvc
 					}
 					if (iNdEx + skippy) > postIndex {
@@ -2228,10 +2228,7 @@ func (m *ContainerMetadata) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {
@@ -2281,10 +2278,7 @@ func (m *AddContainerResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {
@@ -2334,10 +2328,7 @@ func (m *Empty) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {
@@ -2419,10 +2410,7 @@ func (m *RemovePodRequest) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {
@@ -2472,10 +2460,7 @@ func (m *RemovePodResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {
@@ -2637,7 +2622,7 @@ func (m *ListAndWatchResponse) Unmarshal(dAtA []byte) error {
 					if err != nil {
 						return err
 					}
-					if skippy < 0 {
+					if (skippy < 0) || (iNdEx+skippy) < 0 {
 						return ErrInvalidLengthAdvisorSvc
 					}
 					if (iNdEx + skippy) > postIndex {
@@ -2688,10 +2673,7 @@ func (m *ListAndWatchResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {
@@ -2853,7 +2835,7 @@ func (m *CalculationEntries) Unmarshal(dAtA []byte) error {
 					if err != nil {
 						return err
 					}
-					if skippy < 0 {
+					if (skippy < 0) || (iNdEx+skippy) < 0 {
 						return ErrInvalidLengthAdvisorSvc
 					}
 					if (iNdEx + skippy) > postIndex {
@@ -2870,10 +2852,7 @@ func (m *CalculationEntries) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {
@@ -2991,10 +2970,7 @@ func (m *CalculationInfo) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {
@@ -3154,7 +3130,7 @@ func (m *CalculationResult) Unmarshal(dAtA []byte) error {
 					if err != nil {
 						return err
 					}
-					if skippy < 0 {
+					if (skippy < 0) || (iNdEx+skippy) < 0 {
 						return ErrInvalidLengthAdvisorSvc
 					}
 					if (iNdEx + skippy) > postIndex {
@@ -3171,10 +3147,7 @@ func (m *CalculationResult) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {
@@ -3258,10 +3231,7 @@ func (m *ListContainersResponse) Unmarshal(dAtA []byte) error {
 			if err != nil {
 				return err
 			}
-			if skippy < 0 {
-				return ErrInvalidLengthAdvisorSvc
-			}
-			if (iNdEx + skippy) < 0 {
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
 				return ErrInvalidLengthAdvisorSvc
 			}
 			if (iNdEx + skippy) > l {