fix(qrm): add EnableReclaimNUMABinding config to support make all rec…

…laim pods to non-RNB by default
kubewharf · Nov 25, 2024 · 0d1210f · 0d1210f
1 parent 5c1dceb
commit 0d1210f
Show file tree

Hide file tree

Showing 8 changed files with 25 additions and 9 deletions.
diff --git a/cmd/katalyst-agent/app/options/qrm/qrm_base.go b/cmd/katalyst-agent/app/options/qrm/qrm_base.go
@@ -30,6 +30,7 @@ type GenericQRMPluginOptions struct {
 	UseKubeletReservedConfig bool
 	PodAnnotationKeptKeys    []string
 	PodLabelKeptKeys         []string
+	EnableReclaimNUMABinding bool
 }
 
 func NewGenericQRMPluginOptions() *GenericQRMPluginOptions {
@@ -57,6 +58,8 @@ func (o *GenericQRMPluginOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 		o.PodAnnotationKeptKeys, "pod annotation keys will be kept in qrm state")
 	fs.StringSliceVar(&o.PodLabelKeptKeys, "pod-label-kept-keys",
 		o.PodLabelKeptKeys, "pod label keys will be kept in qrm state")
+	fs.BoolVar(&o.EnableReclaimNUMABinding, "enable-reclaim-numa-binding",
+		o.EnableReclaimNUMABinding, "if set true, reclaim pod will be allocated on a specific NUMA node best-effort, otherwise, reclaim pod will be allocated on multi NUMA nodes")
 }
 
 func (o *GenericQRMPluginOptions) ApplyTo(conf *qrmconfig.GenericQRMPluginConfiguration) error {
@@ -67,6 +70,7 @@ func (o *GenericQRMPluginOptions) ApplyTo(conf *qrmconfig.GenericQRMPluginConfig
 	conf.UseKubeletReservedConfig = o.UseKubeletReservedConfig
 	conf.PodAnnotationKeptKeys = append(conf.PodAnnotationKeptKeys, o.PodAnnotationKeptKeys...)
 	conf.PodLabelKeptKeys = append(conf.PodLabelKeptKeys, o.PodLabelKeptKeys...)
+	conf.EnableReclaimNUMABinding = o.EnableReclaimNUMABinding
 	return nil
 }
 

diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go
@@ -123,6 +123,7 @@ type DynamicPolicy struct {
 	cpuAdvisorSocketAbsPath                   string
 	cpuPluginSocketAbsPath                    string
 	extraStateFileAbsPath                     string
+	enableReclaimNUMABinding                  bool
 	enableCPUIdle                             bool
 	enableSyncingCPUIdle                      bool
 	reclaimRelativeRootCgroupPath             string
@@ -200,6 +201,7 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration
 		dynamicConfig:                 conf.DynamicAgentConfiguration,
 		cpuAdvisorSocketAbsPath:       conf.CPUAdvisorSocketAbsPath,
 		cpuPluginSocketAbsPath:        conf.CPUPluginSocketAbsPath,
+		enableReclaimNUMABinding:      conf.EnableReclaimNUMABinding,
 		enableCPUAdvisor:              conf.CPUQRMPluginConfig.EnableCPUAdvisor,
 		cpuNUMAHintPreferPolicy:       conf.CPUQRMPluginConfig.CPUNUMAHintPreferPolicy,
 		cpuNUMAHintPreferLowThreshold: conf.CPUQRMPluginConfig.CPUNUMAHintPreferLowThreshold,

diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_hint_handlers.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_hint_handlers.go
@@ -97,7 +97,8 @@ func (p *DynamicPolicy) reclaimedCoresHintHandler(ctx context.Context,
 		return nil, fmt.Errorf("not support inplace update resize for reclaimed cores")
 	}
 
-	if qosutil.AnnotationsIndicateNUMABinding(req.Annotations) {
+	if qosutil.AnnotationsIndicateNUMABinding(req.Annotations) &&
+		p.enableReclaimNUMABinding {
 		return p.reclaimedCoresWithNUMABindingHintHandler(ctx, req)
 	}
 

diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_test.go
@@ -127,6 +127,7 @@ func getTestDynamicPolicyWithoutInitialization(topology *machine.CPUTopology, st
 		advisorValidator:          validator.NewCPUAdvisorValidator(stateImpl, machineInfo),
 		reservedReclaimedCPUsSize: general.Max(reservedReclaimedCPUsSize, topology.NumNUMANodes),
 		reservedCPUs:              reservedCPUs,
+		enableReclaimNUMABinding:  true,
 		emitter:                   metrics.DummyMetrics{},
 		podDebugAnnoKeys:          []string{podDebugAnnoKey},
 	}

diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go
@@ -164,6 +164,7 @@ type DynamicPolicy struct {
 	enableEvictingLogCache  bool
 	logCacheEvictionManager logcache.Manager
 
+	enableReclaimNUMABinding                      bool
 	enableNonBindingShareCoresMemoryResourceCheck bool
 
 	enableNUMAAllocationReactor                   bool
@@ -234,6 +235,7 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration
 		enableOOMPriority:          conf.EnableOOMPriority,
 		oomPriorityMapPinnedPath:   conf.OOMPriorityPinnedMapAbsPath,
 		enableEvictingLogCache:     conf.EnableEvictingLogCache,
+		enableReclaimNUMABinding:   conf.EnableReclaimNUMABinding,
 		enableNonBindingShareCoresMemoryResourceCheck: conf.EnableNonBindingShareCoresMemoryResourceCheck,
 		enableNUMAAllocationReactor:                   conf.EnableNUMAAllocationReactor,
 		numaBindResultResourceAllocationAnnotationKey: conf.NUMABindResultResourceAllocationAnnotationKey,

diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_hint_handlers.go
@@ -99,7 +99,8 @@ func (p *DynamicPolicy) reclaimedCoresHintHandler(ctx context.Context,
 		return nil, fmt.Errorf("not support inplace update resize for reclaimed cores")
 	}
 
-	if qosutil.AnnotationsIndicateNUMABinding(req.Annotations) {
+	if qosutil.AnnotationsIndicateNUMABinding(req.Annotations) &&
+		p.enableReclaimNUMABinding {
 		return p.reclaimedCoresWithNUMABindingHintHandler(ctx, req)
 	}
 

diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_test.go
@@ -130,13 +130,14 @@ func getTestDynamicPolicyWithInitialization(topology *machine.CPUTopology, machi
 	}
 
 	policyImplement := &DynamicPolicy{
-		topology:         topology,
-		qosConfig:        qosConfig,
-		state:            stateImpl,
-		emitter:          metrics.DummyMetrics{},
-		migratingMemory:  make(map[string]map[string]bool),
-		stopCh:           make(chan struct{}),
-		podDebugAnnoKeys: []string{podDebugAnnoKey},
+		topology:                 topology,
+		qosConfig:                qosConfig,
+		state:                    stateImpl,
+		emitter:                  metrics.DummyMetrics{},
+		migratingMemory:          make(map[string]map[string]bool),
+		stopCh:                   make(chan struct{}),
+		podDebugAnnoKeys:         []string{podDebugAnnoKey},
+		enableReclaimNUMABinding: true,
 		enableNonBindingShareCoresMemoryResourceCheck: true,
 		numaBindResultResourceAllocationAnnotationKey: coreconsts.QRMResourceAnnotationKeyNUMABindResult,
 	}

diff --git a/pkg/config/agent/qrm/qrm_base.go b/pkg/config/agent/qrm/qrm_base.go
@@ -28,6 +28,10 @@ type GenericQRMPluginConfiguration struct {
 	PodAnnotationKeptKeys []string
 	// PodLabelKeptKeys indicates pod label keys will be kept in qrm state
 	PodLabelKeptKeys []string
+	// EnableReclaimNUMABinding indicates whether to enable NUMA Binding for reclaim pods
+	// if this flag is set to true, reclaim pod will be allocated on a specific NUMA node
+	// best-effort, otherwise, reclaim pod will be allocated on multi NUMA nodes
+	EnableReclaimNUMABinding bool
 }
 
 type QRMPluginsConfiguration struct {