Skip to content

Commit

Permalink
feat(qrm): adding support to cpu.weight
Browse files Browse the repository at this point in the history
Signed-off-by: Robin Lu <[email protected]>
  • Loading branch information
lubinszARM committed Nov 13, 2024
1 parent 3269ac6 commit 596a871
Show file tree
Hide file tree
Showing 8 changed files with 375 additions and 0 deletions.
16 changes: 16 additions & 0 deletions cmd/katalyst-agent/app/options/qrm/cpu_plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type CPUOptions struct {

CPUDynamicPolicyOptions
CPUNativePolicyOptions
CPUWeightOptions
}

type CPUDynamicPolicyOptions struct {
Expand All @@ -48,6 +49,11 @@ type CPUNativePolicyOptions struct {
CPUAllocationOption string
}

type CPUWeightOptions struct {
EnableSettingCPUWeight bool
CPUWeightConfigFile string
}

func NewCPUOptions() *CPUOptions {
return &CPUOptions{
PolicyName: "dynamic",
Expand All @@ -70,6 +76,10 @@ func NewCPUOptions() *CPUOptions {
EnableFullPhysicalCPUsOnly: false,
CPUAllocationOption: "packed",
},
CPUWeightOptions: CPUWeightOptions{
EnableSettingCPUWeight: false,
CPUWeightConfigFile: "",
},
}
}

Expand Down Expand Up @@ -103,6 +113,10 @@ func (o *CPUOptions) AddFlags(fss *cliflag.NamedFlagSets) {
fs.BoolVar(&o.EnableFullPhysicalCPUsOnly, "enable-full-physical-cpus-only",
o.EnableFullPhysicalCPUsOnly, "if set true, we will enable extra allocation restrictions to "+
"avoid different containers to possibly end up on the same core.")
fs.BoolVar(&o.EnableSettingCPUWeight, "enable-setting-cpu-weight",
o.EnableSettingCPUWeight, "if set true, we will enable cpu weight related features")
fs.StringVar(&o.CPUWeightConfigFile, "qrm-cpu-weight-config-file",
o.CPUWeightConfigFile, "cpu weight config file")
}

func (o *CPUOptions) ApplyTo(conf *qrmconfig.CPUQRMPluginConfig) error {
Expand All @@ -118,5 +132,7 @@ func (o *CPUOptions) ApplyTo(conf *qrmconfig.CPUQRMPluginConfig) error {
conf.CPUAllocationOption = o.CPUAllocationOption
conf.CPUNUMAHintPreferPolicy = o.CPUNUMAHintPreferPolicy
conf.CPUNUMAHintPreferLowThreshold = o.CPUNUMAHintPreferLowThreshold
conf.EnableSettingCPUWeight = o.EnableSettingCPUWeight
conf.CPUWeightConfigFile = o.CPUWeightConfigFile
return nil
}
1 change: 1 addition & 0 deletions pkg/agent/qrm-plugins/cpu/consts/consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ const (
CheckCPUSet = CPUPluginDynamicPolicyName + "_check_cpuset"
SyncCPUIdle = CPUPluginDynamicPolicyName + "_sync_cpu_idle"
CommunicateWithAdvisor = CPUPluginDynamicPolicyName + "_communicate_with_advisor"
SyncCPUWeight = CPUPluginDynamicPolicyName + "_sync_cpu_weight"
)

const (
Expand Down
12 changes: 12 additions & 0 deletions pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/cpueviction"
"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/state"
"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/dynamicpolicy/validator"
"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/handlers/cpuweight"
cpuutil "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/util"
"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util"
"github.com/kubewharf/katalyst-core/pkg/agent/utilcomponent/periodicalhandler"
Expand Down Expand Up @@ -133,6 +134,7 @@ type DynamicPolicy struct {
transitionPeriod time.Duration
cpuNUMAHintPreferPolicy string
cpuNUMAHintPreferLowThreshold float64
enableSettingCPUWeight bool
}

func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration,
Expand Down Expand Up @@ -197,6 +199,7 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration
enableCPUAdvisor: conf.CPUQRMPluginConfig.EnableCPUAdvisor,
cpuNUMAHintPreferPolicy: conf.CPUQRMPluginConfig.CPUNUMAHintPreferPolicy,
cpuNUMAHintPreferLowThreshold: conf.CPUQRMPluginConfig.CPUNUMAHintPreferLowThreshold,
enableSettingCPUWeight: conf.EnableSettingCPUWeight,
reservedCPUs: reservedCPUs,
extraStateFileAbsPath: conf.ExtraStateFileAbsPath,
enableSyncingCPUIdle: conf.CPUQRMPluginConfig.EnableSyncingCPUIdle,
Expand Down Expand Up @@ -332,6 +335,15 @@ func (p *DynamicPolicy) Start() (err error) {
p.cpuAdvisorSocketAbsPath, p.cpuPluginSocketAbsPath)
}

if p.enableSettingCPUWeight {
general.Infof("setCPUWeight enabled")
err = periodicalhandler.RegisterPeriodicalHandlerWithHealthz(cpuconsts.SyncCPUWeight, general.HealthzCheckStateNotReady,
qrm.QRMCPUPluginPeriodicalHandlerGroupName, cpuweight.SetCPUWeight, 1800*time.Second, healthCheckTolerationTimes)
if err != nil {
general.Errorf("start %v failed,err:%v", cpuconsts.SyncCPUWeight, err)
}
}

general.Infof("start dynamic policy cpu plugin with sys-advisor")
err = p.initAdvisorClientConn()
if err != nil {
Expand Down
21 changes: 21 additions & 0 deletions pkg/agent/qrm-plugins/cpu/handlers/cpuweight/const.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
Copyright 2022 The Katalyst Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpuweight

const (
metricNameCPUWeight = "async_handler_cpu_weight"
)
99 changes: 99 additions & 0 deletions pkg/agent/qrm-plugins/cpu/handlers/cpuweight/cpuweight_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
//go:build linux
// +build linux

/*
Copyright 2022 The Katalyst Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpuweight

import (
"k8s.io/apimachinery/pkg/util/errors"

cpuconsts "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/cpu/consts"
coreconfig "github.com/kubewharf/katalyst-core/pkg/config"
dynamicconfig "github.com/kubewharf/katalyst-core/pkg/config/agent/dynamic"
"github.com/kubewharf/katalyst-core/pkg/metaserver"
"github.com/kubewharf/katalyst-core/pkg/metrics"
cgroupcm "github.com/kubewharf/katalyst-core/pkg/util/cgroup/common"
cgroupmgr "github.com/kubewharf/katalyst-core/pkg/util/cgroup/manager"
"github.com/kubewharf/katalyst-core/pkg/util/general"
)

func applyCPUWeightCgroupLevelConfig(conf *coreconfig.Configuration, emitter metrics.MetricEmitter) {
if conf.CPUWeightConfigFile == "" {
general.Errorf("CPUWeightConfigFile isn't configured")
return
}

cpuWightCgroupLevelConfigs := make(map[string]uint64)
err := general.LoadJsonConfig(conf.CPUWeightConfigFile, &cpuWightCgroupLevelConfigs)
if err != nil {
general.Errorf("load CPUWeightCgroupLevelConfig failed with error: %v", err)
return
}

for relativeCgPath, weight := range cpuWightCgroupLevelConfigs {
err := cgroupmgr.ApplyCPUWithRelativePath(relativeCgPath, &cgroupcm.CPUData{
Shares: weight,
})
if err != nil {
general.Errorf("ApplyCPUWeightWithRelativePath in relativeCgPath: %s failed with error: %v",
relativeCgPath, err)
} else {
general.Infof("ApplyCPUWeightWithRelativePath weight: %d in relativeCgPath: %s successfully",
weight, relativeCgPath)
_ = emitter.StoreInt64(metricNameCPUWeight, int64(weight), metrics.MetricTypeNameRaw,
metrics.ConvertMapToTags(map[string]string{
"cgPath": relativeCgPath,
})...)

}
}
}

func SetCPUWeight(conf *coreconfig.Configuration,
_ interface{}, _ *dynamicconfig.DynamicAgentConfiguration,
emitter metrics.MetricEmitter, metaServer *metaserver.MetaServer,
) {
general.Infof("called")

var errList []error
defer func() {
_ = general.UpdateHealthzStateByError(cpuconsts.SyncCPUWeight, errors.NewAggregate(errList))
}()

if conf == nil {
general.Errorf("nil extraConf")
return
} else if emitter == nil {
general.Errorf("nil emitter")
return
} else if metaServer == nil {
general.Errorf("nil metaServer")
return
}

// SettingCPUWeight featuregate.
if !conf.EnableSettingCPUWeight {
general.Infof("SetCPUWeight disabled")
return
}

// checking cgroup-level cpu.weight configuration.
if len(conf.CPUWeightConfigFile) > 0 {
applyCPUWeightCgroupLevelConfig(conf, emitter)
}
}
Loading

0 comments on commit 596a871

Please sign in to comment.